sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.69 2012/06/07 22:57:26 emax Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.5";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  84         /* required last entry */
  85         {0, 0, 0, 0, 0}
  86 };
  87
  88 /*********************************************************************
  89  *  Table of branding strings
  90  *********************************************************************/
  91
  92 static char    *ixgbe_strings[] = {
  93         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  94 };
  95
  96 /*********************************************************************
  97  *  Function prototypes
  98  *********************************************************************/
  99 static int      ixgbe_probe(device_t);
 100 static int      ixgbe_attach(device_t);
 101 static int      ixgbe_detach(device_t);
 102 static int      ixgbe_shutdown(device_t);
 103 static void     ixgbe_start(struct ifnet *);
 104 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 105 #if 0 /* __FreeBSD_version >= 800000 */
 106 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 107 static int      ixgbe_mq_start_locked(struct ifnet *,
 108                     struct tx_ring *, struct mbuf *);
 109 static void     ixgbe_qflush(struct ifnet *);
 110 #endif
 111 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 112 static void     ixgbe_init(void *);
 113 static void     ixgbe_init_locked(struct adapter *);
 114 static void     ixgbe_stop(void *);
 115 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 116 static int      ixgbe_media_change(struct ifnet *);
 117 static void     ixgbe_identify_hardware(struct adapter *);
 118 static int      ixgbe_allocate_pci_resources(struct adapter *);
 119 static int      ixgbe_allocate_msix(struct adapter *);
 120 static int      ixgbe_allocate_legacy(struct adapter *);
 121 static int      ixgbe_allocate_queues(struct adapter *);
 122 #if 0   /* HAVE_MSIX */
 123 static int      ixgbe_setup_msix(struct adapter *);
 124 #endif
 125 static void     ixgbe_free_pci_resources(struct adapter *);
 126 static void     ixgbe_local_timer(void *);
 127 static int      ixgbe_setup_interface(device_t, struct adapter *);
 128 static void     ixgbe_config_link(struct adapter *);
 129
 130 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 131 static int      ixgbe_setup_transmit_structures(struct adapter *);
 132 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 133 static void     ixgbe_initialize_transmit_units(struct adapter *);
 134 static void     ixgbe_free_transmit_structures(struct adapter *);
 135 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 136
 137 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 138 static int      ixgbe_setup_receive_structures(struct adapter *);
 139 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 140 static void     ixgbe_initialize_receive_units(struct adapter *);
 141 static void     ixgbe_free_receive_structures(struct adapter *);
 142 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 143 #if 0   /* NET_LRO */
 144 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 145 #endif
 146
 147 static void     ixgbe_enable_intr(struct adapter *);
 148 static void     ixgbe_disable_intr(struct adapter *);
 149 static void     ixgbe_update_stats_counters(struct adapter *);
 150 static bool     ixgbe_txeof(struct tx_ring *);
 151 static bool     ixgbe_rxeof(struct ix_queue *, int);
 152 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 153 static void     ixgbe_set_promisc(struct adapter *);
 154 static void     ixgbe_set_multi(struct adapter *);
 155 static void     ixgbe_update_link_status(struct adapter *);
 156 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 157 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 158 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 159 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 160 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 162                     struct ixgbe_dma_alloc *, int);
 163 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 164 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 165                     const char *, int *, int);
 166 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 167 #if 0   /* NET_TSO */
 168 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 169 #endif
 170 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 171 static void     ixgbe_configure_ivars(struct adapter *);
 172 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 173
 174 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 175 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 176 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 177
 178 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 179
 180 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 181 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 182                     struct mbuf *, u32);
 183
 184 /* Support for pluggable optic modules */
 185 static bool     ixgbe_sfp_probe(struct adapter *);
 186 static void     ixgbe_setup_optics(struct adapter *);
 187
 188 /* Legacy (single vector interrupt handler */
 189 static void     ixgbe_legacy_irq(void *);
 190
 191 /* The MSI/X Interrupt handlers */
 192 static void     ixgbe_msix_que(void *);
 193 static void     ixgbe_msix_link(void *);
 194
 195 /* Deferred interrupt tasklets */
 196 static void     ixgbe_handle_que(void *, int);
 197 static void     ixgbe_handle_link(void *, int);
 198 static void     ixgbe_handle_msf(void *, int);
 199 static void     ixgbe_handle_mod(void *, int);
 200
 201 #ifdef IXGBE_FDIR
 202 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 203 static void     ixgbe_reinit_fdir(void *, int);
 204 #endif
 205
 206 /*********************************************************************
 207  *  FreeBSD Device Interface Entry Points
 208  *********************************************************************/
 209
 210 static device_method_t ixgbe_methods[] = {
 211         /* Device interface */
 212         DEVMETHOD(device_probe, ixgbe_probe),
 213         DEVMETHOD(device_attach, ixgbe_attach),
 214         DEVMETHOD(device_detach, ixgbe_detach),
 215         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 216         {0, 0}
 217 };
 218
 219 static driver_t ixgbe_driver = {
 220         "ix", ixgbe_methods, sizeof(struct adapter),
 221 };
 222
 223 devclass_t ixgbe_devclass;
 224 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 225
 226 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 227 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 228
 229 /*
 230 ** TUNEABLE PARAMETERS:
 231 */
 232
 233 /*
 234 ** AIM: Adaptive Interrupt Moderation
 235 ** which means that the interrupt rate
 236 ** is varied over time based on the
 237 ** traffic for that interrupt vector
 238 */
 239 static int ixgbe_enable_aim = TRUE;
 240 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 241
 242 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 243 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 244
 245 /* How many packets rxeof tries to clean at a time */
 246 static int ixgbe_rx_process_limit = 128;
 247 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 248
 249 /* Flow control setting, default to full */
 250 static int ixgbe_flow_control = ixgbe_fc_full;
 251 TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control);
 252
 253 /*
 254 ** Smart speed setting, default to on
 255 ** this only works as a compile option
 256 ** right now as its during attach, set
 257 ** this to 'ixgbe_smart_speed_off' to
 258 ** disable.
 259 */
 260 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 261
 262 /*
 263  * MSIX should be the default for best performance,
 264  * but this allows it to be forced off for testing.
 265  */
 266 static int ixgbe_enable_msix = 1;
 267 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 268
 269 /*
 270  * Header split: this causes the hardware to DMA
 271  * the header into a separate mbuf from the payload,
 272  * it can be a performance win in some workloads, but
 273  * in others it actually hurts, its off by default.
 274  */
 275 static int ixgbe_header_split = FALSE;
 276 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 277
 278 /*
 279  * Number of Queues, can be set to 0,
 280  * it then autoconfigures based on the
 281  * number of cpus with a max of 8. This
 282  * can be overriden manually here.
 283  */
 284 static int ixgbe_num_queues = 0;
 285 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 286
 287 /*
 288 ** Number of TX descriptors per ring,
 289 ** setting higher than RX as this seems
 290 ** the better performing choice.
 291 */
 292 static int ixgbe_txd = PERFORM_TXD;
 293 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 294
 295 /* Number of RX descriptors per ring */
 296 static int ixgbe_rxd = PERFORM_RXD;
 297 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 298
 299 /* Keep running tab on them for sanity check */
 300 static int ixgbe_total_ports;
 301
 302 #ifdef IXGBE_FDIR
 303 /*
 304 ** For Flow Director: this is the
 305 ** number of TX packets we sample
 306 ** for the filter pool, this means
 307 ** every 20th packet will be probed.
 308 **
 309 ** This feature can be disabled by
 310 ** setting this to 0.
 311 */
 312 static int atr_sample_rate = 20;
 313 /*
 314 ** Flow Director actually 'steals'
 315 ** part of the packet buffer as its
 316 ** filter pool, this variable controls
 317 ** how much it uses:
 318 **  0 = 64K, 1 = 128K, 2 = 256K
 319 */
 320 static int fdir_pballoc = 1;
 321 #endif
 322
 323 #ifdef DEV_NETMAP
 324 /*
 325  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 326  * be a reference on how to implement netmap support in a driver.
 327  * Additional comments are in ixgbe_netmap.h .
 328  *
 329  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 330  * that extend the standard driver.
 331  */
 332 #include <dev/netmap/ixgbe_netmap.h>
 333 #endif /* DEV_NETMAP */
 334
 335 /*********************************************************************
 336  *  Device identification routine
 337  *
 338  *  ixgbe_probe determines if the driver should be loaded on
 339  *  adapter based on PCI vendor/device id of the adapter.
 340  *
 341  *  return BUS_PROBE_DEFAULT on success, positive on failure
 342  *********************************************************************/
 343
 344 static int
 345 ixgbe_probe(device_t dev)
 346 {
 347         ixgbe_vendor_info_t *ent;
 348
 349         u16     pci_vendor_id = 0;
 350         u16     pci_device_id = 0;
 351         u16     pci_subvendor_id = 0;
 352         u16     pci_subdevice_id = 0;
 353         char    adapter_name[256];
 354
 355         INIT_DEBUGOUT("ixgbe_probe: begin");
 356
 357         pci_vendor_id = pci_get_vendor(dev);
 358         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 359                 return (ENXIO);
 360
 361         pci_device_id = pci_get_device(dev);
 362         pci_subvendor_id = pci_get_subvendor(dev);
 363         pci_subdevice_id = pci_get_subdevice(dev);
 364
 365         ent = ixgbe_vendor_info_array;
 366         while (ent->vendor_id != 0) {
 367                 if ((pci_vendor_id == ent->vendor_id) &&
 368                     (pci_device_id == ent->device_id) &&
 369
 370                     ((pci_subvendor_id == ent->subvendor_id) ||
 371                      (ent->subvendor_id == 0)) &&
 372
 373                     ((pci_subdevice_id == ent->subdevice_id) ||
 374                      (ent->subdevice_id == 0))) {
 375                         ksprintf(adapter_name, "%s, Version - %s",
 376                                 ixgbe_strings[ent->index],
 377                                 ixgbe_driver_version);
 378                         device_set_desc_copy(dev, adapter_name);
 379                         ++ixgbe_total_ports;
 380                         return (BUS_PROBE_DEFAULT);
 381                 }
 382                 ent++;
 383         }
 384         return (ENXIO);
 385 }
 386
 387 /*********************************************************************
 388  *  Device initialization routine
 389  *
 390  *  The attach entry point is called when the driver is being loaded.
 391  *  This routine identifies the type of hardware, allocates all resources
 392  *  and initializes the hardware.
 393  *
 394  *  return 0 on success, positive on failure
 395  *********************************************************************/
 396
 397 static int
 398 ixgbe_attach(device_t dev)
 399 {
 400         struct adapter *adapter;
 401         struct ixgbe_hw *hw;
 402         int             error = 0;
 403         u16             csum;
 404         u32             ctrl_ext;
 405
 406         INIT_DEBUGOUT("ixgbe_attach: begin");
 407
 408         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 409                 device_printf(dev, "Disabled by device hint\n");
 410                 return (ENXIO);
 411         }
 412
 413         /* Allocate, clear, and link in our adapter structure */
 414         adapter = device_get_softc(dev);
 415         adapter->dev = adapter->osdep.dev = dev;
 416         hw = &adapter->hw;
 417
 418         /* Core Lock Init*/
 419         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 420         spin_init(&adapter->mcast_spin);
 421
 422         /* SYSCTL APIs */
 423
 424         sysctl_ctx_init(&adapter->sysctl_ctx);
 425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 428         if (adapter->sysctl_tree == NULL) {
 429                 device_printf(adapter->dev, "can't add sysctl node\n");
 430                 return (EINVAL);
 431         }
 432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 436
 437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 441
 442         /*
 443         ** Allow a kind of speed control by forcing the autoneg
 444         ** advertised speed list to only a certain value, this
 445         ** supports 1G on 82599 devices, and 100Mb on x540.
 446         */
 447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 451
 452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 456
 457         /* Set up the timer callout */
 458         /* XXX: shouldn't this be a spin lock ? */
 459         lockinit(&adapter->core_lock, "ixgbe core lock", 0, LK_CANRECURSE);
 460         callout_init(&adapter->timer);
 461
 462         /* Determine hardware revision */
 463         ixgbe_identify_hardware(adapter);
 464
 465         /* Do base PCI setup - map BAR0 */
 466         if (ixgbe_allocate_pci_resources(adapter)) {
 467                 device_printf(dev, "Allocation of PCI resources failed\n");
 468                 error = ENXIO;
 469                 goto err_out;
 470         }
 471
 472         /* Do descriptor calc and sanity checks */
 473         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 474             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 475                 device_printf(dev, "TXD config issue, using default!\n");
 476                 adapter->num_tx_desc = DEFAULT_TXD;
 477         } else
 478                 adapter->num_tx_desc = ixgbe_txd;
 479
 480         /*
 481         ** With many RX rings it is easy to exceed the
 482         ** system mbuf allocation. Tuning nmbclusters
 483         ** can alleviate this.
 484         */
 485         if (nmbclusters > 0 ) {
 486                 int s;
 487                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 488                 if (s > nmbclusters) {
 489                         device_printf(dev, "RX Descriptors exceed "
 490                             "system mbuf max, using default instead!\n");
 491                         ixgbe_rxd = DEFAULT_RXD;
 492                 }
 493         }
 494
 495         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 496             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 497                 device_printf(dev, "RXD config issue, using default!\n");
 498                 adapter->num_rx_desc = DEFAULT_RXD;
 499         } else
 500                 adapter->num_rx_desc = ixgbe_rxd;
 501
 502         /* Allocate our TX/RX Queues */
 503         if (ixgbe_allocate_queues(adapter)) {
 504                 error = ENOMEM;
 505                 goto err_out;
 506         }
 507
 508         /* Allocate multicast array memory. */
 509         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 510             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 511         if (adapter->mta == NULL) {
 512                 device_printf(dev, "Can not allocate multicast setup array\n");
 513                 error = ENOMEM;
 514                 goto err_late;
 515         }
 516
 517         /* Initialize the shared code */
 518         error = ixgbe_init_shared_code(hw);
 519         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 520                 /*
 521                 ** No optics in this port, set up
 522                 ** so the timer routine will probe
 523                 ** for later insertion.
 524                 */
 525                 adapter->sfp_probe = TRUE;
 526                 error = 0;
 527         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 528                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 529                 error = EIO;
 530                 goto err_late;
 531         } else if (error) {
 532                 device_printf(dev,"Unable to initialize the shared code\n");
 533                 error = EIO;
 534                 goto err_late;
 535         }
 536
 537         /* Make sure we have a good EEPROM before we read from it */
 538         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 539                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 540                 error = EIO;
 541                 goto err_late;
 542         }
 543
 544         /* Get Hardware Flow Control setting */
 545         hw->fc.requested_mode = ixgbe_fc_full;
 546         adapter->fc = hw->fc.requested_mode;
 547         hw->fc.pause_time = IXGBE_FC_PAUSE;
 548         hw->fc.low_water = IXGBE_FC_LO;
 549         hw->fc.high_water[0] = IXGBE_FC_HI;
 550         hw->fc.send_xon = TRUE;
 551
 552         error = ixgbe_init_hw(hw);
 553         if (error == IXGBE_ERR_EEPROM_VERSION) {
 554                 device_printf(dev, "This device is a pre-production adapter/"
 555                     "LOM.  Please be aware there may be issues associated "
 556                     "with your hardware.\n If you are experiencing problems "
 557                     "please contact your Intel or hardware representative "
 558                     "who provided you with this hardware.\n");
 559         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED)
 560                 device_printf(dev,"Unsupported SFP+ Module\n");
 561
 562         if (error) {
 563                 error = EIO;
 564                 device_printf(dev,"Hardware Initialization Failure\n");
 565                 goto err_late;
 566         }
 567
 568         /* Detect and set physical type */
 569         ixgbe_setup_optics(adapter);
 570
 571         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 572                 error = ixgbe_allocate_msix(adapter);
 573         else
 574                 error = ixgbe_allocate_legacy(adapter);
 575         if (error)
 576                 goto err_late;
 577
 578         /* Setup OS specific network interface */
 579         if (ixgbe_setup_interface(dev, adapter) != 0)
 580                 goto err_late;
 581
 582         /* Sysctl for limiting the amount of work done in the taskqueue */
 583         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 584             "max number of rx packets to process", &adapter->rx_process_limit,
 585             ixgbe_rx_process_limit);
 586
 587         /* Initialize statistics */
 588         ixgbe_update_stats_counters(adapter);
 589
 590         /* Register for VLAN events */
 591         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 592             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 593         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 594             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 595
 596         /* Print PCIE bus type/speed/width info */
 597         ixgbe_get_bus_info(hw);
 598         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 599             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 600             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 601             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 602             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 603             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 604             ("Unknown"));
 605
 606         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 607             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 608                 device_printf(dev, "PCI-Express bandwidth available"
 609                     " for this card\n     is not sufficient for"
 610                     " optimal performance.\n");
 611                 device_printf(dev, "For optimal performance a x8 "
 612                     "PCIE, or x4 PCIE 2 slot is required.\n");
 613         }
 614
 615         /* let hardware know driver is loaded */
 616         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 617         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 618         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 619
 620         ixgbe_add_hw_stats(adapter);
 621
 622 #ifdef DEV_NETMAP
 623         ixgbe_netmap_attach(adapter);
 624 #endif /* DEV_NETMAP */
 625         INIT_DEBUGOUT("ixgbe_attach: end");
 626         return (0);
 627 err_late:
 628         ixgbe_free_transmit_structures(adapter);
 629         ixgbe_free_receive_structures(adapter);
 630 err_out:
 631         if (adapter->ifp != NULL)
 632                 if_free(adapter->ifp);
 633         ixgbe_free_pci_resources(adapter);
 634         kfree(adapter->mta, M_DEVBUF);
 635         return (error);
 636
 637 }
 638
 639 /*********************************************************************
 640  *  Device removal routine
 641  *
 642  *  The detach entry point is called when the driver is being removed.
 643  *  This routine stops the adapter and deallocates all the resources
 644  *  that were allocated for driver operation.
 645  *
 646  *  return 0 on success, positive on failure
 647  *********************************************************************/
 648
 649 static int
 650 ixgbe_detach(device_t dev)
 651 {
 652         struct adapter *adapter = device_get_softc(dev);
 653         struct ix_queue *que = adapter->queues;
 654         u32     ctrl_ext;
 655
 656         INIT_DEBUGOUT("ixgbe_detach: begin");
 657
 658 #ifdef NET_VLAN
 659         /* Make sure VLANS are not using driver */
 660         if (adapter->ifp->if_vlantrunk != NULL) {
 661                 device_printf(dev,"Vlan in use, detach first\n");
 662                 return (EBUSY);
 663         }
 664 #endif
 665
 666         IXGBE_CORE_LOCK(adapter);
 667         ixgbe_stop(adapter);
 668         IXGBE_CORE_UNLOCK(adapter);
 669
 670         for (int i = 0; i < adapter->num_queues; i++, que++) {
 671                 if (que->tq) {
 672                         taskqueue_drain(que->tq, &que->que_task);
 673                         taskqueue_free(que->tq);
 674                 }
 675         }
 676
 677         /* Drain the Link queue */
 678         if (adapter->tq) {
 679                 taskqueue_drain(adapter->tq, &adapter->link_task);
 680                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 681                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 682 #ifdef IXGBE_FDIR
 683                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 684 #endif
 685                 taskqueue_free(adapter->tq);
 686         }
 687
 688         /* let hardware know driver is unloading */
 689         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 690         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 691         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 692
 693         /* Unregister VLAN events */
 694 #ifdef NET_VLAN
 695         if (adapter->vlan_attach != NULL)
 696                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 697         if (adapter->vlan_detach != NULL)
 698                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 699 #endif
 700
 701         ether_ifdetach(adapter->ifp);
 702         callout_stop(&adapter->timer);
 703         lockuninit(&adapter->core_lock);
 704 #ifdef DEV_NETMAP
 705         netmap_detach(adapter->ifp);
 706 #endif /* DEV_NETMAP */
 707         ixgbe_free_pci_resources(adapter);
 708         bus_generic_detach(dev);
 709         if_free(adapter->ifp);
 710
 711         ixgbe_free_transmit_structures(adapter);
 712         ixgbe_free_receive_structures(adapter);
 713         kfree(adapter->mta, M_DEVBUF);
 714         sysctl_ctx_free(&adapter->sysctl_ctx);
 715
 716         spin_uninit(&adapter->mcast_spin);
 717         IXGBE_CORE_LOCK_DESTROY(adapter);
 718         return (0);
 719 }
 720
 721 /*********************************************************************
 722  *
 723  *  Shutdown entry point
 724  *
 725  **********************************************************************/
 726
 727 static int
 728 ixgbe_shutdown(device_t dev)
 729 {
 730         struct adapter *adapter = device_get_softc(dev);
 731         IXGBE_CORE_LOCK(adapter);
 732         ixgbe_stop(adapter);
 733         IXGBE_CORE_UNLOCK(adapter);
 734         return (0);
 735 }
 736
 737
 738 /*********************************************************************
 739  *  Transmit entry point
 740  *
 741  *  ixgbe_start is called by the stack to initiate a transmit.
 742  *  The driver will remain in this routine as long as there are
 743  *  packets to transmit and transmit resources are available.
 744  *  In case resources are not available stack is notified and
 745  *  the packet is requeued.
 746  **********************************************************************/
 747
 748 static void
 749 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 750 {
 751         struct mbuf    *m_head;
 752         struct adapter *adapter = txr->adapter;
 753
 754         IXGBE_TX_LOCK_ASSERT(txr);
 755
 756         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 757                 return;
 758         if (!adapter->link_active)
 759                 return;
 760
 761         while (!ifq_is_empty(&ifp->if_snd)) {
 762                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 763                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 764                         break;
 765                 }
 766
 767                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 768                 if (m_head == NULL)
 769                         break;
 770
 771                 if (ixgbe_xmit(txr, &m_head)) {
 772 #if 0 /* XXX: prepend to an ALTQ queue ? */
 773                         if (m_head != NULL)
 774                                 IF_PREPEND(&ifp->if_snd, m_head);
 775 #endif
 776                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 777                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 778                         break;
 779                 }
 780                 /* Send a copy of the frame to the BPF listener */
 781                 ETHER_BPF_MTAP(ifp, m_head);
 782
 783                 /* Set watchdog on */
 784                 txr->watchdog_time = ticks;
 785                 txr->queue_status = IXGBE_QUEUE_WORKING;
 786
 787         }
 788         return;
 789 }
 790
 791 /*
 792  * Legacy TX start - called by the stack, this
 793  * always uses the first tx ring, and should
 794  * not be used with multiqueue tx enabled.
 795  */
 796 static void
 797 ixgbe_start(struct ifnet *ifp)
 798 {
 799         struct adapter *adapter = ifp->if_softc;
 800         struct tx_ring  *txr = adapter->tx_rings;
 801
 802         if (ifp->if_flags & IFF_RUNNING) {
 803                 IXGBE_TX_LOCK(txr);
 804                 ixgbe_start_locked(txr, ifp);
 805                 IXGBE_TX_UNLOCK(txr);
 806         }
 807         return;
 808 }
 809
 810 #if 0 /* __FreeBSD_version >= 800000 */
 811 /*
 812 ** Multiqueue Transmit driver
 813 **
 814 */
 815 static int
 816 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 817 {
 818         struct adapter  *adapter = ifp->if_softc;
 819         struct ix_queue *que;
 820         struct tx_ring  *txr;
 821         int             i = 0, err = 0;
 822
 823         /* Which queue to use */
 824         if ((m->m_flags & M_FLOWID) != 0)
 825                 i = m->m_pkthdr.flowid % adapter->num_queues;
 826         else
 827                 i = curcpu % adapter->num_queues;
 828
 829         txr = &adapter->tx_rings[i];
 830         que = &adapter->queues[i];
 831
 832         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 833             IXGBE_TX_TRYLOCK(txr)) {
 834                 err = ixgbe_mq_start_locked(ifp, txr, m);
 835                 IXGBE_TX_UNLOCK(txr);
 836         } else {
 837                 err = drbr_enqueue(ifp, txr->br, m);
 838                 taskqueue_enqueue(que->tq, &que->que_task);
 839         }
 840
 841         return (err);
 842 }
 843
 844 static int
 845 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 846 {
 847         struct adapter  *adapter = txr->adapter;
 848         struct mbuf     *next;
 849         int             enqueued, err = 0;
 850
 851         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 852             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 853             adapter->link_active == 0) {
 854                 if (m != NULL)
 855                         err = drbr_enqueue(ifp, txr->br, m);
 856                 return (err);
 857         }
 858
 859         enqueued = 0;
 860         if (m == NULL) {
 861                 next = drbr_dequeue(ifp, txr->br);
 862         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 863                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 864                         return (err);
 865                 next = drbr_dequeue(ifp, txr->br);
 866         } else
 867                 next = m;
 868
 869         /* Process the queue */
 870         while (next != NULL) {
 871                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 872                         if (next != NULL)
 873                                 err = drbr_enqueue(ifp, txr->br, next);
 874                         break;
 875                 }
 876                 enqueued++;
 877                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 878                 /* Send a copy of the frame to the BPF listener */
 879                 ETHER_BPF_MTAP(ifp, next);
 880                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 881                         break;
 882                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 883                         ixgbe_txeof(txr);
 884                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 885                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 886                         break;
 887                 }
 888                 next = drbr_dequeue(ifp, txr->br);
 889         }
 890
 891         if (enqueued > 0) {
 892                 /* Set watchdog on */
 893                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 894                 txr->watchdog_time = ticks;
 895         }
 896
 897         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 898                 ixgbe_txeof(txr);
 899
 900         return (err);
 901 }
 902
 903 /*
 904 ** Flush all ring buffers
 905 */
 906 static void
 907 ixgbe_qflush(struct ifnet *ifp)
 908 {
 909         struct adapter  *adapter = ifp->if_softc;
 910         struct tx_ring  *txr = adapter->tx_rings;
 911         struct mbuf     *m;
 912
 913         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 914                 IXGBE_TX_LOCK(txr);
 915                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 916                         m_freem(m);
 917                 IXGBE_TX_UNLOCK(txr);
 918         }
 919         if_qflush(ifp);
 920 }
 921 #endif /* __FreeBSD_version >= 800000 */
 922
 923 /*********************************************************************
 924  *  Ioctl entry point
 925  *
 926  *  ixgbe_ioctl is called when the user wants to configure the
 927  *  interface.
 928  *
 929  *  return 0 on success, positive on failure
 930  **********************************************************************/
 931
 932 static int
 933 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 934 {
 935         struct adapter  *adapter = ifp->if_softc;
 936         struct ifreq    *ifr = (struct ifreq *) data;
 937 #if defined(INET) || defined(INET6)
 938         struct ifaddr *ifa = (struct ifaddr *)data;
 939         bool            avoid_reset = FALSE;
 940 #endif
 941         int             error = 0;
 942
 943         switch (command) {
 944
 945         case SIOCSIFADDR:
 946 #ifdef INET
 947                 if (ifa->ifa_addr->sa_family == AF_INET)
 948                         avoid_reset = TRUE;
 949 #endif
 950 #ifdef INET6
 951                 if (ifa->ifa_addr->sa_family == AF_INET6)
 952                         avoid_reset = TRUE;
 953 #endif
 954 #if defined(INET) || defined(INET6)
 955                 /*
 956                 ** Calling init results in link renegotiation,
 957                 ** so we avoid doing it when possible.
 958                 */
 959                 if (avoid_reset) {
 960                         ifp->if_flags |= IFF_UP;
 961                         if (!(ifp->if_flags & IFF_RUNNING))
 962                                 ixgbe_init(adapter);
 963                         if (!(ifp->if_flags & IFF_NOARP))
 964                                 arp_ifinit(ifp, ifa);
 965                 } else
 966                         error = ether_ioctl(ifp, command, data);
 967 #endif
 968                 break;
 969         case SIOCSIFMTU:
 970                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 971                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 972                         error = EINVAL;
 973                 } else {
 974                         IXGBE_CORE_LOCK(adapter);
 975                         ifp->if_mtu = ifr->ifr_mtu;
 976                         adapter->max_frame_size =
 977                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 978                         ixgbe_init_locked(adapter);
 979                         IXGBE_CORE_UNLOCK(adapter);
 980                 }
 981                 break;
 982         case SIOCSIFFLAGS:
 983                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 984                 IXGBE_CORE_LOCK(adapter);
 985                 if (ifp->if_flags & IFF_UP) {
 986                         if ((ifp->if_flags & IFF_RUNNING)) {
 987                                 if ((ifp->if_flags ^ adapter->if_flags) &
 988                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 989                                         ixgbe_set_promisc(adapter);
 990                                 }
 991                         } else
 992                                 ixgbe_init_locked(adapter);
 993                 } else
 994                         if (ifp->if_flags & IFF_RUNNING)
 995                                 ixgbe_stop(adapter);
 996                 adapter->if_flags = ifp->if_flags;
 997                 IXGBE_CORE_UNLOCK(adapter);
 998                 break;
 999         case SIOCADDMULTI:
1000         case SIOCDELMULTI:
1001                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
1002                 if (ifp->if_flags & IFF_RUNNING) {
1003                         IXGBE_CORE_LOCK(adapter);
1004                         ixgbe_disable_intr(adapter);
1005                         ixgbe_set_multi(adapter);
1006                         ixgbe_enable_intr(adapter);
1007                         IXGBE_CORE_UNLOCK(adapter);
1008                 }
1009                 break;
1010         case SIOCSIFMEDIA:
1011         case SIOCGIFMEDIA:
1012                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1013                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1014                 break;
1015         case SIOCSIFCAP:
1016         {
1017                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1018                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1019                 if (mask & IFCAP_HWCSUM)
1020                         ifp->if_capenable ^= IFCAP_HWCSUM;
1021 #if 0 /* NET_TSO */
1022                 if (mask & IFCAP_TSO4)
1023                         ifp->if_capenable ^= IFCAP_TSO4;
1024                 if (mask & IFCAP_TSO6)
1025                         ifp->if_capenable ^= IFCAP_TSO6;
1026 #endif
1027 #if 0 /* NET_LRO */
1028                 if (mask & IFCAP_LRO)
1029                         ifp->if_capenable ^= IFCAP_LRO;
1030 #endif
1031                 if (mask & IFCAP_VLAN_HWTAGGING)
1032                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1033                 if (mask & IFCAP_VLAN_HWFILTER)
1034                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1035 #if 0 /* NET_TSO */
1036                 if (mask & IFCAP_VLAN_HWTSO)
1037                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1038 #endif
1039                 if (ifp->if_flags & IFF_RUNNING) {
1040                         IXGBE_CORE_LOCK(adapter);
1041                         ixgbe_init_locked(adapter);
1042                         IXGBE_CORE_UNLOCK(adapter);
1043                 }
1044 #if 0
1045                 VLAN_CAPABILITIES(ifp);
1046 #endif
1047                 break;
1048         }
1049
1050         default:
1051                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1052                 error = ether_ioctl(ifp, command, data);
1053                 break;
1054         }
1055
1056         return (error);
1057 }
1058
1059 /*********************************************************************
1060  *  Init entry point
1061  *
1062  *  This routine is used in two ways. It is used by the stack as
1063  *  init entry point in network interface structure. It is also used
1064  *  by the driver as a hw/sw initialization routine to get to a
1065  *  consistent state.
1066  *
1067  *  return 0 on success, positive on failure
1068  **********************************************************************/
1069 #define IXGBE_MHADD_MFS_SHIFT 16
1070
1071 static void
1072 ixgbe_init_locked(struct adapter *adapter)
1073 {
1074         struct ifnet   *ifp = adapter->ifp;
1075         device_t        dev = adapter->dev;
1076         struct ixgbe_hw *hw = &adapter->hw;
1077         u32             k, txdctl, mhadd, gpie;
1078         u32             rxdctl, rxctrl;
1079
1080         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1081         INIT_DEBUGOUT("ixgbe_init: begin");
1082         hw->adapter_stopped = FALSE;
1083         ixgbe_stop_adapter(hw);
1084         callout_stop(&adapter->timer);
1085
1086         /* reprogram the RAR[0] in case user changed it. */
1087         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1088
1089         /* Get the latest mac address, User can use a LAA */
1090         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1091               IXGBE_ETH_LENGTH_OF_ADDRESS);
1092         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1093         hw->addr_ctrl.rar_used_count = 1;
1094
1095         /* Set the various hardware offload abilities */
1096         ifp->if_hwassist = 0;
1097 #if 0 /* NET_TSO */
1098         if (ifp->if_capenable & IFCAP_TSO)
1099                 ifp->if_hwassist |= CSUM_TSO;
1100 #endif
1101         if (ifp->if_capenable & IFCAP_TXCSUM) {
1102                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1103 #if 0
1104                 if (hw->mac.type != ixgbe_mac_82598EB)
1105                         ifp->if_hwassist |= CSUM_SCTP;
1106 #endif
1107         }
1108
1109         /* Prepare transmit descriptors and buffers */
1110         if (ixgbe_setup_transmit_structures(adapter)) {
1111                 device_printf(dev,"Could not setup transmit structures\n");
1112                 ixgbe_stop(adapter);
1113                 return;
1114         }
1115
1116         ixgbe_init_hw(hw);
1117         ixgbe_initialize_transmit_units(adapter);
1118
1119         /* Setup Multicast table */
1120         ixgbe_set_multi(adapter);
1121
1122         /*
1123         ** Determine the correct mbuf pool
1124         ** for doing jumbo/headersplit
1125         */
1126         if (adapter->max_frame_size <= 2048)
1127                 adapter->rx_mbuf_sz = MCLBYTES;
1128         else if (adapter->max_frame_size <= 4096)
1129                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1130         else if (adapter->max_frame_size <= 9216)
1131                 adapter->rx_mbuf_sz = MJUM9BYTES;
1132         else
1133                 adapter->rx_mbuf_sz = MJUM16BYTES;
1134
1135         /* Prepare receive descriptors and buffers */
1136         if (ixgbe_setup_receive_structures(adapter)) {
1137                 device_printf(dev,"Could not setup receive structures\n");
1138                 ixgbe_stop(adapter);
1139                 return;
1140         }
1141
1142         /* Configure RX settings */
1143         ixgbe_initialize_receive_units(adapter);
1144
1145         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1146
1147         /* Enable Fan Failure Interrupt */
1148         gpie |= IXGBE_SDP1_GPIEN;
1149
1150         /* Add for Module detection */
1151         if (hw->mac.type == ixgbe_mac_82599EB)
1152                 gpie |= IXGBE_SDP2_GPIEN;
1153
1154         /* Thermal Failure Detection */
1155         if (hw->mac.type == ixgbe_mac_X540)
1156                 gpie |= IXGBE_SDP0_GPIEN;
1157
1158         if (adapter->msix > 1) {
1159                 /* Enable Enhanced MSIX mode */
1160                 gpie |= IXGBE_GPIE_MSIX_MODE;
1161                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1162                     IXGBE_GPIE_OCD;
1163         }
1164         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1165
1166         /* Set MTU size */
1167         if (ifp->if_mtu > ETHERMTU) {
1168                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1169                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1170                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1171                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1172         }
1173
1174         /* Now enable all the queues */
1175
1176         for (int i = 0; i < adapter->num_queues; i++) {
1177                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1178                 txdctl |= IXGBE_TXDCTL_ENABLE;
1179                 /* Set WTHRESH to 8, burst writeback */
1180                 txdctl |= (8 << 16);
1181                 /*
1182                  * When the internal queue falls below PTHRESH (32),
1183                  * start prefetching as long as there are at least
1184                  * HTHRESH (1) buffers ready. The values are taken
1185                  * from the Intel linux driver 3.8.21.
1186                  * Prefetching enables tx line rate even with 1 queue.
1187                  */
1188                 txdctl |= (32 << 0) | (1 << 8);
1189                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1190         }
1191
1192         for (int i = 0; i < adapter->num_queues; i++) {
1193                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1194                 if (hw->mac.type == ixgbe_mac_82598EB) {
1195                         /*
1196                         ** PTHRESH = 21
1197                         ** HTHRESH = 4
1198                         ** WTHRESH = 8
1199                         */
1200                         rxdctl &= ~0x3FFFFF;
1201                         rxdctl |= 0x080420;
1202                 }
1203                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1204                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1205                 for (k = 0; k < 10; k++) {
1206                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1207                             IXGBE_RXDCTL_ENABLE)
1208                                 break;
1209                         else
1210                                 msec_delay(1);
1211                 }
1212                 wmb();
1213 #ifdef DEV_NETMAP
1214                 /*
1215                  * In netmap mode, we must preserve the buffers made
1216                  * available to userspace before the if_init()
1217                  * (this is true by default on the TX side, because
1218                  * init makes all buffers available to userspace).
1219                  *
1220                  * netmap_reset() and the device specific routines
1221                  * (e.g. ixgbe_setup_receive_rings()) map these
1222                  * buffers at the end of the NIC ring, so here we
1223                  * must set the RDT (tail) register to make sure
1224                  * they are not overwritten.
1225                  *
1226                  * In this driver the NIC ring starts at RDH = 0,
1227                  * RDT points to the last slot available for reception (?),
1228                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1229                  */
1230                 if (ifp->if_capenable & IFCAP_NETMAP) {
1231                         struct netmap_adapter *na = NA(adapter->ifp);
1232                         struct netmap_kring *kring = &na->rx_rings[i];
1233                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1234
1235                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1236                 } else
1237 #endif /* DEV_NETMAP */
1238                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1239         }
1240
1241         /* Set up VLAN support and filter */
1242         ixgbe_setup_vlan_hw_support(adapter);
1243
1244         /* Enable Receive engine */
1245         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1246         if (hw->mac.type == ixgbe_mac_82598EB)
1247                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1248         rxctrl |= IXGBE_RXCTRL_RXEN;
1249         ixgbe_enable_rx_dma(hw, rxctrl);
1250
1251         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1252
1253         /* Set up MSI/X routing */
1254         if (ixgbe_enable_msix)  {
1255                 ixgbe_configure_ivars(adapter);
1256                 /* Set up auto-mask */
1257                 if (hw->mac.type == ixgbe_mac_82598EB)
1258                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1259                 else {
1260                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1261                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1262                 }
1263         } else {  /* Simple settings for Legacy/MSI */
1264                 ixgbe_set_ivar(adapter, 0, 0, 0);
1265                 ixgbe_set_ivar(adapter, 0, 0, 1);
1266                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1267         }
1268
1269 #ifdef IXGBE_FDIR
1270         /* Init Flow director */
1271         if (hw->mac.type != ixgbe_mac_82598EB) {
1272                 u32 hdrm = 64 << fdir_pballoc;
1273
1274                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1275                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1276         }
1277 #endif
1278
1279         /*
1280         ** Check on any SFP devices that
1281         ** need to be kick-started
1282         */
1283         if (hw->phy.type == ixgbe_phy_none) {
1284                 int err = hw->phy.ops.identify(hw);
1285                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1286                         device_printf(dev,
1287                             "Unsupported SFP+ module type was detected.\n");
1288                         return;
1289                 }
1290         }
1291
1292         /* Set moderation on the Link interrupt */
1293         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1294
1295         /* Config/Enable Link */
1296         ixgbe_config_link(adapter);
1297
1298         /* And now turn on interrupts */
1299         ixgbe_enable_intr(adapter);
1300
1301         /* Now inform the stack we're ready */
1302         ifp->if_flags |= IFF_RUNNING;
1303         ifp->if_flags &= ~IFF_OACTIVE;
1304
1305         return;
1306 }
1307
1308 static void
1309 ixgbe_init(void *arg)
1310 {
1311         struct adapter *adapter = arg;
1312
1313         IXGBE_CORE_LOCK(adapter);
1314         ixgbe_init_locked(adapter);
1315         IXGBE_CORE_UNLOCK(adapter);
1316         return;
1317 }
1318
1319
1320 /*
1321 **
1322 ** MSIX Interrupt Handlers and Tasklets
1323 **
1324 */
1325
1326 static inline void
1327 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1328 {
1329         struct ixgbe_hw *hw = &adapter->hw;
1330         u64     queue = (u64)(1 << vector);
1331         u32     mask;
1332
1333         if (hw->mac.type == ixgbe_mac_82598EB) {
1334                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1335                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1336         } else {
1337                 mask = (queue & 0xFFFFFFFF);
1338                 if (mask)
1339                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1340                 mask = (queue >> 32);
1341                 if (mask)
1342                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1343         }
1344 }
1345
1346 static inline void
1347 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1348 {
1349         struct ixgbe_hw *hw = &adapter->hw;
1350         u64     queue = (u64)(1 << vector);
1351         u32     mask;
1352
1353         if (hw->mac.type == ixgbe_mac_82598EB) {
1354                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1355                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1356         } else {
1357                 mask = (queue & 0xFFFFFFFF);
1358                 if (mask)
1359                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1360                 mask = (queue >> 32);
1361                 if (mask)
1362                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1363         }
1364 }
1365
1366 static inline void
1367 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1368 {
1369         u32 mask;
1370
1371         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1372                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1373                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1374         } else {
1375                 mask = (queues & 0xFFFFFFFF);
1376                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1377                 mask = (queues >> 32);
1378                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1379         }
1380 }
1381
1382
1383 static void
1384 ixgbe_handle_que(void *context, int pending)
1385 {
1386         struct ix_queue *que = context;
1387         struct adapter  *adapter = que->adapter;
1388         struct tx_ring  *txr = que->txr;
1389         struct ifnet    *ifp = adapter->ifp;
1390         bool            more;
1391
1392         if (ifp->if_flags & IFF_RUNNING) {
1393                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1394                 IXGBE_TX_LOCK(txr);
1395                 ixgbe_txeof(txr);
1396 #if 0 /*__FreeBSD_version >= 800000*/
1397                 if (!drbr_empty(ifp, txr->br))
1398                         ixgbe_mq_start_locked(ifp, txr, NULL);
1399 #else
1400                 if (!ifq_is_empty(&ifp->if_snd))
1401                         ixgbe_start_locked(txr, ifp);
1402 #endif
1403                 IXGBE_TX_UNLOCK(txr);
1404                 if (more) {
1405                         taskqueue_enqueue(que->tq, &que->que_task);
1406                         return;
1407                 }
1408         }
1409
1410         /* Reenable this interrupt */
1411         ixgbe_enable_queue(adapter, que->msix);
1412         return;
1413 }
1414
1415
1416 /*********************************************************************
1417  *
1418  *  Legacy Interrupt Service routine
1419  *
1420  **********************************************************************/
1421
1422 static void
1423 ixgbe_legacy_irq(void *arg)
1424 {
1425         struct ix_queue *que = arg;
1426         struct adapter  *adapter = que->adapter;
1427         struct ixgbe_hw *hw = &adapter->hw;
1428         struct          tx_ring *txr = adapter->tx_rings;
1429         bool            more_tx, more_rx;
1430         u32             reg_eicr, loop = MAX_LOOP;
1431
1432
1433         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1434
1435         ++que->irqs;
1436         if (reg_eicr == 0) {
1437                 ixgbe_enable_intr(adapter);
1438                 return;
1439         }
1440
1441         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1442
1443         IXGBE_TX_LOCK(txr);
1444         do {
1445                 more_tx = ixgbe_txeof(txr);
1446         } while (loop-- && more_tx);
1447         IXGBE_TX_UNLOCK(txr);
1448
1449         if (more_rx || more_tx)
1450                 taskqueue_enqueue(que->tq, &que->que_task);
1451
1452         /* Check for fan failure */
1453         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1454             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1455                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1456                     "REPLACE IMMEDIATELY!!\n");
1457                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1458         }
1459
1460         /* Link status change */
1461         if (reg_eicr & IXGBE_EICR_LSC)
1462                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1463
1464         ixgbe_enable_intr(adapter);
1465         return;
1466 }
1467
1468
1469 /*********************************************************************
1470  *
1471  *  MSIX Queue Interrupt Service routine
1472  *
1473  **********************************************************************/
1474 void
1475 ixgbe_msix_que(void *arg)
1476 {
1477         struct ix_queue *que = arg;
1478         struct adapter  *adapter = que->adapter;
1479         struct tx_ring  *txr = que->txr;
1480         struct rx_ring  *rxr = que->rxr;
1481         bool            more_tx, more_rx;
1482         u32             newitr = 0;
1483
1484         ixgbe_disable_queue(adapter, que->msix);
1485         ++que->irqs;
1486
1487         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1488
1489         IXGBE_TX_LOCK(txr);
1490         more_tx = ixgbe_txeof(txr);
1491         /*
1492         ** Make certain that if the stack
1493         ** has anything queued the task gets
1494         ** scheduled to handle it.
1495         */
1496 #if 0
1497 #if __FreeBSD_version < 800000
1498         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1499 #else
1500         if (!drbr_empty(adapter->ifp, txr->br))
1501 #endif
1502 #endif
1503         if (!ifq_is_empty(&adapter->ifp->if_snd))
1504                 more_tx = 1;
1505         IXGBE_TX_UNLOCK(txr);
1506
1507         /* Do AIM now? */
1508
1509         if (ixgbe_enable_aim == FALSE)
1510                 goto no_calc;
1511         /*
1512         ** Do Adaptive Interrupt Moderation:
1513         **  - Write out last calculated setting
1514         **  - Calculate based on average size over
1515         **    the last interval.
1516         */
1517         if (que->eitr_setting)
1518                 IXGBE_WRITE_REG(&adapter->hw,
1519                     IXGBE_EITR(que->msix), que->eitr_setting);
1520
1521         que->eitr_setting = 0;
1522
1523         /* Idle, do nothing */
1524         if ((txr->bytes == 0) && (rxr->bytes == 0))
1525                 goto no_calc;
1526
1527         if ((txr->bytes) && (txr->packets))
1528                 newitr = txr->bytes/txr->packets;
1529         if ((rxr->bytes) && (rxr->packets))
1530                 newitr = max(newitr,
1531                     (rxr->bytes / rxr->packets));
1532         newitr += 24; /* account for hardware frame, crc */
1533
1534         /* set an upper boundary */
1535         newitr = min(newitr, 3000);
1536
1537         /* Be nice to the mid range */
1538         if ((newitr > 300) && (newitr < 1200))
1539                 newitr = (newitr / 3);
1540         else
1541                 newitr = (newitr / 2);
1542
1543         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1544                 newitr |= newitr << 16;
1545         else
1546                 newitr |= IXGBE_EITR_CNT_WDIS;
1547
1548         /* save for next interrupt */
1549         que->eitr_setting = newitr;
1550
1551         /* Reset state */
1552         txr->bytes = 0;
1553         txr->packets = 0;
1554         rxr->bytes = 0;
1555         rxr->packets = 0;
1556
1557 no_calc:
1558         if (more_tx || more_rx)
1559                 taskqueue_enqueue(que->tq, &que->que_task);
1560         else /* Reenable this interrupt */
1561                 ixgbe_enable_queue(adapter, que->msix);
1562         return;
1563 }
1564
1565
1566 static void
1567 ixgbe_msix_link(void *arg)
1568 {
1569         struct adapter  *adapter = arg;
1570         struct ixgbe_hw *hw = &adapter->hw;
1571         u32             reg_eicr;
1572
1573         ++adapter->link_irq;
1574
1575         /* First get the cause */
1576         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1577         /* Clear interrupt with write */
1578         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1579
1580         /* Link status change */
1581         if (reg_eicr & IXGBE_EICR_LSC)
1582                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1583
1584         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1585 #ifdef IXGBE_FDIR
1586                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1587                         /* This is probably overkill :) */
1588                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1589                                 return;
1590                         /* Clear the interrupt */
1591                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1592                         /* Turn off the interface */
1593                         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1594                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1595                 } else
1596 #endif
1597                 if (reg_eicr & IXGBE_EICR_ECC) {
1598                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1599                             "Please Reboot!!\n");
1600                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1601                 } else
1602
1603                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1604                         /* Clear the interrupt */
1605                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1606                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1607                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1608                         /* Clear the interrupt */
1609                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1610                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1611                 }
1612         }
1613
1614         /* Check for fan failure */
1615         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1616             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1617                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1618                     "REPLACE IMMEDIATELY!!\n");
1619                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1620         }
1621
1622         /* Check for over temp condition */
1623         if ((hw->mac.type == ixgbe_mac_X540) &&
1624             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1625                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1626                     "PHY IS SHUT DOWN!!\n");
1627                 device_printf(adapter->dev, "System shutdown required\n");
1628                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1629         }
1630
1631         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1632         return;
1633 }
1634
1635 /*********************************************************************
1636  *
1637  *  Media Ioctl callback
1638  *
1639  *  This routine is called whenever the user queries the status of
1640  *  the interface using ifconfig.
1641  *
1642  **********************************************************************/
1643 static void
1644 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1645 {
1646         struct adapter *adapter = ifp->if_softc;
1647
1648         INIT_DEBUGOUT("ixgbe_media_status: begin");
1649         IXGBE_CORE_LOCK(adapter);
1650         ixgbe_update_link_status(adapter);
1651
1652         ifmr->ifm_status = IFM_AVALID;
1653         ifmr->ifm_active = IFM_ETHER;
1654
1655         if (!adapter->link_active) {
1656                 IXGBE_CORE_UNLOCK(adapter);
1657                 return;
1658         }
1659
1660         ifmr->ifm_status |= IFM_ACTIVE;
1661
1662         switch (adapter->link_speed) {
1663                 case IXGBE_LINK_SPEED_100_FULL:
1664                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1665                         break;
1666                 case IXGBE_LINK_SPEED_1GB_FULL:
1667                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1668                         break;
1669                 case IXGBE_LINK_SPEED_10GB_FULL:
1670                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1671                         break;
1672         }
1673
1674         IXGBE_CORE_UNLOCK(adapter);
1675
1676         return;
1677 }
1678
1679 /*********************************************************************
1680  *
1681  *  Media Ioctl callback
1682  *
1683  *  This routine is called when the user changes speed/duplex using
1684  *  media/mediopt option with ifconfig.
1685  *
1686  **********************************************************************/
1687 static int
1688 ixgbe_media_change(struct ifnet * ifp)
1689 {
1690         struct adapter *adapter = ifp->if_softc;
1691         struct ifmedia *ifm = &adapter->media;
1692
1693         INIT_DEBUGOUT("ixgbe_media_change: begin");
1694
1695         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1696                 return (EINVAL);
1697
1698         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1699         case IFM_AUTO:
1700                 adapter->hw.phy.autoneg_advertised =
1701                     IXGBE_LINK_SPEED_100_FULL |
1702                     IXGBE_LINK_SPEED_1GB_FULL |
1703                     IXGBE_LINK_SPEED_10GB_FULL;
1704                 break;
1705         default:
1706                 device_printf(adapter->dev, "Only auto media type\n");
1707                 return (EINVAL);
1708         }
1709
1710         return (0);
1711 }
1712
1713 /*********************************************************************
1714  *
1715  *  This routine maps the mbufs to tx descriptors, allowing the
1716  *  TX engine to transmit the packets.
1717  *      - return 0 on success, positive on failure
1718  *
1719  **********************************************************************/
1720
1721 static int
1722 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1723 {
1724         struct adapter  *adapter = txr->adapter;
1725         u32             olinfo_status = 0, cmd_type_len;
1726         u32             paylen = 0;
1727         int             i, j, error, nsegs;
1728         int             first, last = 0;
1729         struct mbuf     *m_head;
1730         bus_dma_segment_t segs[1];
1731         bus_dmamap_t    map;
1732         struct ixgbe_tx_buf *txbuf;
1733         union ixgbe_adv_tx_desc *txd = NULL;
1734
1735         m_head = *m_headp;
1736
1737         /* Basic descriptor defines */
1738         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1739             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1740
1741         if (m_head->m_flags & M_VLANTAG)
1742                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1743
1744         /*
1745          * Important to capture the first descriptor
1746          * used because it will contain the index of
1747          * the one we tell the hardware to report back
1748          */
1749         first = txr->next_avail_desc;
1750         txbuf = &txr->tx_buffers[first];
1751         map = txbuf->map;
1752
1753         /*
1754          * Map the packet for DMA.
1755          */
1756         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1757             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1758
1759         if (error == EFBIG) {
1760                 struct mbuf *m;
1761
1762                 m = m_defrag(*m_headp, MB_DONTWAIT);
1763                 if (m == NULL) {
1764                         adapter->mbuf_defrag_failed++;
1765                         m_freem(*m_headp);
1766                         *m_headp = NULL;
1767                         return (ENOBUFS);
1768                 }
1769                 *m_headp = m;
1770
1771                 /* Try it again */
1772                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1773                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1774
1775                 if (error == ENOMEM) {
1776                         adapter->no_tx_dma_setup++;
1777                         return (error);
1778                 } else if (error != 0) {
1779                         adapter->no_tx_dma_setup++;
1780                         m_freem(*m_headp);
1781                         *m_headp = NULL;
1782                         return (error);
1783                 }
1784         } else if (error == ENOMEM) {
1785                 adapter->no_tx_dma_setup++;
1786                 return (error);
1787         } else if (error != 0) {
1788                 adapter->no_tx_dma_setup++;
1789                 m_freem(*m_headp);
1790                 *m_headp = NULL;
1791                 return (error);
1792         }
1793
1794         /* Make certain there are enough descriptors */
1795         if (nsegs > txr->tx_avail - 2) {
1796                 txr->no_desc_avail++;
1797                 error = ENOBUFS;
1798                 goto xmit_fail;
1799         }
1800         m_head = *m_headp;
1801
1802         /*
1803         ** Set up the appropriate offload context
1804         ** this becomes the first descriptor of
1805         ** a packet.
1806         */
1807 #if 0 /* NET_TSO */
1808         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1809                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1810                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1811                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1812                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1813                         ++adapter->tso_tx;
1814                 } else
1815                         return (ENXIO);
1816         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1817 #endif
1818         if (ixgbe_tx_ctx_setup(txr, m_head))
1819                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1820
1821 #ifdef IXGBE_IEEE1588
1822         /* This is changing soon to an mtag detection */
1823         if (we detect this mbuf has a TSTAMP mtag)
1824                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1825 #endif
1826
1827 #ifdef IXGBE_FDIR
1828         /* Do the flow director magic */
1829         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1830                 ++txr->atr_count;
1831                 if (txr->atr_count >= atr_sample_rate) {
1832                         ixgbe_atr(txr, m_head);
1833                         txr->atr_count = 0;
1834                 }
1835         }
1836 #endif
1837         /* Record payload length */
1838         if (paylen == 0)
1839                 olinfo_status |= m_head->m_pkthdr.len <<
1840                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1841
1842         i = txr->next_avail_desc;
1843         for (j = 0; j < nsegs; j++) {
1844                 bus_size_t seglen;
1845                 bus_addr_t segaddr;
1846
1847                 txbuf = &txr->tx_buffers[i];
1848                 txd = &txr->tx_base[i];
1849                 seglen = segs[j].ds_len;
1850                 segaddr = htole64(segs[j].ds_addr);
1851
1852                 txd->read.buffer_addr = segaddr;
1853                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1854                     cmd_type_len |seglen);
1855                 txd->read.olinfo_status = htole32(olinfo_status);
1856                 last = i; /* descriptor that will get completion IRQ */
1857
1858                 if (++i == adapter->num_tx_desc)
1859                         i = 0;
1860
1861                 txbuf->m_head = NULL;
1862                 txbuf->eop_index = -1;
1863         }
1864
1865         txd->read.cmd_type_len |=
1866             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1867         txr->tx_avail -= nsegs;
1868         txr->next_avail_desc = i;
1869
1870         txbuf->m_head = m_head;
1871         /* Swap the dma map between the first and last descriptor */
1872         txr->tx_buffers[first].map = txbuf->map;
1873         txbuf->map = map;
1874         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1875
1876         /* Set the index of the descriptor that will be marked done */
1877         txbuf = &txr->tx_buffers[first];
1878         txbuf->eop_index = last;
1879
1880         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1881             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1882         /*
1883          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1884          * hardware that this frame is available to transmit.
1885          */
1886         ++txr->total_packets;
1887         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1888
1889         return (0);
1890
1891 xmit_fail:
1892         bus_dmamap_unload(txr->txtag, txbuf->map);
1893         return (error);
1894
1895 }
1896
1897 static void
1898 ixgbe_set_promisc(struct adapter *adapter)
1899 {
1900         u_int32_t       reg_rctl;
1901         struct ifnet   *ifp = adapter->ifp;
1902
1903         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1904         reg_rctl &= (~IXGBE_FCTRL_UPE);
1905         reg_rctl &= (~IXGBE_FCTRL_MPE);
1906         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1907
1908         if (ifp->if_flags & IFF_PROMISC) {
1909                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1910                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1911         } else if (ifp->if_flags & IFF_ALLMULTI) {
1912                 reg_rctl |= IXGBE_FCTRL_MPE;
1913                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1914                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1915         }
1916         return;
1917 }
1918
1919
1920 /*********************************************************************
1921  *  Multicast Update
1922  *
1923  *  This routine is called whenever multicast address list is updated.
1924  *
1925  **********************************************************************/
1926 #define IXGBE_RAR_ENTRIES 16
1927
1928 static void
1929 ixgbe_set_multi(struct adapter *adapter)
1930 {
1931         u32     fctrl;
1932         u8      *mta;
1933         u8      *update_ptr;
1934         struct  ifmultiaddr *ifma;
1935         int     mcnt = 0;
1936         struct ifnet   *ifp = adapter->ifp;
1937
1938         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1939
1940         mta = adapter->mta;
1941         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1942             MAX_NUM_MULTICAST_ADDRESSES);
1943
1944         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1945         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1946         if (ifp->if_flags & IFF_PROMISC)
1947                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1948         else if (ifp->if_flags & IFF_ALLMULTI) {
1949                 fctrl |= IXGBE_FCTRL_MPE;
1950                 fctrl &= ~IXGBE_FCTRL_UPE;
1951         } else
1952                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1953
1954         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1955
1956         spin_lock(&adapter->mcast_spin);
1957         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1958                 if (ifma->ifma_addr->sa_family != AF_LINK)
1959                         continue;
1960                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1961                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1962                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1963                 mcnt++;
1964         }
1965         spin_unlock(&adapter->mcast_spin);
1966
1967         update_ptr = mta;
1968         ixgbe_update_mc_addr_list(&adapter->hw,
1969             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1970
1971         return;
1972 }
1973
1974 /*
1975  * This is an iterator function now needed by the multicast
1976  * shared code. It simply feeds the shared code routine the
1977  * addresses in the array of ixgbe_set_multi() one by one.
1978  */
1979 static u8 *
1980 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1981 {
1982         u8 *addr = *update_ptr;
1983         u8 *newptr;
1984         *vmdq = 0;
1985
1986         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1987         *update_ptr = newptr;
1988         return addr;
1989 }
1990
1991
1992 /*********************************************************************
1993  *  Timer routine
1994  *
1995  *  This routine checks for link status,updates statistics,
1996  *  and runs the watchdog check.
1997  *
1998  **********************************************************************/
1999
2000 static void
2001 ixgbe_local_timer(void *arg)
2002 {
2003         struct adapter  *adapter = arg;
2004         device_t        dev = adapter->dev;
2005         struct ifnet    *ifp = adapter->ifp;
2006         struct ix_queue *que = adapter->queues;
2007         struct tx_ring  *txr = adapter->tx_rings;
2008         int             hung, busy, paused;
2009
2010         lockmgr(&adapter->core_lock, LK_EXCLUSIVE);
2011         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2012         hung = busy = paused = 0;
2013
2014         /* Check for pluggable optics */
2015         if (adapter->sfp_probe)
2016                 if (!ixgbe_sfp_probe(adapter))
2017                         goto out; /* Nothing to do */
2018
2019         ixgbe_update_link_status(adapter);
2020         ixgbe_update_stats_counters(adapter);
2021
2022         /*
2023          * If the interface has been paused
2024          * then don't do the watchdog check
2025          */
2026         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2027                 paused = 1;
2028
2029         /*
2030         ** Check the TX queues status
2031         **      - central locked handling of OACTIVE
2032         **      - watchdog only if all queues show hung
2033         */
2034         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2035                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2036                     (paused == 0))
2037                         ++hung;
2038                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2039                         ++busy;
2040                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2041                         taskqueue_enqueue(que->tq, &que->que_task);
2042         }
2043         /* Only truely watchdog if all queues show hung */
2044         if (hung == adapter->num_queues)
2045                 goto watchdog;
2046         /* Only turn off the stack flow when ALL are depleted */
2047         if (busy == adapter->num_queues)
2048                 ifp->if_flags |= IFF_OACTIVE;
2049         else if ((ifp->if_flags & IFF_OACTIVE) &&
2050             (busy < adapter->num_queues))
2051                 ifp->if_flags &= ~IFF_OACTIVE;
2052
2053 out:
2054         ixgbe_rearm_queues(adapter, adapter->que_mask);
2055         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2056         lockmgr(&adapter->core_lock, LK_RELEASE);
2057         return;
2058
2059 watchdog:
2060         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2061         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2062             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2063             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2064         device_printf(dev,"TX(%d) desc avail = %d,"
2065             "Next TX to Clean = %d\n",
2066             txr->me, txr->tx_avail, txr->next_to_clean);
2067         adapter->ifp->if_flags &= ~IFF_RUNNING;
2068         adapter->watchdog_events++;
2069         ixgbe_init_locked(adapter);
2070
2071         lockmgr(&adapter->core_lock, LK_RELEASE);
2072 }
2073
2074 /*
2075 ** Note: this routine updates the OS on the link state
2076 **      the real check of the hardware only happens with
2077 **      a link interrupt.
2078 */
2079 static void
2080 ixgbe_update_link_status(struct adapter *adapter)
2081 {
2082         struct ifnet    *ifp = adapter->ifp;
2083         struct tx_ring *txr = adapter->tx_rings;
2084         device_t dev = adapter->dev;
2085
2086
2087         if (adapter->link_up){
2088                 if (adapter->link_active == FALSE) {
2089                         if (bootverbose)
2090                                 device_printf(dev,"Link is up %d Gbps %s \n",
2091                                     ((adapter->link_speed == 128)? 10:1),
2092                                     "Full Duplex");
2093                         adapter->link_active = TRUE;
2094                         ifp->if_link_state = LINK_STATE_UP;
2095                         if_link_state_change(ifp);
2096                 }
2097         } else { /* Link down */
2098                 if (adapter->link_active == TRUE) {
2099                         if (bootverbose)
2100                                 device_printf(dev,"Link is Down\n");
2101                         ifp->if_link_state = LINK_STATE_DOWN;
2102                         if_link_state_change(ifp);
2103                         adapter->link_active = FALSE;
2104                         for (int i = 0; i < adapter->num_queues;
2105                             i++, txr++)
2106                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2107                 }
2108         }
2109
2110         return;
2111 }
2112
2113
2114 /*********************************************************************
2115  *
2116  *  This routine disables all traffic on the adapter by issuing a
2117  *  global reset on the MAC and deallocates TX/RX buffers.
2118  *
2119  **********************************************************************/
2120
2121 static void
2122 ixgbe_stop(void *arg)
2123 {
2124         struct ifnet   *ifp;
2125         struct adapter *adapter = arg;
2126         struct ixgbe_hw *hw = &adapter->hw;
2127         ifp = adapter->ifp;
2128
2129         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2130
2131         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2132         ixgbe_disable_intr(adapter);
2133         callout_stop(&adapter->timer);
2134
2135         /* Let the stack know...*/
2136         ifp->if_flags &= ~IFF_RUNNING;
2137         ifp->if_flags |= IFF_OACTIVE;
2138
2139         ixgbe_reset_hw(hw);
2140         hw->adapter_stopped = FALSE;
2141         ixgbe_stop_adapter(hw);
2142         /* Turn off the laser */
2143         if (hw->phy.multispeed_fiber)
2144                 ixgbe_disable_tx_laser(hw);
2145
2146         /* reprogram the RAR[0] in case user changed it. */
2147         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2148
2149         return;
2150 }
2151
2152
2153 /*********************************************************************
2154  *
2155  *  Determine hardware revision.
2156  *
2157  **********************************************************************/
2158 static void
2159 ixgbe_identify_hardware(struct adapter *adapter)
2160 {
2161         device_t        dev = adapter->dev;
2162         struct ixgbe_hw *hw = &adapter->hw;
2163
2164         /* Save off the information about this board */
2165         hw->vendor_id = pci_get_vendor(dev);
2166         hw->device_id = pci_get_device(dev);
2167         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2168         hw->subsystem_vendor_id =
2169             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2170         hw->subsystem_device_id =
2171             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2172
2173         /* We need this here to set the num_segs below */
2174         ixgbe_set_mac_type(hw);
2175
2176         /* Pick up the 82599 and VF settings */
2177         if (hw->mac.type != ixgbe_mac_82598EB) {
2178                 hw->phy.smart_speed = ixgbe_smart_speed;
2179                 adapter->num_segs = IXGBE_82599_SCATTER;
2180         } else
2181                 adapter->num_segs = IXGBE_82598_SCATTER;
2182
2183         return;
2184 }
2185
2186 /*********************************************************************
2187  *
2188  *  Determine optic type
2189  *
2190  **********************************************************************/
2191 static void
2192 ixgbe_setup_optics(struct adapter *adapter)
2193 {
2194         struct ixgbe_hw *hw = &adapter->hw;
2195         int             layer;
2196
2197         layer = ixgbe_get_supported_physical_layer(hw);
2198
2199         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2200                 adapter->optics = IFM_10G_T;
2201                 return;
2202         }
2203
2204         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2205                 adapter->optics = IFM_1000_T;
2206                 return;
2207         }
2208
2209         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2210             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2211                 adapter->optics = IFM_10G_LR;
2212                 return;
2213         }
2214
2215         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2216                 adapter->optics = IFM_10G_SR;
2217                 return;
2218         }
2219
2220         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2221                 adapter->optics = IFM_10G_TWINAX;
2222                 return;
2223         }
2224
2225         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2226             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2227                 adapter->optics = IFM_10G_CX4;
2228                 return;
2229         }
2230
2231         /* If we get here just set the default */
2232         adapter->optics = IFM_ETHER | IFM_AUTO;
2233         return;
2234 }
2235
2236 /*********************************************************************
2237  *
2238  *  Setup the Legacy or MSI Interrupt handler
2239  *
2240  **********************************************************************/
2241 static int
2242 ixgbe_allocate_legacy(struct adapter *adapter)
2243 {
2244         device_t dev = adapter->dev;
2245         struct          ix_queue *que = adapter->queues;
2246         int error, rid = 0;
2247
2248         /* MSI RID at 1 */
2249         if (adapter->msix == 1)
2250                 rid = 1;
2251
2252         /* We allocate a single interrupt resource */
2253         adapter->res = bus_alloc_resource_any(dev,
2254             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2255         if (adapter->res == NULL) {
2256                 device_printf(dev, "Unable to allocate bus resource: "
2257                     "interrupt\n");
2258                 return (ENXIO);
2259         }
2260
2261         /*
2262          * Try allocating a fast interrupt and the associated deferred
2263          * processing contexts.
2264          */
2265         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2266         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2267             taskqueue_thread_enqueue, &que->tq);
2268         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2269             device_get_nameunit(adapter->dev));
2270
2271         /* Tasklets for Link, SFP and Multispeed Fiber */
2272         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2273         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2274         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2275 #ifdef IXGBE_FDIR
2276         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2277 #endif
2278         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2279             taskqueue_thread_enqueue, &adapter->tq);
2280         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2281             device_get_nameunit(adapter->dev));
2282
2283         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2284             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2285                 device_printf(dev, "Failed to register fast interrupt "
2286                     "handler: %d\n", error);
2287                 taskqueue_free(que->tq);
2288                 taskqueue_free(adapter->tq);
2289                 que->tq = NULL;
2290                 adapter->tq = NULL;
2291                 return (error);
2292         }
2293         /* For simplicity in the handlers */
2294         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2295
2296         return (0);
2297 }
2298
2299
2300 /*********************************************************************
2301  *
2302  *  Setup MSIX Interrupt resources and handlers
2303  *
2304  **********************************************************************/
2305 static int
2306 ixgbe_allocate_msix(struct adapter *adapter)
2307 {
2308         device_t        dev = adapter->dev;
2309         struct          ix_queue *que = adapter->queues;
2310         int             error, rid, vector = 0;
2311
2312         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2313                 rid = vector + 1;
2314                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2315                     RF_SHAREABLE | RF_ACTIVE);
2316                 if (que->res == NULL) {
2317                         device_printf(dev,"Unable to allocate"
2318                             " bus resource: que interrupt [%d]\n", vector);
2319                         return (ENXIO);
2320                 }
2321                 /* Set the handler function */
2322                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2323                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2324                 if (error) {
2325                         que->res = NULL;
2326                         device_printf(dev, "Failed to register QUE handler");
2327                         return (error);
2328                 }
2329 #if 0 /* __FreeBSD_version >= 800504 */
2330                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2331 #endif
2332                 que->msix = vector;
2333                 adapter->que_mask |= (u64)(1 << que->msix);
2334                 /*
2335                 ** Bind the msix vector, and thus the
2336                 ** ring to the corresponding cpu.
2337                 */
2338 #if 0 /* XXX */
2339                 if (adapter->num_queues > 1)
2340                         bus_bind_intr(dev, que->res, i);
2341 #endif
2342
2343                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2344                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2345                     taskqueue_thread_enqueue, &que->tq);
2346                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2347                     device_get_nameunit(adapter->dev));
2348         }
2349
2350         /* and Link */
2351         rid = vector + 1;
2352         adapter->res = bus_alloc_resource_any(dev,
2353             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2354         if (!adapter->res) {
2355                 device_printf(dev,"Unable to allocate"
2356             " bus resource: Link interrupt [%d]\n", rid);
2357                 return (ENXIO);
2358         }
2359         /* Set the link handler function */
2360         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2361             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2362         if (error) {
2363                 adapter->res = NULL;
2364                 device_printf(dev, "Failed to register LINK handler");
2365                 return (error);
2366         }
2367 #if 0 /* __FreeBSD_version >= 800504 */
2368         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2369 #endif
2370         adapter->linkvec = vector;
2371         /* Tasklets for Link, SFP and Multispeed Fiber */
2372         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2373         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2374         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2375 #ifdef IXGBE_FDIR
2376         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2377 #endif
2378         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2379             taskqueue_thread_enqueue, &adapter->tq);
2380         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2381             device_get_nameunit(adapter->dev));
2382
2383         return (0);
2384 }
2385
2386 #if 0   /* HAVE_MSIX */
2387 /*
2388  * Setup Either MSI/X or MSI
2389  */
2390 static int
2391 ixgbe_setup_msix(struct adapter *adapter)
2392 {
2393         device_t dev = adapter->dev;
2394         int rid, want, queues, msgs;
2395
2396         /* Override by tuneable */
2397         if (ixgbe_enable_msix == 0)
2398                 goto msi;
2399
2400         /* First try MSI/X */
2401         rid = PCIR_BAR(MSIX_82598_BAR);
2402         adapter->msix_mem = bus_alloc_resource_any(dev,
2403             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2404         if (!adapter->msix_mem) {
2405                 rid += 4;       /* 82599 maps in higher BAR */
2406                 adapter->msix_mem = bus_alloc_resource_any(dev,
2407                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2408         }
2409         if (!adapter->msix_mem) {
2410                 /* May not be enabled */
2411                 device_printf(adapter->dev,
2412                     "Unable to map MSIX table \n");
2413                 goto msi;
2414         }
2415
2416         msgs = pci_msix_count(dev);
2417         if (msgs == 0) { /* system has msix disabled */
2418                 bus_release_resource(dev, SYS_RES_MEMORY,
2419                     rid, adapter->msix_mem);
2420                 adapter->msix_mem = NULL;
2421                 goto msi;
2422         }
2423
2424         /* Figure out a reasonable auto config value */
2425         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2426
2427         if (ixgbe_num_queues != 0)
2428                 queues = ixgbe_num_queues;
2429         /* Set max queues to 8 when autoconfiguring */
2430         else if ((ixgbe_num_queues == 0) && (queues > 8))
2431                 queues = 8;
2432
2433         /*
2434         ** Want one vector (RX/TX pair) per queue
2435         ** plus an additional for Link.
2436         */
2437         want = queues + 1;
2438         if (msgs >= want)
2439                 msgs = want;
2440         else {
2441                 device_printf(adapter->dev,
2442                     "MSIX Configuration Problem, "
2443                     "%d vectors but %d queues wanted!\n",
2444                     msgs, want);
2445                 return (0); /* Will go to Legacy setup */
2446         }
2447         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2448                 device_printf(adapter->dev,
2449                     "Using MSIX interrupts with %d vectors\n", msgs);
2450                 adapter->num_queues = queues;
2451                 return (msgs);
2452         }
2453 msi:
2454         msgs = pci_msi_count(dev);
2455         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2456                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2457         else
2458                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2459         return (msgs);
2460 }
2461 #endif
2462
2463
2464 static int
2465 ixgbe_allocate_pci_resources(struct adapter *adapter)
2466 {
2467         int             rid;
2468         device_t        dev = adapter->dev;
2469
2470         rid = PCIR_BAR(0);
2471         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2472             &rid, RF_ACTIVE);
2473
2474         if (!(adapter->pci_mem)) {
2475                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2476                 return (ENXIO);
2477         }
2478
2479         adapter->osdep.mem_bus_space_tag =
2480                 rman_get_bustag(adapter->pci_mem);
2481         adapter->osdep.mem_bus_space_handle =
2482                 rman_get_bushandle(adapter->pci_mem);
2483         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2484
2485         /* Legacy defaults */
2486         adapter->num_queues = 1;
2487         adapter->hw.back = &adapter->osdep;
2488
2489         /*
2490         ** Now setup MSI or MSI/X, should
2491         ** return us the number of supported
2492         ** vectors. (Will be 1 for MSI)
2493         */
2494 #if 0   /* HAVE_MSIX */
2495         adapter->msix = ixgbe_setup_msix(adapter);
2496 #endif
2497         return (0);
2498 }
2499
2500 static void
2501 ixgbe_free_pci_resources(struct adapter * adapter)
2502 {
2503         struct          ix_queue *que = adapter->queues;
2504         device_t        dev = adapter->dev;
2505         int             rid, memrid;
2506
2507         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2508                 memrid = PCIR_BAR(MSIX_82598_BAR);
2509         else
2510                 memrid = PCIR_BAR(MSIX_82599_BAR);
2511
2512         /*
2513         ** There is a slight possibility of a failure mode
2514         ** in attach that will result in entering this function
2515         ** before interrupt resources have been initialized, and
2516         ** in that case we do not want to execute the loops below
2517         ** We can detect this reliably by the state of the adapter
2518         ** res pointer.
2519         */
2520         if (adapter->res == NULL)
2521                 goto mem;
2522
2523         /*
2524         **  Release all msix queue resources:
2525         */
2526         for (int i = 0; i < adapter->num_queues; i++, que++) {
2527                 rid = que->msix + 1;
2528                 if (que->tag != NULL) {
2529                         bus_teardown_intr(dev, que->res, que->tag);
2530                         que->tag = NULL;
2531                 }
2532                 if (que->res != NULL)
2533                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2534         }
2535
2536
2537         /* Clean the Legacy or Link interrupt last */
2538         if (adapter->linkvec) /* we are doing MSIX */
2539                 rid = adapter->linkvec + 1;
2540         else
2541                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2542
2543         if (adapter->tag != NULL) {
2544                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2545                 adapter->tag = NULL;
2546         }
2547         if (adapter->res != NULL)
2548                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2549
2550 mem:
2551         if (adapter->msix)
2552                 pci_release_msi(dev);
2553
2554         if (adapter->msix_mem != NULL)
2555                 bus_release_resource(dev, SYS_RES_MEMORY,
2556                     memrid, adapter->msix_mem);
2557
2558         if (adapter->pci_mem != NULL)
2559                 bus_release_resource(dev, SYS_RES_MEMORY,
2560                     PCIR_BAR(0), adapter->pci_mem);
2561
2562         return;
2563 }
2564
2565 /*********************************************************************
2566  *
2567  *  Setup networking device structure and register an interface.
2568  *
2569  **********************************************************************/
2570 static int
2571 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2572 {
2573         struct ixgbe_hw *hw = &adapter->hw;
2574         struct ifnet   *ifp;
2575
2576         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2577
2578         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2579         if (ifp == NULL) {
2580                 device_printf(dev, "can not allocate ifnet structure\n");
2581                 return (-1);
2582         }
2583         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2584         ifp->if_baudrate = 1000000000;
2585         ifp->if_init = ixgbe_init;
2586         ifp->if_softc = adapter;
2587         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2588         ifp->if_ioctl = ixgbe_ioctl;
2589         ifp->if_start = ixgbe_start;
2590 #if 0 /* __FreeBSD_version >= 800000 */
2591         ifp->if_transmit = ixgbe_mq_start;
2592         ifp->if_qflush = ixgbe_qflush;
2593 #endif
2594         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2595
2596         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2597
2598         adapter->max_frame_size =
2599             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2600
2601         /*
2602          * Tell the upper layer(s) we support long frames.
2603          */
2604         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2605
2606 #if 0 /* NET_TSO */
2607         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2608 #endif
2609         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2610         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2611         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2612 #if 0 /* NET_TSO */
2613                              |  IFCAP_VLAN_HWTSO
2614 #endif
2615                              |  IFCAP_VLAN_MTU;
2616         ifp->if_capenable = ifp->if_capabilities;
2617
2618         /* Don't enable LRO by default */
2619 #if 0 /* NET_LRO */
2620         ifp->if_capabilities |= IFCAP_LRO;
2621 #endif
2622
2623         /*
2624         ** Don't turn this on by default, if vlans are
2625         ** created on another pseudo device (eg. lagg)
2626         ** then vlan events are not passed thru, breaking
2627         ** operation, but with HW FILTER off it works. If
2628         ** using vlans directly on the ixgbe driver you can
2629         ** enable this and get full hardware tag filtering.
2630         */
2631         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2632
2633         /*
2634          * Specify the media types supported by this adapter and register
2635          * callbacks to update media and link information
2636          */
2637         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2638                      ixgbe_media_status);
2639         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2640         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2641         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2642                 ifmedia_add(&adapter->media,
2643                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2644                 ifmedia_add(&adapter->media,
2645                     IFM_ETHER | IFM_1000_T, 0, NULL);
2646         }
2647         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2648         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2649
2650         return (0);
2651 }
2652
2653 static void
2654 ixgbe_config_link(struct adapter *adapter)
2655 {
2656         struct ixgbe_hw *hw = &adapter->hw;
2657         u32     autoneg, err = 0;
2658         bool    sfp, negotiate;
2659
2660         sfp = ixgbe_is_sfp(hw);
2661
2662         if (sfp) {
2663                 if (hw->phy.multispeed_fiber) {
2664                         hw->mac.ops.setup_sfp(hw);
2665                         ixgbe_enable_tx_laser(hw);
2666                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2667                 } else
2668                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2669         } else {
2670                 if (hw->mac.ops.check_link)
2671                         err = ixgbe_check_link(hw, &autoneg,
2672                             &adapter->link_up, FALSE);
2673                 if (err)
2674                         goto out;
2675                 autoneg = hw->phy.autoneg_advertised;
2676                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2677                         err  = hw->mac.ops.get_link_capabilities(hw,
2678                             &autoneg, &negotiate);
2679                 if (err)
2680                         goto out;
2681                 if (hw->mac.ops.setup_link)
2682                         err = hw->mac.ops.setup_link(hw, autoneg,
2683                             negotiate, adapter->link_up);
2684         }
2685 out:
2686         return;
2687 }
2688
2689 /********************************************************************
2690  * Manage DMA'able memory.
2691  *******************************************************************/
2692 static void
2693 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2694 {
2695         if (error)
2696                 return;
2697         *(bus_addr_t *) arg = segs->ds_addr;
2698         return;
2699 }
2700
2701 static int
2702 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2703                 struct ixgbe_dma_alloc *dma, int mapflags)
2704 {
2705         device_t dev = adapter->dev;
2706         int             r;
2707
2708         r = bus_dma_tag_create(NULL,    /* parent */
2709                                DBA_ALIGN, 0,    /* alignment, bounds */
2710                                BUS_SPACE_MAXADDR,       /* lowaddr */
2711                                BUS_SPACE_MAXADDR,       /* highaddr */
2712                                NULL, NULL,      /* filter, filterarg */
2713                                size,    /* maxsize */
2714                                1,       /* nsegments */
2715                                size,    /* maxsegsize */
2716                                BUS_DMA_ALLOCNOW,        /* flags */
2717                                &dma->dma_tag);
2718         if (r != 0) {
2719                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2720                        "error %u\n", r);
2721                 goto fail_0;
2722         }
2723         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2724                              BUS_DMA_NOWAIT, &dma->dma_map);
2725         if (r != 0) {
2726                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2727                        "error %u\n", r);
2728                 goto fail_1;
2729         }
2730         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2731                             size,
2732                             ixgbe_dmamap_cb,
2733                             &dma->dma_paddr,
2734                             mapflags | BUS_DMA_NOWAIT);
2735         if (r != 0) {
2736                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2737                        "error %u\n", r);
2738                 goto fail_2;
2739         }
2740         dma->dma_size = size;
2741         return (0);
2742 fail_2:
2743         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2744 fail_1:
2745         bus_dma_tag_destroy(dma->dma_tag);
2746 fail_0:
2747         dma->dma_map = NULL;
2748         dma->dma_tag = NULL;
2749         return (r);
2750 }
2751
2752 static void
2753 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2754 {
2755         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2756             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2757         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2758         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2759         bus_dma_tag_destroy(dma->dma_tag);
2760 }
2761
2762
2763 /*********************************************************************
2764  *
2765  *  Allocate memory for the transmit and receive rings, and then
2766  *  the descriptors associated with each, called only once at attach.
2767  *
2768  **********************************************************************/
2769 static int
2770 ixgbe_allocate_queues(struct adapter *adapter)
2771 {
2772         device_t        dev = adapter->dev;
2773         struct ix_queue *que;
2774         struct tx_ring  *txr;
2775         struct rx_ring  *rxr;
2776         int rsize, tsize, error = IXGBE_SUCCESS;
2777         int txconf = 0, rxconf = 0;
2778
2779         /* First allocate the top level queue structs */
2780         if (!(adapter->queues =
2781             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2782             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2783                 device_printf(dev, "Unable to allocate queue memory\n");
2784                 error = ENOMEM;
2785                 goto fail;
2786         }
2787
2788         /* First allocate the TX ring struct memory */
2789         if (!(adapter->tx_rings =
2790             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2791             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2792                 device_printf(dev, "Unable to allocate TX ring memory\n");
2793                 error = ENOMEM;
2794                 goto tx_fail;
2795         }
2796
2797         /* Next allocate the RX */
2798         if (!(adapter->rx_rings =
2799             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2800             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2801                 device_printf(dev, "Unable to allocate RX ring memory\n");
2802                 error = ENOMEM;
2803                 goto rx_fail;
2804         }
2805
2806         /* For the ring itself */
2807         tsize = roundup2(adapter->num_tx_desc *
2808             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2809
2810         /*
2811          * Now set up the TX queues, txconf is needed to handle the
2812          * possibility that things fail midcourse and we need to
2813          * undo memory gracefully
2814          */
2815         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2816                 /* Set up some basics */
2817                 txr = &adapter->tx_rings[i];
2818                 txr->adapter = adapter;
2819                 txr->me = i;
2820
2821                 /* Initialize the TX side lock */
2822                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2823                     device_get_nameunit(dev), txr->me);
2824                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2825
2826                 if (ixgbe_dma_malloc(adapter, tsize,
2827                         &txr->txdma, BUS_DMA_NOWAIT)) {
2828                         device_printf(dev,
2829                             "Unable to allocate TX Descriptor memory\n");
2830                         error = ENOMEM;
2831                         goto err_tx_desc;
2832                 }
2833                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2834                 bzero((void *)txr->tx_base, tsize);
2835
2836                 /* Now allocate transmit buffers for the ring */
2837                 if (ixgbe_allocate_transmit_buffers(txr)) {
2838                         device_printf(dev,
2839                             "Critical Failure setting up transmit buffers\n");
2840                         error = ENOMEM;
2841                         goto err_tx_desc;
2842                 }
2843 #if 0 /* __FreeBSD_version >= 800000 */
2844                 /* Allocate a buf ring */
2845                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2846                     M_WAITOK, &txr->tx_mtx);
2847                 if (txr->br == NULL) {
2848                         device_printf(dev,
2849                             "Critical Failure setting up buf ring\n");
2850                         error = ENOMEM;
2851                         goto err_tx_desc;
2852                 }
2853 #endif
2854         }
2855
2856         /*
2857          * Next the RX queues...
2858          */
2859         rsize = roundup2(adapter->num_rx_desc *
2860             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2861         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2862                 rxr = &adapter->rx_rings[i];
2863                 /* Set up some basics */
2864                 rxr->adapter = adapter;
2865                 rxr->me = i;
2866
2867                 /* Initialize the RX side lock */
2868                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2869                     device_get_nameunit(dev), rxr->me);
2870                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2871
2872                 if (ixgbe_dma_malloc(adapter, rsize,
2873                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2874                         device_printf(dev,
2875                             "Unable to allocate RxDescriptor memory\n");
2876                         error = ENOMEM;
2877                         goto err_rx_desc;
2878                 }
2879                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2880                 bzero((void *)rxr->rx_base, rsize);
2881
2882                 /* Allocate receive buffers for the ring*/
2883                 if (ixgbe_allocate_receive_buffers(rxr)) {
2884                         device_printf(dev,
2885                             "Critical Failure setting up receive buffers\n");
2886                         error = ENOMEM;
2887                         goto err_rx_desc;
2888                 }
2889         }
2890
2891         /*
2892         ** Finally set up the queue holding structs
2893         */
2894         for (int i = 0; i < adapter->num_queues; i++) {
2895                 que = &adapter->queues[i];
2896                 que->adapter = adapter;
2897                 que->txr = &adapter->tx_rings[i];
2898                 que->rxr = &adapter->rx_rings[i];
2899         }
2900
2901         return (0);
2902
2903 err_rx_desc:
2904         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2905                 ixgbe_dma_free(adapter, &rxr->rxdma);
2906 err_tx_desc:
2907         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2908                 ixgbe_dma_free(adapter, &txr->txdma);
2909         kfree(adapter->rx_rings, M_DEVBUF);
2910 rx_fail:
2911         kfree(adapter->tx_rings, M_DEVBUF);
2912 tx_fail:
2913         kfree(adapter->queues, M_DEVBUF);
2914 fail:
2915         return (error);
2916 }
2917
2918 /*********************************************************************
2919  *
2920  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2921  *  the information needed to transmit a packet on the wire. This is
2922  *  called only once at attach, setup is done every reset.
2923  *
2924  **********************************************************************/
2925 static int
2926 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2927 {
2928         struct adapter *adapter = txr->adapter;
2929         device_t dev = adapter->dev;
2930         struct ixgbe_tx_buf *txbuf;
2931         int error, i;
2932
2933         /*
2934          * Setup DMA descriptor areas.
2935          */
2936         if ((error = bus_dma_tag_create(
2937                                NULL,    /* parent */
2938                                1, 0,            /* alignment, bounds */
2939                                BUS_SPACE_MAXADDR,       /* lowaddr */
2940                                BUS_SPACE_MAXADDR,       /* highaddr */
2941                                NULL, NULL,              /* filter, filterarg */
2942                                IXGBE_TSO_SIZE,          /* maxsize */
2943                                adapter->num_segs,       /* nsegments */
2944                                PAGE_SIZE,               /* maxsegsize */
2945                                0,                       /* flags */
2946                                &txr->txtag))) {
2947                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2948                 goto fail;
2949         }
2950
2951         if (!(txr->tx_buffers =
2952             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2953             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2954                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2955                 error = ENOMEM;
2956                 goto fail;
2957         }
2958
2959         /* Create the descriptor buffer dma maps */
2960         txbuf = txr->tx_buffers;
2961         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2962                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2963                 if (error != 0) {
2964                         device_printf(dev, "Unable to create TX DMA map\n");
2965                         goto fail;
2966                 }
2967         }
2968
2969         return 0;
2970 fail:
2971         /* We free all, it handles case where we are in the middle */
2972         ixgbe_free_transmit_structures(adapter);
2973         return (error);
2974 }
2975
2976 /*********************************************************************
2977  *
2978  *  Initialize a transmit ring.
2979  *
2980  **********************************************************************/
2981 static void
2982 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2983 {
2984         struct adapter *adapter = txr->adapter;
2985         struct ixgbe_tx_buf *txbuf;
2986         int i;
2987 #ifdef DEV_NETMAP
2988         struct netmap_adapter *na = NA(adapter->ifp);
2989         struct netmap_slot *slot;
2990 #endif /* DEV_NETMAP */
2991
2992         /* Clear the old ring contents */
2993         IXGBE_TX_LOCK(txr);
2994 #ifdef DEV_NETMAP
2995         /*
2996          * (under lock): if in netmap mode, do some consistency
2997          * checks and set slot to entry 0 of the netmap ring.
2998          */
2999         slot = netmap_reset(na, NR_TX, txr->me, 0);
3000 #endif /* DEV_NETMAP */
3001         bzero((void *)txr->tx_base,
3002               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3003         /* Reset indices */
3004         txr->next_avail_desc = 0;
3005         txr->next_to_clean = 0;
3006
3007         /* Free any existing tx buffers. */
3008         txbuf = txr->tx_buffers;
3009         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3010                 if (txbuf->m_head != NULL) {
3011                         bus_dmamap_sync(txr->txtag, txbuf->map,
3012                             BUS_DMASYNC_POSTWRITE);
3013                         bus_dmamap_unload(txr->txtag, txbuf->map);
3014                         m_freem(txbuf->m_head);
3015                         txbuf->m_head = NULL;
3016                 }
3017 #ifdef DEV_NETMAP
3018                 /*
3019                  * In netmap mode, set the map for the packet buffer.
3020                  * NOTE: Some drivers (not this one) also need to set
3021                  * the physical buffer address in the NIC ring.
3022                  * Slots in the netmap ring (indexed by "si") are
3023                  * kring->nkr_hwofs positions "ahead" wrt the
3024                  * corresponding slot in the NIC ring. In some drivers
3025                  * (not here) nkr_hwofs can be negative. Function
3026                  * netmap_idx_n2k() handles wraparounds properly.
3027                  */
3028                 if (slot) {
3029                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3030                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3031                 }
3032 #endif /* DEV_NETMAP */
3033                 /* Clear the EOP index */
3034                 txbuf->eop_index = -1;
3035         }
3036
3037 #ifdef IXGBE_FDIR
3038         /* Set the rate at which we sample packets */
3039         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3040                 txr->atr_sample = atr_sample_rate;
3041 #endif
3042
3043         /* Set number of descriptors available */
3044         txr->tx_avail = adapter->num_tx_desc;
3045
3046         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3047             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3048         IXGBE_TX_UNLOCK(txr);
3049 }
3050
3051 /*********************************************************************
3052  *
3053  *  Initialize all transmit rings.
3054  *
3055  **********************************************************************/
3056 static int
3057 ixgbe_setup_transmit_structures(struct adapter *adapter)
3058 {
3059         struct tx_ring *txr = adapter->tx_rings;
3060
3061         for (int i = 0; i < adapter->num_queues; i++, txr++)
3062                 ixgbe_setup_transmit_ring(txr);
3063
3064         return (0);
3065 }
3066
3067 /*********************************************************************
3068  *
3069  *  Enable transmit unit.
3070  *
3071  **********************************************************************/
3072 static void
3073 ixgbe_initialize_transmit_units(struct adapter *adapter)
3074 {
3075         struct tx_ring  *txr = adapter->tx_rings;
3076         struct ixgbe_hw *hw = &adapter->hw;
3077
3078         /* Setup the Base and Length of the Tx Descriptor Ring */
3079
3080         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3081                 u64     tdba = txr->txdma.dma_paddr;
3082                 u32     txctrl;
3083
3084                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3085                        (tdba & 0x00000000ffffffffULL));
3086                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3087                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3088                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3089
3090                 /* Setup the HW Tx Head and Tail descriptor pointers */
3091                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3092                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3093
3094                 /* Setup Transmit Descriptor Cmd Settings */
3095                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3096                 txr->queue_status = IXGBE_QUEUE_IDLE;
3097
3098                 /* Disable Head Writeback */
3099                 switch (hw->mac.type) {
3100                 case ixgbe_mac_82598EB:
3101                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3102                         break;
3103                 case ixgbe_mac_82599EB:
3104                 case ixgbe_mac_X540:
3105                 default:
3106                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3107                         break;
3108                 }
3109                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
3110                 switch (hw->mac.type) {
3111                 case ixgbe_mac_82598EB:
3112                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3113                         break;
3114                 case ixgbe_mac_82599EB:
3115                 case ixgbe_mac_X540:
3116                 default:
3117                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3118                         break;
3119                 }
3120
3121         }
3122
3123         if (hw->mac.type != ixgbe_mac_82598EB) {
3124                 u32 dmatxctl, rttdcs;
3125                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3126                 dmatxctl |= IXGBE_DMATXCTL_TE;
3127                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3128                 /* Disable arbiter to set MTQC */
3129                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3130                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3131                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3132                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3133                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3134                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3135         }
3136
3137         return;
3138 }
3139
3140 /*********************************************************************
3141  *
3142  *  Free all transmit rings.
3143  *
3144  **********************************************************************/
3145 static void
3146 ixgbe_free_transmit_structures(struct adapter *adapter)
3147 {
3148         struct tx_ring *txr = adapter->tx_rings;
3149
3150         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3151                 IXGBE_TX_LOCK(txr);
3152                 ixgbe_free_transmit_buffers(txr);
3153                 ixgbe_dma_free(adapter, &txr->txdma);
3154                 IXGBE_TX_UNLOCK(txr);
3155                 IXGBE_TX_LOCK_DESTROY(txr);
3156         }
3157         kfree(adapter->tx_rings, M_DEVBUF);
3158 }
3159
3160 /*********************************************************************
3161  *
3162  *  Free transmit ring related data structures.
3163  *
3164  **********************************************************************/
3165 static void
3166 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3167 {
3168         struct adapter *adapter = txr->adapter;
3169         struct ixgbe_tx_buf *tx_buffer;
3170         int             i;
3171
3172         INIT_DEBUGOUT("free_transmit_ring: begin");
3173
3174         if (txr->tx_buffers == NULL)
3175                 return;
3176
3177         tx_buffer = txr->tx_buffers;
3178         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3179                 if (tx_buffer->m_head != NULL) {
3180                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3181                             BUS_DMASYNC_POSTWRITE);
3182                         bus_dmamap_unload(txr->txtag,
3183                             tx_buffer->map);
3184                         m_freem(tx_buffer->m_head);
3185                         tx_buffer->m_head = NULL;
3186                         if (tx_buffer->map != NULL) {
3187                                 bus_dmamap_destroy(txr->txtag,
3188                                     tx_buffer->map);
3189                                 tx_buffer->map = NULL;
3190                         }
3191                 } else if (tx_buffer->map != NULL) {
3192                         bus_dmamap_unload(txr->txtag,
3193                             tx_buffer->map);
3194                         bus_dmamap_destroy(txr->txtag,
3195                             tx_buffer->map);
3196                         tx_buffer->map = NULL;
3197                 }
3198         }
3199 #if 0 /* __FreeBSD_version >= 800000 */
3200         if (txr->br != NULL)
3201                 buf_ring_free(txr->br, M_DEVBUF);
3202 #endif
3203         if (txr->tx_buffers != NULL) {
3204                 kfree(txr->tx_buffers, M_DEVBUF);
3205                 txr->tx_buffers = NULL;
3206         }
3207         if (txr->txtag != NULL) {
3208                 bus_dma_tag_destroy(txr->txtag);
3209                 txr->txtag = NULL;
3210         }
3211         return;
3212 }
3213
3214 /*********************************************************************
3215  *
3216  *  Advanced Context Descriptor setup for VLAN or CSUM
3217  *
3218  **********************************************************************/
3219
3220 static bool
3221 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3222 {
3223         struct adapter *adapter = txr->adapter;
3224         struct ixgbe_adv_tx_context_desc *TXD;
3225         struct ixgbe_tx_buf        *tx_buffer;
3226         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3227         struct ether_vlan_header *eh;
3228         struct ip *ip;
3229         struct ip6_hdr *ip6;
3230         int  ehdrlen, ip_hlen = 0;
3231         u16     etype;
3232         u8      ipproto = 0;
3233         bool    offload = TRUE;
3234         int ctxd = txr->next_avail_desc;
3235 #ifdef NET_VLAN
3236         u16 vtag = 0;
3237 #endif
3238
3239
3240         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3241                 offload = FALSE;
3242
3243         tx_buffer = &txr->tx_buffers[ctxd];
3244         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3245
3246         /*
3247         ** In advanced descriptors the vlan tag must
3248         ** be placed into the descriptor itself.
3249         */
3250 #ifdef NET_VLAN
3251         if (mp->m_flags & M_VLANTAG) {
3252                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3253                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3254         } else if (offload == FALSE)
3255                 return FALSE;
3256 #endif
3257
3258         /*
3259          * Determine where frame payload starts.
3260          * Jump over vlan headers if already present,
3261          * helpful for QinQ too.
3262          */
3263         eh = mtod(mp, struct ether_vlan_header *);
3264         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3265                 etype = ntohs(eh->evl_proto);
3266                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3267         } else {
3268                 etype = ntohs(eh->evl_encap_proto);
3269                 ehdrlen = ETHER_HDR_LEN;
3270         }
3271
3272         /* Set the ether header length */
3273         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3274
3275         switch (etype) {
3276                 case ETHERTYPE_IP:
3277                         ip = (struct ip *)(mp->m_data + ehdrlen);
3278                         ip_hlen = ip->ip_hl << 2;
3279                         ipproto = ip->ip_p;
3280                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3281                         break;
3282                 case ETHERTYPE_IPV6:
3283                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3284                         ip_hlen = sizeof(struct ip6_hdr);
3285                         /* XXX-BZ this will go badly in case of ext hdrs. */
3286                         ipproto = ip6->ip6_nxt;
3287                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3288                         break;
3289                 default:
3290                         offload = FALSE;
3291                         break;
3292         }
3293
3294         vlan_macip_lens |= ip_hlen;
3295         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3296
3297         switch (ipproto) {
3298                 case IPPROTO_TCP:
3299                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3300                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3301                         break;
3302
3303                 case IPPROTO_UDP:
3304                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3305                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3306                         break;
3307
3308 #if 0
3309                 case IPPROTO_SCTP:
3310                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3311                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3312                         break;
3313 #endif
3314                 default:
3315                         offload = FALSE;
3316                         break;
3317         }
3318
3319         /* Now copy bits into descriptor */
3320         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3321         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3322         TXD->seqnum_seed = htole32(0);
3323         TXD->mss_l4len_idx = htole32(0);
3324
3325         tx_buffer->m_head = NULL;
3326         tx_buffer->eop_index = -1;
3327
3328         /* We've consumed the first desc, adjust counters */
3329         if (++ctxd == adapter->num_tx_desc)
3330                 ctxd = 0;
3331         txr->next_avail_desc = ctxd;
3332         --txr->tx_avail;
3333
3334         return (offload);
3335 }
3336
3337 /**********************************************************************
3338  *
3339  *  Setup work for hardware segmentation offload (TSO) on
3340  *  adapters using advanced tx descriptors
3341  *
3342  **********************************************************************/
3343 #if 0   /* NET_TSO */
3344 static bool
3345 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3346     u32 *olinfo_status)
3347 {
3348         struct adapter *adapter = txr->adapter;
3349         struct ixgbe_adv_tx_context_desc *TXD;
3350         struct ixgbe_tx_buf        *tx_buffer;
3351 #ifdef NET_VLAN
3352         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3353         u16 vtag = 0, eh_type;
3354 #else
3355         u16 eh_type;
3356         u32 type_tucmd_mlhl = 0;
3357 #endif
3358         u32 mss_l4len_idx = 0, len;
3359         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3360         struct ether_vlan_header *eh;
3361 #ifdef INET6
3362         struct ip6_hdr *ip6;
3363 #endif
3364 #ifdef INET
3365         struct ip *ip;
3366 #endif
3367         struct tcphdr *th;
3368
3369
3370         /*
3371          * Determine where frame payload starts.
3372          * Jump over vlan headers if already present
3373          */
3374         eh = mtod(mp, struct ether_vlan_header *);
3375         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3376                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3377                 eh_type = eh->evl_proto;
3378         } else {
3379                 ehdrlen = ETHER_HDR_LEN;
3380                 eh_type = eh->evl_encap_proto;
3381         }
3382
3383         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3384         len = ehdrlen + sizeof(struct tcphdr);
3385         switch (ntohs(eh_type)) {
3386 #ifdef INET6
3387         case ETHERTYPE_IPV6:
3388                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3389                         return FALSE;
3390                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3391                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3392                 if (ip6->ip6_nxt != IPPROTO_TCP)
3393                         return FALSE;
3394                 ip_hlen = sizeof(struct ip6_hdr);
3395                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3396                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3397                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3398                 break;
3399 #endif
3400 #ifdef INET
3401         case ETHERTYPE_IP:
3402                 if (mp->m_len < len + sizeof(struct ip))
3403                         return FALSE;
3404                 ip = (struct ip *)(mp->m_data + ehdrlen);
3405                 if (ip->ip_p != IPPROTO_TCP)
3406                         return FALSE;
3407                 ip->ip_sum = 0;
3408                 ip_hlen = ip->ip_hl << 2;
3409                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3410                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3411                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3412                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3413                 /* Tell transmit desc to also do IPv4 checksum. */
3414                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3415                 break;
3416 #endif
3417         default:
3418                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3419                     __func__, ntohs(eh_type));
3420                 break;
3421         }
3422
3423         ctxd = txr->next_avail_desc;
3424         tx_buffer = &txr->tx_buffers[ctxd];
3425         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3426
3427         tcp_hlen = th->th_off << 2;
3428
3429         /* This is used in the transmit desc in encap */
3430         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3431
3432         /* VLAN MACLEN IPLEN */
3433 #ifdef NET_VLAN
3434         if (mp->m_flags & M_VLANTAG) {
3435                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3436                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3437         }
3438
3439         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3440         vlan_macip_lens |= ip_hlen;
3441         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3442 #endif
3443
3444         /* ADV DTYPE TUCMD */
3445         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3446         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3447         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3448
3449         /* MSS L4LEN IDX */
3450         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3451         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3452         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3453
3454         TXD->seqnum_seed = htole32(0);
3455         tx_buffer->m_head = NULL;
3456         tx_buffer->eop_index = -1;
3457
3458         if (++ctxd == adapter->num_tx_desc)
3459                 ctxd = 0;
3460
3461         txr->tx_avail--;
3462         txr->next_avail_desc = ctxd;
3463         return TRUE;
3464 }
3465 #endif
3466
3467 #ifdef IXGBE_FDIR
3468 /*
3469 ** This routine parses packet headers so that Flow
3470 ** Director can make a hashed filter table entry
3471 ** allowing traffic flows to be identified and kept
3472 ** on the same cpu.  This would be a performance
3473 ** hit, but we only do it at IXGBE_FDIR_RATE of
3474 ** packets.
3475 */
3476 static void
3477 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3478 {
3479         struct adapter                  *adapter = txr->adapter;
3480         struct ix_queue                 *que;
3481         struct ip                       *ip;
3482         struct tcphdr                   *th;
3483         struct udphdr                   *uh;
3484         struct ether_vlan_header        *eh;
3485         union ixgbe_atr_hash_dword      input = {.dword = 0};
3486         union ixgbe_atr_hash_dword      common = {.dword = 0};
3487         int                             ehdrlen, ip_hlen;
3488         u16                             etype;
3489
3490         eh = mtod(mp, struct ether_vlan_header *);
3491         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3492                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3493                 etype = eh->evl_proto;
3494         } else {
3495                 ehdrlen = ETHER_HDR_LEN;
3496                 etype = eh->evl_encap_proto;
3497         }
3498
3499         /* Only handling IPv4 */
3500         if (etype != htons(ETHERTYPE_IP))
3501                 return;
3502
3503         ip = (struct ip *)(mp->m_data + ehdrlen);
3504         ip_hlen = ip->ip_hl << 2;
3505
3506         /* check if we're UDP or TCP */
3507         switch (ip->ip_p) {
3508         case IPPROTO_TCP:
3509                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3510                 /* src and dst are inverted */
3511                 common.port.dst ^= th->th_sport;
3512                 common.port.src ^= th->th_dport;
3513                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3514                 break;
3515         case IPPROTO_UDP:
3516                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3517                 /* src and dst are inverted */
3518                 common.port.dst ^= uh->uh_sport;
3519                 common.port.src ^= uh->uh_dport;
3520                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3521                 break;
3522         default:
3523                 return;
3524         }
3525
3526         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3527         if (mp->m_pkthdr.ether_vtag)
3528                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3529         else
3530                 common.flex_bytes ^= etype;
3531         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3532
3533         que = &adapter->queues[txr->me];
3534         /*
3535         ** This assumes the Rx queue and Tx
3536         ** queue are bound to the same CPU
3537         */
3538         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3539             input, common, que->msix);
3540 }
3541 #endif /* IXGBE_FDIR */
3542
3543 /**********************************************************************
3544  *
3545  *  Examine each tx_buffer in the used queue. If the hardware is done
3546  *  processing the packet then free associated resources. The
3547  *  tx_buffer is put back on the free queue.
3548  *
3549  **********************************************************************/
3550 static bool
3551 ixgbe_txeof(struct tx_ring *txr)
3552 {
3553         struct adapter  *adapter = txr->adapter;
3554         struct ifnet    *ifp = adapter->ifp;
3555         u32     first, last, done, processed;
3556         struct ixgbe_tx_buf *tx_buffer;
3557         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3558
3559         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3560
3561 #ifdef DEV_NETMAP
3562         if (ifp->if_capenable & IFCAP_NETMAP) {
3563                 struct netmap_adapter *na = NA(ifp);
3564                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3565
3566                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3567
3568                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3569                     BUS_DMASYNC_POSTREAD);
3570                 /*
3571                  * In netmap mode, all the work is done in the context
3572                  * of the client thread. Interrupt handlers only wake up
3573                  * clients, which may be sleeping on individual rings
3574                  * or on a global resource for all rings.
3575                  * To implement tx interrupt mitigation, we wake up the client
3576                  * thread roughly every half ring, even if the NIC interrupts
3577                  * more frequently. This is implemented as follows:
3578                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3579                  *   the slot that should wake up the thread (nkr_num_slots
3580                  *   means the user thread should not be woken up);
3581                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3582                  *   or the slot has the DD bit set.
3583                  *
3584                  * When the driver has separate locks, we need to
3585                  * release and re-acquire txlock to avoid deadlocks.
3586                  * XXX see if we can find a better way.
3587                  */
3588                 if (!netmap_mitigate ||
3589                     (kring->nr_kflags < kring->nkr_num_slots &&
3590                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3591                         kring->nr_kflags = kring->nkr_num_slots;
3592                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3593                         IXGBE_TX_UNLOCK(txr);
3594                         IXGBE_CORE_LOCK(adapter);
3595                         selwakeuppri(&na->tx_si, PI_NET);
3596                         IXGBE_CORE_UNLOCK(adapter);
3597                         IXGBE_TX_LOCK(txr);
3598                 }
3599                 return FALSE;
3600         }
3601 #endif /* DEV_NETMAP */
3602
3603         if (txr->tx_avail == adapter->num_tx_desc) {
3604                 txr->queue_status = IXGBE_QUEUE_IDLE;
3605                 return FALSE;
3606         }
3607
3608         processed = 0;
3609         first = txr->next_to_clean;
3610         tx_buffer = &txr->tx_buffers[first];
3611         /* For cleanup we just use legacy struct */
3612         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3613         last = tx_buffer->eop_index;
3614         if (last == -1)
3615                 return FALSE;
3616         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3617
3618         /*
3619         ** Get the index of the first descriptor
3620         ** BEYOND the EOP and call that 'done'.
3621         ** I do this so the comparison in the
3622         ** inner while loop below can be simple
3623         */
3624         if (++last == adapter->num_tx_desc) last = 0;
3625         done = last;
3626
3627         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3628             BUS_DMASYNC_POSTREAD);
3629         /*
3630         ** Only the EOP descriptor of a packet now has the DD
3631         ** bit set, this is what we look for...
3632         */
3633         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3634                 /* We clean the range of the packet */
3635                 while (first != done) {
3636                         tx_desc->upper.data = 0;
3637                         tx_desc->lower.data = 0;
3638                         tx_desc->buffer_addr = 0;
3639                         ++txr->tx_avail;
3640                         ++processed;
3641
3642                         if (tx_buffer->m_head) {
3643                                 txr->bytes +=
3644                                     tx_buffer->m_head->m_pkthdr.len;
3645                                 bus_dmamap_sync(txr->txtag,
3646                                     tx_buffer->map,
3647                                     BUS_DMASYNC_POSTWRITE);
3648                                 bus_dmamap_unload(txr->txtag,
3649                                     tx_buffer->map);
3650                                 m_freem(tx_buffer->m_head);
3651                                 tx_buffer->m_head = NULL;
3652                                 tx_buffer->map = NULL;
3653                         }
3654                         tx_buffer->eop_index = -1;
3655                         txr->watchdog_time = ticks;
3656
3657                         if (++first == adapter->num_tx_desc)
3658                                 first = 0;
3659
3660                         tx_buffer = &txr->tx_buffers[first];
3661                         tx_desc =
3662                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3663                 }
3664                 ++txr->packets;
3665                 ++ifp->if_opackets;
3666                 /* See if there is more work now */
3667                 last = tx_buffer->eop_index;
3668                 if (last != -1) {
3669                         eop_desc =
3670                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3671                         /* Get next done point */
3672                         if (++last == adapter->num_tx_desc) last = 0;
3673                         done = last;
3674                 } else
3675                         break;
3676         }
3677         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3678             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3679
3680         txr->next_to_clean = first;
3681
3682         /*
3683         ** Watchdog calculation, we know there's
3684         ** work outstanding or the first return
3685         ** would have been taken, so none processed
3686         ** for too long indicates a hang.
3687         */
3688         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3689                 txr->queue_status = IXGBE_QUEUE_HUNG;
3690
3691         /* With a minimum free clear the depleted state bit.  */
3692         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3693                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3694
3695         if (txr->tx_avail == adapter->num_tx_desc) {
3696                 txr->queue_status = IXGBE_QUEUE_IDLE;
3697                 return (FALSE);
3698         }
3699
3700         return TRUE;
3701 }
3702
3703 /*********************************************************************
3704  *
3705  *  Refresh mbuf buffers for RX descriptor rings
3706  *   - now keeps its own state so discards due to resource
3707  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3708  *     it just returns, keeping its placeholder, thus it can simply
3709  *     be recalled to try again.
3710  *
3711  **********************************************************************/
3712 static void
3713 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3714 {
3715         struct adapter          *adapter = rxr->adapter;
3716         bus_dma_segment_t       hseg[1];
3717         bus_dma_segment_t       pseg[1];
3718         struct ixgbe_rx_buf     *rxbuf;
3719         struct mbuf             *mh, *mp;
3720         int                     i, j, nsegs, error;
3721         bool                    refreshed = FALSE;
3722
3723         i = j = rxr->next_to_refresh;
3724         /* Control the loop with one beyond */
3725         if (++j == adapter->num_rx_desc)
3726                 j = 0;
3727
3728         while (j != limit) {
3729                 rxbuf = &rxr->rx_buffers[i];
3730                 if (rxr->hdr_split == FALSE)
3731                         goto no_split;
3732
3733                 if (rxbuf->m_head == NULL) {
3734                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3735                         if (mh == NULL)
3736                                 goto update;
3737                 } else
3738                         mh = rxbuf->m_head;
3739
3740                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3741                 mh->m_len = MHLEN;
3742                 mh->m_flags |= M_PKTHDR;
3743                 /* Get the memory mapping */
3744                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3745                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3746                 if (error != 0) {
3747                         kprintf("Refresh mbufs: hdr dmamap load"
3748                             " failure - %d\n", error);
3749                         m_free(mh);
3750                         rxbuf->m_head = NULL;
3751                         goto update;
3752                 }
3753                 rxbuf->m_head = mh;
3754                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3755                     BUS_DMASYNC_PREREAD);
3756                 rxr->rx_base[i].read.hdr_addr =
3757                     htole64(hseg[0].ds_addr);
3758
3759 no_split:
3760                 if (rxbuf->m_pack == NULL) {
3761                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3762                             M_PKTHDR, adapter->rx_mbuf_sz);
3763                         if (mp == NULL)
3764                                 goto update;
3765                 } else
3766                         mp = rxbuf->m_pack;
3767
3768                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3769                 /* Get the memory mapping */
3770                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3771                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3772                 if (error != 0) {
3773                         kprintf("Refresh mbufs: payload dmamap load"
3774                             " failure - %d\n", error);
3775                         m_free(mp);
3776                         rxbuf->m_pack = NULL;
3777                         goto update;
3778                 }
3779                 rxbuf->m_pack = mp;
3780                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3781                     BUS_DMASYNC_PREREAD);
3782                 rxr->rx_base[i].read.pkt_addr =
3783                     htole64(pseg[0].ds_addr);
3784
3785                 refreshed = TRUE;
3786                 /* Next is precalculated */
3787                 i = j;
3788                 rxr->next_to_refresh = i;
3789                 if (++j == adapter->num_rx_desc)
3790                         j = 0;
3791         }
3792 update:
3793         if (refreshed) /* Update hardware tail index */
3794                 IXGBE_WRITE_REG(&adapter->hw,
3795                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3796         return;
3797 }
3798
3799 /*********************************************************************
3800  *
3801  *  Allocate memory for rx_buffer structures. Since we use one
3802  *  rx_buffer per received packet, the maximum number of rx_buffer's
3803  *  that we'll need is equal to the number of receive descriptors
3804  *  that we've allocated.
3805  *
3806  **********************************************************************/
3807 static int
3808 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3809 {
3810         struct  adapter         *adapter = rxr->adapter;
3811         device_t                dev = adapter->dev;
3812         struct ixgbe_rx_buf     *rxbuf;
3813         int                     i, bsize, error;
3814
3815         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3816         if (!(rxr->rx_buffers =
3817             (struct ixgbe_rx_buf *) kmalloc(bsize,
3818             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3819                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3820                 error = ENOMEM;
3821                 goto fail;
3822         }
3823
3824         if ((error = bus_dma_tag_create(NULL,   /* parent */
3825                                    1, 0,        /* alignment, bounds */
3826                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3827                                    BUS_SPACE_MAXADDR,   /* highaddr */
3828                                    NULL, NULL,          /* filter, filterarg */
3829                                    MSIZE,               /* maxsize */
3830                                    1,                   /* nsegments */
3831                                    MSIZE,               /* maxsegsize */
3832                                    0,                   /* flags */
3833                                    &rxr->htag))) {
3834                 device_printf(dev, "Unable to create RX DMA tag\n");
3835                 goto fail;
3836         }
3837
3838         if ((error = bus_dma_tag_create(NULL,   /* parent */
3839                                    1, 0,        /* alignment, bounds */
3840                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3841                                    BUS_SPACE_MAXADDR,   /* highaddr */
3842                                    NULL, NULL,          /* filter, filterarg */
3843                                    MJUM16BYTES,         /* maxsize */
3844                                    1,                   /* nsegments */
3845                                    MJUM16BYTES,         /* maxsegsize */
3846                                    0,                   /* flags */
3847                                    &rxr->ptag))) {
3848                 device_printf(dev, "Unable to create RX DMA tag\n");
3849                 goto fail;
3850         }
3851
3852         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3853                 rxbuf = &rxr->rx_buffers[i];
3854                 error = bus_dmamap_create(rxr->htag,
3855                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3856                 if (error) {
3857                         device_printf(dev, "Unable to create RX head map\n");
3858                         goto fail;
3859                 }
3860                 error = bus_dmamap_create(rxr->ptag,
3861                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3862                 if (error) {
3863                         device_printf(dev, "Unable to create RX pkt map\n");
3864                         goto fail;
3865                 }
3866         }
3867
3868         return (0);
3869
3870 fail:
3871         /* Frees all, but can handle partial completion */
3872         ixgbe_free_receive_structures(adapter);
3873         return (error);
3874 }
3875
3876 /*
3877 ** Used to detect a descriptor that has
3878 ** been merged by Hardware RSC.
3879 */
3880 static inline u32
3881 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3882 {
3883         return (le32toh(rx->wb.lower.lo_dword.data) &
3884             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3885 }
3886
3887 /*********************************************************************
3888  *
3889  *  Initialize Hardware RSC (LRO) feature on 82599
3890  *  for an RX ring, this is toggled by the LRO capability
3891  *  even though it is transparent to the stack.
3892  *
3893  **********************************************************************/
3894 #if 0   /* NET_LRO */
3895 static void
3896 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3897 {
3898         struct  adapter         *adapter = rxr->adapter;
3899         struct  ixgbe_hw        *hw = &adapter->hw;
3900         u32                     rscctrl, rdrxctl;
3901
3902         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3903         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3904 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3905         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3906 #endif /* DEV_NETMAP */
3907         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3908         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3909         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3910
3911         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3912         rscctrl |= IXGBE_RSCCTL_RSCEN;
3913         /*
3914         ** Limit the total number of descriptors that
3915         ** can be combined, so it does not exceed 64K
3916         */
3917         if (adapter->rx_mbuf_sz == MCLBYTES)
3918                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3919         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3920                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3921         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3922                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3923         else  /* Using 16K cluster */
3924                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3925
3926         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3927
3928         /* Enable TCP header recognition */
3929         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3930             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3931             IXGBE_PSRTYPE_TCPHDR));
3932
3933         /* Disable RSC for ACK packets */
3934         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3935             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3936
3937         rxr->hw_rsc = TRUE;
3938 }
3939 #endif
3940
3941 static void
3942 ixgbe_free_receive_ring(struct rx_ring *rxr)
3943 {
3944         struct  adapter         *adapter;
3945         struct ixgbe_rx_buf       *rxbuf;
3946         int i;
3947
3948         adapter = rxr->adapter;
3949         for (i = 0; i < adapter->num_rx_desc; i++) {
3950                 rxbuf = &rxr->rx_buffers[i];
3951                 if (rxbuf->m_head != NULL) {
3952                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3953                             BUS_DMASYNC_POSTREAD);
3954                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3955                         rxbuf->m_head->m_flags |= M_PKTHDR;
3956                         m_freem(rxbuf->m_head);
3957                 }
3958                 if (rxbuf->m_pack != NULL) {
3959                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3960                             BUS_DMASYNC_POSTREAD);
3961                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3962                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3963                         m_freem(rxbuf->m_pack);
3964                 }
3965                 rxbuf->m_head = NULL;
3966                 rxbuf->m_pack = NULL;
3967         }
3968 }
3969
3970
3971 /*********************************************************************
3972  *
3973  *  Initialize a receive ring and its buffers.
3974  *
3975  **********************************************************************/
3976 static int
3977 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3978 {
3979         struct  adapter         *adapter;
3980         struct ifnet            *ifp;
3981         device_t                dev;
3982         struct ixgbe_rx_buf     *rxbuf;
3983         bus_dma_segment_t       pseg[1], hseg[1];
3984 #if 0   /* NET_LRO */
3985         struct lro_ctrl         *lro = &rxr->lro;
3986 #endif
3987         int                     rsize, nsegs, error = 0;
3988 #ifdef DEV_NETMAP
3989         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3990         struct netmap_slot *slot;
3991 #endif /* DEV_NETMAP */
3992
3993         adapter = rxr->adapter;
3994         ifp = adapter->ifp;
3995         dev = adapter->dev;
3996
3997         /* Clear the ring contents */
3998         IXGBE_RX_LOCK(rxr);
3999 #ifdef DEV_NETMAP
4000         /* same as in ixgbe_setup_transmit_ring() */
4001         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4002 #endif /* DEV_NETMAP */
4003         rsize = roundup2(adapter->num_rx_desc *
4004             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4005         bzero((void *)rxr->rx_base, rsize);
4006
4007         /* Free current RX buffer structs and their mbufs */
4008         ixgbe_free_receive_ring(rxr);
4009
4010         /* Configure header split? */
4011         if (ixgbe_header_split)
4012                 rxr->hdr_split = TRUE;
4013
4014         /* Now replenish the mbufs */
4015         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4016                 struct mbuf     *mh, *mp;
4017
4018                 rxbuf = &rxr->rx_buffers[j];
4019 #ifdef DEV_NETMAP
4020                 /*
4021                  * In netmap mode, fill the map and set the buffer
4022                  * address in the NIC ring, considering the offset
4023                  * between the netmap and NIC rings (see comment in
4024                  * ixgbe_setup_transmit_ring() ). No need to allocate
4025                  * an mbuf, so end the block with a continue;
4026                  */
4027                 if (slot) {
4028                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4029                         uint64_t paddr;
4030                         void *addr;
4031
4032                         addr = PNMB(slot + sj, &paddr);
4033                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4034                         /* Update descriptor */
4035                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4036                         continue;
4037                 }
4038 #endif /* DEV_NETMAP */
4039                 /*
4040                 ** Don't allocate mbufs if not
4041                 ** doing header split, its wasteful
4042                 */
4043                 if (rxr->hdr_split == FALSE)
4044                         goto skip_head;
4045
4046                 /* First the header */
4047                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4048                 if (rxbuf->m_head == NULL) {
4049                         error = ENOBUFS;
4050                         goto fail;
4051                 }
4052                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4053                 mh = rxbuf->m_head;
4054                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4055                 mh->m_flags |= M_PKTHDR;
4056                 /* Get the memory mapping */
4057                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4058                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4059                     &nsegs, BUS_DMA_NOWAIT);
4060
4061                 if (error != 0) /* Nothing elegant to do here */
4062                         goto fail;
4063                 bus_dmamap_sync(rxr->htag,
4064                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4065                 /* Update descriptor */
4066                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4067
4068 skip_head:
4069                 /* Now the payload cluster */
4070                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4071                     M_PKTHDR, adapter->rx_mbuf_sz);
4072                 if (rxbuf->m_pack == NULL) {
4073                         error = ENOBUFS;
4074                         goto fail;
4075                 }
4076                 mp = rxbuf->m_pack;
4077                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4078                 /* Get the memory mapping */
4079                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4080                     rxbuf->pmap, mp, hseg, 1,
4081                     &nsegs, BUS_DMA_NOWAIT);
4082                 if (error != 0)
4083                         goto fail;
4084                 bus_dmamap_sync(rxr->ptag,
4085                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4086                 /* Update descriptor */
4087                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4088         }
4089
4090
4091         /* Setup our descriptor indices */
4092         rxr->next_to_check = 0;
4093         rxr->next_to_refresh = 0;
4094         rxr->lro_enabled = FALSE;
4095         rxr->rx_split_packets = 0;
4096         rxr->rx_bytes = 0;
4097         rxr->discard = FALSE;
4098         rxr->vtag_strip = FALSE;
4099
4100         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4101             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4102
4103         /*
4104         ** Now set up the LRO interface:
4105         ** 82598 uses software LRO, the
4106         ** 82599 and X540 use a hardware assist.
4107         */
4108 #if 0 /* NET_LRO */
4109         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4110             (ifp->if_capenable & IFCAP_RXCSUM) &&
4111             (ifp->if_capenable & IFCAP_LRO))
4112                 ixgbe_setup_hw_rsc(rxr);
4113         else if (ifp->if_capenable & IFCAP_LRO) {
4114                 int err = tcp_lro_init(lro);
4115                 if (err) {
4116                         device_printf(dev, "LRO Initialization failed!\n");
4117                         goto fail;
4118                 }
4119                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4120                 rxr->lro_enabled = TRUE;
4121                 lro->ifp = adapter->ifp;
4122         }
4123 #endif
4124
4125         IXGBE_RX_UNLOCK(rxr);
4126         return (0);
4127
4128 fail:
4129         ixgbe_free_receive_ring(rxr);
4130         IXGBE_RX_UNLOCK(rxr);
4131         return (error);
4132 }
4133
4134 /*********************************************************************
4135  *
4136  *  Initialize all receive rings.
4137  *
4138  **********************************************************************/
4139 static int
4140 ixgbe_setup_receive_structures(struct adapter *adapter)
4141 {
4142         struct rx_ring *rxr = adapter->rx_rings;
4143         int j;
4144
4145         for (j = 0; j < adapter->num_queues; j++, rxr++)
4146                 if (ixgbe_setup_receive_ring(rxr))
4147                         goto fail;
4148
4149         return (0);
4150 fail:
4151         /*
4152          * Free RX buffers allocated so far, we will only handle
4153          * the rings that completed, the failing case will have
4154          * cleaned up for itself. 'j' failed, so its the terminus.
4155          */
4156         for (int i = 0; i < j; ++i) {
4157                 rxr = &adapter->rx_rings[i];
4158                 ixgbe_free_receive_ring(rxr);
4159         }
4160
4161         return (ENOBUFS);
4162 }
4163
4164 /*********************************************************************
4165  *
4166  *  Setup receive registers and features.
4167  *
4168  **********************************************************************/
4169 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4170
4171 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4172
4173 static void
4174 ixgbe_initialize_receive_units(struct adapter *adapter)
4175 {
4176         struct  rx_ring *rxr = adapter->rx_rings;
4177         struct ixgbe_hw *hw = &adapter->hw;
4178         struct ifnet   *ifp = adapter->ifp;
4179         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4180         u32             reta, mrqc = 0, hlreg, random[10];
4181
4182
4183         /*
4184          * Make sure receives are disabled while
4185          * setting up the descriptor ring
4186          */
4187         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4188         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4189             rxctrl & ~IXGBE_RXCTRL_RXEN);
4190
4191         /* Enable broadcasts */
4192         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4193         fctrl |= IXGBE_FCTRL_BAM;
4194         fctrl |= IXGBE_FCTRL_DPF;
4195         fctrl |= IXGBE_FCTRL_PMCF;
4196         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4197
4198         /* Set for Jumbo Frames? */
4199         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4200         if (ifp->if_mtu > ETHERMTU)
4201                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4202         else
4203                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4204 #ifdef DEV_NETMAP
4205         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4206         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4207                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4208         else
4209                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4210 #endif /* DEV_NETMAP */
4211         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4212
4213         bufsz = (adapter->rx_mbuf_sz +
4214             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4215
4216         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4217                 u64 rdba = rxr->rxdma.dma_paddr;
4218
4219                 /* Setup the Base and Length of the Rx Descriptor Ring */
4220                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4221                                (rdba & 0x00000000ffffffffULL));
4222                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4223                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4224                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4225
4226                 /* Set up the SRRCTL register */
4227                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4228                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4229                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4230                 srrctl |= bufsz;
4231                 if (rxr->hdr_split) {
4232                         /* Use a standard mbuf for the header */
4233                         srrctl |= ((IXGBE_RX_HDR <<
4234                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4235                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4236                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4237                 } else
4238                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4239                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4240
4241                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4242                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4243                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4244         }
4245
4246         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4247                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4248                               IXGBE_PSRTYPE_UDPHDR |
4249                               IXGBE_PSRTYPE_IPV4HDR |
4250                               IXGBE_PSRTYPE_IPV6HDR;
4251                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4252         }
4253
4254         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4255
4256         /* Setup RSS */
4257         if (adapter->num_queues > 1) {
4258                 int i, j;
4259                 reta = 0;
4260
4261                 /* set up random bits */
4262                 karc4rand(&random, sizeof(random));
4263
4264                 /* Set up the redirection table */
4265                 for (i = 0, j = 0; i < 128; i++, j++) {
4266                         if (j == adapter->num_queues) j = 0;
4267                         reta = (reta << 8) | (j * 0x11);
4268                         if ((i & 3) == 3)
4269                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4270                 }
4271
4272                 /* Now fill our hash function seeds */
4273                 for (int i = 0; i < 10; i++)
4274                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4275
4276                 /* Perform hash on these packet types */
4277                 mrqc = IXGBE_MRQC_RSSEN
4278                      | IXGBE_MRQC_RSS_FIELD_IPV4
4279                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4280                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4281                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4282                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4283                      | IXGBE_MRQC_RSS_FIELD_IPV6
4284                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4285                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4286                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4287                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4288
4289                 /* RSS and RX IPP Checksum are mutually exclusive */
4290                 rxcsum |= IXGBE_RXCSUM_PCSD;
4291         }
4292
4293         if (ifp->if_capenable & IFCAP_RXCSUM)
4294                 rxcsum |= IXGBE_RXCSUM_PCSD;
4295
4296         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4297                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4298
4299         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4300
4301         return;
4302 }
4303
4304 /*********************************************************************
4305  *
4306  *  Free all receive rings.
4307  *
4308  **********************************************************************/
4309 static void
4310 ixgbe_free_receive_structures(struct adapter *adapter)
4311 {
4312         struct rx_ring *rxr = adapter->rx_rings;
4313
4314         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4315 #if 0   /* NET_LRO */
4316                 struct lro_ctrl         *lro = &rxr->lro;
4317 #endif
4318                 ixgbe_free_receive_buffers(rxr);
4319                 /* Free LRO memory */
4320 #if 0   /* NET_LRO */
4321                 tcp_lro_free(lro);
4322 #endif
4323                 /* Free the ring memory as well */
4324                 ixgbe_dma_free(adapter, &rxr->rxdma);
4325         }
4326
4327         kfree(adapter->rx_rings, M_DEVBUF);
4328 }
4329
4330
4331 /*********************************************************************
4332  *
4333  *  Free receive ring data structures
4334  *
4335  **********************************************************************/
4336 static void
4337 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4338 {
4339         struct adapter          *adapter = rxr->adapter;
4340         struct ixgbe_rx_buf     *rxbuf;
4341
4342         INIT_DEBUGOUT("free_receive_structures: begin");
4343
4344         /* Cleanup any existing buffers */
4345         if (rxr->rx_buffers != NULL) {
4346                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4347                         rxbuf = &rxr->rx_buffers[i];
4348                         if (rxbuf->m_head != NULL) {
4349                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4350                                     BUS_DMASYNC_POSTREAD);
4351                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4352                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4353                                 m_freem(rxbuf->m_head);
4354                         }
4355                         if (rxbuf->m_pack != NULL) {
4356                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4357                                     BUS_DMASYNC_POSTREAD);
4358                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4359                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4360                                 m_freem(rxbuf->m_pack);
4361                         }
4362                         rxbuf->m_head = NULL;
4363                         rxbuf->m_pack = NULL;
4364                         if (rxbuf->hmap != NULL) {
4365                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4366                                 rxbuf->hmap = NULL;
4367                         }
4368                         if (rxbuf->pmap != NULL) {
4369                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4370                                 rxbuf->pmap = NULL;
4371                         }
4372                 }
4373                 if (rxr->rx_buffers != NULL) {
4374                         kfree(rxr->rx_buffers, M_DEVBUF);
4375                         rxr->rx_buffers = NULL;
4376                 }
4377         }
4378
4379         if (rxr->htag != NULL) {
4380                 bus_dma_tag_destroy(rxr->htag);
4381                 rxr->htag = NULL;
4382         }
4383         if (rxr->ptag != NULL) {
4384                 bus_dma_tag_destroy(rxr->ptag);
4385                 rxr->ptag = NULL;
4386         }
4387
4388         return;
4389 }
4390
4391 static __inline void
4392 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4393 {
4394
4395         /*
4396          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4397          * should be computed by hardware. Also it should not have VLAN tag in
4398          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4399          */
4400 #if 0   /* NET_LRO */
4401         if (rxr->lro_enabled &&
4402             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4403             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4404             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4405             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4406             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4407             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4408             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4409             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4410                 /*
4411                  * Send to the stack if:
4412                  **  - LRO not enabled, or
4413                  **  - no LRO resources, or
4414                  **  - lro enqueue fails
4415                  */
4416                 if (rxr->lro.lro_cnt != 0)
4417                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4418                                 return;
4419         }
4420 #endif
4421         IXGBE_RX_UNLOCK(rxr);
4422         (*ifp->if_input)(ifp, m);
4423         IXGBE_RX_LOCK(rxr);
4424 }
4425
4426 static __inline void
4427 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4428 {
4429         struct ixgbe_rx_buf     *rbuf;
4430
4431         rbuf = &rxr->rx_buffers[i];
4432
4433         if (rbuf->fmp != NULL) {/* Partial chain ? */
4434                 rbuf->fmp->m_flags |= M_PKTHDR;
4435                 m_freem(rbuf->fmp);
4436                 rbuf->fmp = NULL;
4437         }
4438
4439         /*
4440         ** With advanced descriptors the writeback
4441         ** clobbers the buffer addrs, so its easier
4442         ** to just free the existing mbufs and take
4443         ** the normal refresh path to get new buffers
4444         ** and mapping.
4445         */
4446         if (rbuf->m_head) {
4447                 m_free(rbuf->m_head);
4448                 rbuf->m_head = NULL;
4449         }
4450
4451         if (rbuf->m_pack) {
4452                 m_free(rbuf->m_pack);
4453                 rbuf->m_pack = NULL;
4454         }
4455
4456         return;
4457 }
4458
4459
4460 /*********************************************************************
4461  *
4462  *  This routine executes in interrupt context. It replenishes
4463  *  the mbufs in the descriptor and sends data which has been
4464  *  dma'ed into host memory to upper layer.
4465  *
4466  *  We loop at most count times if count is > 0, or until done if
4467  *  count < 0.
4468  *
4469  *  Return TRUE for more work, FALSE for all clean.
4470  *********************************************************************/
4471 static bool
4472 ixgbe_rxeof(struct ix_queue *que, int count)
4473 {
4474         struct adapter          *adapter = que->adapter;
4475         struct rx_ring          *rxr = que->rxr;
4476         struct ifnet            *ifp = adapter->ifp;
4477 #if 0   /* NET_LRO */
4478         struct lro_ctrl         *lro = &rxr->lro;
4479         struct lro_entry        *queued;
4480 #endif
4481         int                     i, nextp, processed = 0;
4482         u32                     staterr = 0;
4483         union ixgbe_adv_rx_desc *cur;
4484         struct ixgbe_rx_buf     *rbuf, *nbuf;
4485
4486         IXGBE_RX_LOCK(rxr);
4487
4488 #ifdef DEV_NETMAP
4489         if (ifp->if_capenable & IFCAP_NETMAP) {
4490                 /*
4491                  * Same as the txeof routine: only wakeup clients on intr.
4492                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4493                  * mitigation (ixgbe_rxsync() will not look for new packets
4494                  * unless NKR_PENDINTR is set).
4495                  */
4496                 struct netmap_adapter *na = NA(ifp);
4497
4498                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4499                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4500                 IXGBE_RX_UNLOCK(rxr);
4501                 IXGBE_CORE_LOCK(adapter);
4502                 selwakeuppri(&na->rx_si, PI_NET);
4503                 IXGBE_CORE_UNLOCK(adapter);
4504                 return (FALSE);
4505         }
4506 #endif /* DEV_NETMAP */
4507         for (i = rxr->next_to_check; count != 0;) {
4508                 struct mbuf     *sendmp, *mh, *mp;
4509                 u32             rsc, ptype;
4510                 u16             hlen, plen, hdr;
4511 #ifdef NET_VLAN
4512                 u16             vtag = 0;
4513 #endif
4514                 bool            eop;
4515
4516                 /* Sync the ring. */
4517                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4518                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4519
4520                 cur = &rxr->rx_base[i];
4521                 staterr = le32toh(cur->wb.upper.status_error);
4522
4523                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4524                         break;
4525                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4526                         break;
4527
4528                 count--;
4529                 sendmp = NULL;
4530                 nbuf = NULL;
4531                 rsc = 0;
4532                 cur->wb.upper.status_error = 0;
4533                 rbuf = &rxr->rx_buffers[i];
4534                 mh = rbuf->m_head;
4535                 mp = rbuf->m_pack;
4536
4537                 plen = le16toh(cur->wb.upper.length);
4538                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4539                     IXGBE_RXDADV_PKTTYPE_MASK;
4540                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4541                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4542
4543                 /* Process vlan info */
4544 #ifdef NET_VLAN
4545                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4546                         vtag = le16toh(cur->wb.upper.vlan);
4547 #endif
4548
4549                 /* Make sure bad packets are discarded */
4550                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4551                     (rxr->discard)) {
4552                         ifp->if_ierrors++;
4553                         rxr->rx_discarded++;
4554                         if (eop)
4555                                 rxr->discard = FALSE;
4556                         else
4557                                 rxr->discard = TRUE;
4558                         ixgbe_rx_discard(rxr, i);
4559                         goto next_desc;
4560                 }
4561
4562                 /*
4563                 ** On 82599 which supports a hardware
4564                 ** LRO (called HW RSC), packets need
4565                 ** not be fragmented across sequential
4566                 ** descriptors, rather the next descriptor
4567                 ** is indicated in bits of the descriptor.
4568                 ** This also means that we might proceses
4569                 ** more than one packet at a time, something
4570                 ** that has never been true before, it
4571                 ** required eliminating global chain pointers
4572                 ** in favor of what we are doing here.  -jfv
4573                 */
4574                 if (!eop) {
4575                         /*
4576                         ** Figure out the next descriptor
4577                         ** of this frame.
4578                         */
4579                         if (rxr->hw_rsc == TRUE) {
4580                                 rsc = ixgbe_rsc_count(cur);
4581                                 rxr->rsc_num += (rsc - 1);
4582                         }
4583                         if (rsc) { /* Get hardware index */
4584                                 nextp = ((staterr &
4585                                     IXGBE_RXDADV_NEXTP_MASK) >>
4586                                     IXGBE_RXDADV_NEXTP_SHIFT);
4587                         } else { /* Just sequential */
4588                                 nextp = i + 1;
4589                                 if (nextp == adapter->num_rx_desc)
4590                                         nextp = 0;
4591                         }
4592                         nbuf = &rxr->rx_buffers[nextp];
4593                         prefetch(nbuf);
4594                 }
4595                 /*
4596                 ** The header mbuf is ONLY used when header
4597                 ** split is enabled, otherwise we get normal
4598                 ** behavior, ie, both header and payload
4599                 ** are DMA'd into the payload buffer.
4600                 **
4601                 ** Rather than using the fmp/lmp global pointers
4602                 ** we now keep the head of a packet chain in the
4603                 ** buffer struct and pass this along from one
4604                 ** descriptor to the next, until we get EOP.
4605                 */
4606                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4607                         /* This must be an initial descriptor */
4608                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4609                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4610                         if (hlen > IXGBE_RX_HDR)
4611                                 hlen = IXGBE_RX_HDR;
4612                         mh->m_len = hlen;
4613                         mh->m_flags |= M_PKTHDR;
4614                         mh->m_next = NULL;
4615                         mh->m_pkthdr.len = mh->m_len;
4616                         /* Null buf pointer so it is refreshed */
4617                         rbuf->m_head = NULL;
4618                         /*
4619                         ** Check the payload length, this
4620                         ** could be zero if its a small
4621                         ** packet.
4622                         */
4623                         if (plen > 0) {
4624                                 mp->m_len = plen;
4625                                 mp->m_next = NULL;
4626                                 mp->m_flags &= ~M_PKTHDR;
4627                                 mh->m_next = mp;
4628                                 mh->m_pkthdr.len += mp->m_len;
4629                                 /* Null buf pointer so it is refreshed */
4630                                 rbuf->m_pack = NULL;
4631                                 rxr->rx_split_packets++;
4632                         }
4633                         /*
4634                         ** Now create the forward
4635                         ** chain so when complete
4636                         ** we wont have to.
4637                         */
4638                         if (eop == 0) {
4639                                 /* stash the chain head */
4640                                 nbuf->fmp = mh;
4641                                 /* Make forward chain */
4642                                 if (plen)
4643                                         mp->m_next = nbuf->m_pack;
4644                                 else
4645                                         mh->m_next = nbuf->m_pack;
4646                         } else {
4647                                 /* Singlet, prepare to send */
4648                                 sendmp = mh;
4649                                 /* If hardware handled vtag */
4650 #ifdef NET_VLAN
4651                                 if (vtag) {
4652                                         sendmp->m_pkthdr.ether_vtag = vtag;
4653                                         sendmp->m_flags |= M_VLANTAG;
4654                                 }
4655 #endif
4656                         }
4657                 } else {
4658                         /*
4659                         ** Either no header split, or a
4660                         ** secondary piece of a fragmented
4661                         ** split packet.
4662                         */
4663                         mp->m_len = plen;
4664                         /*
4665                         ** See if there is a stored head
4666                         ** that determines what we are
4667                         */
4668                         sendmp = rbuf->fmp;
4669                         rbuf->m_pack = rbuf->fmp = NULL;
4670
4671                         if (sendmp != NULL) {  /* secondary frag */
4672                                 mp->m_flags &= ~M_PKTHDR;
4673                                 sendmp->m_pkthdr.len += mp->m_len;
4674                         } else {
4675                                 /* first desc of a non-ps chain */
4676                                 sendmp = mp;
4677                                 sendmp->m_flags |= M_PKTHDR;
4678                                 sendmp->m_pkthdr.len = mp->m_len;
4679 #ifdef NET_VLAN
4680                                 if (staterr & IXGBE_RXD_STAT_VP) {
4681                                         sendmp->m_pkthdr.ether_vtag = vtag;
4682                                         sendmp->m_flags |= M_VLANTAG;
4683                                 }
4684 #endif
4685                         }
4686                         /* Pass the head pointer on */
4687                         if (eop == 0) {
4688                                 nbuf->fmp = sendmp;
4689                                 sendmp = NULL;
4690                                 mp->m_next = nbuf->m_pack;
4691                         }
4692                 }
4693                 ++processed;
4694                 /* Sending this frame? */
4695                 if (eop) {
4696                         sendmp->m_pkthdr.rcvif = ifp;
4697                         ifp->if_ipackets++;
4698                         rxr->rx_packets++;
4699                         /* capture data for AIM */
4700                         rxr->bytes += sendmp->m_pkthdr.len;
4701                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4702                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4703                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4704 #if 0 /* __FreeBSD_version >= 800000 */
4705                         sendmp->m_pkthdr.flowid = que->msix;
4706                         sendmp->m_flags |= M_FLOWID;
4707 #endif
4708                 }
4709 next_desc:
4710                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4711                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4712
4713                 /* Advance our pointers to the next descriptor. */
4714                 if (++i == adapter->num_rx_desc)
4715                         i = 0;
4716
4717                 /* Now send to the stack or do LRO */
4718                 if (sendmp != NULL) {
4719                         rxr->next_to_check = i;
4720                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4721                         i = rxr->next_to_check;
4722                 }
4723
4724                /* Every 8 descriptors we go to refresh mbufs */
4725                 if (processed == 8) {
4726                         ixgbe_refresh_mbufs(rxr, i);
4727                         processed = 0;
4728                 }
4729         }
4730
4731         /* Refresh any remaining buf structs */
4732         if (ixgbe_rx_unrefreshed(rxr))
4733                 ixgbe_refresh_mbufs(rxr, i);
4734
4735         rxr->next_to_check = i;
4736
4737         /*
4738          * Flush any outstanding LRO work
4739          */
4740 #if 0   /* NET_LRO */
4741         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4742                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4743                 tcp_lro_flush(lro, queued);
4744         }
4745 #endif
4746
4747         IXGBE_RX_UNLOCK(rxr);
4748
4749         /*
4750         ** We still have cleaning to do?
4751         ** Schedule another interrupt if so.
4752         */
4753         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4754                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4755                 return (TRUE);
4756         }
4757
4758         return (FALSE);
4759 }
4760
4761
4762 /*********************************************************************
4763  *
4764  *  Verify that the hardware indicated that the checksum is valid.
4765  *  Inform the stack about the status of checksum so that stack
4766  *  doesn't spend time verifying the checksum.
4767  *
4768  *********************************************************************/
4769 static void
4770 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4771 {
4772         u16     status = (u16) staterr;
4773         u8      errors = (u8) (staterr >> 24);
4774         bool    sctp = FALSE;
4775
4776         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4777             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4778                 sctp = TRUE;
4779
4780         if (status & IXGBE_RXD_STAT_IPCS) {
4781                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4782                         /* IP Checksum Good */
4783                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4784                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4785
4786                 } else
4787                         mp->m_pkthdr.csum_flags = 0;
4788         }
4789         if (status & IXGBE_RXD_STAT_L4CS) {
4790                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4791 #if 0
4792                 if (sctp)
4793                         type = CSUM_SCTP_VALID;
4794 #endif
4795                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4796                         mp->m_pkthdr.csum_flags |= type;
4797                         if (!sctp)
4798                                 mp->m_pkthdr.csum_data = htons(0xffff);
4799                 }
4800         }
4801         return;
4802 }
4803
4804
4805 /*
4806 ** This routine is run via an vlan config EVENT,
4807 ** it enables us to use the HW Filter table since
4808 ** we can get the vlan id. This just creates the
4809 ** entry in the soft version of the VFTA, init will
4810 ** repopulate the real table.
4811 */
4812 static void
4813 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4814 {
4815         struct adapter  *adapter = ifp->if_softc;
4816         u16             index, bit;
4817
4818         if (ifp->if_softc !=  arg)   /* Not our event */
4819                 return;
4820
4821         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4822                 return;
4823
4824         IXGBE_CORE_LOCK(adapter);
4825         index = (vtag >> 5) & 0x7F;
4826         bit = vtag & 0x1F;
4827         adapter->shadow_vfta[index] |= (1 << bit);
4828         ++adapter->num_vlans;
4829         ixgbe_init_locked(adapter);
4830         IXGBE_CORE_UNLOCK(adapter);
4831 }
4832
4833 /*
4834 ** This routine is run via an vlan
4835 ** unconfig EVENT, remove our entry
4836 ** in the soft vfta.
4837 */
4838 static void
4839 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4840 {
4841         struct adapter  *adapter = ifp->if_softc;
4842         u16             index, bit;
4843
4844         if (ifp->if_softc !=  arg)
4845                 return;
4846
4847         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4848                 return;
4849
4850         IXGBE_CORE_LOCK(adapter);
4851         index = (vtag >> 5) & 0x7F;
4852         bit = vtag & 0x1F;
4853         adapter->shadow_vfta[index] &= ~(1 << bit);
4854         --adapter->num_vlans;
4855         /* Re-init to load the changes */
4856         ixgbe_init_locked(adapter);
4857         IXGBE_CORE_UNLOCK(adapter);
4858 }
4859
4860 static void
4861 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4862 {
4863 #ifdef NET_VLAN
4864         struct ifnet    *ifp = adapter->ifp;
4865         struct ixgbe_hw *hw = &adapter->hw;
4866         struct rx_ring  *rxr;
4867         u32             ctrl;
4868
4869         /*
4870         ** We get here thru init_locked, meaning
4871         ** a soft reset, this has already cleared
4872         ** the VFTA and other state, so if there
4873         ** have been no vlan's registered do nothing.
4874         */
4875         if (adapter->num_vlans == 0)
4876                 return;
4877
4878         /*
4879         ** A soft reset zero's out the VFTA, so
4880         ** we need to repopulate it now.
4881         */
4882         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4883                 if (adapter->shadow_vfta[i] != 0)
4884                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4885                             adapter->shadow_vfta[i]);
4886
4887         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4888         /* Enable the Filter Table if enabled */
4889         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4890                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4891                 ctrl |= IXGBE_VLNCTRL_VFE;
4892         }
4893         if (hw->mac.type == ixgbe_mac_82598EB)
4894                 ctrl |= IXGBE_VLNCTRL_VME;
4895         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4896
4897         /* Setup the queues for vlans */
4898         for (int i = 0; i < adapter->num_queues; i++) {
4899                 rxr = &adapter->rx_rings[i];
4900                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4901                 if (hw->mac.type != ixgbe_mac_82598EB) {
4902                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4903                         ctrl |= IXGBE_RXDCTL_VME;
4904                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4905                 }
4906                 rxr->vtag_strip = TRUE;
4907         }
4908 #endif
4909 }
4910
4911 static void
4912 ixgbe_enable_intr(struct adapter *adapter)
4913 {
4914         struct ixgbe_hw *hw = &adapter->hw;
4915         struct ix_queue *que = adapter->queues;
4916         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4917
4918
4919         /* Enable Fan Failure detection */
4920         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4921                     mask |= IXGBE_EIMS_GPI_SDP1;
4922         else {
4923                     mask |= IXGBE_EIMS_ECC;
4924                     mask |= IXGBE_EIMS_GPI_SDP0;
4925                     mask |= IXGBE_EIMS_GPI_SDP1;
4926                     mask |= IXGBE_EIMS_GPI_SDP2;
4927 #ifdef IXGBE_FDIR
4928                     mask |= IXGBE_EIMS_FLOW_DIR;
4929 #endif
4930         }
4931
4932         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4933
4934         /* With RSS we use auto clear */
4935         if (adapter->msix_mem) {
4936                 mask = IXGBE_EIMS_ENABLE_MASK;
4937                 /* Don't autoclear Link */
4938                 mask &= ~IXGBE_EIMS_OTHER;
4939                 mask &= ~IXGBE_EIMS_LSC;
4940                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4941         }
4942
4943         /*
4944         ** Now enable all queues, this is done separately to
4945         ** allow for handling the extended (beyond 32) MSIX
4946         ** vectors that can be used by 82599
4947         */
4948         for (int i = 0; i < adapter->num_queues; i++, que++)
4949                 ixgbe_enable_queue(adapter, que->msix);
4950
4951         IXGBE_WRITE_FLUSH(hw);
4952
4953         return;
4954 }
4955
4956 static void
4957 ixgbe_disable_intr(struct adapter *adapter)
4958 {
4959         if (adapter->msix_mem)
4960                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4961         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4962                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4963         } else {
4964                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4965                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4966                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4967         }
4968         IXGBE_WRITE_FLUSH(&adapter->hw);
4969         return;
4970 }
4971
4972 u16
4973 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4974 {
4975         u16 value;
4976
4977         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4978             reg, 2);
4979
4980         return (value);
4981 }
4982
4983 void
4984 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4985 {
4986         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4987             reg, value, 2);
4988
4989         return;
4990 }
4991
4992 /*
4993 ** Setup the correct IVAR register for a particular MSIX interrupt
4994 **   (yes this is all very magic and confusing :)
4995 **  - entry is the register array entry
4996 **  - vector is the MSIX vector for this queue
4997 **  - type is RX/TX/MISC
4998 */
4999 static void
5000 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5001 {
5002         struct ixgbe_hw *hw = &adapter->hw;
5003         u32 ivar, index;
5004
5005         vector |= IXGBE_IVAR_ALLOC_VAL;
5006
5007         switch (hw->mac.type) {
5008
5009         case ixgbe_mac_82598EB:
5010                 if (type == -1)
5011                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5012                 else
5013                         entry += (type * 64);
5014                 index = (entry >> 2) & 0x1F;
5015                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5016                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5017                 ivar |= (vector << (8 * (entry & 0x3)));
5018                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5019                 break;
5020
5021         case ixgbe_mac_82599EB:
5022         case ixgbe_mac_X540:
5023                 if (type == -1) { /* MISC IVAR */
5024                         index = (entry & 1) * 8;
5025                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5026                         ivar &= ~(0xFF << index);
5027                         ivar |= (vector << index);
5028                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5029                 } else {        /* RX/TX IVARS */
5030                         index = (16 * (entry & 1)) + (8 * type);
5031                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5032                         ivar &= ~(0xFF << index);
5033                         ivar |= (vector << index);
5034                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5035                 }
5036
5037         default:
5038                 break;
5039         }
5040 }
5041
5042 static void
5043 ixgbe_configure_ivars(struct adapter *adapter)
5044 {
5045         struct  ix_queue *que = adapter->queues;
5046         u32 newitr;
5047
5048         if (ixgbe_max_interrupt_rate > 0)
5049                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5050         else
5051                 newitr = 0;
5052
5053         for (int i = 0; i < adapter->num_queues; i++, que++) {
5054                 /* First the RX queue entry */
5055                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5056                 /* ... and the TX */
5057                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5058                 /* Set an Initial EITR value */
5059                 IXGBE_WRITE_REG(&adapter->hw,
5060                     IXGBE_EITR(que->msix), newitr);
5061         }
5062
5063         /* For the Link interrupt */
5064         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5065 }
5066
5067 /*
5068 ** ixgbe_sfp_probe - called in the local timer to
5069 ** determine if a port had optics inserted.
5070 */
5071 static bool ixgbe_sfp_probe(struct adapter *adapter)
5072 {
5073         struct ixgbe_hw *hw = &adapter->hw;
5074         device_t        dev = adapter->dev;
5075         bool            result = FALSE;
5076
5077         if ((hw->phy.type == ixgbe_phy_nl) &&
5078             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5079                 s32 ret = hw->phy.ops.identify_sfp(hw);
5080                 if (ret)
5081                         goto out;
5082                 ret = hw->phy.ops.reset(hw);
5083                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5084                         device_printf(dev,"Unsupported SFP+ module detected!");
5085                         kprintf(" Reload driver with supported module.\n");
5086                         adapter->sfp_probe = FALSE;
5087                         goto out;
5088                 } else
5089                         device_printf(dev,"SFP+ module detected!\n");
5090                 /* We now have supported optics */
5091                 adapter->sfp_probe = FALSE;
5092                 /* Set the optics type so system reports correctly */
5093                 ixgbe_setup_optics(adapter);
5094                 result = TRUE;
5095         }
5096 out:
5097         return (result);
5098 }
5099
5100 /*
5101 ** Tasklet handler for MSIX Link interrupts
5102 **  - do outside interrupt since it might sleep
5103 */
5104 static void
5105 ixgbe_handle_link(void *context, int pending)
5106 {
5107         struct adapter  *adapter = context;
5108
5109         ixgbe_check_link(&adapter->hw,
5110             &adapter->link_speed, &adapter->link_up, 0);
5111         ixgbe_update_link_status(adapter);
5112 }
5113
5114 /*
5115 ** Tasklet for handling SFP module interrupts
5116 */
5117 static void
5118 ixgbe_handle_mod(void *context, int pending)
5119 {
5120         struct adapter  *adapter = context;
5121         struct ixgbe_hw *hw = &adapter->hw;
5122         device_t        dev = adapter->dev;
5123         u32 err;
5124
5125         err = hw->phy.ops.identify_sfp(hw);
5126         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5127                 device_printf(dev,
5128                     "Unsupported SFP+ module type was detected.\n");
5129                 return;
5130         }
5131         err = hw->mac.ops.setup_sfp(hw);
5132         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5133                 device_printf(dev,
5134                     "Setup failure - unsupported SFP+ module type.\n");
5135                 return;
5136         }
5137         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5138         return;
5139 }
5140
5141
5142 /*
5143 ** Tasklet for handling MSF (multispeed fiber) interrupts
5144 */
5145 static void
5146 ixgbe_handle_msf(void *context, int pending)
5147 {
5148         struct adapter  *adapter = context;
5149         struct ixgbe_hw *hw = &adapter->hw;
5150         u32 autoneg;
5151         bool negotiate;
5152
5153         autoneg = hw->phy.autoneg_advertised;
5154         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5155                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5156         if (hw->mac.ops.setup_link)
5157                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5158         return;
5159 }
5160
5161 #ifdef IXGBE_FDIR
5162 /*
5163 ** Tasklet for reinitializing the Flow Director filter table
5164 */
5165 static void
5166 ixgbe_reinit_fdir(void *context, int pending)
5167 {
5168         struct adapter  *adapter = context;
5169         struct ifnet   *ifp = adapter->ifp;
5170
5171         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5172                 return;
5173         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5174         adapter->fdir_reinit = 0;
5175         /* Restart the interface */
5176         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5177         return;
5178 }
5179 #endif
5180
5181 /**********************************************************************
5182  *
5183  *  Update the board statistics counters.
5184  *
5185  **********************************************************************/
5186 static void
5187 ixgbe_update_stats_counters(struct adapter *adapter)
5188 {
5189         struct ifnet   *ifp = adapter->ifp;
5190         struct ixgbe_hw *hw = &adapter->hw;
5191         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5192         u64  total_missed_rx = 0;
5193
5194         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5195         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5196         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5197         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5198
5199         for (int i = 0; i < 8; i++) {
5200                 u32 mp;
5201                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5202                 /* missed_rx tallies misses for the gprc workaround */
5203                 missed_rx += mp;
5204                 /* global total per queue */
5205                 adapter->stats.mpc[i] += mp;
5206                 /* Running comprehensive total for stats display */
5207                 total_missed_rx += adapter->stats.mpc[i];
5208                 if (hw->mac.type == ixgbe_mac_82598EB)
5209                         adapter->stats.rnbc[i] +=
5210                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5211                 adapter->stats.pxontxc[i] +=
5212                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5213                 adapter->stats.pxonrxc[i] +=
5214                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5215                 adapter->stats.pxofftxc[i] +=
5216                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5217                 adapter->stats.pxoffrxc[i] +=
5218                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5219                 adapter->stats.pxon2offc[i] +=
5220                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5221         }
5222         for (int i = 0; i < 16; i++) {
5223                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5224                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5225                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5226                 adapter->stats.qbrc[i] +=
5227                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5228                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5229                 adapter->stats.qbtc[i] +=
5230                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5231                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5232         }
5233         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5234         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5235         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5236
5237         /* Hardware workaround, gprc counts missed packets */
5238         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5239         adapter->stats.gprc -= missed_rx;
5240
5241         if (hw->mac.type != ixgbe_mac_82598EB) {
5242                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5243                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5244                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5245                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5246                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5247                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5248                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5249                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5250         } else {
5251                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5252                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5253                 /* 82598 only has a counter in the high register */
5254                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5255                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5256                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5257         }
5258
5259         /*
5260          * Workaround: mprc hardware is incorrectly counting
5261          * broadcasts, so for now we subtract those.
5262          */
5263         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5264         adapter->stats.bprc += bprc;
5265         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5266         if (hw->mac.type == ixgbe_mac_82598EB)
5267                 adapter->stats.mprc -= bprc;
5268
5269         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5270         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5271         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5272         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5273         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5274         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5275
5276         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5277         adapter->stats.lxontxc += lxon;
5278         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5279         adapter->stats.lxofftxc += lxoff;
5280         total = lxon + lxoff;
5281
5282         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5283         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5284         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5285         adapter->stats.gptc -= total;
5286         adapter->stats.mptc -= total;
5287         adapter->stats.ptc64 -= total;
5288         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5289
5290         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5291         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5292         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5293         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5294         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5295         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5296         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5297         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5298         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5299         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5300         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5301         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5302         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5303         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5304         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5305         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5306         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5307         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5308         /* Only read FCOE on 82599 */
5309         if (hw->mac.type != ixgbe_mac_82598EB) {
5310                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5311                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5312                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5313                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5314                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5315         }
5316
5317         /* Fill out the OS statistics structure */
5318         ifp->if_ipackets = adapter->stats.gprc;
5319         ifp->if_opackets = adapter->stats.gptc;
5320         ifp->if_ibytes = adapter->stats.gorc;
5321         ifp->if_obytes = adapter->stats.gotc;
5322         ifp->if_imcasts = adapter->stats.mprc;
5323         ifp->if_collisions = 0;
5324
5325         /* Rx Errors */
5326         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5327                 adapter->stats.rlec;
5328 }
5329
5330 /** ixgbe_sysctl_tdh_handler - Handler function
5331  *  Retrieves the TDH value from the hardware
5332  */
5333 static int
5334 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5335 {
5336         int error;
5337
5338         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5339         if (!txr) return 0;
5340
5341         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5342         error = sysctl_handle_int(oidp, &val, 0, req);
5343         if (error || !req->newptr)
5344                 return error;
5345         return 0;
5346 }
5347
5348 /** ixgbe_sysctl_tdt_handler - Handler function
5349  *  Retrieves the TDT value from the hardware
5350  */
5351 static int
5352 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5353 {
5354         int error;
5355
5356         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5357         if (!txr) return 0;
5358
5359         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5360         error = sysctl_handle_int(oidp, &val, 0, req);
5361         if (error || !req->newptr)
5362                 return error;
5363         return 0;
5364 }
5365
5366 /** ixgbe_sysctl_rdh_handler - Handler function
5367  *  Retrieves the RDH value from the hardware
5368  */
5369 static int
5370 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5371 {
5372         int error;
5373
5374         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5375         if (!rxr) return 0;
5376
5377         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5378         error = sysctl_handle_int(oidp, &val, 0, req);
5379         if (error || !req->newptr)
5380                 return error;
5381         return 0;
5382 }
5383
5384 /** ixgbe_sysctl_rdt_handler - Handler function
5385  *  Retrieves the RDT value from the hardware
5386  */
5387 static int
5388 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5389 {
5390         int error;
5391
5392         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5393         if (!rxr) return 0;
5394
5395         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5396         error = sysctl_handle_int(oidp, &val, 0, req);
5397         if (error || !req->newptr)
5398                 return error;
5399         return 0;
5400 }
5401
5402 static int
5403 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5404 {
5405         int error;
5406         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5407         unsigned int reg, usec, rate;
5408
5409         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5410         usec = ((reg & 0x0FF8) >> 3);
5411         if (usec > 0)
5412                 rate = 500000 / usec;
5413         else
5414                 rate = 0;
5415         error = sysctl_handle_int(oidp, &rate, 0, req);
5416         if (error || !req->newptr)
5417                 return error;
5418         reg &= ~0xfff; /* default, no limitation */
5419         ixgbe_max_interrupt_rate = 0;
5420         if (rate > 0 && rate < 500000) {
5421                 if (rate < 1000)
5422                         rate = 1000;
5423                 ixgbe_max_interrupt_rate = rate;
5424                 reg |= ((4000000/rate) & 0xff8 );
5425         }
5426         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5427         return 0;
5428 }
5429
5430 /*
5431  * Add sysctl variables, one per statistic, to the system.
5432  */
5433 static void
5434 ixgbe_add_hw_stats(struct adapter *adapter)
5435 {
5436         struct tx_ring *txr = adapter->tx_rings;
5437         struct rx_ring *rxr = adapter->rx_rings;
5438
5439         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5440         struct sysctl_oid *tree = adapter->sysctl_tree;
5441         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5442         struct ixgbe_hw_stats *stats = &adapter->stats;
5443
5444         struct sysctl_oid *stat_node, *queue_node;
5445         struct sysctl_oid_list *stat_list, *queue_list;
5446
5447 #define QUEUE_NAME_LEN 32
5448         char namebuf[QUEUE_NAME_LEN];
5449
5450         /* Driver Statistics */
5451         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5452                         CTLFLAG_RD, &adapter->dropped_pkts,
5453                         "Driver dropped packets");
5454         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5455                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5456                         "m_defrag() failed");
5457         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5458                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5459                         "Driver tx dma failure in xmit");
5460         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5461                         CTLFLAG_RD, &adapter->watchdog_events,
5462                         "Watchdog timeouts");
5463 #if 0   /* NET_TSO */
5464         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5465                         CTLFLAG_RD, &adapter->tso_tx,
5466                         "TSO");
5467 #endif
5468         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5469                         CTLFLAG_RD, &adapter->link_irq,
5470                         "Link MSIX IRQ Handled");
5471
5472         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5473         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5474                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5475                                             CTLFLAG_RD, NULL, "Queue Name");
5476                 queue_list = SYSCTL_CHILDREN(queue_node);
5477
5478                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5479                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5480                                 sizeof(&adapter->queues[i]),
5481                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5482                                 "Interrupt Rate");
5483                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5484                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5485                                 "irqs on this queue");
5486                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5487                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5488                                 ixgbe_sysctl_tdh_handler, "IU",
5489                                 "Transmit Descriptor Head");
5490                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5491                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5492                                 ixgbe_sysctl_tdt_handler, "IU",
5493                                 "Transmit Descriptor Tail");
5494                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5495                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5496                                 "Queue No Descriptor Available");
5497                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5498                                 CTLFLAG_RD, &txr->total_packets, 0,
5499                                 "Queue Packets Transmitted");
5500         }
5501
5502         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5503         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5504                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5505                                             CTLFLAG_RD, NULL, "Queue Name");
5506                 queue_list = SYSCTL_CHILDREN(queue_node);
5507
5508 #if 0   /* NET_LRO */
5509                 struct lro_ctrl *lro = &rxr->lro;
5510 #endif
5511
5512         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5513                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5514                                             CTLFLAG_RD, NULL, "Queue Name");
5515                 queue_list = SYSCTL_CHILDREN(queue_node);
5516
5517                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5518                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5519                                 ixgbe_sysctl_rdh_handler, "IU",
5520                                 "Receive Descriptor Head");
5521                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5522                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5523                                 ixgbe_sysctl_rdt_handler, "IU",
5524                                 "Receive Descriptor Tail");
5525                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5526                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5527                                 "Queue Packets Received");
5528                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5529                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5530                                 "Queue Bytes Received");
5531 #if 0   /* NET_LRO */
5532                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5533                                 CTLFLAG_RD, &lro->lro_queued, 0,
5534                                 "LRO Queued");
5535                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5536                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5537                                 "LRO Flushed");
5538 #endif
5539         }
5540
5541         /* MAC stats get the own sub node */
5542
5543         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5544                                     CTLFLAG_RD, NULL, "MAC Statistics");
5545         stat_list = SYSCTL_CHILDREN(stat_node);
5546
5547         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5548                         CTLFLAG_RD, &stats->crcerrs, 0,
5549                         "CRC Errors");
5550         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5551                         CTLFLAG_RD, &stats->illerrc, 0,
5552                         "Illegal Byte Errors");
5553         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5554                         CTLFLAG_RD, &stats->errbc, 0,
5555                         "Byte Errors");
5556         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5557                         CTLFLAG_RD, &stats->mspdc, 0,
5558                         "MAC Short Packets Discarded");
5559         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5560                         CTLFLAG_RD, &stats->mlfc, 0,
5561                         "MAC Local Faults");
5562         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5563                         CTLFLAG_RD, &stats->mrfc, 0,
5564                         "MAC Remote Faults");
5565         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5566                         CTLFLAG_RD, &stats->rlec, 0,
5567                         "Receive Length Errors");
5568         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5569                         CTLFLAG_RD, &stats->lxontxc, 0,
5570                         "Link XON Transmitted");
5571         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5572                         CTLFLAG_RD, &stats->lxonrxc, 0,
5573                         "Link XON Received");
5574         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5575                         CTLFLAG_RD, &stats->lxofftxc, 0,
5576                         "Link XOFF Transmitted");
5577         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5578                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5579                         "Link XOFF Received");
5580
5581         /* Packet Reception Stats */
5582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5583                         CTLFLAG_RD, &stats->tor, 0,
5584                         "Total Octets Received");
5585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5586                         CTLFLAG_RD, &stats->gorc, 0,
5587                         "Good Octets Received");
5588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5589                         CTLFLAG_RD, &stats->tpr, 0,
5590                         "Total Packets Received");
5591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5592                         CTLFLAG_RD, &stats->gprc, 0,
5593                         "Good Packets Received");
5594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5595                         CTLFLAG_RD, &stats->mprc, 0,
5596                         "Multicast Packets Received");
5597         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5598                         CTLFLAG_RD, &stats->bprc, 0,
5599                         "Broadcast Packets Received");
5600         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5601                         CTLFLAG_RD, &stats->prc64, 0,
5602                         "64 byte frames received ");
5603         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5604                         CTLFLAG_RD, &stats->prc127, 0,
5605                         "65-127 byte frames received");
5606         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5607                         CTLFLAG_RD, &stats->prc255, 0,
5608                         "128-255 byte frames received");
5609         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5610                         CTLFLAG_RD, &stats->prc511, 0,
5611                         "256-511 byte frames received");
5612         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5613                         CTLFLAG_RD, &stats->prc1023, 0,
5614                         "512-1023 byte frames received");
5615         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5616                         CTLFLAG_RD, &stats->prc1522, 0,
5617                         "1023-1522 byte frames received");
5618         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5619                         CTLFLAG_RD, &stats->ruc, 0,
5620                         "Receive Undersized");
5621         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5622                         CTLFLAG_RD, &stats->rfc, 0,
5623                         "Fragmented Packets Received ");
5624         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5625                         CTLFLAG_RD, &stats->roc, 0,
5626                         "Oversized Packets Received");
5627         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5628                         CTLFLAG_RD, &stats->rjc, 0,
5629                         "Received Jabber");
5630         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5631                         CTLFLAG_RD, &stats->mngprc, 0,
5632                         "Management Packets Received");
5633         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5634                         CTLFLAG_RD, &stats->mngptc, 0,
5635                         "Management Packets Dropped");
5636         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5637                         CTLFLAG_RD, &stats->xec, 0,
5638                         "Checksum Errors");
5639
5640         /* Packet Transmission Stats */
5641         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5642                         CTLFLAG_RD, &stats->gotc, 0,
5643                         "Good Octets Transmitted");
5644         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5645                         CTLFLAG_RD, &stats->tpt, 0,
5646                         "Total Packets Transmitted");
5647         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5648                         CTLFLAG_RD, &stats->gptc, 0,
5649                         "Good Packets Transmitted");
5650         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5651                         CTLFLAG_RD, &stats->bptc, 0,
5652                         "Broadcast Packets Transmitted");
5653         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5654                         CTLFLAG_RD, &stats->mptc, 0,
5655                         "Multicast Packets Transmitted");
5656         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5657                         CTLFLAG_RD, &stats->mngptc, 0,
5658                         "Management Packets Transmitted");
5659         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5660                         CTLFLAG_RD, &stats->ptc64, 0,
5661                         "64 byte frames transmitted ");
5662         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5663                         CTLFLAG_RD, &stats->ptc127, 0,
5664                         "65-127 byte frames transmitted");
5665         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5666                         CTLFLAG_RD, &stats->ptc255, 0,
5667                         "128-255 byte frames transmitted");
5668         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5669                         CTLFLAG_RD, &stats->ptc511, 0,
5670                         "256-511 byte frames transmitted");
5671         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5672                         CTLFLAG_RD, &stats->ptc1023, 0,
5673                         "512-1023 byte frames transmitted");
5674         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5675                         CTLFLAG_RD, &stats->ptc1522, 0,
5676                         "1024-1522 byte frames transmitted");
5677
5678         /* FC Stats */
5679         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5680                 CTLFLAG_RD, &stats->fccrc, 0,
5681                 "FC CRC Errors");
5682         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5683                 CTLFLAG_RD, &stats->fclast, 0,
5684                 "FC Last Error");
5685         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5686                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5687                 "FCoE Packets Dropped");
5688         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5689                 CTLFLAG_RD, &stats->fcoeprc, 0,
5690                 "FCoE Packets Received");
5691         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5692                 CTLFLAG_RD, &stats->fcoeptc, 0,
5693                 "FCoE Packets Transmitted");
5694         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5695                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5696                 "FCoE DWords Received");
5697         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5698                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5699                 "FCoE DWords Transmitted");
5700 }
5701
5702 /*
5703 ** Set flow control using sysctl:
5704 ** Flow control values:
5705 **      0 - off
5706 **      1 - rx pause
5707 **      2 - tx pause
5708 **      3 - full
5709 */
5710 static int
5711 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5712 {
5713         int error, last;
5714         struct adapter *adapter = (struct adapter *) arg1;
5715
5716         last = adapter->fc;
5717         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5718         if ((error) || (req->newptr == NULL))
5719                 return (error);
5720
5721         /* Don't bother if it's not changed */
5722         if (adapter->fc == last)
5723                 return (0);
5724
5725         switch (adapter->fc) {
5726                 case ixgbe_fc_rx_pause:
5727                 case ixgbe_fc_tx_pause:
5728                 case ixgbe_fc_full:
5729                         adapter->hw.fc.requested_mode = adapter->fc;
5730                         break;
5731                 case ixgbe_fc_none:
5732                 default:
5733                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5734         }
5735
5736         ixgbe_fc_enable(&adapter->hw, 0);
5737         return error;
5738 }
5739
5740 static void
5741 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5742         const char *description, int *limit, int value)
5743 {
5744         *limit = value;
5745         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5746             SYSCTL_CHILDREN(adapter->sysctl_tree),
5747             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5748 }
5749
5750 /*
5751 ** Control link advertise speed:
5752 **      0 - normal
5753 **      1 - advertise only 1G
5754 **      2 - advertise 100Mb
5755 */
5756 static int
5757 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5758 {
5759         int                     error = 0;
5760         struct adapter          *adapter;
5761         device_t                dev;
5762         struct ixgbe_hw         *hw;
5763         ixgbe_link_speed        speed, last;
5764
5765         adapter = (struct adapter *) arg1;
5766         dev = adapter->dev;
5767         hw = &adapter->hw;
5768         last = hw->phy.autoneg_advertised;
5769
5770         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5771
5772         if ((error) || (adapter->advertise == -1))
5773                 return (error);
5774
5775         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5776             (hw->phy.multispeed_fiber)))
5777                 return (error);
5778
5779         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5780                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5781                 return (error);
5782         }
5783
5784         if (adapter->advertise == 1)
5785                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5786         else if (adapter->advertise == 2)
5787                 speed = IXGBE_LINK_SPEED_100_FULL;
5788         else
5789                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5790                         IXGBE_LINK_SPEED_10GB_FULL;
5791
5792         if (speed == last) /* no change */
5793                 return (error);
5794
5795         hw->mac.autotry_restart = TRUE;
5796         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5797
5798         return (error);
5799 }
5800
5801 /*
5802 ** Thermal Shutdown Trigger
5803 **   - cause a Thermal Overtemp IRQ
5804 */
5805 static int
5806 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5807 {
5808         int             error, fire = 0;
5809         struct adapter  *adapter = (struct adapter *) arg1;
5810         struct ixgbe_hw *hw = &adapter->hw;
5811
5812
5813         if (hw->mac.type != ixgbe_mac_X540)
5814                 return (0);
5815
5816         error = sysctl_handle_int(oidp, &fire, 0, req);
5817         if ((error) || (req->newptr == NULL))
5818                 return (error);
5819
5820         if (fire) {
5821                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5822                 reg |= IXGBE_EICR_TS;
5823                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5824         }
5825
5826         return (0);
5827 }