sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 #if 0   /* HAVE_MSIX */
 125 static int      ixgbe_setup_msix(struct adapter *);
 126 #endif
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 #if 0   /* NET_LRO */
 146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 147 #endif
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *, int);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 167                     const char *, int *, int);
 168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 169 #if 0   /* NET_TSO */
 170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 171 #endif
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_que(void *, int);
 199 static void     ixgbe_handle_link(void *, int);
 200 static void     ixgbe_handle_msf(void *, int);
 201 static void     ixgbe_handle_mod(void *, int);
 202
 203 #ifdef IXGBE_FDIR
 204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 205 static void     ixgbe_reinit_fdir(void *, int);
 206 #endif
 207
 208 /*********************************************************************
 209  *  FreeBSD Device Interface Entry Points
 210  *********************************************************************/
 211
 212 static device_method_t ixgbe_methods[] = {
 213         /* Device interface */
 214         DEVMETHOD(device_probe, ixgbe_probe),
 215         DEVMETHOD(device_attach, ixgbe_attach),
 216         DEVMETHOD(device_detach, ixgbe_detach),
 217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 218         {0, 0}
 219 };
 220
 221 static driver_t ixgbe_driver = {
 222         "ix", ixgbe_methods, sizeof(struct adapter),
 223 };
 224
 225 devclass_t ixgbe_devclass;
 226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 227
 228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 230
 231 /*
 232 ** TUNEABLE PARAMETERS:
 233 */
 234
 235 /*
 236 ** AIM: Adaptive Interrupt Moderation
 237 ** which means that the interrupt rate
 238 ** is varied over time based on the
 239 ** traffic for that interrupt vector
 240 */
 241 static int ixgbe_enable_aim = TRUE;
 242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 243
 244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 246
 247 /* How many packets rxeof tries to clean at a time */
 248 static int ixgbe_rx_process_limit = 128;
 249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 static int ixgbe_msi_enable = 1;
 261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 262
 263 /*
 264  * MSIX should be the default for best performance,
 265  * but this allows it to be forced off for testing.
 266  */
 267 static int ixgbe_enable_msix = 1;
 268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 269
 270 /*
 271  * Header split: this causes the hardware to DMA
 272  * the header into a separate mbuf from the payload,
 273  * it can be a performance win in some workloads, but
 274  * in others it actually hurts, its off by default.
 275  */
 276 static int ixgbe_header_split = FALSE;
 277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 278
 279 /*
 280  * Number of Queues, can be set to 0,
 281  * it then autoconfigures based on the
 282  * number of cpus with a max of 8. This
 283  * can be overriden manually here.
 284  */
 285 static int ixgbe_num_queues = 0;
 286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 287
 288 /*
 289 ** Number of TX descriptors per ring,
 290 ** setting higher than RX as this seems
 291 ** the better performing choice.
 292 */
 293 static int ixgbe_txd = PERFORM_TXD;
 294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 295
 296 /* Number of RX descriptors per ring */
 297 static int ixgbe_rxd = PERFORM_RXD;
 298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** For Flow Director: this is the
 306 ** number of TX packets we sample
 307 ** for the filter pool, this means
 308 ** every 20th packet will be probed.
 309 **
 310 ** This feature can be disabled by
 311 ** setting this to 0.
 312 */
 313 static int atr_sample_rate = 20;
 314 /*
 315 ** Flow Director actually 'steals'
 316 ** part of the packet buffer as its
 317 ** filter pool, this variable controls
 318 ** how much it uses:
 319 **  0 = 64K, 1 = 128K, 2 = 256K
 320 */
 321 static int fdir_pballoc = 1;
 322 #endif
 323
 324 #ifdef DEV_NETMAP
 325 /*
 326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 327  * be a reference on how to implement netmap support in a driver.
 328  * Additional comments are in ixgbe_netmap.h .
 329  *
 330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 331  * that extend the standard driver.
 332  */
 333 #include <dev/netmap/ixgbe_netmap.h>
 334 #endif /* DEV_NETMAP */
 335
 336 /*********************************************************************
 337  *  Device identification routine
 338  *
 339  *  ixgbe_probe determines if the driver should be loaded on
 340  *  adapter based on PCI vendor/device id of the adapter.
 341  *
 342  *  return BUS_PROBE_DEFAULT on success, positive on failure
 343  *********************************************************************/
 344
 345 static int
 346 ixgbe_probe(device_t dev)
 347 {
 348         ixgbe_vendor_info_t *ent;
 349
 350         u16     pci_vendor_id = 0;
 351         u16     pci_device_id = 0;
 352         u16     pci_subvendor_id = 0;
 353         u16     pci_subdevice_id = 0;
 354         char    adapter_name[256];
 355
 356         INIT_DEBUGOUT("ixgbe_probe: begin");
 357
 358         pci_vendor_id = pci_get_vendor(dev);
 359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 360                 return (ENXIO);
 361
 362         pci_device_id = pci_get_device(dev);
 363         pci_subvendor_id = pci_get_subvendor(dev);
 364         pci_subdevice_id = pci_get_subdevice(dev);
 365
 366         ent = ixgbe_vendor_info_array;
 367         while (ent->vendor_id != 0) {
 368                 if ((pci_vendor_id == ent->vendor_id) &&
 369                     (pci_device_id == ent->device_id) &&
 370
 371                     ((pci_subvendor_id == ent->subvendor_id) ||
 372                      (ent->subvendor_id == 0)) &&
 373
 374                     ((pci_subdevice_id == ent->subdevice_id) ||
 375                      (ent->subdevice_id == 0))) {
 376                         ksprintf(adapter_name, "%s, Version - %s",
 377                                 ixgbe_strings[ent->index],
 378                                 ixgbe_driver_version);
 379                         device_set_desc_copy(dev, adapter_name);
 380                         ++ixgbe_total_ports;
 381                         return (BUS_PROBE_DEFAULT);
 382                 }
 383                 ent++;
 384         }
 385         return (ENXIO);
 386 }
 387
 388 /*********************************************************************
 389  *  Device initialization routine
 390  *
 391  *  The attach entry point is called when the driver is being loaded.
 392  *  This routine identifies the type of hardware, allocates all resources
 393  *  and initializes the hardware.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 ixgbe_attach(device_t dev)
 400 {
 401         struct adapter *adapter;
 402         struct ixgbe_hw *hw;
 403         int             error = 0;
 404         u16             csum;
 405         u32             ctrl_ext;
 406
 407         INIT_DEBUGOUT("ixgbe_attach: begin");
 408
 409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 410                 device_printf(dev, "Disabled by device hint\n");
 411                 return (ENXIO);
 412         }
 413
 414         /* Allocate, clear, and link in our adapter structure */
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         hw = &adapter->hw;
 418
 419         /* Core Lock Init*/
 420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 421         spin_init(&adapter->mcast_spin);
 422
 423         /* SYSCTL APIs */
 424
 425         sysctl_ctx_init(&adapter->sysctl_ctx);
 426         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 427             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 428             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 429         if (adapter->sysctl_tree == NULL) {
 430                 device_printf(adapter->dev, "can't add sysctl node\n");
 431                 return (EINVAL);
 432         }
 433         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 434                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 435                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 436                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 437
 438         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 439                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 440                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 441                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 442
 443         /*
 444         ** Allow a kind of speed control by forcing the autoneg
 445         ** advertised speed list to only a certain value, this
 446         ** supports 1G on 82599 devices, and 100Mb on x540.
 447         */
 448         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 449                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 450                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 451                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 452
 453         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 454                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 455                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 456                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 457
 458         /* Set up the timer callout */
 459         callout_init(&adapter->timer);
 460
 461         /* Determine hardware revision */
 462         ixgbe_identify_hardware(adapter);
 463
 464         /* Do base PCI setup - map BAR0 */
 465         if (ixgbe_allocate_pci_resources(adapter)) {
 466                 device_printf(dev, "Allocation of PCI resources failed\n");
 467                 error = ENXIO;
 468                 goto err_out;
 469         }
 470
 471         /* Do descriptor calc and sanity checks */
 472         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 473             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 474                 device_printf(dev, "TXD config issue, using default!\n");
 475                 adapter->num_tx_desc = DEFAULT_TXD;
 476         } else
 477                 adapter->num_tx_desc = ixgbe_txd;
 478
 479         /*
 480         ** With many RX rings it is easy to exceed the
 481         ** system mbuf allocation. Tuning nmbclusters
 482         ** can alleviate this.
 483         */
 484         if (nmbclusters > 0 ) {
 485                 int s;
 486                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 487                 if (s > nmbclusters) {
 488                         device_printf(dev, "RX Descriptors exceed "
 489                             "system mbuf max, using default instead!\n");
 490                         ixgbe_rxd = DEFAULT_RXD;
 491                 }
 492         }
 493
 494         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 495             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 496                 device_printf(dev, "RXD config issue, using default!\n");
 497                 adapter->num_rx_desc = DEFAULT_RXD;
 498         } else
 499                 adapter->num_rx_desc = ixgbe_rxd;
 500
 501         /* Allocate our TX/RX Queues */
 502         if (ixgbe_allocate_queues(adapter)) {
 503                 error = ENOMEM;
 504                 goto err_out;
 505         }
 506
 507         /* Allocate multicast array memory. */
 508         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 509             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 510         if (adapter->mta == NULL) {
 511                 device_printf(dev, "Can not allocate multicast setup array\n");
 512                 error = ENOMEM;
 513                 goto err_late;
 514         }
 515
 516         /* Initialize the shared code */
 517         error = ixgbe_init_shared_code(hw);
 518         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 519                 /*
 520                 ** No optics in this port, set up
 521                 ** so the timer routine will probe
 522                 ** for later insertion.
 523                 */
 524                 adapter->sfp_probe = TRUE;
 525                 error = 0;
 526         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 527                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 528                 error = EIO;
 529                 goto err_late;
 530         } else if (error) {
 531                 device_printf(dev,"Unable to initialize the shared code\n");
 532                 error = EIO;
 533                 goto err_late;
 534         }
 535
 536         /* Make sure we have a good EEPROM before we read from it */
 537         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 538                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 539                 error = EIO;
 540                 goto err_late;
 541         }
 542
 543         error = ixgbe_init_hw(hw);
 544         switch (error) {
 545         case IXGBE_ERR_EEPROM_VERSION:
 546                 device_printf(dev, "This device is a pre-production adapter/"
 547                     "LOM.  Please be aware there may be issues associated "
 548                     "with your hardware.\n If you are experiencing problems "
 549                     "please contact your Intel or hardware representative "
 550                     "who provided you with this hardware.\n");
 551                 break;
 552         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 553                 device_printf(dev,"Unsupported SFP+ Module\n");
 554                 error = EIO;
 555                 device_printf(dev,"Hardware Initialization Failure\n");
 556                 goto err_late;
 557         case IXGBE_ERR_SFP_NOT_PRESENT:
 558                 device_printf(dev,"No SFP+ Module found\n");
 559                 /* falls thru */
 560         default:
 561                 break;
 562         }
 563
 564         /* Detect and set physical type */
 565         ixgbe_setup_optics(adapter);
 566
 567         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 568                 error = ixgbe_allocate_msix(adapter);
 569         else
 570                 error = ixgbe_allocate_legacy(adapter);
 571         if (error)
 572                 goto err_late;
 573
 574         /* Setup OS specific network interface */
 575         if (ixgbe_setup_interface(dev, adapter) != 0)
 576                 goto err_late;
 577
 578         /* Sysctl for limiting the amount of work done in the taskqueue */
 579         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 580             "max number of rx packets to process", &adapter->rx_process_limit,
 581             ixgbe_rx_process_limit);
 582
 583         /* Initialize statistics */
 584         ixgbe_update_stats_counters(adapter);
 585
 586         /* Register for VLAN events */
 587         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 588             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 589         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 590             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 591
 592         /* Print PCIE bus type/speed/width info */
 593         ixgbe_get_bus_info(hw);
 594         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 595             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 596             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 597             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 598             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 599             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 600             ("Unknown"));
 601
 602         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 603             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 604                 device_printf(dev, "PCI-Express bandwidth available"
 605                     " for this card\n     is not sufficient for"
 606                     " optimal performance.\n");
 607                 device_printf(dev, "For optimal performance a x8 "
 608                     "PCIE, or x4 PCIE 2 slot is required.\n");
 609         }
 610
 611         /* let hardware know driver is loaded */
 612         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 613         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 614         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 615
 616         ixgbe_add_hw_stats(adapter);
 617
 618 #ifdef DEV_NETMAP
 619         ixgbe_netmap_attach(adapter);
 620 #endif /* DEV_NETMAP */
 621         INIT_DEBUGOUT("ixgbe_attach: end");
 622         return (0);
 623 err_late:
 624         ixgbe_free_transmit_structures(adapter);
 625         ixgbe_free_receive_structures(adapter);
 626 err_out:
 627         if (adapter->ifp != NULL)
 628                 if_free(adapter->ifp);
 629         ixgbe_free_pci_resources(adapter);
 630         kfree(adapter->mta, M_DEVBUF);
 631         return (error);
 632
 633 }
 634
 635 /*********************************************************************
 636  *  Device removal routine
 637  *
 638  *  The detach entry point is called when the driver is being removed.
 639  *  This routine stops the adapter and deallocates all the resources
 640  *  that were allocated for driver operation.
 641  *
 642  *  return 0 on success, positive on failure
 643  *********************************************************************/
 644
 645 static int
 646 ixgbe_detach(device_t dev)
 647 {
 648         struct adapter *adapter = device_get_softc(dev);
 649         struct ix_queue *que = adapter->queues;
 650         u32     ctrl_ext;
 651
 652         INIT_DEBUGOUT("ixgbe_detach: begin");
 653
 654 #ifdef NET_VLAN
 655         /* Make sure VLANS are not using driver */
 656         if (adapter->ifp->if_vlantrunk != NULL) {
 657                 device_printf(dev,"Vlan in use, detach first\n");
 658                 return (EBUSY);
 659         }
 660 #endif
 661
 662         IXGBE_CORE_LOCK(adapter);
 663         ixgbe_stop(adapter);
 664         IXGBE_CORE_UNLOCK(adapter);
 665
 666         for (int i = 0; i < adapter->num_queues; i++, que++) {
 667                 if (que->tq) {
 668                         taskqueue_drain(que->tq, &que->que_task);
 669                         taskqueue_free(que->tq);
 670                 }
 671         }
 672
 673         /* Drain the Link queue */
 674         if (adapter->tq) {
 675                 taskqueue_drain(adapter->tq, &adapter->link_task);
 676                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 677                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 678 #ifdef IXGBE_FDIR
 679                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 680 #endif
 681                 taskqueue_free(adapter->tq);
 682         }
 683
 684         /* let hardware know driver is unloading */
 685         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 686         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 687         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 688
 689         /* Unregister VLAN events */
 690 #ifdef NET_VLAN
 691         if (adapter->vlan_attach != NULL)
 692                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 693         if (adapter->vlan_detach != NULL)
 694                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 695 #endif
 696
 697         ether_ifdetach(adapter->ifp);
 698         callout_stop(&adapter->timer);
 699         lockuninit(&adapter->core_lock);
 700 #ifdef DEV_NETMAP
 701         netmap_detach(adapter->ifp);
 702 #endif /* DEV_NETMAP */
 703         ixgbe_free_pci_resources(adapter);
 704         bus_generic_detach(dev);
 705         if_free(adapter->ifp);
 706
 707         ixgbe_free_transmit_structures(adapter);
 708         ixgbe_free_receive_structures(adapter);
 709         kfree(adapter->mta, M_DEVBUF);
 710         sysctl_ctx_free(&adapter->sysctl_ctx);
 711
 712         spin_uninit(&adapter->mcast_spin);
 713         IXGBE_CORE_LOCK_DESTROY(adapter);
 714         return (0);
 715 }
 716
 717 /*********************************************************************
 718  *
 719  *  Shutdown entry point
 720  *
 721  **********************************************************************/
 722
 723 static int
 724 ixgbe_shutdown(device_t dev)
 725 {
 726         struct adapter *adapter = device_get_softc(dev);
 727         IXGBE_CORE_LOCK(adapter);
 728         ixgbe_stop(adapter);
 729         IXGBE_CORE_UNLOCK(adapter);
 730         return (0);
 731 }
 732
 733
 734 /*********************************************************************
 735  *  Transmit entry point
 736  *
 737  *  ixgbe_start is called by the stack to initiate a transmit.
 738  *  The driver will remain in this routine as long as there are
 739  *  packets to transmit and transmit resources are available.
 740  *  In case resources are not available stack is notified and
 741  *  the packet is requeued.
 742  **********************************************************************/
 743
 744 static void
 745 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 746 {
 747         struct mbuf    *m_head;
 748         struct adapter *adapter = txr->adapter;
 749
 750         IXGBE_TX_LOCK_ASSERT(txr);
 751
 752         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 753                 return;
 754         if (!adapter->link_active)
 755                 return;
 756
 757         while (!ifq_is_empty(&ifp->if_snd)) {
 758                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 759                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 760                         break;
 761                 }
 762
 763                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 764                 if (m_head == NULL)
 765                         break;
 766
 767                 if (ixgbe_xmit(txr, &m_head)) {
 768 #if 0 /* XXX: prepend to an ALTQ queue ? */
 769                         if (m_head != NULL)
 770                                 IF_PREPEND(&ifp->if_snd, m_head);
 771 #endif
 772                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 773                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 774                         break;
 775                 }
 776                 /* Send a copy of the frame to the BPF listener */
 777                 ETHER_BPF_MTAP(ifp, m_head);
 778
 779                 /* Set watchdog on */
 780                 txr->watchdog_time = ticks;
 781                 txr->queue_status = IXGBE_QUEUE_WORKING;
 782
 783         }
 784         return;
 785 }
 786
 787 /*
 788  * Legacy TX start - called by the stack, this
 789  * always uses the first tx ring, and should
 790  * not be used with multiqueue tx enabled.
 791  */
 792 static void
 793 ixgbe_start(struct ifnet *ifp)
 794 {
 795         struct adapter *adapter = ifp->if_softc;
 796         struct tx_ring  *txr = adapter->tx_rings;
 797
 798         if (ifp->if_flags & IFF_RUNNING) {
 799                 IXGBE_TX_LOCK(txr);
 800                 ixgbe_start_locked(txr, ifp);
 801                 IXGBE_TX_UNLOCK(txr);
 802         }
 803         return;
 804 }
 805
 806 #if 0 /* __FreeBSD_version >= 800000 */
 807 /*
 808 ** Multiqueue Transmit driver
 809 **
 810 */
 811 static int
 812 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 813 {
 814         struct adapter  *adapter = ifp->if_softc;
 815         struct ix_queue *que;
 816         struct tx_ring  *txr;
 817         int             i = 0, err = 0;
 818
 819         /* Which queue to use */
 820         if ((m->m_flags & M_FLOWID) != 0)
 821                 i = m->m_pkthdr.flowid % adapter->num_queues;
 822         else
 823                 i = curcpu % adapter->num_queues;
 824
 825         txr = &adapter->tx_rings[i];
 826         que = &adapter->queues[i];
 827
 828         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 829             IXGBE_TX_TRYLOCK(txr)) {
 830                 err = ixgbe_mq_start_locked(ifp, txr, m);
 831                 IXGBE_TX_UNLOCK(txr);
 832         } else {
 833                 err = drbr_enqueue(ifp, txr->br, m);
 834                 taskqueue_enqueue(que->tq, &que->que_task);
 835         }
 836
 837         return (err);
 838 }
 839
 840 static int
 841 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 842 {
 843         struct adapter  *adapter = txr->adapter;
 844         struct mbuf     *next;
 845         int             enqueued, err = 0;
 846
 847         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 848             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 849             adapter->link_active == 0) {
 850                 if (m != NULL)
 851                         err = drbr_enqueue(ifp, txr->br, m);
 852                 return (err);
 853         }
 854
 855         enqueued = 0;
 856         if (m == NULL) {
 857                 next = drbr_dequeue(ifp, txr->br);
 858         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 859                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 860                         return (err);
 861                 next = drbr_dequeue(ifp, txr->br);
 862         } else
 863                 next = m;
 864
 865         /* Process the queue */
 866         while (next != NULL) {
 867                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 868                         if (next != NULL)
 869                                 err = drbr_enqueue(ifp, txr->br, next);
 870                         break;
 871                 }
 872                 enqueued++;
 873                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 874                 /* Send a copy of the frame to the BPF listener */
 875                 ETHER_BPF_MTAP(ifp, next);
 876                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 877                         break;
 878                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 879                         ixgbe_txeof(txr);
 880                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 881                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 882                         break;
 883                 }
 884                 next = drbr_dequeue(ifp, txr->br);
 885         }
 886
 887         if (enqueued > 0) {
 888                 /* Set watchdog on */
 889                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 890                 txr->watchdog_time = ticks;
 891         }
 892
 893         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 894                 ixgbe_txeof(txr);
 895
 896         return (err);
 897 }
 898
 899 /*
 900 ** Flush all ring buffers
 901 */
 902 static void
 903 ixgbe_qflush(struct ifnet *ifp)
 904 {
 905         struct adapter  *adapter = ifp->if_softc;
 906         struct tx_ring  *txr = adapter->tx_rings;
 907         struct mbuf     *m;
 908
 909         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 910                 IXGBE_TX_LOCK(txr);
 911                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 912                         m_freem(m);
 913                 IXGBE_TX_UNLOCK(txr);
 914         }
 915         if_qflush(ifp);
 916 }
 917 #endif /* __FreeBSD_version >= 800000 */
 918
 919 /*********************************************************************
 920  *  Ioctl entry point
 921  *
 922  *  ixgbe_ioctl is called when the user wants to configure the
 923  *  interface.
 924  *
 925  *  return 0 on success, positive on failure
 926  **********************************************************************/
 927
 928 static int
 929 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 930 {
 931         struct adapter  *adapter = ifp->if_softc;
 932         struct ifreq    *ifr = (struct ifreq *) data;
 933 #if defined(INET) || defined(INET6)
 934         struct ifaddr *ifa = (struct ifaddr *)data;
 935         bool            avoid_reset = FALSE;
 936 #endif
 937         int             error = 0;
 938
 939         switch (command) {
 940
 941         case SIOCSIFADDR:
 942 #ifdef INET
 943                 if (ifa->ifa_addr->sa_family == AF_INET)
 944                         avoid_reset = TRUE;
 945 #endif
 946 #ifdef INET6
 947                 if (ifa->ifa_addr->sa_family == AF_INET6)
 948                         avoid_reset = TRUE;
 949 #endif
 950 #if defined(INET) || defined(INET6)
 951                 /*
 952                 ** Calling init results in link renegotiation,
 953                 ** so we avoid doing it when possible.
 954                 */
 955                 if (avoid_reset) {
 956                         ifp->if_flags |= IFF_UP;
 957                         if (!(ifp->if_flags & IFF_RUNNING))
 958                                 ixgbe_init(adapter);
 959                         if (!(ifp->if_flags & IFF_NOARP))
 960                                 arp_ifinit(ifp, ifa);
 961                 } else
 962                         error = ether_ioctl(ifp, command, data);
 963 #endif
 964                 break;
 965         case SIOCSIFMTU:
 966                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 967                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 968                         error = EINVAL;
 969                 } else {
 970                         IXGBE_CORE_LOCK(adapter);
 971                         ifp->if_mtu = ifr->ifr_mtu;
 972                         adapter->max_frame_size =
 973                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 974                         ixgbe_init_locked(adapter);
 975                         IXGBE_CORE_UNLOCK(adapter);
 976                 }
 977                 break;
 978         case SIOCSIFFLAGS:
 979                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 980                 IXGBE_CORE_LOCK(adapter);
 981                 if (ifp->if_flags & IFF_UP) {
 982                         if ((ifp->if_flags & IFF_RUNNING)) {
 983                                 if ((ifp->if_flags ^ adapter->if_flags) &
 984                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 985                                         ixgbe_set_promisc(adapter);
 986                                 }
 987                         } else
 988                                 ixgbe_init_locked(adapter);
 989                 } else
 990                         if (ifp->if_flags & IFF_RUNNING)
 991                                 ixgbe_stop(adapter);
 992                 adapter->if_flags = ifp->if_flags;
 993                 IXGBE_CORE_UNLOCK(adapter);
 994                 break;
 995         case SIOCADDMULTI:
 996         case SIOCDELMULTI:
 997                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 998                 if (ifp->if_flags & IFF_RUNNING) {
 999                         IXGBE_CORE_LOCK(adapter);
1000                         ixgbe_disable_intr(adapter);
1001                         ixgbe_set_multi(adapter);
1002                         ixgbe_enable_intr(adapter);
1003                         IXGBE_CORE_UNLOCK(adapter);
1004                 }
1005                 break;
1006         case SIOCSIFMEDIA:
1007         case SIOCGIFMEDIA:
1008                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1009                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1010                 break;
1011         case SIOCSIFCAP:
1012         {
1013                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1014                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1015                 if (mask & IFCAP_HWCSUM)
1016                         ifp->if_capenable ^= IFCAP_HWCSUM;
1017 #if 0 /* NET_TSO */
1018                 if (mask & IFCAP_TSO4)
1019                         ifp->if_capenable ^= IFCAP_TSO4;
1020                 if (mask & IFCAP_TSO6)
1021                         ifp->if_capenable ^= IFCAP_TSO6;
1022 #endif
1023 #if 0 /* NET_LRO */
1024                 if (mask & IFCAP_LRO)
1025                         ifp->if_capenable ^= IFCAP_LRO;
1026 #endif
1027                 if (mask & IFCAP_VLAN_HWTAGGING)
1028                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1029                 if (mask & IFCAP_VLAN_HWFILTER)
1030                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1031 #if 0 /* NET_TSO */
1032                 if (mask & IFCAP_VLAN_HWTSO)
1033                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1034 #endif
1035                 if (ifp->if_flags & IFF_RUNNING) {
1036                         IXGBE_CORE_LOCK(adapter);
1037                         ixgbe_init_locked(adapter);
1038                         IXGBE_CORE_UNLOCK(adapter);
1039                 }
1040 #if 0
1041                 VLAN_CAPABILITIES(ifp);
1042 #endif
1043                 break;
1044         }
1045
1046         default:
1047                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1048                 error = ether_ioctl(ifp, command, data);
1049                 break;
1050         }
1051
1052         return (error);
1053 }
1054
1055 /*********************************************************************
1056  *  Init entry point
1057  *
1058  *  This routine is used in two ways. It is used by the stack as
1059  *  init entry point in network interface structure. It is also used
1060  *  by the driver as a hw/sw initialization routine to get to a
1061  *  consistent state.
1062  *
1063  *  return 0 on success, positive on failure
1064  **********************************************************************/
1065 #define IXGBE_MHADD_MFS_SHIFT 16
1066
1067 static void
1068 ixgbe_init_locked(struct adapter *adapter)
1069 {
1070         struct ifnet   *ifp = adapter->ifp;
1071         device_t        dev = adapter->dev;
1072         struct ixgbe_hw *hw = &adapter->hw;
1073         u32             k, txdctl, mhadd, gpie;
1074         u32             rxdctl, rxctrl;
1075
1076         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1077         INIT_DEBUGOUT("ixgbe_init: begin");
1078         hw->adapter_stopped = FALSE;
1079         ixgbe_stop_adapter(hw);
1080         callout_stop(&adapter->timer);
1081
1082         /* reprogram the RAR[0] in case user changed it. */
1083         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1084
1085         /* Get the latest mac address, User can use a LAA */
1086         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1087               IXGBE_ETH_LENGTH_OF_ADDRESS);
1088         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1089         hw->addr_ctrl.rar_used_count = 1;
1090
1091         /* Set the various hardware offload abilities */
1092         ifp->if_hwassist = 0;
1093 #if 0 /* NET_TSO */
1094         if (ifp->if_capenable & IFCAP_TSO)
1095                 ifp->if_hwassist |= CSUM_TSO;
1096 #endif
1097         if (ifp->if_capenable & IFCAP_TXCSUM) {
1098                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1099 #if 0
1100                 if (hw->mac.type != ixgbe_mac_82598EB)
1101                         ifp->if_hwassist |= CSUM_SCTP;
1102 #endif
1103         }
1104
1105         /* Prepare transmit descriptors and buffers */
1106         if (ixgbe_setup_transmit_structures(adapter)) {
1107                 device_printf(dev,"Could not setup transmit structures\n");
1108                 ixgbe_stop(adapter);
1109                 return;
1110         }
1111
1112         ixgbe_init_hw(hw);
1113         ixgbe_initialize_transmit_units(adapter);
1114
1115         /* Setup Multicast table */
1116         ixgbe_set_multi(adapter);
1117
1118         /*
1119         ** Determine the correct mbuf pool
1120         ** for doing jumbo/headersplit
1121         */
1122         if (adapter->max_frame_size <= 2048)
1123                 adapter->rx_mbuf_sz = MCLBYTES;
1124         else if (adapter->max_frame_size <= 4096)
1125                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1126         else if (adapter->max_frame_size <= 9216)
1127                 adapter->rx_mbuf_sz = MJUM9BYTES;
1128         else
1129                 adapter->rx_mbuf_sz = MJUM16BYTES;
1130
1131         /* Prepare receive descriptors and buffers */
1132         if (ixgbe_setup_receive_structures(adapter)) {
1133                 device_printf(dev,"Could not setup receive structures\n");
1134                 ixgbe_stop(adapter);
1135                 return;
1136         }
1137
1138         /* Configure RX settings */
1139         ixgbe_initialize_receive_units(adapter);
1140
1141         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1142
1143         /* Enable Fan Failure Interrupt */
1144         gpie |= IXGBE_SDP1_GPIEN;
1145
1146         /* Add for Module detection */
1147         if (hw->mac.type == ixgbe_mac_82599EB)
1148                 gpie |= IXGBE_SDP2_GPIEN;
1149
1150         /* Thermal Failure Detection */
1151         if (hw->mac.type == ixgbe_mac_X540)
1152                 gpie |= IXGBE_SDP0_GPIEN;
1153
1154         if (adapter->msix > 1) {
1155                 /* Enable Enhanced MSIX mode */
1156                 gpie |= IXGBE_GPIE_MSIX_MODE;
1157                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1158                     IXGBE_GPIE_OCD;
1159         }
1160         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1161
1162         /* Set MTU size */
1163         if (ifp->if_mtu > ETHERMTU) {
1164                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1165                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1166                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1167                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1168         }
1169
1170         /* Now enable all the queues */
1171
1172         for (int i = 0; i < adapter->num_queues; i++) {
1173                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1174                 txdctl |= IXGBE_TXDCTL_ENABLE;
1175                 /* Set WTHRESH to 8, burst writeback */
1176                 txdctl |= (8 << 16);
1177                 /*
1178                  * When the internal queue falls below PTHRESH (32),
1179                  * start prefetching as long as there are at least
1180                  * HTHRESH (1) buffers ready. The values are taken
1181                  * from the Intel linux driver 3.8.21.
1182                  * Prefetching enables tx line rate even with 1 queue.
1183                  */
1184                 txdctl |= (32 << 0) | (1 << 8);
1185                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1186         }
1187
1188         for (int i = 0; i < adapter->num_queues; i++) {
1189                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1190                 if (hw->mac.type == ixgbe_mac_82598EB) {
1191                         /*
1192                         ** PTHRESH = 21
1193                         ** HTHRESH = 4
1194                         ** WTHRESH = 8
1195                         */
1196                         rxdctl &= ~0x3FFFFF;
1197                         rxdctl |= 0x080420;
1198                 }
1199                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1200                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1201                 for (k = 0; k < 10; k++) {
1202                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1203                             IXGBE_RXDCTL_ENABLE)
1204                                 break;
1205                         else
1206                                 msec_delay(1);
1207                 }
1208                 wmb();
1209 #ifdef DEV_NETMAP
1210                 /*
1211                  * In netmap mode, we must preserve the buffers made
1212                  * available to userspace before the if_init()
1213                  * (this is true by default on the TX side, because
1214                  * init makes all buffers available to userspace).
1215                  *
1216                  * netmap_reset() and the device specific routines
1217                  * (e.g. ixgbe_setup_receive_rings()) map these
1218                  * buffers at the end of the NIC ring, so here we
1219                  * must set the RDT (tail) register to make sure
1220                  * they are not overwritten.
1221                  *
1222                  * In this driver the NIC ring starts at RDH = 0,
1223                  * RDT points to the last slot available for reception (?),
1224                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1225                  */
1226                 if (ifp->if_capenable & IFCAP_NETMAP) {
1227                         struct netmap_adapter *na = NA(adapter->ifp);
1228                         struct netmap_kring *kring = &na->rx_rings[i];
1229                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1230
1231                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1232                 } else
1233 #endif /* DEV_NETMAP */
1234                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1235         }
1236
1237         /* Set up VLAN support and filter */
1238         ixgbe_setup_vlan_hw_support(adapter);
1239
1240         /* Enable Receive engine */
1241         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1242         if (hw->mac.type == ixgbe_mac_82598EB)
1243                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1244         rxctrl |= IXGBE_RXCTRL_RXEN;
1245         ixgbe_enable_rx_dma(hw, rxctrl);
1246
1247         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1248
1249         /* Set up MSI/X routing */
1250         if (ixgbe_enable_msix)  {
1251                 ixgbe_configure_ivars(adapter);
1252                 /* Set up auto-mask */
1253                 if (hw->mac.type == ixgbe_mac_82598EB)
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1255                 else {
1256                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1257                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1258                 }
1259         } else {  /* Simple settings for Legacy/MSI */
1260                 ixgbe_set_ivar(adapter, 0, 0, 0);
1261                 ixgbe_set_ivar(adapter, 0, 0, 1);
1262                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1263         }
1264
1265 #ifdef IXGBE_FDIR
1266         /* Init Flow director */
1267         if (hw->mac.type != ixgbe_mac_82598EB) {
1268                 u32 hdrm = 32 << fdir_pballoc;
1269
1270                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1271                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1272         }
1273 #endif
1274
1275         /*
1276         ** Check on any SFP devices that
1277         ** need to be kick-started
1278         */
1279         if (hw->phy.type == ixgbe_phy_none) {
1280                 int err = hw->phy.ops.identify(hw);
1281                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1282                         device_printf(dev,
1283                             "Unsupported SFP+ module type was detected.\n");
1284                         return;
1285                 }
1286         }
1287
1288         /* Set moderation on the Link interrupt */
1289         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1290
1291         /* Config/Enable Link */
1292         ixgbe_config_link(adapter);
1293
1294         /* Hardware Packet Buffer & Flow Control setup */
1295         {
1296                 u32 rxpb, frame, size, tmp;
1297
1298                 frame = adapter->max_frame_size;
1299
1300                 /* Calculate High Water */
1301                 if (hw->mac.type == ixgbe_mac_X540)
1302                         tmp = IXGBE_DV_X540(frame, frame);
1303                 else
1304                         tmp = IXGBE_DV(frame, frame);
1305                 size = IXGBE_BT2KB(tmp);
1306                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1307                 hw->fc.high_water[0] = rxpb - size;
1308
1309                 /* Now calculate Low Water */
1310                 if (hw->mac.type == ixgbe_mac_X540)
1311                         tmp = IXGBE_LOW_DV_X540(frame);
1312                 else
1313                         tmp = IXGBE_LOW_DV(frame);
1314                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1315
1316                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1317                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1318                 hw->fc.send_xon = TRUE;
1319         }
1320         /* Initialize the FC settings */
1321         ixgbe_start_hw(hw);
1322
1323         /* And now turn on interrupts */
1324         ixgbe_enable_intr(adapter);
1325
1326         /* Now inform the stack we're ready */
1327         ifp->if_flags |= IFF_RUNNING;
1328         ifp->if_flags &= ~IFF_OACTIVE;
1329
1330         return;
1331 }
1332
1333 static void
1334 ixgbe_init(void *arg)
1335 {
1336         struct adapter *adapter = arg;
1337
1338         IXGBE_CORE_LOCK(adapter);
1339         ixgbe_init_locked(adapter);
1340         IXGBE_CORE_UNLOCK(adapter);
1341         return;
1342 }
1343
1344
1345 /*
1346 **
1347 ** MSIX Interrupt Handlers and Tasklets
1348 **
1349 */
1350
1351 static inline void
1352 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1353 {
1354         struct ixgbe_hw *hw = &adapter->hw;
1355         u64     queue = (u64)(1 << vector);
1356         u32     mask;
1357
1358         if (hw->mac.type == ixgbe_mac_82598EB) {
1359                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1360                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1361         } else {
1362                 mask = (queue & 0xFFFFFFFF);
1363                 if (mask)
1364                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1365                 mask = (queue >> 32);
1366                 if (mask)
1367                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1368         }
1369 }
1370
1371 static inline void
1372 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1373 {
1374         struct ixgbe_hw *hw = &adapter->hw;
1375         u64     queue = (u64)(1 << vector);
1376         u32     mask;
1377
1378         if (hw->mac.type == ixgbe_mac_82598EB) {
1379                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1380                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1381         } else {
1382                 mask = (queue & 0xFFFFFFFF);
1383                 if (mask)
1384                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1385                 mask = (queue >> 32);
1386                 if (mask)
1387                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1388         }
1389 }
1390
1391 static inline void
1392 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1393 {
1394         u32 mask;
1395
1396         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1397                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1398                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1399         } else {
1400                 mask = (queues & 0xFFFFFFFF);
1401                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1402                 mask = (queues >> 32);
1403                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1404         }
1405 }
1406
1407
1408 static void
1409 ixgbe_handle_que(void *context, int pending)
1410 {
1411         struct ix_queue *que = context;
1412         struct adapter  *adapter = que->adapter;
1413         struct tx_ring  *txr = que->txr;
1414         struct ifnet    *ifp = adapter->ifp;
1415         bool            more;
1416
1417         if (ifp->if_flags & IFF_RUNNING) {
1418                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1419                 IXGBE_TX_LOCK(txr);
1420                 ixgbe_txeof(txr);
1421 #if 0 /*__FreeBSD_version >= 800000*/
1422                 if (!drbr_empty(ifp, txr->br))
1423                         ixgbe_mq_start_locked(ifp, txr, NULL);
1424 #else
1425                 if (!ifq_is_empty(&ifp->if_snd))
1426                         ixgbe_start_locked(txr, ifp);
1427 #endif
1428                 IXGBE_TX_UNLOCK(txr);
1429                 if (more) {
1430                         taskqueue_enqueue(que->tq, &que->que_task);
1431                         return;
1432                 }
1433         }
1434
1435         /* Reenable this interrupt */
1436         ixgbe_enable_queue(adapter, que->msix);
1437         return;
1438 }
1439
1440
1441 /*********************************************************************
1442  *
1443  *  Legacy Interrupt Service routine
1444  *
1445  **********************************************************************/
1446
1447 static void
1448 ixgbe_legacy_irq(void *arg)
1449 {
1450         struct ix_queue *que = arg;
1451         struct adapter  *adapter = que->adapter;
1452         struct ixgbe_hw *hw = &adapter->hw;
1453         struct          tx_ring *txr = adapter->tx_rings;
1454         bool            more_tx, more_rx;
1455         u32             reg_eicr, loop = MAX_LOOP;
1456
1457
1458         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1459
1460         ++que->irqs;
1461         if (reg_eicr == 0) {
1462                 ixgbe_enable_intr(adapter);
1463                 return;
1464         }
1465
1466         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1467
1468         IXGBE_TX_LOCK(txr);
1469         do {
1470                 more_tx = ixgbe_txeof(txr);
1471         } while (loop-- && more_tx);
1472         IXGBE_TX_UNLOCK(txr);
1473
1474         if (more_rx || more_tx)
1475                 taskqueue_enqueue(que->tq, &que->que_task);
1476
1477         /* Check for fan failure */
1478         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1479             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1480                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1481                     "REPLACE IMMEDIATELY!!\n");
1482                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1483         }
1484
1485         /* Link status change */
1486         if (reg_eicr & IXGBE_EICR_LSC)
1487                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1488
1489         ixgbe_enable_intr(adapter);
1490         return;
1491 }
1492
1493
1494 /*********************************************************************
1495  *
1496  *  MSIX Queue Interrupt Service routine
1497  *
1498  **********************************************************************/
1499 void
1500 ixgbe_msix_que(void *arg)
1501 {
1502         struct ix_queue *que = arg;
1503         struct adapter  *adapter = que->adapter;
1504         struct tx_ring  *txr = que->txr;
1505         struct rx_ring  *rxr = que->rxr;
1506         bool            more_tx, more_rx;
1507         u32             newitr = 0;
1508
1509         ixgbe_disable_queue(adapter, que->msix);
1510         ++que->irqs;
1511
1512         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1513
1514         IXGBE_TX_LOCK(txr);
1515         more_tx = ixgbe_txeof(txr);
1516         /*
1517         ** Make certain that if the stack
1518         ** has anything queued the task gets
1519         ** scheduled to handle it.
1520         */
1521 #if 0
1522 #if __FreeBSD_version < 800000
1523         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1524 #else
1525         if (!drbr_empty(adapter->ifp, txr->br))
1526 #endif
1527 #endif
1528         if (!ifq_is_empty(&adapter->ifp->if_snd))
1529                 more_tx = 1;
1530         IXGBE_TX_UNLOCK(txr);
1531
1532         /* Do AIM now? */
1533
1534         if (ixgbe_enable_aim == FALSE)
1535                 goto no_calc;
1536         /*
1537         ** Do Adaptive Interrupt Moderation:
1538         **  - Write out last calculated setting
1539         **  - Calculate based on average size over
1540         **    the last interval.
1541         */
1542         if (que->eitr_setting)
1543                 IXGBE_WRITE_REG(&adapter->hw,
1544                     IXGBE_EITR(que->msix), que->eitr_setting);
1545
1546         que->eitr_setting = 0;
1547
1548         /* Idle, do nothing */
1549         if ((txr->bytes == 0) && (rxr->bytes == 0))
1550                 goto no_calc;
1551
1552         if ((txr->bytes) && (txr->packets))
1553                 newitr = txr->bytes/txr->packets;
1554         if ((rxr->bytes) && (rxr->packets))
1555                 newitr = max(newitr,
1556                     (rxr->bytes / rxr->packets));
1557         newitr += 24; /* account for hardware frame, crc */
1558
1559         /* set an upper boundary */
1560         newitr = min(newitr, 3000);
1561
1562         /* Be nice to the mid range */
1563         if ((newitr > 300) && (newitr < 1200))
1564                 newitr = (newitr / 3);
1565         else
1566                 newitr = (newitr / 2);
1567
1568         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1569                 newitr |= newitr << 16;
1570         else
1571                 newitr |= IXGBE_EITR_CNT_WDIS;
1572
1573         /* save for next interrupt */
1574         que->eitr_setting = newitr;
1575
1576         /* Reset state */
1577         txr->bytes = 0;
1578         txr->packets = 0;
1579         rxr->bytes = 0;
1580         rxr->packets = 0;
1581
1582 no_calc:
1583         if (more_tx || more_rx)
1584                 taskqueue_enqueue(que->tq, &que->que_task);
1585         else /* Reenable this interrupt */
1586                 ixgbe_enable_queue(adapter, que->msix);
1587         return;
1588 }
1589
1590
1591 static void
1592 ixgbe_msix_link(void *arg)
1593 {
1594         struct adapter  *adapter = arg;
1595         struct ixgbe_hw *hw = &adapter->hw;
1596         u32             reg_eicr;
1597
1598         ++adapter->link_irq;
1599
1600         /* First get the cause */
1601         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1602         /* Clear interrupt with write */
1603         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1604
1605         /* Link status change */
1606         if (reg_eicr & IXGBE_EICR_LSC)
1607                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1608
1609         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1610 #ifdef IXGBE_FDIR
1611                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1612                         /* This is probably overkill :) */
1613                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1614                                 return;
1615                         /* Disable the interrupt */
1616                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1617                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1618                 } else
1619 #endif
1620                 if (reg_eicr & IXGBE_EICR_ECC) {
1621                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1622                             "Please Reboot!!\n");
1623                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1624                 } else
1625
1626                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1627                         /* Clear the interrupt */
1628                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1629                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1630                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1631                         /* Clear the interrupt */
1632                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1633                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1634                 }
1635         }
1636
1637         /* Check for fan failure */
1638         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1639             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1640                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1641                     "REPLACE IMMEDIATELY!!\n");
1642                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1643         }
1644
1645         /* Check for over temp condition */
1646         if ((hw->mac.type == ixgbe_mac_X540) &&
1647             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1648                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1649                     "PHY IS SHUT DOWN!!\n");
1650                 device_printf(adapter->dev, "System shutdown required\n");
1651                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1652         }
1653
1654         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1655         return;
1656 }
1657
1658 /*********************************************************************
1659  *
1660  *  Media Ioctl callback
1661  *
1662  *  This routine is called whenever the user queries the status of
1663  *  the interface using ifconfig.
1664  *
1665  **********************************************************************/
1666 static void
1667 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1668 {
1669         struct adapter *adapter = ifp->if_softc;
1670
1671         INIT_DEBUGOUT("ixgbe_media_status: begin");
1672         IXGBE_CORE_LOCK(adapter);
1673         ixgbe_update_link_status(adapter);
1674
1675         ifmr->ifm_status = IFM_AVALID;
1676         ifmr->ifm_active = IFM_ETHER;
1677
1678         if (!adapter->link_active) {
1679                 IXGBE_CORE_UNLOCK(adapter);
1680                 return;
1681         }
1682
1683         ifmr->ifm_status |= IFM_ACTIVE;
1684
1685         switch (adapter->link_speed) {
1686                 case IXGBE_LINK_SPEED_100_FULL:
1687                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1688                         break;
1689                 case IXGBE_LINK_SPEED_1GB_FULL:
1690                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1691                         break;
1692                 case IXGBE_LINK_SPEED_10GB_FULL:
1693                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1694                         break;
1695         }
1696
1697         IXGBE_CORE_UNLOCK(adapter);
1698
1699         return;
1700 }
1701
1702 /*********************************************************************
1703  *
1704  *  Media Ioctl callback
1705  *
1706  *  This routine is called when the user changes speed/duplex using
1707  *  media/mediopt option with ifconfig.
1708  *
1709  **********************************************************************/
1710 static int
1711 ixgbe_media_change(struct ifnet * ifp)
1712 {
1713         struct adapter *adapter = ifp->if_softc;
1714         struct ifmedia *ifm = &adapter->media;
1715
1716         INIT_DEBUGOUT("ixgbe_media_change: begin");
1717
1718         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1719                 return (EINVAL);
1720
1721         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1722         case IFM_AUTO:
1723                 adapter->hw.phy.autoneg_advertised =
1724                     IXGBE_LINK_SPEED_100_FULL |
1725                     IXGBE_LINK_SPEED_1GB_FULL |
1726                     IXGBE_LINK_SPEED_10GB_FULL;
1727                 break;
1728         default:
1729                 device_printf(adapter->dev, "Only auto media type\n");
1730                 return (EINVAL);
1731         }
1732
1733         return (0);
1734 }
1735
1736 /*********************************************************************
1737  *
1738  *  This routine maps the mbufs to tx descriptors, allowing the
1739  *  TX engine to transmit the packets.
1740  *      - return 0 on success, positive on failure
1741  *
1742  **********************************************************************/
1743
1744 static int
1745 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1746 {
1747         struct adapter  *adapter = txr->adapter;
1748         u32             olinfo_status = 0, cmd_type_len;
1749         u32             paylen = 0;
1750         int             i, j, error, nsegs;
1751         int             first, last = 0;
1752         struct mbuf     *m_head;
1753         bus_dma_segment_t segs[1];
1754         bus_dmamap_t    map;
1755         struct ixgbe_tx_buf *txbuf;
1756         union ixgbe_adv_tx_desc *txd = NULL;
1757
1758         m_head = *m_headp;
1759
1760         /* Basic descriptor defines */
1761         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1762             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1763
1764         if (m_head->m_flags & M_VLANTAG)
1765                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1766
1767         /*
1768          * Important to capture the first descriptor
1769          * used because it will contain the index of
1770          * the one we tell the hardware to report back
1771          */
1772         first = txr->next_avail_desc;
1773         txbuf = &txr->tx_buffers[first];
1774         map = txbuf->map;
1775
1776         /*
1777          * Map the packet for DMA.
1778          */
1779         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1780             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1781
1782         if (error == EFBIG) {
1783                 struct mbuf *m;
1784
1785                 m = m_defrag(*m_headp, MB_DONTWAIT);
1786                 if (m == NULL) {
1787                         adapter->mbuf_defrag_failed++;
1788                         m_freem(*m_headp);
1789                         *m_headp = NULL;
1790                         return (ENOBUFS);
1791                 }
1792                 *m_headp = m;
1793
1794                 /* Try it again */
1795                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1796                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1797
1798                 if (error == ENOMEM) {
1799                         adapter->no_tx_dma_setup++;
1800                         return (error);
1801                 } else if (error != 0) {
1802                         adapter->no_tx_dma_setup++;
1803                         m_freem(*m_headp);
1804                         *m_headp = NULL;
1805                         return (error);
1806                 }
1807         } else if (error == ENOMEM) {
1808                 adapter->no_tx_dma_setup++;
1809                 return (error);
1810         } else if (error != 0) {
1811                 adapter->no_tx_dma_setup++;
1812                 m_freem(*m_headp);
1813                 *m_headp = NULL;
1814                 return (error);
1815         }
1816
1817         /* Make certain there are enough descriptors */
1818         if (nsegs > txr->tx_avail - 2) {
1819                 txr->no_desc_avail++;
1820                 error = ENOBUFS;
1821                 goto xmit_fail;
1822         }
1823         m_head = *m_headp;
1824
1825         /*
1826         ** Set up the appropriate offload context
1827         ** this becomes the first descriptor of
1828         ** a packet.
1829         */
1830 #if 0 /* NET_TSO */
1831         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1832                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1833                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1834                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1835                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1836                         ++adapter->tso_tx;
1837                 } else
1838                         return (ENXIO);
1839         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1840 #endif
1841         if (ixgbe_tx_ctx_setup(txr, m_head))
1842                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1843
1844 #ifdef IXGBE_IEEE1588
1845         /* This is changing soon to an mtag detection */
1846         if (we detect this mbuf has a TSTAMP mtag)
1847                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1848 #endif
1849
1850 #ifdef IXGBE_FDIR
1851         /* Do the flow director magic */
1852         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1853                 ++txr->atr_count;
1854                 if (txr->atr_count >= atr_sample_rate) {
1855                         ixgbe_atr(txr, m_head);
1856                         txr->atr_count = 0;
1857                 }
1858         }
1859 #endif
1860         /* Record payload length */
1861         if (paylen == 0)
1862                 olinfo_status |= m_head->m_pkthdr.len <<
1863                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1864
1865         i = txr->next_avail_desc;
1866         for (j = 0; j < nsegs; j++) {
1867                 bus_size_t seglen;
1868                 bus_addr_t segaddr;
1869
1870                 txbuf = &txr->tx_buffers[i];
1871                 txd = &txr->tx_base[i];
1872                 seglen = segs[j].ds_len;
1873                 segaddr = htole64(segs[j].ds_addr);
1874
1875                 txd->read.buffer_addr = segaddr;
1876                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1877                     cmd_type_len |seglen);
1878                 txd->read.olinfo_status = htole32(olinfo_status);
1879                 last = i; /* descriptor that will get completion IRQ */
1880
1881                 if (++i == adapter->num_tx_desc)
1882                         i = 0;
1883
1884                 txbuf->m_head = NULL;
1885                 txbuf->eop_index = -1;
1886         }
1887
1888         txd->read.cmd_type_len |=
1889             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1890         txr->tx_avail -= nsegs;
1891         txr->next_avail_desc = i;
1892
1893         txbuf->m_head = m_head;
1894         /* Swap the dma map between the first and last descriptor */
1895         txr->tx_buffers[first].map = txbuf->map;
1896         txbuf->map = map;
1897         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1898
1899         /* Set the index of the descriptor that will be marked done */
1900         txbuf = &txr->tx_buffers[first];
1901         txbuf->eop_index = last;
1902
1903         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1904             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1905         /*
1906          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1907          * hardware that this frame is available to transmit.
1908          */
1909         ++txr->total_packets;
1910         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1911
1912         return (0);
1913
1914 xmit_fail:
1915         bus_dmamap_unload(txr->txtag, txbuf->map);
1916         return (error);
1917
1918 }
1919
1920 static void
1921 ixgbe_set_promisc(struct adapter *adapter)
1922 {
1923         u_int32_t       reg_rctl;
1924         struct ifnet   *ifp = adapter->ifp;
1925
1926         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1927         reg_rctl &= (~IXGBE_FCTRL_UPE);
1928         reg_rctl &= (~IXGBE_FCTRL_MPE);
1929         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1930
1931         if (ifp->if_flags & IFF_PROMISC) {
1932                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1933                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1934         } else if (ifp->if_flags & IFF_ALLMULTI) {
1935                 reg_rctl |= IXGBE_FCTRL_MPE;
1936                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1937                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1938         }
1939         return;
1940 }
1941
1942
1943 /*********************************************************************
1944  *  Multicast Update
1945  *
1946  *  This routine is called whenever multicast address list is updated.
1947  *
1948  **********************************************************************/
1949 #define IXGBE_RAR_ENTRIES 16
1950
1951 static void
1952 ixgbe_set_multi(struct adapter *adapter)
1953 {
1954         u32     fctrl;
1955         u8      *mta;
1956         u8      *update_ptr;
1957         struct  ifmultiaddr *ifma;
1958         int     mcnt = 0;
1959         struct ifnet   *ifp = adapter->ifp;
1960
1961         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1962
1963         mta = adapter->mta;
1964         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1965             MAX_NUM_MULTICAST_ADDRESSES);
1966
1967         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1968         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969         if (ifp->if_flags & IFF_PROMISC)
1970                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1971         else if (ifp->if_flags & IFF_ALLMULTI) {
1972                 fctrl |= IXGBE_FCTRL_MPE;
1973                 fctrl &= ~IXGBE_FCTRL_UPE;
1974         } else
1975                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1976
1977         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1978
1979         spin_lock(&adapter->mcast_spin);
1980         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1981                 if (ifma->ifma_addr->sa_family != AF_LINK)
1982                         continue;
1983                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1984                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1985                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1986                 mcnt++;
1987         }
1988         spin_unlock(&adapter->mcast_spin);
1989
1990         update_ptr = mta;
1991         ixgbe_update_mc_addr_list(&adapter->hw,
1992             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1993
1994         return;
1995 }
1996
1997 /*
1998  * This is an iterator function now needed by the multicast
1999  * shared code. It simply feeds the shared code routine the
2000  * addresses in the array of ixgbe_set_multi() one by one.
2001  */
2002 static u8 *
2003 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2004 {
2005         u8 *addr = *update_ptr;
2006         u8 *newptr;
2007         *vmdq = 0;
2008
2009         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2010         *update_ptr = newptr;
2011         return addr;
2012 }
2013
2014
2015 /*********************************************************************
2016  *  Timer routine
2017  *
2018  *  This routine checks for link status,updates statistics,
2019  *  and runs the watchdog check.
2020  *
2021  **********************************************************************/
2022
2023 static void
2024 ixgbe_local_timer(void *arg)
2025 {
2026         struct adapter  *adapter = arg;
2027         device_t        dev = adapter->dev;
2028         struct ifnet    *ifp = adapter->ifp;
2029         struct ix_queue *que = adapter->queues;
2030         struct tx_ring  *txr = adapter->tx_rings;
2031         int             hung, busy, paused;
2032
2033         IXGBE_CORE_LOCK(adapter);
2034         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2035         hung = busy = paused = 0;
2036
2037         /* Check for pluggable optics */
2038         if (adapter->sfp_probe)
2039                 if (!ixgbe_sfp_probe(adapter))
2040                         goto out; /* Nothing to do */
2041
2042         ixgbe_update_link_status(adapter);
2043         ixgbe_update_stats_counters(adapter);
2044
2045         /*
2046          * If the interface has been paused
2047          * then don't do the watchdog check
2048          */
2049         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2050                 paused = 1;
2051
2052         /*
2053         ** Check the TX queues status
2054         **      - central locked handling of OACTIVE
2055         **      - watchdog only if all queues show hung
2056         */
2057         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2058                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2059                     (paused == 0))
2060                         ++hung;
2061                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2062                         ++busy;
2063                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2064                         taskqueue_enqueue(que->tq, &que->que_task);
2065         }
2066         /* Only truely watchdog if all queues show hung */
2067         if (hung == adapter->num_queues)
2068                 goto watchdog;
2069         /* Only turn off the stack flow when ALL are depleted */
2070         if (busy == adapter->num_queues)
2071                 ifp->if_flags |= IFF_OACTIVE;
2072         else if ((ifp->if_flags & IFF_OACTIVE) &&
2073             (busy < adapter->num_queues))
2074                 ifp->if_flags &= ~IFF_OACTIVE;
2075
2076 out:
2077         ixgbe_rearm_queues(adapter, adapter->que_mask);
2078         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2079         IXGBE_CORE_UNLOCK(adapter);
2080         return;
2081
2082 watchdog:
2083         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2084         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2085             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2086             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2087         device_printf(dev,"TX(%d) desc avail = %d,"
2088             "Next TX to Clean = %d\n",
2089             txr->me, txr->tx_avail, txr->next_to_clean);
2090         adapter->ifp->if_flags &= ~IFF_RUNNING;
2091         adapter->watchdog_events++;
2092         ixgbe_init_locked(adapter);
2093
2094         IXGBE_CORE_UNLOCK(adapter);
2095 }
2096
2097 /*
2098 ** Note: this routine updates the OS on the link state
2099 **      the real check of the hardware only happens with
2100 **      a link interrupt.
2101 */
2102 static void
2103 ixgbe_update_link_status(struct adapter *adapter)
2104 {
2105         struct ifnet    *ifp = adapter->ifp;
2106         struct tx_ring *txr = adapter->tx_rings;
2107         device_t dev = adapter->dev;
2108
2109
2110         if (adapter->link_up){
2111                 if (adapter->link_active == FALSE) {
2112                         if (bootverbose)
2113                                 device_printf(dev,"Link is up %d Gbps %s \n",
2114                                     ((adapter->link_speed == 128)? 10:1),
2115                                     "Full Duplex");
2116                         adapter->link_active = TRUE;
2117                         /* Update any Flow Control changes */
2118                         ixgbe_fc_enable(&adapter->hw);
2119                         ifp->if_link_state = LINK_STATE_UP;
2120                         if_link_state_change(ifp);
2121                 }
2122         } else { /* Link down */
2123                 if (adapter->link_active == TRUE) {
2124                         if (bootverbose)
2125                                 device_printf(dev,"Link is Down\n");
2126                         ifp->if_link_state = LINK_STATE_DOWN;
2127                         if_link_state_change(ifp);
2128                         adapter->link_active = FALSE;
2129                         for (int i = 0; i < adapter->num_queues;
2130                             i++, txr++)
2131                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2132                 }
2133         }
2134
2135         return;
2136 }
2137
2138
2139 /*********************************************************************
2140  *
2141  *  This routine disables all traffic on the adapter by issuing a
2142  *  global reset on the MAC and deallocates TX/RX buffers.
2143  *
2144  **********************************************************************/
2145
2146 static void
2147 ixgbe_stop(void *arg)
2148 {
2149         struct ifnet   *ifp;
2150         struct adapter *adapter = arg;
2151         struct ixgbe_hw *hw = &adapter->hw;
2152         ifp = adapter->ifp;
2153
2154         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2155
2156         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2157         ixgbe_disable_intr(adapter);
2158         callout_stop(&adapter->timer);
2159
2160         /* Let the stack know...*/
2161         ifp->if_flags &= ~IFF_RUNNING;
2162         ifp->if_flags |= IFF_OACTIVE;
2163
2164         ixgbe_reset_hw(hw);
2165         hw->adapter_stopped = FALSE;
2166         ixgbe_stop_adapter(hw);
2167         /* Turn off the laser */
2168         if (hw->phy.multispeed_fiber)
2169                 ixgbe_disable_tx_laser(hw);
2170
2171         /* reprogram the RAR[0] in case user changed it. */
2172         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2173
2174         return;
2175 }
2176
2177
2178 /*********************************************************************
2179  *
2180  *  Determine hardware revision.
2181  *
2182  **********************************************************************/
2183 static void
2184 ixgbe_identify_hardware(struct adapter *adapter)
2185 {
2186         device_t        dev = adapter->dev;
2187         struct ixgbe_hw *hw = &adapter->hw;
2188
2189         /* Save off the information about this board */
2190         hw->vendor_id = pci_get_vendor(dev);
2191         hw->device_id = pci_get_device(dev);
2192         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2193         hw->subsystem_vendor_id =
2194             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2195         hw->subsystem_device_id =
2196             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2197
2198         /* We need this here to set the num_segs below */
2199         ixgbe_set_mac_type(hw);
2200
2201         /* Pick up the 82599 and VF settings */
2202         if (hw->mac.type != ixgbe_mac_82598EB) {
2203                 hw->phy.smart_speed = ixgbe_smart_speed;
2204                 adapter->num_segs = IXGBE_82599_SCATTER;
2205         } else
2206                 adapter->num_segs = IXGBE_82598_SCATTER;
2207
2208         return;
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Determine optic type
2214  *
2215  **********************************************************************/
2216 static void
2217 ixgbe_setup_optics(struct adapter *adapter)
2218 {
2219         struct ixgbe_hw *hw = &adapter->hw;
2220         int             layer;
2221
2222         layer = ixgbe_get_supported_physical_layer(hw);
2223
2224         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2225                 adapter->optics = IFM_10G_T;
2226                 return;
2227         }
2228
2229         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2230                 adapter->optics = IFM_1000_T;
2231                 return;
2232         }
2233
2234         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2235             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2236                 adapter->optics = IFM_10G_LR;
2237                 return;
2238         }
2239
2240         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2241                 adapter->optics = IFM_10G_SR;
2242                 return;
2243         }
2244
2245         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2246                 adapter->optics = IFM_10G_TWINAX;
2247                 return;
2248         }
2249
2250         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2251             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2252                 adapter->optics = IFM_10G_CX4;
2253                 return;
2254         }
2255
2256         /* If we get here just set the default */
2257         adapter->optics = IFM_ETHER | IFM_AUTO;
2258         return;
2259 }
2260
2261 /*********************************************************************
2262  *
2263  *  Setup the Legacy or MSI Interrupt handler
2264  *
2265  **********************************************************************/
2266 static int
2267 ixgbe_allocate_legacy(struct adapter *adapter)
2268 {
2269         device_t dev = adapter->dev;
2270         struct          ix_queue *que = adapter->queues;
2271         int error, rid = 0;
2272         unsigned int intr_flags;
2273
2274         /* MSI RID at 1 */
2275         if (adapter->msix == 1)
2276                 rid = 1;
2277
2278         /* Try allocating a MSI interrupt first */
2279         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2280                 &rid, &intr_flags);
2281
2282         /* We allocate a single interrupt resource */
2283         adapter->res = bus_alloc_resource_any(dev,
2284             SYS_RES_IRQ, &rid, intr_flags);
2285         if (adapter->res == NULL) {
2286                 device_printf(dev, "Unable to allocate bus resource: "
2287                     "interrupt\n");
2288                 return (ENXIO);
2289         }
2290
2291         /*
2292          * Try allocating a fast interrupt and the associated deferred
2293          * processing contexts.
2294          */
2295         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2296         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2297             taskqueue_thread_enqueue, &que->tq);
2298         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2299             device_get_nameunit(adapter->dev));
2300
2301         /* Tasklets for Link, SFP and Multispeed Fiber */
2302         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2303         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2304         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2305 #ifdef IXGBE_FDIR
2306         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2307 #endif
2308         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2309             taskqueue_thread_enqueue, &adapter->tq);
2310         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2311             device_get_nameunit(adapter->dev));
2312
2313         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2314             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2315                 device_printf(dev, "Failed to register fast interrupt "
2316                     "handler: %d\n", error);
2317                 taskqueue_free(que->tq);
2318                 taskqueue_free(adapter->tq);
2319                 que->tq = NULL;
2320                 adapter->tq = NULL;
2321                 return (error);
2322         }
2323         /* For simplicity in the handlers */
2324         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2325
2326         return (0);
2327 }
2328
2329
2330 /*********************************************************************
2331  *
2332  *  Setup MSIX Interrupt resources and handlers
2333  *
2334  **********************************************************************/
2335 static int
2336 ixgbe_allocate_msix(struct adapter *adapter)
2337 {
2338         device_t        dev = adapter->dev;
2339         struct          ix_queue *que = adapter->queues;
2340         int             error, rid, vector = 0;
2341
2342         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2343                 rid = vector + 1;
2344                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2345                     RF_SHAREABLE | RF_ACTIVE);
2346                 if (que->res == NULL) {
2347                         device_printf(dev,"Unable to allocate"
2348                             " bus resource: que interrupt [%d]\n", vector);
2349                         return (ENXIO);
2350                 }
2351                 /* Set the handler function */
2352                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2353                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2354                 if (error) {
2355                         que->res = NULL;
2356                         device_printf(dev, "Failed to register QUE handler");
2357                         return (error);
2358                 }
2359 #if 0 /* __FreeBSD_version >= 800504 */
2360                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2361 #endif
2362                 que->msix = vector;
2363                 adapter->que_mask |= (u64)(1 << que->msix);
2364                 /*
2365                 ** Bind the msix vector, and thus the
2366                 ** ring to the corresponding cpu.
2367                 */
2368 #if 0 /* XXX */
2369                 if (adapter->num_queues > 1)
2370                         bus_bind_intr(dev, que->res, i);
2371 #endif
2372
2373                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2374                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2375                     taskqueue_thread_enqueue, &que->tq);
2376                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2377                     device_get_nameunit(adapter->dev));
2378         }
2379
2380         /* and Link */
2381         rid = vector + 1;
2382         adapter->res = bus_alloc_resource_any(dev,
2383             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2384         if (!adapter->res) {
2385                 device_printf(dev,"Unable to allocate"
2386             " bus resource: Link interrupt [%d]\n", rid);
2387                 return (ENXIO);
2388         }
2389         /* Set the link handler function */
2390         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2391             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2392         if (error) {
2393                 adapter->res = NULL;
2394                 device_printf(dev, "Failed to register LINK handler");
2395                 return (error);
2396         }
2397 #if 0 /* __FreeBSD_version >= 800504 */
2398         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2399 #endif
2400         adapter->linkvec = vector;
2401         /* Tasklets for Link, SFP and Multispeed Fiber */
2402         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2403         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2404         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2405 #ifdef IXGBE_FDIR
2406         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2407 #endif
2408         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2409             taskqueue_thread_enqueue, &adapter->tq);
2410         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2411             device_get_nameunit(adapter->dev));
2412
2413         return (0);
2414 }
2415
2416 #if 0   /* HAVE_MSIX */
2417 /*
2418  * Setup Either MSI/X or MSI
2419  */
2420 static int
2421 ixgbe_setup_msix(struct adapter *adapter)
2422 {
2423         device_t dev = adapter->dev;
2424         int rid, want, queues, msgs;
2425
2426         /* Override by tuneable */
2427         if (ixgbe_enable_msix == 0)
2428                 goto msi;
2429
2430         /* First try MSI/X */
2431         rid = PCIR_BAR(MSIX_82598_BAR);
2432         adapter->msix_mem = bus_alloc_resource_any(dev,
2433             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2434         if (!adapter->msix_mem) {
2435                 rid += 4;       /* 82599 maps in higher BAR */
2436                 adapter->msix_mem = bus_alloc_resource_any(dev,
2437                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2438         }
2439         if (!adapter->msix_mem) {
2440                 /* May not be enabled */
2441                 device_printf(adapter->dev,
2442                     "Unable to map MSIX table \n");
2443                 goto msi;
2444         }
2445
2446         msgs = pci_msix_count(dev);
2447         if (msgs == 0) { /* system has msix disabled */
2448                 bus_release_resource(dev, SYS_RES_MEMORY,
2449                     rid, adapter->msix_mem);
2450                 adapter->msix_mem = NULL;
2451                 goto msi;
2452         }
2453
2454         /* Figure out a reasonable auto config value */
2455         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2456
2457         if (ixgbe_num_queues != 0)
2458                 queues = ixgbe_num_queues;
2459         /* Set max queues to 8 when autoconfiguring */
2460         else if ((ixgbe_num_queues == 0) && (queues > 8))
2461                 queues = 8;
2462
2463         /*
2464         ** Want one vector (RX/TX pair) per queue
2465         ** plus an additional for Link.
2466         */
2467         want = queues + 1;
2468         if (msgs >= want)
2469                 msgs = want;
2470         else {
2471                 device_printf(adapter->dev,
2472                     "MSIX Configuration Problem, "
2473                     "%d vectors but %d queues wanted!\n",
2474                     msgs, want);
2475                 return (0); /* Will go to Legacy setup */
2476         }
2477         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2478                 device_printf(adapter->dev,
2479                     "Using MSIX interrupts with %d vectors\n", msgs);
2480                 adapter->num_queues = queues;
2481                 return (msgs);
2482         }
2483 msi:
2484         msgs = pci_msi_count(dev);
2485         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2486                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2487         else
2488                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2489         return (msgs);
2490 }
2491 #endif
2492
2493
2494 static int
2495 ixgbe_allocate_pci_resources(struct adapter *adapter)
2496 {
2497         int             rid;
2498         device_t        dev = adapter->dev;
2499
2500         rid = PCIR_BAR(0);
2501         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2502             &rid, RF_ACTIVE);
2503
2504         if (!(adapter->pci_mem)) {
2505                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2506                 return (ENXIO);
2507         }
2508
2509         adapter->osdep.mem_bus_space_tag =
2510                 rman_get_bustag(adapter->pci_mem);
2511         adapter->osdep.mem_bus_space_handle =
2512                 rman_get_bushandle(adapter->pci_mem);
2513         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2514
2515         /* Legacy defaults */
2516         adapter->num_queues = 1;
2517         adapter->hw.back = &adapter->osdep;
2518
2519         /*
2520         ** Now setup MSI or MSI/X, should
2521         ** return us the number of supported
2522         ** vectors. (Will be 1 for MSI)
2523         */
2524 #if 0   /* HAVE_MSIX */
2525         adapter->msix = ixgbe_setup_msix(adapter);
2526 #endif
2527         return (0);
2528 }
2529
2530 static void
2531 ixgbe_free_pci_resources(struct adapter * adapter)
2532 {
2533         struct          ix_queue *que = adapter->queues;
2534         device_t        dev = adapter->dev;
2535         int             rid, memrid;
2536
2537         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2538                 memrid = PCIR_BAR(MSIX_82598_BAR);
2539         else
2540                 memrid = PCIR_BAR(MSIX_82599_BAR);
2541
2542         /*
2543         ** There is a slight possibility of a failure mode
2544         ** in attach that will result in entering this function
2545         ** before interrupt resources have been initialized, and
2546         ** in that case we do not want to execute the loops below
2547         ** We can detect this reliably by the state of the adapter
2548         ** res pointer.
2549         */
2550         if (adapter->res == NULL)
2551                 goto mem;
2552
2553         /*
2554         **  Release all msix queue resources:
2555         */
2556         for (int i = 0; i < adapter->num_queues; i++, que++) {
2557                 rid = que->msix + 1;
2558                 if (que->tag != NULL) {
2559                         bus_teardown_intr(dev, que->res, que->tag);
2560                         que->tag = NULL;
2561                 }
2562                 if (que->res != NULL)
2563                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2564         }
2565
2566
2567         /* Clean the Legacy or Link interrupt last */
2568         if (adapter->linkvec) /* we are doing MSIX */
2569                 rid = adapter->linkvec + 1;
2570         else
2571                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2572
2573         if (adapter->tag != NULL) {
2574                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2575                 adapter->tag = NULL;
2576         }
2577         if (adapter->res != NULL)
2578                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2579         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2580                 pci_release_msi(adapter->dev);
2581
2582 mem:
2583         if (adapter->msix)
2584                 pci_release_msi(dev);
2585
2586         if (adapter->msix_mem != NULL)
2587                 bus_release_resource(dev, SYS_RES_MEMORY,
2588                     memrid, adapter->msix_mem);
2589
2590         if (adapter->pci_mem != NULL)
2591                 bus_release_resource(dev, SYS_RES_MEMORY,
2592                     PCIR_BAR(0), adapter->pci_mem);
2593
2594         return;
2595 }
2596
2597 /*********************************************************************
2598  *
2599  *  Setup networking device structure and register an interface.
2600  *
2601  **********************************************************************/
2602 static int
2603 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2604 {
2605         struct ixgbe_hw *hw = &adapter->hw;
2606         struct ifnet   *ifp;
2607
2608         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2609
2610         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2611         if (ifp == NULL) {
2612                 device_printf(dev, "can not allocate ifnet structure\n");
2613                 return (-1);
2614         }
2615         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2616         ifp->if_baudrate = 1000000000;
2617         ifp->if_init = ixgbe_init;
2618         ifp->if_softc = adapter;
2619         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2620         ifp->if_ioctl = ixgbe_ioctl;
2621         ifp->if_start = ixgbe_start;
2622 #if 0 /* __FreeBSD_version >= 800000 */
2623         ifp->if_transmit = ixgbe_mq_start;
2624         ifp->if_qflush = ixgbe_qflush;
2625 #endif
2626         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2627
2628         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2629
2630         adapter->max_frame_size =
2631             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2632
2633         /*
2634          * Tell the upper layer(s) we support long frames.
2635          */
2636         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2637
2638 #if 0 /* NET_TSO */
2639         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2640 #endif
2641         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2642         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2643         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2644 #if 0 /* NET_TSO */
2645                              |  IFCAP_VLAN_HWTSO
2646 #endif
2647                              |  IFCAP_VLAN_MTU;
2648         ifp->if_capenable = ifp->if_capabilities;
2649
2650         /* Don't enable LRO by default */
2651 #if 0 /* NET_LRO */
2652         ifp->if_capabilities |= IFCAP_LRO;
2653 #endif
2654
2655         /*
2656         ** Don't turn this on by default, if vlans are
2657         ** created on another pseudo device (eg. lagg)
2658         ** then vlan events are not passed thru, breaking
2659         ** operation, but with HW FILTER off it works. If
2660         ** using vlans directly on the ixgbe driver you can
2661         ** enable this and get full hardware tag filtering.
2662         */
2663         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2664
2665         /*
2666          * Specify the media types supported by this adapter and register
2667          * callbacks to update media and link information
2668          */
2669         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2670                      ixgbe_media_status);
2671         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2672         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2673         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2674                 ifmedia_add(&adapter->media,
2675                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2676                 ifmedia_add(&adapter->media,
2677                     IFM_ETHER | IFM_1000_T, 0, NULL);
2678         }
2679         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2680         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2681
2682         return (0);
2683 }
2684
2685 static void
2686 ixgbe_config_link(struct adapter *adapter)
2687 {
2688         struct ixgbe_hw *hw = &adapter->hw;
2689         u32     autoneg, err = 0;
2690         bool    sfp, negotiate;
2691
2692         sfp = ixgbe_is_sfp(hw);
2693
2694         if (sfp) {
2695                 if (hw->phy.multispeed_fiber) {
2696                         hw->mac.ops.setup_sfp(hw);
2697                         ixgbe_enable_tx_laser(hw);
2698                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2699                 } else
2700                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2701         } else {
2702                 if (hw->mac.ops.check_link)
2703                         err = ixgbe_check_link(hw, &autoneg,
2704                             &adapter->link_up, FALSE);
2705                 if (err)
2706                         goto out;
2707                 autoneg = hw->phy.autoneg_advertised;
2708                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2709                         err  = hw->mac.ops.get_link_capabilities(hw,
2710                             &autoneg, &negotiate);
2711                 if (err)
2712                         goto out;
2713                 if (hw->mac.ops.setup_link)
2714                         err = hw->mac.ops.setup_link(hw, autoneg,
2715                             negotiate, adapter->link_up);
2716         }
2717 out:
2718         return;
2719 }
2720
2721 /********************************************************************
2722  * Manage DMA'able memory.
2723  *******************************************************************/
2724 static void
2725 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2726 {
2727         if (error)
2728                 return;
2729         *(bus_addr_t *) arg = segs->ds_addr;
2730         return;
2731 }
2732
2733 static int
2734 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2735                 struct ixgbe_dma_alloc *dma, int mapflags)
2736 {
2737         device_t dev = adapter->dev;
2738         int             r;
2739
2740         r = bus_dma_tag_create(NULL,    /* parent */
2741                                DBA_ALIGN, 0,    /* alignment, bounds */
2742                                BUS_SPACE_MAXADDR,       /* lowaddr */
2743                                BUS_SPACE_MAXADDR,       /* highaddr */
2744                                NULL, NULL,      /* filter, filterarg */
2745                                size,    /* maxsize */
2746                                1,       /* nsegments */
2747                                size,    /* maxsegsize */
2748                                BUS_DMA_ALLOCNOW,        /* flags */
2749                                &dma->dma_tag);
2750         if (r != 0) {
2751                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2752                        "error %u\n", r);
2753                 goto fail_0;
2754         }
2755         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2756                              BUS_DMA_NOWAIT, &dma->dma_map);
2757         if (r != 0) {
2758                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2759                        "error %u\n", r);
2760                 goto fail_1;
2761         }
2762         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2763                             size,
2764                             ixgbe_dmamap_cb,
2765                             &dma->dma_paddr,
2766                             mapflags | BUS_DMA_NOWAIT);
2767         if (r != 0) {
2768                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2769                        "error %u\n", r);
2770                 goto fail_2;
2771         }
2772         dma->dma_size = size;
2773         return (0);
2774 fail_2:
2775         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2776 fail_1:
2777         bus_dma_tag_destroy(dma->dma_tag);
2778 fail_0:
2779         dma->dma_map = NULL;
2780         dma->dma_tag = NULL;
2781         return (r);
2782 }
2783
2784 static void
2785 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2786 {
2787         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2788             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2789         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2790         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2791         bus_dma_tag_destroy(dma->dma_tag);
2792 }
2793
2794
2795 /*********************************************************************
2796  *
2797  *  Allocate memory for the transmit and receive rings, and then
2798  *  the descriptors associated with each, called only once at attach.
2799  *
2800  **********************************************************************/
2801 static int
2802 ixgbe_allocate_queues(struct adapter *adapter)
2803 {
2804         device_t        dev = adapter->dev;
2805         struct ix_queue *que;
2806         struct tx_ring  *txr;
2807         struct rx_ring  *rxr;
2808         int rsize, tsize, error = IXGBE_SUCCESS;
2809         int txconf = 0, rxconf = 0;
2810
2811         /* First allocate the top level queue structs */
2812         if (!(adapter->queues =
2813             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2814             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2815                 device_printf(dev, "Unable to allocate queue memory\n");
2816                 error = ENOMEM;
2817                 goto fail;
2818         }
2819
2820         /* First allocate the TX ring struct memory */
2821         if (!(adapter->tx_rings =
2822             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2823             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2824                 device_printf(dev, "Unable to allocate TX ring memory\n");
2825                 error = ENOMEM;
2826                 goto tx_fail;
2827         }
2828
2829         /* Next allocate the RX */
2830         if (!(adapter->rx_rings =
2831             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2832             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2833                 device_printf(dev, "Unable to allocate RX ring memory\n");
2834                 error = ENOMEM;
2835                 goto rx_fail;
2836         }
2837
2838         /* For the ring itself */
2839         tsize = roundup2(adapter->num_tx_desc *
2840             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2841
2842         /*
2843          * Now set up the TX queues, txconf is needed to handle the
2844          * possibility that things fail midcourse and we need to
2845          * undo memory gracefully
2846          */
2847         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2848                 /* Set up some basics */
2849                 txr = &adapter->tx_rings[i];
2850                 txr->adapter = adapter;
2851                 txr->me = i;
2852
2853                 /* Initialize the TX side lock */
2854                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2855                     device_get_nameunit(dev), txr->me);
2856                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2857
2858                 if (ixgbe_dma_malloc(adapter, tsize,
2859                         &txr->txdma, BUS_DMA_NOWAIT)) {
2860                         device_printf(dev,
2861                             "Unable to allocate TX Descriptor memory\n");
2862                         error = ENOMEM;
2863                         goto err_tx_desc;
2864                 }
2865                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2866                 bzero((void *)txr->tx_base, tsize);
2867
2868                 /* Now allocate transmit buffers for the ring */
2869                 if (ixgbe_allocate_transmit_buffers(txr)) {
2870                         device_printf(dev,
2871                             "Critical Failure setting up transmit buffers\n");
2872                         error = ENOMEM;
2873                         goto err_tx_desc;
2874                 }
2875 #if 0 /* __FreeBSD_version >= 800000 */
2876                 /* Allocate a buf ring */
2877                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2878                     M_WAITOK, &txr->tx_mtx);
2879                 if (txr->br == NULL) {
2880                         device_printf(dev,
2881                             "Critical Failure setting up buf ring\n");
2882                         error = ENOMEM;
2883                         goto err_tx_desc;
2884                 }
2885 #endif
2886         }
2887
2888         /*
2889          * Next the RX queues...
2890          */
2891         rsize = roundup2(adapter->num_rx_desc *
2892             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2893         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2894                 rxr = &adapter->rx_rings[i];
2895                 /* Set up some basics */
2896                 rxr->adapter = adapter;
2897                 rxr->me = i;
2898
2899                 /* Initialize the RX side lock */
2900                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2901                     device_get_nameunit(dev), rxr->me);
2902                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2903
2904                 if (ixgbe_dma_malloc(adapter, rsize,
2905                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2906                         device_printf(dev,
2907                             "Unable to allocate RxDescriptor memory\n");
2908                         error = ENOMEM;
2909                         goto err_rx_desc;
2910                 }
2911                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2912                 bzero((void *)rxr->rx_base, rsize);
2913
2914                 /* Allocate receive buffers for the ring*/
2915                 if (ixgbe_allocate_receive_buffers(rxr)) {
2916                         device_printf(dev,
2917                             "Critical Failure setting up receive buffers\n");
2918                         error = ENOMEM;
2919                         goto err_rx_desc;
2920                 }
2921         }
2922
2923         /*
2924         ** Finally set up the queue holding structs
2925         */
2926         for (int i = 0; i < adapter->num_queues; i++) {
2927                 que = &adapter->queues[i];
2928                 que->adapter = adapter;
2929                 que->txr = &adapter->tx_rings[i];
2930                 que->rxr = &adapter->rx_rings[i];
2931         }
2932
2933         return (0);
2934
2935 err_rx_desc:
2936         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2937                 ixgbe_dma_free(adapter, &rxr->rxdma);
2938 err_tx_desc:
2939         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2940                 ixgbe_dma_free(adapter, &txr->txdma);
2941         kfree(adapter->rx_rings, M_DEVBUF);
2942 rx_fail:
2943         kfree(adapter->tx_rings, M_DEVBUF);
2944 tx_fail:
2945         kfree(adapter->queues, M_DEVBUF);
2946 fail:
2947         return (error);
2948 }
2949
2950 /*********************************************************************
2951  *
2952  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2953  *  the information needed to transmit a packet on the wire. This is
2954  *  called only once at attach, setup is done every reset.
2955  *
2956  **********************************************************************/
2957 static int
2958 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2959 {
2960         struct adapter *adapter = txr->adapter;
2961         device_t dev = adapter->dev;
2962         struct ixgbe_tx_buf *txbuf;
2963         int error, i;
2964
2965         /*
2966          * Setup DMA descriptor areas.
2967          */
2968         if ((error = bus_dma_tag_create(
2969                                NULL,    /* parent */
2970                                1, 0,            /* alignment, bounds */
2971                                BUS_SPACE_MAXADDR,       /* lowaddr */
2972                                BUS_SPACE_MAXADDR,       /* highaddr */
2973                                NULL, NULL,              /* filter, filterarg */
2974                                IXGBE_TSO_SIZE,          /* maxsize */
2975                                adapter->num_segs,       /* nsegments */
2976                                PAGE_SIZE,               /* maxsegsize */
2977                                0,                       /* flags */
2978                                &txr->txtag))) {
2979                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2980                 goto fail;
2981         }
2982
2983         if (!(txr->tx_buffers =
2984             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2985             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2986                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2987                 error = ENOMEM;
2988                 goto fail;
2989         }
2990
2991         /* Create the descriptor buffer dma maps */
2992         txbuf = txr->tx_buffers;
2993         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2994                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2995                 if (error != 0) {
2996                         device_printf(dev, "Unable to create TX DMA map\n");
2997                         goto fail;
2998                 }
2999         }
3000
3001         return 0;
3002 fail:
3003         /* We free all, it handles case where we are in the middle */
3004         ixgbe_free_transmit_structures(adapter);
3005         return (error);
3006 }
3007
3008 /*********************************************************************
3009  *
3010  *  Initialize a transmit ring.
3011  *
3012  **********************************************************************/
3013 static void
3014 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3015 {
3016         struct adapter *adapter = txr->adapter;
3017         struct ixgbe_tx_buf *txbuf;
3018         int i;
3019 #ifdef DEV_NETMAP
3020         struct netmap_adapter *na = NA(adapter->ifp);
3021         struct netmap_slot *slot;
3022 #endif /* DEV_NETMAP */
3023
3024         /* Clear the old ring contents */
3025         IXGBE_TX_LOCK(txr);
3026 #ifdef DEV_NETMAP
3027         /*
3028          * (under lock): if in netmap mode, do some consistency
3029          * checks and set slot to entry 0 of the netmap ring.
3030          */
3031         slot = netmap_reset(na, NR_TX, txr->me, 0);
3032 #endif /* DEV_NETMAP */
3033         bzero((void *)txr->tx_base,
3034               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3035         /* Reset indices */
3036         txr->next_avail_desc = 0;
3037         txr->next_to_clean = 0;
3038
3039         /* Free any existing tx buffers. */
3040         txbuf = txr->tx_buffers;
3041         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3042                 if (txbuf->m_head != NULL) {
3043                         bus_dmamap_sync(txr->txtag, txbuf->map,
3044                             BUS_DMASYNC_POSTWRITE);
3045                         bus_dmamap_unload(txr->txtag, txbuf->map);
3046                         m_freem(txbuf->m_head);
3047                         txbuf->m_head = NULL;
3048                 }
3049 #ifdef DEV_NETMAP
3050                 /*
3051                  * In netmap mode, set the map for the packet buffer.
3052                  * NOTE: Some drivers (not this one) also need to set
3053                  * the physical buffer address in the NIC ring.
3054                  * Slots in the netmap ring (indexed by "si") are
3055                  * kring->nkr_hwofs positions "ahead" wrt the
3056                  * corresponding slot in the NIC ring. In some drivers
3057                  * (not here) nkr_hwofs can be negative. Function
3058                  * netmap_idx_n2k() handles wraparounds properly.
3059                  */
3060                 if (slot) {
3061                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3062                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3063                 }
3064 #endif /* DEV_NETMAP */
3065                 /* Clear the EOP index */
3066                 txbuf->eop_index = -1;
3067         }
3068
3069 #ifdef IXGBE_FDIR
3070         /* Set the rate at which we sample packets */
3071         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3072                 txr->atr_sample = atr_sample_rate;
3073 #endif
3074
3075         /* Set number of descriptors available */
3076         txr->tx_avail = adapter->num_tx_desc;
3077
3078         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3079             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3080         IXGBE_TX_UNLOCK(txr);
3081 }
3082
3083 /*********************************************************************
3084  *
3085  *  Initialize all transmit rings.
3086  *
3087  **********************************************************************/
3088 static int
3089 ixgbe_setup_transmit_structures(struct adapter *adapter)
3090 {
3091         struct tx_ring *txr = adapter->tx_rings;
3092
3093         for (int i = 0; i < adapter->num_queues; i++, txr++)
3094                 ixgbe_setup_transmit_ring(txr);
3095
3096         return (0);
3097 }
3098
3099 /*********************************************************************
3100  *
3101  *  Enable transmit unit.
3102  *
3103  **********************************************************************/
3104 static void
3105 ixgbe_initialize_transmit_units(struct adapter *adapter)
3106 {
3107         struct tx_ring  *txr = adapter->tx_rings;
3108         struct ixgbe_hw *hw = &adapter->hw;
3109
3110         /* Setup the Base and Length of the Tx Descriptor Ring */
3111
3112         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3113                 u64     tdba = txr->txdma.dma_paddr;
3114                 u32     txctrl;
3115
3116                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3117                        (tdba & 0x00000000ffffffffULL));
3118                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3119                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3120                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3121
3122                 /* Setup the HW Tx Head and Tail descriptor pointers */
3123                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3124                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3125
3126                 /* Setup Transmit Descriptor Cmd Settings */
3127                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3128                 txr->queue_status = IXGBE_QUEUE_IDLE;
3129
3130                 /* Disable Head Writeback */
3131                 switch (hw->mac.type) {
3132                 case ixgbe_mac_82598EB:
3133                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3134                         break;
3135                 case ixgbe_mac_82599EB:
3136                 case ixgbe_mac_X540:
3137                 default:
3138                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3139                         break;
3140                 }
3141                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3142                 switch (hw->mac.type) {
3143                 case ixgbe_mac_82598EB:
3144                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3145                         break;
3146                 case ixgbe_mac_82599EB:
3147                 case ixgbe_mac_X540:
3148                 default:
3149                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3150                         break;
3151                 }
3152
3153         }
3154
3155         if (hw->mac.type != ixgbe_mac_82598EB) {
3156                 u32 dmatxctl, rttdcs;
3157                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3158                 dmatxctl |= IXGBE_DMATXCTL_TE;
3159                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3160                 /* Disable arbiter to set MTQC */
3161                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3162                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3163                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3164                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3165                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3166                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3167         }
3168
3169         return;
3170 }
3171
3172 /*********************************************************************
3173  *
3174  *  Free all transmit rings.
3175  *
3176  **********************************************************************/
3177 static void
3178 ixgbe_free_transmit_structures(struct adapter *adapter)
3179 {
3180         struct tx_ring *txr = adapter->tx_rings;
3181
3182         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3183                 IXGBE_TX_LOCK(txr);
3184                 ixgbe_free_transmit_buffers(txr);
3185                 ixgbe_dma_free(adapter, &txr->txdma);
3186                 IXGBE_TX_UNLOCK(txr);
3187                 IXGBE_TX_LOCK_DESTROY(txr);
3188         }
3189         kfree(adapter->tx_rings, M_DEVBUF);
3190 }
3191
3192 /*********************************************************************
3193  *
3194  *  Free transmit ring related data structures.
3195  *
3196  **********************************************************************/
3197 static void
3198 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3199 {
3200         struct adapter *adapter = txr->adapter;
3201         struct ixgbe_tx_buf *tx_buffer;
3202         int             i;
3203
3204         INIT_DEBUGOUT("free_transmit_ring: begin");
3205
3206         if (txr->tx_buffers == NULL)
3207                 return;
3208
3209         tx_buffer = txr->tx_buffers;
3210         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3211                 if (tx_buffer->m_head != NULL) {
3212                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3213                             BUS_DMASYNC_POSTWRITE);
3214                         bus_dmamap_unload(txr->txtag,
3215                             tx_buffer->map);
3216                         m_freem(tx_buffer->m_head);
3217                         tx_buffer->m_head = NULL;
3218                         if (tx_buffer->map != NULL) {
3219                                 bus_dmamap_destroy(txr->txtag,
3220                                     tx_buffer->map);
3221                                 tx_buffer->map = NULL;
3222                         }
3223                 } else if (tx_buffer->map != NULL) {
3224                         bus_dmamap_unload(txr->txtag,
3225                             tx_buffer->map);
3226                         bus_dmamap_destroy(txr->txtag,
3227                             tx_buffer->map);
3228                         tx_buffer->map = NULL;
3229                 }
3230         }
3231 #if 0 /* __FreeBSD_version >= 800000 */
3232         if (txr->br != NULL)
3233                 buf_ring_free(txr->br, M_DEVBUF);
3234 #endif
3235         if (txr->tx_buffers != NULL) {
3236                 kfree(txr->tx_buffers, M_DEVBUF);
3237                 txr->tx_buffers = NULL;
3238         }
3239         if (txr->txtag != NULL) {
3240                 bus_dma_tag_destroy(txr->txtag);
3241                 txr->txtag = NULL;
3242         }
3243         return;
3244 }
3245
3246 /*********************************************************************
3247  *
3248  *  Advanced Context Descriptor setup for VLAN or CSUM
3249  *
3250  **********************************************************************/
3251
3252 static bool
3253 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3254 {
3255         struct adapter *adapter = txr->adapter;
3256         struct ixgbe_adv_tx_context_desc *TXD;
3257         struct ixgbe_tx_buf        *tx_buffer;
3258         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3259         struct ether_vlan_header *eh;
3260         struct ip *ip;
3261         struct ip6_hdr *ip6;
3262         int  ehdrlen, ip_hlen = 0;
3263         u16     etype;
3264         u8      ipproto = 0;
3265         bool    offload = TRUE;
3266         int ctxd = txr->next_avail_desc;
3267 #ifdef NET_VLAN
3268         u16 vtag = 0;
3269 #endif
3270
3271
3272         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3273                 offload = FALSE;
3274
3275         tx_buffer = &txr->tx_buffers[ctxd];
3276         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3277
3278         /*
3279         ** In advanced descriptors the vlan tag must
3280         ** be placed into the descriptor itself.
3281         */
3282 #ifdef NET_VLAN
3283         if (mp->m_flags & M_VLANTAG) {
3284                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3285                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3286         } else if (offload == FALSE)
3287                 return FALSE;
3288 #endif
3289
3290         /*
3291          * Determine where frame payload starts.
3292          * Jump over vlan headers if already present,
3293          * helpful for QinQ too.
3294          */
3295         eh = mtod(mp, struct ether_vlan_header *);
3296         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3297                 etype = ntohs(eh->evl_proto);
3298                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3299         } else {
3300                 etype = ntohs(eh->evl_encap_proto);
3301                 ehdrlen = ETHER_HDR_LEN;
3302         }
3303
3304         /* Set the ether header length */
3305         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3306
3307         switch (etype) {
3308                 case ETHERTYPE_IP:
3309                         ip = (struct ip *)(mp->m_data + ehdrlen);
3310                         ip_hlen = ip->ip_hl << 2;
3311                         ipproto = ip->ip_p;
3312                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3313                         break;
3314                 case ETHERTYPE_IPV6:
3315                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3316                         ip_hlen = sizeof(struct ip6_hdr);
3317                         /* XXX-BZ this will go badly in case of ext hdrs. */
3318                         ipproto = ip6->ip6_nxt;
3319                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3320                         break;
3321                 default:
3322                         offload = FALSE;
3323                         break;
3324         }
3325
3326         vlan_macip_lens |= ip_hlen;
3327         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3328
3329         switch (ipproto) {
3330                 case IPPROTO_TCP:
3331                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3332                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3333                         break;
3334
3335                 case IPPROTO_UDP:
3336                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3337                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3338                         break;
3339
3340 #if 0
3341                 case IPPROTO_SCTP:
3342                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3343                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3344                         break;
3345 #endif
3346                 default:
3347                         offload = FALSE;
3348                         break;
3349         }
3350
3351         /* Now copy bits into descriptor */
3352         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3353         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3354         TXD->seqnum_seed = htole32(0);
3355         TXD->mss_l4len_idx = htole32(0);
3356
3357         tx_buffer->m_head = NULL;
3358         tx_buffer->eop_index = -1;
3359
3360         /* We've consumed the first desc, adjust counters */
3361         if (++ctxd == adapter->num_tx_desc)
3362                 ctxd = 0;
3363         txr->next_avail_desc = ctxd;
3364         --txr->tx_avail;
3365
3366         return (offload);
3367 }
3368
3369 /**********************************************************************
3370  *
3371  *  Setup work for hardware segmentation offload (TSO) on
3372  *  adapters using advanced tx descriptors
3373  *
3374  **********************************************************************/
3375 #if 0   /* NET_TSO */
3376 static bool
3377 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3378     u32 *olinfo_status)
3379 {
3380         struct adapter *adapter = txr->adapter;
3381         struct ixgbe_adv_tx_context_desc *TXD;
3382         struct ixgbe_tx_buf        *tx_buffer;
3383 #ifdef NET_VLAN
3384         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3385         u16 vtag = 0, eh_type;
3386 #else
3387         u16 eh_type;
3388         u32 type_tucmd_mlhl = 0;
3389 #endif
3390         u32 mss_l4len_idx = 0, len;
3391         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3392         struct ether_vlan_header *eh;
3393 #ifdef INET6
3394         struct ip6_hdr *ip6;
3395 #endif
3396 #ifdef INET
3397         struct ip *ip;
3398 #endif
3399         struct tcphdr *th;
3400
3401
3402         /*
3403          * Determine where frame payload starts.
3404          * Jump over vlan headers if already present
3405          */
3406         eh = mtod(mp, struct ether_vlan_header *);
3407         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3408                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3409                 eh_type = eh->evl_proto;
3410         } else {
3411                 ehdrlen = ETHER_HDR_LEN;
3412                 eh_type = eh->evl_encap_proto;
3413         }
3414
3415         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3416         len = ehdrlen + sizeof(struct tcphdr);
3417         switch (ntohs(eh_type)) {
3418 #ifdef INET6
3419         case ETHERTYPE_IPV6:
3420                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3421                         return FALSE;
3422                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3423                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3424                 if (ip6->ip6_nxt != IPPROTO_TCP)
3425                         return FALSE;
3426                 ip_hlen = sizeof(struct ip6_hdr);
3427                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3428                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3429                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3430                 break;
3431 #endif
3432 #ifdef INET
3433         case ETHERTYPE_IP:
3434                 if (mp->m_len < len + sizeof(struct ip))
3435                         return FALSE;
3436                 ip = (struct ip *)(mp->m_data + ehdrlen);
3437                 if (ip->ip_p != IPPROTO_TCP)
3438                         return FALSE;
3439                 ip->ip_sum = 0;
3440                 ip_hlen = ip->ip_hl << 2;
3441                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3442                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3443                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3444                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3445                 /* Tell transmit desc to also do IPv4 checksum. */
3446                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3447                 break;
3448 #endif
3449         default:
3450                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3451                     __func__, ntohs(eh_type));
3452                 break;
3453         }
3454
3455         ctxd = txr->next_avail_desc;
3456         tx_buffer = &txr->tx_buffers[ctxd];
3457         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3458
3459         tcp_hlen = th->th_off << 2;
3460
3461         /* This is used in the transmit desc in encap */
3462         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3463
3464         /* VLAN MACLEN IPLEN */
3465 #ifdef NET_VLAN
3466         if (mp->m_flags & M_VLANTAG) {
3467                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3468                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3469         }
3470
3471         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3472         vlan_macip_lens |= ip_hlen;
3473         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3474 #endif
3475
3476         /* ADV DTYPE TUCMD */
3477         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3478         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3479         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3480
3481         /* MSS L4LEN IDX */
3482         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3483         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3484         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3485
3486         TXD->seqnum_seed = htole32(0);
3487         tx_buffer->m_head = NULL;
3488         tx_buffer->eop_index = -1;
3489
3490         if (++ctxd == adapter->num_tx_desc)
3491                 ctxd = 0;
3492
3493         txr->tx_avail--;
3494         txr->next_avail_desc = ctxd;
3495         return TRUE;
3496 }
3497 #endif
3498
3499 #ifdef IXGBE_FDIR
3500 /*
3501 ** This routine parses packet headers so that Flow
3502 ** Director can make a hashed filter table entry
3503 ** allowing traffic flows to be identified and kept
3504 ** on the same cpu.  This would be a performance
3505 ** hit, but we only do it at IXGBE_FDIR_RATE of
3506 ** packets.
3507 */
3508 static void
3509 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3510 {
3511         struct adapter                  *adapter = txr->adapter;
3512         struct ix_queue                 *que;
3513         struct ip                       *ip;
3514         struct tcphdr                   *th;
3515         struct udphdr                   *uh;
3516         struct ether_vlan_header        *eh;
3517         union ixgbe_atr_hash_dword      input = {.dword = 0};
3518         union ixgbe_atr_hash_dword      common = {.dword = 0};
3519         int                             ehdrlen, ip_hlen;
3520         u16                             etype;
3521
3522         eh = mtod(mp, struct ether_vlan_header *);
3523         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3524                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3525                 etype = eh->evl_proto;
3526         } else {
3527                 ehdrlen = ETHER_HDR_LEN;
3528                 etype = eh->evl_encap_proto;
3529         }
3530
3531         /* Only handling IPv4 */
3532         if (etype != htons(ETHERTYPE_IP))
3533                 return;
3534
3535         ip = (struct ip *)(mp->m_data + ehdrlen);
3536         ip_hlen = ip->ip_hl << 2;
3537
3538         /* check if we're UDP or TCP */
3539         switch (ip->ip_p) {
3540         case IPPROTO_TCP:
3541                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3542                 /* src and dst are inverted */
3543                 common.port.dst ^= th->th_sport;
3544                 common.port.src ^= th->th_dport;
3545                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3546                 break;
3547         case IPPROTO_UDP:
3548                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3549                 /* src and dst are inverted */
3550                 common.port.dst ^= uh->uh_sport;
3551                 common.port.src ^= uh->uh_dport;
3552                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3553                 break;
3554         default:
3555                 return;
3556         }
3557
3558         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3559         if (mp->m_pkthdr.ether_vtag)
3560                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3561         else
3562                 common.flex_bytes ^= etype;
3563         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3564
3565         que = &adapter->queues[txr->me];
3566         /*
3567         ** This assumes the Rx queue and Tx
3568         ** queue are bound to the same CPU
3569         */
3570         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3571             input, common, que->msix);
3572 }
3573 #endif /* IXGBE_FDIR */
3574
3575 /**********************************************************************
3576  *
3577  *  Examine each tx_buffer in the used queue. If the hardware is done
3578  *  processing the packet then free associated resources. The
3579  *  tx_buffer is put back on the free queue.
3580  *
3581  **********************************************************************/
3582 static bool
3583 ixgbe_txeof(struct tx_ring *txr)
3584 {
3585         struct adapter  *adapter = txr->adapter;
3586         struct ifnet    *ifp = adapter->ifp;
3587         u32     first, last, done, processed;
3588         struct ixgbe_tx_buf *tx_buffer;
3589         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3590
3591         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3592
3593 #ifdef DEV_NETMAP
3594         if (ifp->if_capenable & IFCAP_NETMAP) {
3595                 struct netmap_adapter *na = NA(ifp);
3596                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3597
3598                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3599
3600                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3601                     BUS_DMASYNC_POSTREAD);
3602                 /*
3603                  * In netmap mode, all the work is done in the context
3604                  * of the client thread. Interrupt handlers only wake up
3605                  * clients, which may be sleeping on individual rings
3606                  * or on a global resource for all rings.
3607                  * To implement tx interrupt mitigation, we wake up the client
3608                  * thread roughly every half ring, even if the NIC interrupts
3609                  * more frequently. This is implemented as follows:
3610                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3611                  *   the slot that should wake up the thread (nkr_num_slots
3612                  *   means the user thread should not be woken up);
3613                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3614                  *   or the slot has the DD bit set.
3615                  *
3616                  * When the driver has separate locks, we need to
3617                  * release and re-acquire txlock to avoid deadlocks.
3618                  * XXX see if we can find a better way.
3619                  */
3620                 if (!netmap_mitigate ||
3621                     (kring->nr_kflags < kring->nkr_num_slots &&
3622                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3623                         kring->nr_kflags = kring->nkr_num_slots;
3624                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3625                         IXGBE_TX_UNLOCK(txr);
3626                         IXGBE_CORE_LOCK(adapter);
3627                         selwakeuppri(&na->tx_si, PI_NET);
3628                         IXGBE_CORE_UNLOCK(adapter);
3629                         IXGBE_TX_LOCK(txr);
3630                 }
3631                 return FALSE;
3632         }
3633 #endif /* DEV_NETMAP */
3634
3635         if (txr->tx_avail == adapter->num_tx_desc) {
3636                 txr->queue_status = IXGBE_QUEUE_IDLE;
3637                 return FALSE;
3638         }
3639
3640         processed = 0;
3641         first = txr->next_to_clean;
3642         tx_buffer = &txr->tx_buffers[first];
3643         /* For cleanup we just use legacy struct */
3644         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3645         last = tx_buffer->eop_index;
3646         if (last == -1)
3647                 return FALSE;
3648         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3649
3650         /*
3651         ** Get the index of the first descriptor
3652         ** BEYOND the EOP and call that 'done'.
3653         ** I do this so the comparison in the
3654         ** inner while loop below can be simple
3655         */
3656         if (++last == adapter->num_tx_desc) last = 0;
3657         done = last;
3658
3659         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3660             BUS_DMASYNC_POSTREAD);
3661         /*
3662         ** Only the EOP descriptor of a packet now has the DD
3663         ** bit set, this is what we look for...
3664         */
3665         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3666                 /* We clean the range of the packet */
3667                 while (first != done) {
3668                         tx_desc->upper.data = 0;
3669                         tx_desc->lower.data = 0;
3670                         tx_desc->buffer_addr = 0;
3671                         ++txr->tx_avail;
3672                         ++processed;
3673
3674                         if (tx_buffer->m_head) {
3675                                 txr->bytes +=
3676                                     tx_buffer->m_head->m_pkthdr.len;
3677                                 bus_dmamap_sync(txr->txtag,
3678                                     tx_buffer->map,
3679                                     BUS_DMASYNC_POSTWRITE);
3680                                 bus_dmamap_unload(txr->txtag,
3681                                     tx_buffer->map);
3682                                 m_freem(tx_buffer->m_head);
3683                                 tx_buffer->m_head = NULL;
3684                                 tx_buffer->map = NULL;
3685                         }
3686                         tx_buffer->eop_index = -1;
3687                         txr->watchdog_time = ticks;
3688
3689                         if (++first == adapter->num_tx_desc)
3690                                 first = 0;
3691
3692                         tx_buffer = &txr->tx_buffers[first];
3693                         tx_desc =
3694                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3695                 }
3696                 ++txr->packets;
3697                 ++ifp->if_opackets;
3698                 /* See if there is more work now */
3699                 last = tx_buffer->eop_index;
3700                 if (last != -1) {
3701                         eop_desc =
3702                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3703                         /* Get next done point */
3704                         if (++last == adapter->num_tx_desc) last = 0;
3705                         done = last;
3706                 } else
3707                         break;
3708         }
3709         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3710             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3711
3712         txr->next_to_clean = first;
3713
3714         /*
3715         ** Watchdog calculation, we know there's
3716         ** work outstanding or the first return
3717         ** would have been taken, so none processed
3718         ** for too long indicates a hang.
3719         */
3720         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3721                 txr->queue_status = IXGBE_QUEUE_HUNG;
3722
3723         /* With a minimum free clear the depleted state bit.  */
3724         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3725                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3726
3727         if (txr->tx_avail == adapter->num_tx_desc) {
3728                 txr->queue_status = IXGBE_QUEUE_IDLE;
3729                 return (FALSE);
3730         }
3731
3732         return TRUE;
3733 }
3734
3735 /*********************************************************************
3736  *
3737  *  Refresh mbuf buffers for RX descriptor rings
3738  *   - now keeps its own state so discards due to resource
3739  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3740  *     it just returns, keeping its placeholder, thus it can simply
3741  *     be recalled to try again.
3742  *
3743  **********************************************************************/
3744 static void
3745 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3746 {
3747         struct adapter          *adapter = rxr->adapter;
3748         bus_dma_segment_t       hseg[1];
3749         bus_dma_segment_t       pseg[1];
3750         struct ixgbe_rx_buf     *rxbuf;
3751         struct mbuf             *mh, *mp;
3752         int                     i, j, nsegs, error;
3753         bool                    refreshed = FALSE;
3754
3755         i = j = rxr->next_to_refresh;
3756         /* Control the loop with one beyond */
3757         if (++j == adapter->num_rx_desc)
3758                 j = 0;
3759
3760         while (j != limit) {
3761                 rxbuf = &rxr->rx_buffers[i];
3762                 if (rxr->hdr_split == FALSE)
3763                         goto no_split;
3764
3765                 if (rxbuf->m_head == NULL) {
3766                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3767                         if (mh == NULL)
3768                                 goto update;
3769                 } else
3770                         mh = rxbuf->m_head;
3771
3772                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3773                 mh->m_len = MHLEN;
3774                 mh->m_flags |= M_PKTHDR;
3775                 /* Get the memory mapping */
3776                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3777                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3778                 if (error != 0) {
3779                         kprintf("Refresh mbufs: hdr dmamap load"
3780                             " failure - %d\n", error);
3781                         m_free(mh);
3782                         rxbuf->m_head = NULL;
3783                         goto update;
3784                 }
3785                 rxbuf->m_head = mh;
3786                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3787                     BUS_DMASYNC_PREREAD);
3788                 rxr->rx_base[i].read.hdr_addr =
3789                     htole64(hseg[0].ds_addr);
3790
3791 no_split:
3792                 if (rxbuf->m_pack == NULL) {
3793                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3794                             M_PKTHDR, adapter->rx_mbuf_sz);
3795                         if (mp == NULL)
3796                                 goto update;
3797                 } else
3798                         mp = rxbuf->m_pack;
3799
3800                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3801                 /* Get the memory mapping */
3802                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3803                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3804                 if (error != 0) {
3805                         kprintf("Refresh mbufs: payload dmamap load"
3806                             " failure - %d\n", error);
3807                         m_free(mp);
3808                         rxbuf->m_pack = NULL;
3809                         goto update;
3810                 }
3811                 rxbuf->m_pack = mp;
3812                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3813                     BUS_DMASYNC_PREREAD);
3814                 rxr->rx_base[i].read.pkt_addr =
3815                     htole64(pseg[0].ds_addr);
3816
3817                 refreshed = TRUE;
3818                 /* Next is precalculated */
3819                 i = j;
3820                 rxr->next_to_refresh = i;
3821                 if (++j == adapter->num_rx_desc)
3822                         j = 0;
3823         }
3824 update:
3825         if (refreshed) /* Update hardware tail index */
3826                 IXGBE_WRITE_REG(&adapter->hw,
3827                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3828         return;
3829 }
3830
3831 /*********************************************************************
3832  *
3833  *  Allocate memory for rx_buffer structures. Since we use one
3834  *  rx_buffer per received packet, the maximum number of rx_buffer's
3835  *  that we'll need is equal to the number of receive descriptors
3836  *  that we've allocated.
3837  *
3838  **********************************************************************/
3839 static int
3840 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3841 {
3842         struct  adapter         *adapter = rxr->adapter;
3843         device_t                dev = adapter->dev;
3844         struct ixgbe_rx_buf     *rxbuf;
3845         int                     i, bsize, error;
3846
3847         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3848         if (!(rxr->rx_buffers =
3849             (struct ixgbe_rx_buf *) kmalloc(bsize,
3850             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3851                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3852                 error = ENOMEM;
3853                 goto fail;
3854         }
3855
3856         if ((error = bus_dma_tag_create(NULL,   /* parent */
3857                                    1, 0,        /* alignment, bounds */
3858                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3859                                    BUS_SPACE_MAXADDR,   /* highaddr */
3860                                    NULL, NULL,          /* filter, filterarg */
3861                                    MSIZE,               /* maxsize */
3862                                    1,                   /* nsegments */
3863                                    MSIZE,               /* maxsegsize */
3864                                    0,                   /* flags */
3865                                    &rxr->htag))) {
3866                 device_printf(dev, "Unable to create RX DMA tag\n");
3867                 goto fail;
3868         }
3869
3870         if ((error = bus_dma_tag_create(NULL,   /* parent */
3871                                    1, 0,        /* alignment, bounds */
3872                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3873                                    BUS_SPACE_MAXADDR,   /* highaddr */
3874                                    NULL, NULL,          /* filter, filterarg */
3875                                    MJUM16BYTES,         /* maxsize */
3876                                    1,                   /* nsegments */
3877                                    MJUM16BYTES,         /* maxsegsize */
3878                                    0,                   /* flags */
3879                                    &rxr->ptag))) {
3880                 device_printf(dev, "Unable to create RX DMA tag\n");
3881                 goto fail;
3882         }
3883
3884         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3885                 rxbuf = &rxr->rx_buffers[i];
3886                 error = bus_dmamap_create(rxr->htag,
3887                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3888                 if (error) {
3889                         device_printf(dev, "Unable to create RX head map\n");
3890                         goto fail;
3891                 }
3892                 error = bus_dmamap_create(rxr->ptag,
3893                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3894                 if (error) {
3895                         device_printf(dev, "Unable to create RX pkt map\n");
3896                         goto fail;
3897                 }
3898         }
3899
3900         return (0);
3901
3902 fail:
3903         /* Frees all, but can handle partial completion */
3904         ixgbe_free_receive_structures(adapter);
3905         return (error);
3906 }
3907
3908 /*
3909 ** Used to detect a descriptor that has
3910 ** been merged by Hardware RSC.
3911 */
3912 static inline u32
3913 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3914 {
3915         return (le32toh(rx->wb.lower.lo_dword.data) &
3916             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3917 }
3918
3919 /*********************************************************************
3920  *
3921  *  Initialize Hardware RSC (LRO) feature on 82599
3922  *  for an RX ring, this is toggled by the LRO capability
3923  *  even though it is transparent to the stack.
3924  *
3925  **********************************************************************/
3926 #if 0   /* NET_LRO */
3927 static void
3928 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3929 {
3930         struct  adapter         *adapter = rxr->adapter;
3931         struct  ixgbe_hw        *hw = &adapter->hw;
3932         u32                     rscctrl, rdrxctl;
3933
3934         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3935         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3936 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3937         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3938 #endif /* DEV_NETMAP */
3939         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3940         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3941         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3942
3943         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3944         rscctrl |= IXGBE_RSCCTL_RSCEN;
3945         /*
3946         ** Limit the total number of descriptors that
3947         ** can be combined, so it does not exceed 64K
3948         */
3949         if (adapter->rx_mbuf_sz == MCLBYTES)
3950                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3951         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3952                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3953         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3954                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3955         else  /* Using 16K cluster */
3956                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3957
3958         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3959
3960         /* Enable TCP header recognition */
3961         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3962             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3963             IXGBE_PSRTYPE_TCPHDR));
3964
3965         /* Disable RSC for ACK packets */
3966         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3967             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3968
3969         rxr->hw_rsc = TRUE;
3970 }
3971 #endif
3972
3973 static void
3974 ixgbe_free_receive_ring(struct rx_ring *rxr)
3975 {
3976         struct  adapter         *adapter;
3977         struct ixgbe_rx_buf       *rxbuf;
3978         int i;
3979
3980         adapter = rxr->adapter;
3981         for (i = 0; i < adapter->num_rx_desc; i++) {
3982                 rxbuf = &rxr->rx_buffers[i];
3983                 if (rxbuf->m_head != NULL) {
3984                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3985                             BUS_DMASYNC_POSTREAD);
3986                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3987                         rxbuf->m_head->m_flags |= M_PKTHDR;
3988                         m_freem(rxbuf->m_head);
3989                 }
3990                 if (rxbuf->m_pack != NULL) {
3991                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3992                             BUS_DMASYNC_POSTREAD);
3993                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3994                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3995                         m_freem(rxbuf->m_pack);
3996                 }
3997                 rxbuf->m_head = NULL;
3998                 rxbuf->m_pack = NULL;
3999         }
4000 }
4001
4002
4003 /*********************************************************************
4004  *
4005  *  Initialize a receive ring and its buffers.
4006  *
4007  **********************************************************************/
4008 static int
4009 ixgbe_setup_receive_ring(struct rx_ring *rxr)
4010 {
4011         struct  adapter         *adapter;
4012         struct ifnet            *ifp;
4013         device_t                dev;
4014         struct ixgbe_rx_buf     *rxbuf;
4015         bus_dma_segment_t       pseg[1], hseg[1];
4016 #if 0   /* NET_LRO */
4017         struct lro_ctrl         *lro = &rxr->lro;
4018 #endif
4019         int                     rsize, nsegs, error = 0;
4020 #ifdef DEV_NETMAP
4021         struct netmap_adapter *na = NA(rxr->adapter->ifp);
4022         struct netmap_slot *slot;
4023 #endif /* DEV_NETMAP */
4024
4025         adapter = rxr->adapter;
4026         ifp = adapter->ifp;
4027         dev = adapter->dev;
4028
4029         /* Clear the ring contents */
4030         IXGBE_RX_LOCK(rxr);
4031 #ifdef DEV_NETMAP
4032         /* same as in ixgbe_setup_transmit_ring() */
4033         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4034 #endif /* DEV_NETMAP */
4035         rsize = roundup2(adapter->num_rx_desc *
4036             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4037         bzero((void *)rxr->rx_base, rsize);
4038
4039         /* Free current RX buffer structs and their mbufs */
4040         ixgbe_free_receive_ring(rxr);
4041
4042         /* Configure header split? */
4043         if (ixgbe_header_split)
4044                 rxr->hdr_split = TRUE;
4045
4046         /* Now replenish the mbufs */
4047         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4048                 struct mbuf     *mh, *mp;
4049
4050                 rxbuf = &rxr->rx_buffers[j];
4051 #ifdef DEV_NETMAP
4052                 /*
4053                  * In netmap mode, fill the map and set the buffer
4054                  * address in the NIC ring, considering the offset
4055                  * between the netmap and NIC rings (see comment in
4056                  * ixgbe_setup_transmit_ring() ). No need to allocate
4057                  * an mbuf, so end the block with a continue;
4058                  */
4059                 if (slot) {
4060                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4061                         uint64_t paddr;
4062                         void *addr;
4063
4064                         addr = PNMB(slot + sj, &paddr);
4065                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4066                         /* Update descriptor */
4067                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4068                         continue;
4069                 }
4070 #endif /* DEV_NETMAP */
4071                 /*
4072                 ** Don't allocate mbufs if not
4073                 ** doing header split, its wasteful
4074                 */
4075                 if (rxr->hdr_split == FALSE)
4076                         goto skip_head;
4077
4078                 /* First the header */
4079                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4080                 if (rxbuf->m_head == NULL) {
4081                         error = ENOBUFS;
4082                         goto fail;
4083                 }
4084                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4085                 mh = rxbuf->m_head;
4086                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4087                 mh->m_flags |= M_PKTHDR;
4088                 /* Get the memory mapping */
4089                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4090                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4091                     &nsegs, BUS_DMA_NOWAIT);
4092
4093                 if (error != 0) /* Nothing elegant to do here */
4094                         goto fail;
4095                 bus_dmamap_sync(rxr->htag,
4096                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4097                 /* Update descriptor */
4098                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4099
4100 skip_head:
4101                 /* Now the payload cluster */
4102                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4103                     M_PKTHDR, adapter->rx_mbuf_sz);
4104                 if (rxbuf->m_pack == NULL) {
4105                         error = ENOBUFS;
4106                         goto fail;
4107                 }
4108                 mp = rxbuf->m_pack;
4109                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4110                 /* Get the memory mapping */
4111                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4112                     rxbuf->pmap, mp, hseg, 1,
4113                     &nsegs, BUS_DMA_NOWAIT);
4114                 if (error != 0)
4115                         goto fail;
4116                 bus_dmamap_sync(rxr->ptag,
4117                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4118                 /* Update descriptor */
4119                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4120         }
4121
4122
4123         /* Setup our descriptor indices */
4124         rxr->next_to_check = 0;
4125         rxr->next_to_refresh = 0;
4126         rxr->lro_enabled = FALSE;
4127         rxr->rx_split_packets = 0;
4128         rxr->rx_bytes = 0;
4129         rxr->discard = FALSE;
4130         rxr->vtag_strip = FALSE;
4131
4132         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4133             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4134
4135         /*
4136         ** Now set up the LRO interface:
4137         ** 82598 uses software LRO, the
4138         ** 82599 and X540 use a hardware assist.
4139         */
4140 #if 0 /* NET_LRO */
4141         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4142             (ifp->if_capenable & IFCAP_RXCSUM) &&
4143             (ifp->if_capenable & IFCAP_LRO))
4144                 ixgbe_setup_hw_rsc(rxr);
4145         else if (ifp->if_capenable & IFCAP_LRO) {
4146                 int err = tcp_lro_init(lro);
4147                 if (err) {
4148                         device_printf(dev, "LRO Initialization failed!\n");
4149                         goto fail;
4150                 }
4151                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4152                 rxr->lro_enabled = TRUE;
4153                 lro->ifp = adapter->ifp;
4154         }
4155 #endif
4156
4157         IXGBE_RX_UNLOCK(rxr);
4158         return (0);
4159
4160 fail:
4161         ixgbe_free_receive_ring(rxr);
4162         IXGBE_RX_UNLOCK(rxr);
4163         return (error);
4164 }
4165
4166 /*********************************************************************
4167  *
4168  *  Initialize all receive rings.
4169  *
4170  **********************************************************************/
4171 static int
4172 ixgbe_setup_receive_structures(struct adapter *adapter)
4173 {
4174         struct rx_ring *rxr = adapter->rx_rings;
4175         int j;
4176
4177         for (j = 0; j < adapter->num_queues; j++, rxr++)
4178                 if (ixgbe_setup_receive_ring(rxr))
4179                         goto fail;
4180
4181         return (0);
4182 fail:
4183         /*
4184          * Free RX buffers allocated so far, we will only handle
4185          * the rings that completed, the failing case will have
4186          * cleaned up for itself. 'j' failed, so its the terminus.
4187          */
4188         for (int i = 0; i < j; ++i) {
4189                 rxr = &adapter->rx_rings[i];
4190                 ixgbe_free_receive_ring(rxr);
4191         }
4192
4193         return (ENOBUFS);
4194 }
4195
4196 /*********************************************************************
4197  *
4198  *  Setup receive registers and features.
4199  *
4200  **********************************************************************/
4201 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4202
4203 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4204
4205 static void
4206 ixgbe_initialize_receive_units(struct adapter *adapter)
4207 {
4208         struct  rx_ring *rxr = adapter->rx_rings;
4209         struct ixgbe_hw *hw = &adapter->hw;
4210         struct ifnet   *ifp = adapter->ifp;
4211         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4212         u32             reta, mrqc = 0, hlreg, random[10];
4213
4214
4215         /*
4216          * Make sure receives are disabled while
4217          * setting up the descriptor ring
4218          */
4219         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4220         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4221             rxctrl & ~IXGBE_RXCTRL_RXEN);
4222
4223         /* Enable broadcasts */
4224         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4225         fctrl |= IXGBE_FCTRL_BAM;
4226         fctrl |= IXGBE_FCTRL_DPF;
4227         fctrl |= IXGBE_FCTRL_PMCF;
4228         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4229
4230         /* Set for Jumbo Frames? */
4231         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4232         if (ifp->if_mtu > ETHERMTU)
4233                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4234         else
4235                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4236 #ifdef DEV_NETMAP
4237         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4238         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4239                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4240         else
4241                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4242 #endif /* DEV_NETMAP */
4243         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4244
4245         bufsz = (adapter->rx_mbuf_sz +
4246             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4247
4248         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4249                 u64 rdba = rxr->rxdma.dma_paddr;
4250
4251                 /* Setup the Base and Length of the Rx Descriptor Ring */
4252                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4253                                (rdba & 0x00000000ffffffffULL));
4254                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4255                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4256                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4257
4258                 /* Set up the SRRCTL register */
4259                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4260                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4261                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4262                 srrctl |= bufsz;
4263                 if (rxr->hdr_split) {
4264                         /* Use a standard mbuf for the header */
4265                         srrctl |= ((IXGBE_RX_HDR <<
4266                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4267                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4268                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4269                 } else
4270                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4271                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4272
4273                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4274                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4275                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4276         }
4277
4278         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4279                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4280                               IXGBE_PSRTYPE_UDPHDR |
4281                               IXGBE_PSRTYPE_IPV4HDR |
4282                               IXGBE_PSRTYPE_IPV6HDR;
4283                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4284         }
4285
4286         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4287
4288         /* Setup RSS */
4289         if (adapter->num_queues > 1) {
4290                 int i, j;
4291                 reta = 0;
4292
4293                 /* set up random bits */
4294                 karc4rand(&random, sizeof(random));
4295
4296                 /* Set up the redirection table */
4297                 for (i = 0, j = 0; i < 128; i++, j++) {
4298                         if (j == adapter->num_queues) j = 0;
4299                         reta = (reta << 8) | (j * 0x11);
4300                         if ((i & 3) == 3)
4301                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4302                 }
4303
4304                 /* Now fill our hash function seeds */
4305                 for (int i = 0; i < 10; i++)
4306                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4307
4308                 /* Perform hash on these packet types */
4309                 mrqc = IXGBE_MRQC_RSSEN
4310                      | IXGBE_MRQC_RSS_FIELD_IPV4
4311                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4312                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4313                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4314                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4315                      | IXGBE_MRQC_RSS_FIELD_IPV6
4316                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4317                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4318                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4319                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4320
4321                 /* RSS and RX IPP Checksum are mutually exclusive */
4322                 rxcsum |= IXGBE_RXCSUM_PCSD;
4323         }
4324
4325         if (ifp->if_capenable & IFCAP_RXCSUM)
4326                 rxcsum |= IXGBE_RXCSUM_PCSD;
4327
4328         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4329                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4330
4331         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4332
4333         return;
4334 }
4335
4336 /*********************************************************************
4337  *
4338  *  Free all receive rings.
4339  *
4340  **********************************************************************/
4341 static void
4342 ixgbe_free_receive_structures(struct adapter *adapter)
4343 {
4344         struct rx_ring *rxr = adapter->rx_rings;
4345
4346         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4347 #if 0   /* NET_LRO */
4348                 struct lro_ctrl         *lro = &rxr->lro;
4349 #endif
4350                 ixgbe_free_receive_buffers(rxr);
4351                 /* Free LRO memory */
4352 #if 0   /* NET_LRO */
4353                 tcp_lro_free(lro);
4354 #endif
4355                 /* Free the ring memory as well */
4356                 ixgbe_dma_free(adapter, &rxr->rxdma);
4357         }
4358
4359         kfree(adapter->rx_rings, M_DEVBUF);
4360 }
4361
4362
4363 /*********************************************************************
4364  *
4365  *  Free receive ring data structures
4366  *
4367  **********************************************************************/
4368 static void
4369 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4370 {
4371         struct adapter          *adapter = rxr->adapter;
4372         struct ixgbe_rx_buf     *rxbuf;
4373
4374         INIT_DEBUGOUT("free_receive_structures: begin");
4375
4376         /* Cleanup any existing buffers */
4377         if (rxr->rx_buffers != NULL) {
4378                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4379                         rxbuf = &rxr->rx_buffers[i];
4380                         if (rxbuf->m_head != NULL) {
4381                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4382                                     BUS_DMASYNC_POSTREAD);
4383                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4384                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4385                                 m_freem(rxbuf->m_head);
4386                         }
4387                         if (rxbuf->m_pack != NULL) {
4388                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4389                                     BUS_DMASYNC_POSTREAD);
4390                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4391                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4392                                 m_freem(rxbuf->m_pack);
4393                         }
4394                         rxbuf->m_head = NULL;
4395                         rxbuf->m_pack = NULL;
4396                         if (rxbuf->hmap != NULL) {
4397                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4398                                 rxbuf->hmap = NULL;
4399                         }
4400                         if (rxbuf->pmap != NULL) {
4401                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4402                                 rxbuf->pmap = NULL;
4403                         }
4404                 }
4405                 if (rxr->rx_buffers != NULL) {
4406                         kfree(rxr->rx_buffers, M_DEVBUF);
4407                         rxr->rx_buffers = NULL;
4408                 }
4409         }
4410
4411         if (rxr->htag != NULL) {
4412                 bus_dma_tag_destroy(rxr->htag);
4413                 rxr->htag = NULL;
4414         }
4415         if (rxr->ptag != NULL) {
4416                 bus_dma_tag_destroy(rxr->ptag);
4417                 rxr->ptag = NULL;
4418         }
4419
4420         return;
4421 }
4422
4423 static __inline void
4424 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4425 {
4426
4427         /*
4428          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4429          * should be computed by hardware. Also it should not have VLAN tag in
4430          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4431          */
4432 #if 0   /* NET_LRO */
4433         if (rxr->lro_enabled &&
4434             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4435             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4436             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4437             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4438             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4439             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4440             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4441             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4442                 /*
4443                  * Send to the stack if:
4444                  **  - LRO not enabled, or
4445                  **  - no LRO resources, or
4446                  **  - lro enqueue fails
4447                  */
4448                 if (rxr->lro.lro_cnt != 0)
4449                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4450                                 return;
4451         }
4452 #endif
4453         IXGBE_RX_UNLOCK(rxr);
4454         (*ifp->if_input)(ifp, m);
4455         IXGBE_RX_LOCK(rxr);
4456 }
4457
4458 static __inline void
4459 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4460 {
4461         struct ixgbe_rx_buf     *rbuf;
4462
4463         rbuf = &rxr->rx_buffers[i];
4464
4465         if (rbuf->fmp != NULL) {/* Partial chain ? */
4466                 rbuf->fmp->m_flags |= M_PKTHDR;
4467                 m_freem(rbuf->fmp);
4468                 rbuf->fmp = NULL;
4469         }
4470
4471         /*
4472         ** With advanced descriptors the writeback
4473         ** clobbers the buffer addrs, so its easier
4474         ** to just free the existing mbufs and take
4475         ** the normal refresh path to get new buffers
4476         ** and mapping.
4477         */
4478         if (rbuf->m_head) {
4479                 m_free(rbuf->m_head);
4480                 rbuf->m_head = NULL;
4481         }
4482
4483         if (rbuf->m_pack) {
4484                 m_free(rbuf->m_pack);
4485                 rbuf->m_pack = NULL;
4486         }
4487
4488         return;
4489 }
4490
4491
4492 /*********************************************************************
4493  *
4494  *  This routine executes in interrupt context. It replenishes
4495  *  the mbufs in the descriptor and sends data which has been
4496  *  dma'ed into host memory to upper layer.
4497  *
4498  *  We loop at most count times if count is > 0, or until done if
4499  *  count < 0.
4500  *
4501  *  Return TRUE for more work, FALSE for all clean.
4502  *********************************************************************/
4503 static bool
4504 ixgbe_rxeof(struct ix_queue *que, int count)
4505 {
4506         struct adapter          *adapter = que->adapter;
4507         struct rx_ring          *rxr = que->rxr;
4508         struct ifnet            *ifp = adapter->ifp;
4509 #if 0   /* NET_LRO */
4510         struct lro_ctrl         *lro = &rxr->lro;
4511         struct lro_entry        *queued;
4512 #endif
4513         int                     i, nextp, processed = 0;
4514         u32                     staterr = 0;
4515         union ixgbe_adv_rx_desc *cur;
4516         struct ixgbe_rx_buf     *rbuf, *nbuf;
4517
4518         IXGBE_RX_LOCK(rxr);
4519
4520 #ifdef DEV_NETMAP
4521         if (ifp->if_capenable & IFCAP_NETMAP) {
4522                 /*
4523                  * Same as the txeof routine: only wakeup clients on intr.
4524                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4525                  * mitigation (ixgbe_rxsync() will not look for new packets
4526                  * unless NKR_PENDINTR is set).
4527                  */
4528                 struct netmap_adapter *na = NA(ifp);
4529
4530                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4531                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4532                 IXGBE_RX_UNLOCK(rxr);
4533                 IXGBE_CORE_LOCK(adapter);
4534                 selwakeuppri(&na->rx_si, PI_NET);
4535                 IXGBE_CORE_UNLOCK(adapter);
4536                 return (FALSE);
4537         }
4538 #endif /* DEV_NETMAP */
4539         for (i = rxr->next_to_check; count != 0;) {
4540                 struct mbuf     *sendmp, *mh, *mp;
4541                 u32             rsc, ptype;
4542                 u16             hlen, plen, hdr;
4543 #ifdef NET_VLAN
4544                 u16             vtag = 0;
4545 #endif
4546                 bool            eop;
4547
4548                 /* Sync the ring. */
4549                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4550                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4551
4552                 cur = &rxr->rx_base[i];
4553                 staterr = le32toh(cur->wb.upper.status_error);
4554
4555                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4556                         break;
4557                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4558                         break;
4559
4560                 count--;
4561                 sendmp = NULL;
4562                 nbuf = NULL;
4563                 rsc = 0;
4564                 cur->wb.upper.status_error = 0;
4565                 rbuf = &rxr->rx_buffers[i];
4566                 mh = rbuf->m_head;
4567                 mp = rbuf->m_pack;
4568
4569                 plen = le16toh(cur->wb.upper.length);
4570                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4571                     IXGBE_RXDADV_PKTTYPE_MASK;
4572                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4573                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4574
4575                 /* Process vlan info */
4576 #ifdef NET_VLAN
4577                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4578                         vtag = le16toh(cur->wb.upper.vlan);
4579 #endif
4580
4581                 /* Make sure bad packets are discarded */
4582                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4583                     (rxr->discard)) {
4584                         ifp->if_ierrors++;
4585                         rxr->rx_discarded++;
4586                         if (eop)
4587                                 rxr->discard = FALSE;
4588                         else
4589                                 rxr->discard = TRUE;
4590                         ixgbe_rx_discard(rxr, i);
4591                         goto next_desc;
4592                 }
4593
4594                 /*
4595                 ** On 82599 which supports a hardware
4596                 ** LRO (called HW RSC), packets need
4597                 ** not be fragmented across sequential
4598                 ** descriptors, rather the next descriptor
4599                 ** is indicated in bits of the descriptor.
4600                 ** This also means that we might proceses
4601                 ** more than one packet at a time, something
4602                 ** that has never been true before, it
4603                 ** required eliminating global chain pointers
4604                 ** in favor of what we are doing here.  -jfv
4605                 */
4606                 if (!eop) {
4607                         /*
4608                         ** Figure out the next descriptor
4609                         ** of this frame.
4610                         */
4611                         if (rxr->hw_rsc == TRUE) {
4612                                 rsc = ixgbe_rsc_count(cur);
4613                                 rxr->rsc_num += (rsc - 1);
4614                         }
4615                         if (rsc) { /* Get hardware index */
4616                                 nextp = ((staterr &
4617                                     IXGBE_RXDADV_NEXTP_MASK) >>
4618                                     IXGBE_RXDADV_NEXTP_SHIFT);
4619                         } else { /* Just sequential */
4620                                 nextp = i + 1;
4621                                 if (nextp == adapter->num_rx_desc)
4622                                         nextp = 0;
4623                         }
4624                         nbuf = &rxr->rx_buffers[nextp];
4625                         prefetch(nbuf);
4626                 }
4627                 /*
4628                 ** The header mbuf is ONLY used when header
4629                 ** split is enabled, otherwise we get normal
4630                 ** behavior, ie, both header and payload
4631                 ** are DMA'd into the payload buffer.
4632                 **
4633                 ** Rather than using the fmp/lmp global pointers
4634                 ** we now keep the head of a packet chain in the
4635                 ** buffer struct and pass this along from one
4636                 ** descriptor to the next, until we get EOP.
4637                 */
4638                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4639                         /* This must be an initial descriptor */
4640                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4641                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4642                         if (hlen > IXGBE_RX_HDR)
4643                                 hlen = IXGBE_RX_HDR;
4644                         mh->m_len = hlen;
4645                         mh->m_flags |= M_PKTHDR;
4646                         mh->m_next = NULL;
4647                         mh->m_pkthdr.len = mh->m_len;
4648                         /* Null buf pointer so it is refreshed */
4649                         rbuf->m_head = NULL;
4650                         /*
4651                         ** Check the payload length, this
4652                         ** could be zero if its a small
4653                         ** packet.
4654                         */
4655                         if (plen > 0) {
4656                                 mp->m_len = plen;
4657                                 mp->m_next = NULL;
4658                                 mp->m_flags &= ~M_PKTHDR;
4659                                 mh->m_next = mp;
4660                                 mh->m_pkthdr.len += mp->m_len;
4661                                 /* Null buf pointer so it is refreshed */
4662                                 rbuf->m_pack = NULL;
4663                                 rxr->rx_split_packets++;
4664                         }
4665                         /*
4666                         ** Now create the forward
4667                         ** chain so when complete
4668                         ** we wont have to.
4669                         */
4670                         if (eop == 0) {
4671                                 /* stash the chain head */
4672                                 nbuf->fmp = mh;
4673                                 /* Make forward chain */
4674                                 if (plen)
4675                                         mp->m_next = nbuf->m_pack;
4676                                 else
4677                                         mh->m_next = nbuf->m_pack;
4678                         } else {
4679                                 /* Singlet, prepare to send */
4680                                 sendmp = mh;
4681                                 /* If hardware handled vtag */
4682 #ifdef NET_VLAN
4683                                 if (vtag) {
4684                                         sendmp->m_pkthdr.ether_vtag = vtag;
4685                                         sendmp->m_flags |= M_VLANTAG;
4686                                 }
4687 #endif
4688                         }
4689                 } else {
4690                         /*
4691                         ** Either no header split, or a
4692                         ** secondary piece of a fragmented
4693                         ** split packet.
4694                         */
4695                         mp->m_len = plen;
4696                         /*
4697                         ** See if there is a stored head
4698                         ** that determines what we are
4699                         */
4700                         sendmp = rbuf->fmp;
4701                         rbuf->m_pack = rbuf->fmp = NULL;
4702
4703                         if (sendmp != NULL) {  /* secondary frag */
4704                                 mp->m_flags &= ~M_PKTHDR;
4705                                 sendmp->m_pkthdr.len += mp->m_len;
4706                         } else {
4707                                 /* first desc of a non-ps chain */
4708                                 sendmp = mp;
4709                                 sendmp->m_flags |= M_PKTHDR;
4710                                 sendmp->m_pkthdr.len = mp->m_len;
4711 #ifdef NET_VLAN
4712                                 if (staterr & IXGBE_RXD_STAT_VP) {
4713                                         sendmp->m_pkthdr.ether_vtag = vtag;
4714                                         sendmp->m_flags |= M_VLANTAG;
4715                                 }
4716 #endif
4717                         }
4718                         /* Pass the head pointer on */
4719                         if (eop == 0) {
4720                                 nbuf->fmp = sendmp;
4721                                 sendmp = NULL;
4722                                 mp->m_next = nbuf->m_pack;
4723                         }
4724                 }
4725                 ++processed;
4726                 /* Sending this frame? */
4727                 if (eop) {
4728                         sendmp->m_pkthdr.rcvif = ifp;
4729                         ifp->if_ipackets++;
4730                         rxr->rx_packets++;
4731                         /* capture data for AIM */
4732                         rxr->bytes += sendmp->m_pkthdr.len;
4733                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4734                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4735                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4736 #if 0 /* __FreeBSD_version >= 800000 */
4737                         sendmp->m_pkthdr.flowid = que->msix;
4738                         sendmp->m_flags |= M_FLOWID;
4739 #endif
4740                 }
4741 next_desc:
4742                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4743                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4744
4745                 /* Advance our pointers to the next descriptor. */
4746                 if (++i == adapter->num_rx_desc)
4747                         i = 0;
4748
4749                 /* Now send to the stack or do LRO */
4750                 if (sendmp != NULL) {
4751                         rxr->next_to_check = i;
4752                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4753                         i = rxr->next_to_check;
4754                 }
4755
4756                /* Every 8 descriptors we go to refresh mbufs */
4757                 if (processed == 8) {
4758                         ixgbe_refresh_mbufs(rxr, i);
4759                         processed = 0;
4760                 }
4761         }
4762
4763         /* Refresh any remaining buf structs */
4764         if (ixgbe_rx_unrefreshed(rxr))
4765                 ixgbe_refresh_mbufs(rxr, i);
4766
4767         rxr->next_to_check = i;
4768
4769         /*
4770          * Flush any outstanding LRO work
4771          */
4772 #if 0   /* NET_LRO */
4773         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4774                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4775                 tcp_lro_flush(lro, queued);
4776         }
4777 #endif
4778
4779         IXGBE_RX_UNLOCK(rxr);
4780
4781         /*
4782         ** We still have cleaning to do?
4783         ** Schedule another interrupt if so.
4784         */
4785         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4786                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4787                 return (TRUE);
4788         }
4789
4790         return (FALSE);
4791 }
4792
4793
4794 /*********************************************************************
4795  *
4796  *  Verify that the hardware indicated that the checksum is valid.
4797  *  Inform the stack about the status of checksum so that stack
4798  *  doesn't spend time verifying the checksum.
4799  *
4800  *********************************************************************/
4801 static void
4802 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4803 {
4804         u16     status = (u16) staterr;
4805         u8      errors = (u8) (staterr >> 24);
4806         bool    sctp = FALSE;
4807
4808         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4809             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4810                 sctp = TRUE;
4811
4812         if (status & IXGBE_RXD_STAT_IPCS) {
4813                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4814                         /* IP Checksum Good */
4815                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4816                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4817
4818                 } else
4819                         mp->m_pkthdr.csum_flags = 0;
4820         }
4821         if (status & IXGBE_RXD_STAT_L4CS) {
4822                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4823 #if 0
4824                 if (sctp)
4825                         type = CSUM_SCTP_VALID;
4826 #endif
4827                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4828                         mp->m_pkthdr.csum_flags |= type;
4829                         if (!sctp)
4830                                 mp->m_pkthdr.csum_data = htons(0xffff);
4831                 }
4832         }
4833         return;
4834 }
4835
4836
4837 /*
4838 ** This routine is run via an vlan config EVENT,
4839 ** it enables us to use the HW Filter table since
4840 ** we can get the vlan id. This just creates the
4841 ** entry in the soft version of the VFTA, init will
4842 ** repopulate the real table.
4843 */
4844 static void
4845 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4846 {
4847         struct adapter  *adapter = ifp->if_softc;
4848         u16             index, bit;
4849
4850         if (ifp->if_softc !=  arg)   /* Not our event */
4851                 return;
4852
4853         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4854                 return;
4855
4856         IXGBE_CORE_LOCK(adapter);
4857         index = (vtag >> 5) & 0x7F;
4858         bit = vtag & 0x1F;
4859         adapter->shadow_vfta[index] |= (1 << bit);
4860         ++adapter->num_vlans;
4861         ixgbe_init_locked(adapter);
4862         IXGBE_CORE_UNLOCK(adapter);
4863 }
4864
4865 /*
4866 ** This routine is run via an vlan
4867 ** unconfig EVENT, remove our entry
4868 ** in the soft vfta.
4869 */
4870 static void
4871 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4872 {
4873         struct adapter  *adapter = ifp->if_softc;
4874         u16             index, bit;
4875
4876         if (ifp->if_softc !=  arg)
4877                 return;
4878
4879         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4880                 return;
4881
4882         IXGBE_CORE_LOCK(adapter);
4883         index = (vtag >> 5) & 0x7F;
4884         bit = vtag & 0x1F;
4885         adapter->shadow_vfta[index] &= ~(1 << bit);
4886         --adapter->num_vlans;
4887         /* Re-init to load the changes */
4888         ixgbe_init_locked(adapter);
4889         IXGBE_CORE_UNLOCK(adapter);
4890 }
4891
4892 static void
4893 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4894 {
4895 #ifdef NET_VLAN
4896         struct ifnet    *ifp = adapter->ifp;
4897         struct ixgbe_hw *hw = &adapter->hw;
4898         struct rx_ring  *rxr;
4899         u32             ctrl;
4900
4901         /*
4902         ** We get here thru init_locked, meaning
4903         ** a soft reset, this has already cleared
4904         ** the VFTA and other state, so if there
4905         ** have been no vlan's registered do nothing.
4906         */
4907         if (adapter->num_vlans == 0)
4908                 return;
4909
4910         /*
4911         ** A soft reset zero's out the VFTA, so
4912         ** we need to repopulate it now.
4913         */
4914         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4915                 if (adapter->shadow_vfta[i] != 0)
4916                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4917                             adapter->shadow_vfta[i]);
4918
4919         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4920         /* Enable the Filter Table if enabled */
4921         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4922                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4923                 ctrl |= IXGBE_VLNCTRL_VFE;
4924         }
4925         if (hw->mac.type == ixgbe_mac_82598EB)
4926                 ctrl |= IXGBE_VLNCTRL_VME;
4927         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4928
4929         /* Setup the queues for vlans */
4930         for (int i = 0; i < adapter->num_queues; i++) {
4931                 rxr = &adapter->rx_rings[i];
4932                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4933                 if (hw->mac.type != ixgbe_mac_82598EB) {
4934                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4935                         ctrl |= IXGBE_RXDCTL_VME;
4936                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4937                 }
4938                 rxr->vtag_strip = TRUE;
4939         }
4940 #endif
4941 }
4942
4943 static void
4944 ixgbe_enable_intr(struct adapter *adapter)
4945 {
4946         struct ixgbe_hw *hw = &adapter->hw;
4947         struct ix_queue *que = adapter->queues;
4948         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4949
4950
4951         /* Enable Fan Failure detection */
4952         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4953                     mask |= IXGBE_EIMS_GPI_SDP1;
4954         else {
4955                     mask |= IXGBE_EIMS_ECC;
4956                     mask |= IXGBE_EIMS_GPI_SDP0;
4957                     mask |= IXGBE_EIMS_GPI_SDP1;
4958                     mask |= IXGBE_EIMS_GPI_SDP2;
4959 #ifdef IXGBE_FDIR
4960                     mask |= IXGBE_EIMS_FLOW_DIR;
4961 #endif
4962         }
4963
4964         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4965
4966         /* With RSS we use auto clear */
4967         if (adapter->msix_mem) {
4968                 mask = IXGBE_EIMS_ENABLE_MASK;
4969                 /* Don't autoclear Link */
4970                 mask &= ~IXGBE_EIMS_OTHER;
4971                 mask &= ~IXGBE_EIMS_LSC;
4972                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4973         }
4974
4975         /*
4976         ** Now enable all queues, this is done separately to
4977         ** allow for handling the extended (beyond 32) MSIX
4978         ** vectors that can be used by 82599
4979         */
4980         for (int i = 0; i < adapter->num_queues; i++, que++)
4981                 ixgbe_enable_queue(adapter, que->msix);
4982
4983         IXGBE_WRITE_FLUSH(hw);
4984
4985         return;
4986 }
4987
4988 static void
4989 ixgbe_disable_intr(struct adapter *adapter)
4990 {
4991         if (adapter->msix_mem)
4992                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4993         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4994                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4995         } else {
4996                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4997                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4998                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4999         }
5000         IXGBE_WRITE_FLUSH(&adapter->hw);
5001         return;
5002 }
5003
5004 u16
5005 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5006 {
5007         u16 value;
5008
5009         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
5010             reg, 2);
5011
5012         return (value);
5013 }
5014
5015 void
5016 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5017 {
5018         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
5019             reg, value, 2);
5020
5021         return;
5022 }
5023
5024 /*
5025 ** Setup the correct IVAR register for a particular MSIX interrupt
5026 **   (yes this is all very magic and confusing :)
5027 **  - entry is the register array entry
5028 **  - vector is the MSIX vector for this queue
5029 **  - type is RX/TX/MISC
5030 */
5031 static void
5032 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5033 {
5034         struct ixgbe_hw *hw = &adapter->hw;
5035         u32 ivar, index;
5036
5037         vector |= IXGBE_IVAR_ALLOC_VAL;
5038
5039         switch (hw->mac.type) {
5040
5041         case ixgbe_mac_82598EB:
5042                 if (type == -1)
5043                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5044                 else
5045                         entry += (type * 64);
5046                 index = (entry >> 2) & 0x1F;
5047                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5048                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5049                 ivar |= (vector << (8 * (entry & 0x3)));
5050                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5051                 break;
5052
5053         case ixgbe_mac_82599EB:
5054         case ixgbe_mac_X540:
5055                 if (type == -1) { /* MISC IVAR */
5056                         index = (entry & 1) * 8;
5057                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5058                         ivar &= ~(0xFF << index);
5059                         ivar |= (vector << index);
5060                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5061                 } else {        /* RX/TX IVARS */
5062                         index = (16 * (entry & 1)) + (8 * type);
5063                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5064                         ivar &= ~(0xFF << index);
5065                         ivar |= (vector << index);
5066                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5067                 }
5068
5069         default:
5070                 break;
5071         }
5072 }
5073
5074 static void
5075 ixgbe_configure_ivars(struct adapter *adapter)
5076 {
5077         struct  ix_queue *que = adapter->queues;
5078         u32 newitr;
5079
5080         if (ixgbe_max_interrupt_rate > 0)
5081                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5082         else
5083                 newitr = 0;
5084
5085         for (int i = 0; i < adapter->num_queues; i++, que++) {
5086                 /* First the RX queue entry */
5087                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5088                 /* ... and the TX */
5089                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5090                 /* Set an Initial EITR value */
5091                 IXGBE_WRITE_REG(&adapter->hw,
5092                     IXGBE_EITR(que->msix), newitr);
5093         }
5094
5095         /* For the Link interrupt */
5096         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5097 }
5098
5099 /*
5100 ** ixgbe_sfp_probe - called in the local timer to
5101 ** determine if a port had optics inserted.
5102 */
5103 static bool ixgbe_sfp_probe(struct adapter *adapter)
5104 {
5105         struct ixgbe_hw *hw = &adapter->hw;
5106         device_t        dev = adapter->dev;
5107         bool            result = FALSE;
5108
5109         if ((hw->phy.type == ixgbe_phy_nl) &&
5110             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5111                 s32 ret = hw->phy.ops.identify_sfp(hw);
5112                 if (ret)
5113                         goto out;
5114                 ret = hw->phy.ops.reset(hw);
5115                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5116                         device_printf(dev,"Unsupported SFP+ module detected!");
5117                         kprintf(" Reload driver with supported module.\n");
5118                         adapter->sfp_probe = FALSE;
5119                         goto out;
5120                 } else
5121                         device_printf(dev,"SFP+ module detected!\n");
5122                 /* We now have supported optics */
5123                 adapter->sfp_probe = FALSE;
5124                 /* Set the optics type so system reports correctly */
5125                 ixgbe_setup_optics(adapter);
5126                 result = TRUE;
5127         }
5128 out:
5129         return (result);
5130 }
5131
5132 /*
5133 ** Tasklet handler for MSIX Link interrupts
5134 **  - do outside interrupt since it might sleep
5135 */
5136 static void
5137 ixgbe_handle_link(void *context, int pending)
5138 {
5139         struct adapter  *adapter = context;
5140
5141         ixgbe_check_link(&adapter->hw,
5142             &adapter->link_speed, &adapter->link_up, 0);
5143         ixgbe_update_link_status(adapter);
5144 }
5145
5146 /*
5147 ** Tasklet for handling SFP module interrupts
5148 */
5149 static void
5150 ixgbe_handle_mod(void *context, int pending)
5151 {
5152         struct adapter  *adapter = context;
5153         struct ixgbe_hw *hw = &adapter->hw;
5154         device_t        dev = adapter->dev;
5155         u32 err;
5156
5157         err = hw->phy.ops.identify_sfp(hw);
5158         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5159                 device_printf(dev,
5160                     "Unsupported SFP+ module type was detected.\n");
5161                 return;
5162         }
5163         err = hw->mac.ops.setup_sfp(hw);
5164         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5165                 device_printf(dev,
5166                     "Setup failure - unsupported SFP+ module type.\n");
5167                 return;
5168         }
5169         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5170         return;
5171 }
5172
5173
5174 /*
5175 ** Tasklet for handling MSF (multispeed fiber) interrupts
5176 */
5177 static void
5178 ixgbe_handle_msf(void *context, int pending)
5179 {
5180         struct adapter  *adapter = context;
5181         struct ixgbe_hw *hw = &adapter->hw;
5182         u32 autoneg;
5183         bool negotiate;
5184
5185         autoneg = hw->phy.autoneg_advertised;
5186         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5187                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5188         if (hw->mac.ops.setup_link)
5189                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5190         return;
5191 }
5192
5193 #ifdef IXGBE_FDIR
5194 /*
5195 ** Tasklet for reinitializing the Flow Director filter table
5196 */
5197 static void
5198 ixgbe_reinit_fdir(void *context, int pending)
5199 {
5200         struct adapter  *adapter = context;
5201         struct ifnet   *ifp = adapter->ifp;
5202
5203         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5204                 return;
5205         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5206         adapter->fdir_reinit = 0;
5207         /* re-enable flow director interrupts */
5208         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5209         /* Restart the interface */
5210         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5211         return;
5212 }
5213 #endif
5214
5215 /**********************************************************************
5216  *
5217  *  Update the board statistics counters.
5218  *
5219  **********************************************************************/
5220 static void
5221 ixgbe_update_stats_counters(struct adapter *adapter)
5222 {
5223         struct ifnet   *ifp = adapter->ifp;
5224         struct ixgbe_hw *hw = &adapter->hw;
5225         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5226         u64  total_missed_rx = 0;
5227
5228         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5229         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5230         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5231         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5232
5233         for (int i = 0; i < 8; i++) {
5234                 u32 mp;
5235                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5236                 /* missed_rx tallies misses for the gprc workaround */
5237                 missed_rx += mp;
5238                 /* global total per queue */
5239                 adapter->stats.mpc[i] += mp;
5240                 /* Running comprehensive total for stats display */
5241                 total_missed_rx += adapter->stats.mpc[i];
5242                 if (hw->mac.type == ixgbe_mac_82598EB)
5243                         adapter->stats.rnbc[i] +=
5244                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5245                 adapter->stats.pxontxc[i] +=
5246                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5247                 adapter->stats.pxonrxc[i] +=
5248                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5249                 adapter->stats.pxofftxc[i] +=
5250                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5251                 adapter->stats.pxoffrxc[i] +=
5252                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5253                 adapter->stats.pxon2offc[i] +=
5254                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5255         }
5256         for (int i = 0; i < 16; i++) {
5257                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5258                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5259                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5260                 adapter->stats.qbrc[i] +=
5261                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5262                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5263                 adapter->stats.qbtc[i] +=
5264                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5265                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5266         }
5267         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5268         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5269         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5270
5271         /* Hardware workaround, gprc counts missed packets */
5272         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5273         adapter->stats.gprc -= missed_rx;
5274
5275         if (hw->mac.type != ixgbe_mac_82598EB) {
5276                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5277                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5278                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5279                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5280                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5281                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5282                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5283                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5284         } else {
5285                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5286                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5287                 /* 82598 only has a counter in the high register */
5288                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5289                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5290                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5291         }
5292
5293         /*
5294          * Workaround: mprc hardware is incorrectly counting
5295          * broadcasts, so for now we subtract those.
5296          */
5297         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5298         adapter->stats.bprc += bprc;
5299         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5300         if (hw->mac.type == ixgbe_mac_82598EB)
5301                 adapter->stats.mprc -= bprc;
5302
5303         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5304         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5305         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5306         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5307         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5308         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5309
5310         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5311         adapter->stats.lxontxc += lxon;
5312         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5313         adapter->stats.lxofftxc += lxoff;
5314         total = lxon + lxoff;
5315
5316         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5317         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5318         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5319         adapter->stats.gptc -= total;
5320         adapter->stats.mptc -= total;
5321         adapter->stats.ptc64 -= total;
5322         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5323
5324         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5325         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5326         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5327         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5328         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5329         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5330         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5331         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5332         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5333         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5334         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5335         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5336         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5337         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5338         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5339         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5340         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5341         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5342         /* Only read FCOE on 82599 */
5343         if (hw->mac.type != ixgbe_mac_82598EB) {
5344                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5345                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5346                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5347                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5348                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5349         }
5350
5351         /* Fill out the OS statistics structure */
5352         ifp->if_ipackets = adapter->stats.gprc;
5353         ifp->if_opackets = adapter->stats.gptc;
5354         ifp->if_ibytes = adapter->stats.gorc;
5355         ifp->if_obytes = adapter->stats.gotc;
5356         ifp->if_imcasts = adapter->stats.mprc;
5357         ifp->if_collisions = 0;
5358
5359         /* Rx Errors */
5360         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5361                 adapter->stats.rlec;
5362 }
5363
5364 /** ixgbe_sysctl_tdh_handler - Handler function
5365  *  Retrieves the TDH value from the hardware
5366  */
5367 static int
5368 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5369 {
5370         int error;
5371
5372         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5373         if (!txr) return 0;
5374
5375         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5376         error = sysctl_handle_int(oidp, &val, 0, req);
5377         if (error || !req->newptr)
5378                 return error;
5379         return 0;
5380 }
5381
5382 /** ixgbe_sysctl_tdt_handler - Handler function
5383  *  Retrieves the TDT value from the hardware
5384  */
5385 static int
5386 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5387 {
5388         int error;
5389
5390         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5391         if (!txr) return 0;
5392
5393         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5394         error = sysctl_handle_int(oidp, &val, 0, req);
5395         if (error || !req->newptr)
5396                 return error;
5397         return 0;
5398 }
5399
5400 /** ixgbe_sysctl_rdh_handler - Handler function
5401  *  Retrieves the RDH value from the hardware
5402  */
5403 static int
5404 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5405 {
5406         int error;
5407
5408         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5409         if (!rxr) return 0;
5410
5411         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5412         error = sysctl_handle_int(oidp, &val, 0, req);
5413         if (error || !req->newptr)
5414                 return error;
5415         return 0;
5416 }
5417
5418 /** ixgbe_sysctl_rdt_handler - Handler function
5419  *  Retrieves the RDT value from the hardware
5420  */
5421 static int
5422 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5423 {
5424         int error;
5425
5426         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5427         if (!rxr) return 0;
5428
5429         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5430         error = sysctl_handle_int(oidp, &val, 0, req);
5431         if (error || !req->newptr)
5432                 return error;
5433         return 0;
5434 }
5435
5436 static int
5437 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5438 {
5439         int error;
5440         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5441         unsigned int reg, usec, rate;
5442
5443         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5444         usec = ((reg & 0x0FF8) >> 3);
5445         if (usec > 0)
5446                 rate = 500000 / usec;
5447         else
5448                 rate = 0;
5449         error = sysctl_handle_int(oidp, &rate, 0, req);
5450         if (error || !req->newptr)
5451                 return error;
5452         reg &= ~0xfff; /* default, no limitation */
5453         ixgbe_max_interrupt_rate = 0;
5454         if (rate > 0 && rate < 500000) {
5455                 if (rate < 1000)
5456                         rate = 1000;
5457                 ixgbe_max_interrupt_rate = rate;
5458                 reg |= ((4000000/rate) & 0xff8 );
5459         }
5460         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5461         return 0;
5462 }
5463
5464 /*
5465  * Add sysctl variables, one per statistic, to the system.
5466  */
5467 static void
5468 ixgbe_add_hw_stats(struct adapter *adapter)
5469 {
5470         struct tx_ring *txr = adapter->tx_rings;
5471         struct rx_ring *rxr = adapter->rx_rings;
5472
5473         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5474         struct sysctl_oid *tree = adapter->sysctl_tree;
5475         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5476         struct ixgbe_hw_stats *stats = &adapter->stats;
5477
5478         struct sysctl_oid *stat_node, *queue_node;
5479         struct sysctl_oid_list *stat_list, *queue_list;
5480
5481 #define QUEUE_NAME_LEN 32
5482         char namebuf[QUEUE_NAME_LEN];
5483
5484         /* Driver Statistics */
5485         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5486                         CTLFLAG_RD, &adapter->dropped_pkts,
5487                         "Driver dropped packets");
5488         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5489                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5490                         "m_defrag() failed");
5491         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5492                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5493                         "Driver tx dma failure in xmit");
5494         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5495                         CTLFLAG_RD, &adapter->watchdog_events,
5496                         "Watchdog timeouts");
5497 #if 0   /* NET_TSO */
5498         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5499                         CTLFLAG_RD, &adapter->tso_tx,
5500                         "TSO");
5501 #endif
5502         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5503                         CTLFLAG_RD, &adapter->link_irq,
5504                         "Link MSIX IRQ Handled");
5505
5506         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5507         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5508                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5509                                             CTLFLAG_RD, NULL, "Queue Name");
5510                 queue_list = SYSCTL_CHILDREN(queue_node);
5511
5512                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5513                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5514                                 sizeof(&adapter->queues[i]),
5515                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5516                                 "Interrupt Rate");
5517                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5518                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5519                                 "irqs on this queue");
5520                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5521                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5522                                 ixgbe_sysctl_tdh_handler, "IU",
5523                                 "Transmit Descriptor Head");
5524                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5525                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5526                                 ixgbe_sysctl_tdt_handler, "IU",
5527                                 "Transmit Descriptor Tail");
5528                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5529                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5530                                 "Queue No Descriptor Available");
5531                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5532                                 CTLFLAG_RD, &txr->total_packets, 0,
5533                                 "Queue Packets Transmitted");
5534         }
5535
5536         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5537         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5538                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5539                                             CTLFLAG_RD, NULL, "Queue Name");
5540                 queue_list = SYSCTL_CHILDREN(queue_node);
5541
5542 #if 0   /* NET_LRO */
5543                 struct lro_ctrl *lro = &rxr->lro;
5544 #endif
5545
5546         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5547                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5548                                             CTLFLAG_RD, NULL, "Queue Name");
5549                 queue_list = SYSCTL_CHILDREN(queue_node);
5550
5551                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5552                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5553                                 ixgbe_sysctl_rdh_handler, "IU",
5554                                 "Receive Descriptor Head");
5555                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5556                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5557                                 ixgbe_sysctl_rdt_handler, "IU",
5558                                 "Receive Descriptor Tail");
5559                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5560                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5561                                 "Queue Packets Received");
5562                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5563                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5564                                 "Queue Bytes Received");
5565 #if 0   /* NET_LRO */
5566                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5567                                 CTLFLAG_RD, &lro->lro_queued, 0,
5568                                 "LRO Queued");
5569                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5570                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5571                                 "LRO Flushed");
5572 #endif
5573         }
5574
5575         /* MAC stats get the own sub node */
5576
5577         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5578                                     CTLFLAG_RD, NULL, "MAC Statistics");
5579         stat_list = SYSCTL_CHILDREN(stat_node);
5580
5581         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5582                         CTLFLAG_RD, &stats->crcerrs, 0,
5583                         "CRC Errors");
5584         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5585                         CTLFLAG_RD, &stats->illerrc, 0,
5586                         "Illegal Byte Errors");
5587         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5588                         CTLFLAG_RD, &stats->errbc, 0,
5589                         "Byte Errors");
5590         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5591                         CTLFLAG_RD, &stats->mspdc, 0,
5592                         "MAC Short Packets Discarded");
5593         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5594                         CTLFLAG_RD, &stats->mlfc, 0,
5595                         "MAC Local Faults");
5596         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5597                         CTLFLAG_RD, &stats->mrfc, 0,
5598                         "MAC Remote Faults");
5599         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5600                         CTLFLAG_RD, &stats->rlec, 0,
5601                         "Receive Length Errors");
5602         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5603                         CTLFLAG_RD, &stats->lxontxc, 0,
5604                         "Link XON Transmitted");
5605         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5606                         CTLFLAG_RD, &stats->lxonrxc, 0,
5607                         "Link XON Received");
5608         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5609                         CTLFLAG_RD, &stats->lxofftxc, 0,
5610                         "Link XOFF Transmitted");
5611         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5612                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5613                         "Link XOFF Received");
5614
5615         /* Packet Reception Stats */
5616         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5617                         CTLFLAG_RD, &stats->tor, 0,
5618                         "Total Octets Received");
5619         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5620                         CTLFLAG_RD, &stats->gorc, 0,
5621                         "Good Octets Received");
5622         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5623                         CTLFLAG_RD, &stats->tpr, 0,
5624                         "Total Packets Received");
5625         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5626                         CTLFLAG_RD, &stats->gprc, 0,
5627                         "Good Packets Received");
5628         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5629                         CTLFLAG_RD, &stats->mprc, 0,
5630                         "Multicast Packets Received");
5631         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5632                         CTLFLAG_RD, &stats->bprc, 0,
5633                         "Broadcast Packets Received");
5634         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5635                         CTLFLAG_RD, &stats->prc64, 0,
5636                         "64 byte frames received ");
5637         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5638                         CTLFLAG_RD, &stats->prc127, 0,
5639                         "65-127 byte frames received");
5640         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5641                         CTLFLAG_RD, &stats->prc255, 0,
5642                         "128-255 byte frames received");
5643         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5644                         CTLFLAG_RD, &stats->prc511, 0,
5645                         "256-511 byte frames received");
5646         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5647                         CTLFLAG_RD, &stats->prc1023, 0,
5648                         "512-1023 byte frames received");
5649         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5650                         CTLFLAG_RD, &stats->prc1522, 0,
5651                         "1023-1522 byte frames received");
5652         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5653                         CTLFLAG_RD, &stats->ruc, 0,
5654                         "Receive Undersized");
5655         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5656                         CTLFLAG_RD, &stats->rfc, 0,
5657                         "Fragmented Packets Received ");
5658         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5659                         CTLFLAG_RD, &stats->roc, 0,
5660                         "Oversized Packets Received");
5661         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5662                         CTLFLAG_RD, &stats->rjc, 0,
5663                         "Received Jabber");
5664         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5665                         CTLFLAG_RD, &stats->mngprc, 0,
5666                         "Management Packets Received");
5667         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5668                         CTLFLAG_RD, &stats->mngptc, 0,
5669                         "Management Packets Dropped");
5670         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5671                         CTLFLAG_RD, &stats->xec, 0,
5672                         "Checksum Errors");
5673
5674         /* Packet Transmission Stats */
5675         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5676                         CTLFLAG_RD, &stats->gotc, 0,
5677                         "Good Octets Transmitted");
5678         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5679                         CTLFLAG_RD, &stats->tpt, 0,
5680                         "Total Packets Transmitted");
5681         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5682                         CTLFLAG_RD, &stats->gptc, 0,
5683                         "Good Packets Transmitted");
5684         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5685                         CTLFLAG_RD, &stats->bptc, 0,
5686                         "Broadcast Packets Transmitted");
5687         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5688                         CTLFLAG_RD, &stats->mptc, 0,
5689                         "Multicast Packets Transmitted");
5690         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5691                         CTLFLAG_RD, &stats->mngptc, 0,
5692                         "Management Packets Transmitted");
5693         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5694                         CTLFLAG_RD, &stats->ptc64, 0,
5695                         "64 byte frames transmitted ");
5696         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5697                         CTLFLAG_RD, &stats->ptc127, 0,
5698                         "65-127 byte frames transmitted");
5699         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5700                         CTLFLAG_RD, &stats->ptc255, 0,
5701                         "128-255 byte frames transmitted");
5702         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5703                         CTLFLAG_RD, &stats->ptc511, 0,
5704                         "256-511 byte frames transmitted");
5705         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5706                         CTLFLAG_RD, &stats->ptc1023, 0,
5707                         "512-1023 byte frames transmitted");
5708         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5709                         CTLFLAG_RD, &stats->ptc1522, 0,
5710                         "1024-1522 byte frames transmitted");
5711
5712         /* FC Stats */
5713         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5714                 CTLFLAG_RD, &stats->fccrc, 0,
5715                 "FC CRC Errors");
5716         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5717                 CTLFLAG_RD, &stats->fclast, 0,
5718                 "FC Last Error");
5719         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5720                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5721                 "FCoE Packets Dropped");
5722         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5723                 CTLFLAG_RD, &stats->fcoeprc, 0,
5724                 "FCoE Packets Received");
5725         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5726                 CTLFLAG_RD, &stats->fcoeptc, 0,
5727                 "FCoE Packets Transmitted");
5728         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5729                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5730                 "FCoE DWords Received");
5731         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5732                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5733                 "FCoE DWords Transmitted");
5734 }
5735
5736 /*
5737 ** Set flow control using sysctl:
5738 ** Flow control values:
5739 **      0 - off
5740 **      1 - rx pause
5741 **      2 - tx pause
5742 **      3 - full
5743 */
5744 static int
5745 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5746 {
5747         int error, last;
5748         struct adapter *adapter = (struct adapter *) arg1;
5749
5750         last = adapter->fc;
5751         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5752         if ((error) || (req->newptr == NULL))
5753                 return (error);
5754
5755         /* Don't bother if it's not changed */
5756         if (adapter->fc == last)
5757                 return (0);
5758
5759         switch (adapter->fc) {
5760                 case ixgbe_fc_rx_pause:
5761                 case ixgbe_fc_tx_pause:
5762                 case ixgbe_fc_full:
5763                         adapter->hw.fc.requested_mode = adapter->fc;
5764                         break;
5765                 case ixgbe_fc_none:
5766                 default:
5767                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5768         }
5769         /* Don't autoneg if forcing a value */
5770         adapter->hw.fc.disable_fc_autoneg = TRUE;
5771         ixgbe_fc_enable(&adapter->hw);
5772         return error;
5773 }
5774
5775 static void
5776 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5777         const char *description, int *limit, int value)
5778 {
5779         *limit = value;
5780         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5781             SYSCTL_CHILDREN(adapter->sysctl_tree),
5782             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5783 }
5784
5785 /*
5786 ** Control link advertise speed:
5787 **      1 - advertise only 1G
5788 **      2 - advertise 100Mb
5789 **      3 - advertise normal
5790 */
5791 static int
5792 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5793 {
5794         int                     error = 0;
5795         struct adapter          *adapter;
5796         device_t                dev;
5797         struct ixgbe_hw         *hw;
5798         ixgbe_link_speed        speed, last;
5799
5800         adapter = (struct adapter *) arg1;
5801         dev = adapter->dev;
5802         hw = &adapter->hw;
5803         last = adapter->advertise;
5804
5805         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5806         if ((error) || (adapter->advertise == -1))
5807                 return (error);
5808
5809         if (adapter->advertise == last) /* no change */
5810                 return (0);
5811
5812         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5813             (hw->phy.multispeed_fiber)))
5814                 return (error);
5815
5816         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5817                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5818                 return (error);
5819         }
5820
5821         if (adapter->advertise == 1)
5822                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5823         else if (adapter->advertise == 2)
5824                 speed = IXGBE_LINK_SPEED_100_FULL;
5825         else if (adapter->advertise == 3)
5826                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5827                         IXGBE_LINK_SPEED_10GB_FULL;
5828         else /* bogus value */
5829                 return (error);
5830
5831         hw->mac.autotry_restart = TRUE;
5832         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5833
5834         return (error);
5835 }
5836
5837 /*
5838 ** Thermal Shutdown Trigger
5839 **   - cause a Thermal Overtemp IRQ
5840 */
5841 static int
5842 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5843 {
5844         int             error, fire = 0;
5845         struct adapter  *adapter = (struct adapter *) arg1;
5846         struct ixgbe_hw *hw = &adapter->hw;
5847
5848
5849         if (hw->mac.type != ixgbe_mac_X540)
5850                 return (0);
5851
5852         error = sysctl_handle_int(oidp, &fire, 0, req);
5853         if ((error) || (req->newptr == NULL))
5854                 return (error);
5855
5856         if (fire) {
5857                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5858                 reg |= IXGBE_EICR_TS;
5859                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5860         }
5861
5862         return (0);
5863 }