sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 #if 0   /* HAVE_MSIX */
 125 static int      ixgbe_setup_msix(struct adapter *);
 126 #endif
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 #if 0   /* NET_LRO */
 146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 147 #endif
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *, int);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 167                     const char *, int *, int);
 168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 169 #if 0   /* NET_TSO */
 170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 171 #endif
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_que(void *, int);
 199 static void     ixgbe_handle_link(void *, int);
 200 static void     ixgbe_handle_msf(void *, int);
 201 static void     ixgbe_handle_mod(void *, int);
 202
 203 #ifdef IXGBE_FDIR
 204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 205 static void     ixgbe_reinit_fdir(void *, int);
 206 #endif
 207
 208 /*********************************************************************
 209  *  FreeBSD Device Interface Entry Points
 210  *********************************************************************/
 211
 212 static device_method_t ixgbe_methods[] = {
 213         /* Device interface */
 214         DEVMETHOD(device_probe, ixgbe_probe),
 215         DEVMETHOD(device_attach, ixgbe_attach),
 216         DEVMETHOD(device_detach, ixgbe_detach),
 217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 218         {0, 0}
 219 };
 220
 221 static driver_t ixgbe_driver = {
 222         "ix", ixgbe_methods, sizeof(struct adapter),
 223 };
 224
 225 devclass_t ixgbe_devclass;
 226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 227
 228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 230
 231 /*
 232 ** TUNEABLE PARAMETERS:
 233 */
 234
 235 /*
 236 ** AIM: Adaptive Interrupt Moderation
 237 ** which means that the interrupt rate
 238 ** is varied over time based on the
 239 ** traffic for that interrupt vector
 240 */
 241 static int ixgbe_enable_aim = TRUE;
 242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 243
 244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 246
 247 /* How many packets rxeof tries to clean at a time */
 248 static int ixgbe_rx_process_limit = 128;
 249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 static int ixgbe_msi_enable = 1;
 261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 262
 263 /*
 264  * MSIX should be the default for best performance,
 265  * but this allows it to be forced off for testing.
 266  */
 267 static int ixgbe_enable_msix = 1;
 268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 269
 270 /*
 271  * Header split: this causes the hardware to DMA
 272  * the header into a separate mbuf from the payload,
 273  * it can be a performance win in some workloads, but
 274  * in others it actually hurts, its off by default.
 275  */
 276 static int ixgbe_header_split = FALSE;
 277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 278
 279 /*
 280  * Number of Queues, can be set to 0,
 281  * it then autoconfigures based on the
 282  * number of cpus with a max of 8. This
 283  * can be overriden manually here.
 284  */
 285 static int ixgbe_num_queues = 0;
 286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 287
 288 /*
 289 ** Number of TX descriptors per ring,
 290 ** setting higher than RX as this seems
 291 ** the better performing choice.
 292 */
 293 static int ixgbe_txd = PERFORM_TXD;
 294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 295
 296 /* Number of RX descriptors per ring */
 297 static int ixgbe_rxd = PERFORM_RXD;
 298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** For Flow Director: this is the
 306 ** number of TX packets we sample
 307 ** for the filter pool, this means
 308 ** every 20th packet will be probed.
 309 **
 310 ** This feature can be disabled by
 311 ** setting this to 0.
 312 */
 313 static int atr_sample_rate = 20;
 314 /*
 315 ** Flow Director actually 'steals'
 316 ** part of the packet buffer as its
 317 ** filter pool, this variable controls
 318 ** how much it uses:
 319 **  0 = 64K, 1 = 128K, 2 = 256K
 320 */
 321 static int fdir_pballoc = 1;
 322 #endif
 323
 324 #ifdef DEV_NETMAP
 325 /*
 326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 327  * be a reference on how to implement netmap support in a driver.
 328  * Additional comments are in ixgbe_netmap.h .
 329  *
 330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 331  * that extend the standard driver.
 332  */
 333 #include <dev/netmap/ixgbe_netmap.h>
 334 #endif /* DEV_NETMAP */
 335
 336 /*********************************************************************
 337  *  Device identification routine
 338  *
 339  *  ixgbe_probe determines if the driver should be loaded on
 340  *  adapter based on PCI vendor/device id of the adapter.
 341  *
 342  *  return BUS_PROBE_DEFAULT on success, positive on failure
 343  *********************************************************************/
 344
 345 static int
 346 ixgbe_probe(device_t dev)
 347 {
 348         ixgbe_vendor_info_t *ent;
 349
 350         u16     pci_vendor_id = 0;
 351         u16     pci_device_id = 0;
 352         u16     pci_subvendor_id = 0;
 353         u16     pci_subdevice_id = 0;
 354         char    adapter_name[256];
 355
 356         INIT_DEBUGOUT("ixgbe_probe: begin");
 357
 358         pci_vendor_id = pci_get_vendor(dev);
 359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 360                 return (ENXIO);
 361
 362         pci_device_id = pci_get_device(dev);
 363         pci_subvendor_id = pci_get_subvendor(dev);
 364         pci_subdevice_id = pci_get_subdevice(dev);
 365
 366         ent = ixgbe_vendor_info_array;
 367         while (ent->vendor_id != 0) {
 368                 if ((pci_vendor_id == ent->vendor_id) &&
 369                     (pci_device_id == ent->device_id) &&
 370
 371                     ((pci_subvendor_id == ent->subvendor_id) ||
 372                      (ent->subvendor_id == 0)) &&
 373
 374                     ((pci_subdevice_id == ent->subdevice_id) ||
 375                      (ent->subdevice_id == 0))) {
 376                         ksprintf(adapter_name, "%s, Version - %s",
 377                                 ixgbe_strings[ent->index],
 378                                 ixgbe_driver_version);
 379                         device_set_desc_copy(dev, adapter_name);
 380                         ++ixgbe_total_ports;
 381                         return (BUS_PROBE_DEFAULT);
 382                 }
 383                 ent++;
 384         }
 385         return (ENXIO);
 386 }
 387
 388 /*********************************************************************
 389  *  Device initialization routine
 390  *
 391  *  The attach entry point is called when the driver is being loaded.
 392  *  This routine identifies the type of hardware, allocates all resources
 393  *  and initializes the hardware.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 ixgbe_attach(device_t dev)
 400 {
 401         struct adapter *adapter;
 402         struct ixgbe_hw *hw;
 403         int             error = 0;
 404         u16             csum;
 405         u32             ctrl_ext;
 406
 407         INIT_DEBUGOUT("ixgbe_attach: begin");
 408
 409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 410                 device_printf(dev, "Disabled by device hint\n");
 411                 return (ENXIO);
 412         }
 413
 414         /* Allocate, clear, and link in our adapter structure */
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         hw = &adapter->hw;
 418
 419         /* Core Lock Init*/
 420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 421
 422         /* SYSCTL APIs */
 423
 424         sysctl_ctx_init(&adapter->sysctl_ctx);
 425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 428         if (adapter->sysctl_tree == NULL) {
 429                 device_printf(adapter->dev, "can't add sysctl node\n");
 430                 return (EINVAL);
 431         }
 432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 436
 437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 441
 442         /*
 443         ** Allow a kind of speed control by forcing the autoneg
 444         ** advertised speed list to only a certain value, this
 445         ** supports 1G on 82599 devices, and 100Mb on x540.
 446         */
 447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 451
 452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 456
 457         /* Set up the timer callout */
 458         callout_init_mp(&adapter->timer);
 459
 460         /* Determine hardware revision */
 461         ixgbe_identify_hardware(adapter);
 462
 463         /* Do base PCI setup - map BAR0 */
 464         if (ixgbe_allocate_pci_resources(adapter)) {
 465                 device_printf(dev, "Allocation of PCI resources failed\n");
 466                 error = ENXIO;
 467                 goto err_out;
 468         }
 469
 470         /* Do descriptor calc and sanity checks */
 471         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 472             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 473                 device_printf(dev, "TXD config issue, using default!\n");
 474                 adapter->num_tx_desc = DEFAULT_TXD;
 475         } else
 476                 adapter->num_tx_desc = ixgbe_txd;
 477
 478         /*
 479         ** With many RX rings it is easy to exceed the
 480         ** system mbuf allocation. Tuning nmbclusters
 481         ** can alleviate this.
 482         */
 483         if (nmbclusters > 0 ) {
 484                 int s;
 485                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 486                 if (s > nmbclusters) {
 487                         device_printf(dev, "RX Descriptors exceed "
 488                             "system mbuf max, using default instead!\n");
 489                         ixgbe_rxd = DEFAULT_RXD;
 490                 }
 491         }
 492
 493         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 494             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 495                 device_printf(dev, "RXD config issue, using default!\n");
 496                 adapter->num_rx_desc = DEFAULT_RXD;
 497         } else
 498                 adapter->num_rx_desc = ixgbe_rxd;
 499
 500         /* Allocate our TX/RX Queues */
 501         if (ixgbe_allocate_queues(adapter)) {
 502                 error = ENOMEM;
 503                 goto err_out;
 504         }
 505
 506         /* Allocate multicast array memory. */
 507         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 508             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 509         if (adapter->mta == NULL) {
 510                 device_printf(dev, "Can not allocate multicast setup array\n");
 511                 error = ENOMEM;
 512                 goto err_late;
 513         }
 514
 515         /* Initialize the shared code */
 516         error = ixgbe_init_shared_code(hw);
 517         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 518                 /*
 519                 ** No optics in this port, set up
 520                 ** so the timer routine will probe
 521                 ** for later insertion.
 522                 */
 523                 adapter->sfp_probe = TRUE;
 524                 error = 0;
 525         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 526                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 527                 error = EIO;
 528                 goto err_late;
 529         } else if (error) {
 530                 device_printf(dev,"Unable to initialize the shared code\n");
 531                 error = EIO;
 532                 goto err_late;
 533         }
 534
 535         /* Make sure we have a good EEPROM before we read from it */
 536         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 537                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 538                 error = EIO;
 539                 goto err_late;
 540         }
 541
 542         error = ixgbe_init_hw(hw);
 543         switch (error) {
 544         case IXGBE_ERR_EEPROM_VERSION:
 545                 device_printf(dev, "This device is a pre-production adapter/"
 546                     "LOM.  Please be aware there may be issues associated "
 547                     "with your hardware.\n If you are experiencing problems "
 548                     "please contact your Intel or hardware representative "
 549                     "who provided you with this hardware.\n");
 550                 break;
 551         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 552                 device_printf(dev,"Unsupported SFP+ Module\n");
 553                 error = EIO;
 554                 device_printf(dev,"Hardware Initialization Failure\n");
 555                 goto err_late;
 556         case IXGBE_ERR_SFP_NOT_PRESENT:
 557                 device_printf(dev,"No SFP+ Module found\n");
 558                 /* falls thru */
 559         default:
 560                 break;
 561         }
 562
 563         /* Detect and set physical type */
 564         ixgbe_setup_optics(adapter);
 565
 566         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 567                 error = ixgbe_allocate_msix(adapter);
 568         else
 569                 error = ixgbe_allocate_legacy(adapter);
 570         if (error)
 571                 goto err_late;
 572
 573         /* Setup OS specific network interface */
 574         if (ixgbe_setup_interface(dev, adapter) != 0)
 575                 goto err_late;
 576
 577         /* Sysctl for limiting the amount of work done in the taskqueue */
 578         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 579             "max number of rx packets to process", &adapter->rx_process_limit,
 580             ixgbe_rx_process_limit);
 581
 582         /* Initialize statistics */
 583         ixgbe_update_stats_counters(adapter);
 584
 585         /* Register for VLAN events */
 586         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 587             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 588         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 589             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 590
 591         /* Print PCIE bus type/speed/width info */
 592         ixgbe_get_bus_info(hw);
 593         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 594             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 595             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 596             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 597             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 598             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 599             ("Unknown"));
 600
 601         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 602             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 603                 device_printf(dev, "PCI-Express bandwidth available"
 604                     " for this card\n     is not sufficient for"
 605                     " optimal performance.\n");
 606                 device_printf(dev, "For optimal performance a x8 "
 607                     "PCIE, or x4 PCIE 2 slot is required.\n");
 608         }
 609
 610         /* let hardware know driver is loaded */
 611         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 612         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 613         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 614
 615         ixgbe_add_hw_stats(adapter);
 616
 617 #ifdef DEV_NETMAP
 618         ixgbe_netmap_attach(adapter);
 619 #endif /* DEV_NETMAP */
 620         INIT_DEBUGOUT("ixgbe_attach: end");
 621         return (0);
 622 err_late:
 623         ixgbe_free_transmit_structures(adapter);
 624         ixgbe_free_receive_structures(adapter);
 625 err_out:
 626         if (adapter->ifp != NULL)
 627                 if_free(adapter->ifp);
 628         ixgbe_free_pci_resources(adapter);
 629         kfree(adapter->mta, M_DEVBUF);
 630         return (error);
 631
 632 }
 633
 634 /*********************************************************************
 635  *  Device removal routine
 636  *
 637  *  The detach entry point is called when the driver is being removed.
 638  *  This routine stops the adapter and deallocates all the resources
 639  *  that were allocated for driver operation.
 640  *
 641  *  return 0 on success, positive on failure
 642  *********************************************************************/
 643
 644 static int
 645 ixgbe_detach(device_t dev)
 646 {
 647         struct adapter *adapter = device_get_softc(dev);
 648         struct ix_queue *que = adapter->queues;
 649         u32     ctrl_ext;
 650
 651         INIT_DEBUGOUT("ixgbe_detach: begin");
 652
 653 #ifdef NET_VLAN
 654         /* Make sure VLANS are not using driver */
 655         if (adapter->ifp->if_vlantrunk != NULL) {
 656                 device_printf(dev,"Vlan in use, detach first\n");
 657                 return (EBUSY);
 658         }
 659 #endif
 660
 661         IXGBE_CORE_LOCK(adapter);
 662         ixgbe_stop(adapter);
 663         IXGBE_CORE_UNLOCK(adapter);
 664
 665         for (int i = 0; i < adapter->num_queues; i++, que++) {
 666                 if (que->tq) {
 667                         taskqueue_drain(que->tq, &que->que_task);
 668                         taskqueue_free(que->tq);
 669                 }
 670         }
 671
 672         /* Drain the Link queue */
 673         if (adapter->tq) {
 674                 taskqueue_drain(adapter->tq, &adapter->link_task);
 675                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 676                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 677 #ifdef IXGBE_FDIR
 678                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 679 #endif
 680                 taskqueue_free(adapter->tq);
 681         }
 682
 683         /* let hardware know driver is unloading */
 684         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 685         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 686         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 687
 688         /* Unregister VLAN events */
 689 #ifdef NET_VLAN
 690         if (adapter->vlan_attach != NULL)
 691                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 692         if (adapter->vlan_detach != NULL)
 693                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 694 #endif
 695
 696         ether_ifdetach(adapter->ifp);
 697         callout_stop(&adapter->timer);
 698         lockuninit(&adapter->core_lock);
 699 #ifdef DEV_NETMAP
 700         netmap_detach(adapter->ifp);
 701 #endif /* DEV_NETMAP */
 702         ixgbe_free_pci_resources(adapter);
 703         bus_generic_detach(dev);
 704         if_free(adapter->ifp);
 705
 706         ixgbe_free_transmit_structures(adapter);
 707         ixgbe_free_receive_structures(adapter);
 708         kfree(adapter->mta, M_DEVBUF);
 709         sysctl_ctx_free(&adapter->sysctl_ctx);
 710
 711         IXGBE_CORE_LOCK_DESTROY(adapter);
 712         return (0);
 713 }
 714
 715 /*********************************************************************
 716  *
 717  *  Shutdown entry point
 718  *
 719  **********************************************************************/
 720
 721 static int
 722 ixgbe_shutdown(device_t dev)
 723 {
 724         struct adapter *adapter = device_get_softc(dev);
 725         IXGBE_CORE_LOCK(adapter);
 726         ixgbe_stop(adapter);
 727         IXGBE_CORE_UNLOCK(adapter);
 728         return (0);
 729 }
 730
 731
 732 /*********************************************************************
 733  *  Transmit entry point
 734  *
 735  *  ixgbe_start is called by the stack to initiate a transmit.
 736  *  The driver will remain in this routine as long as there are
 737  *  packets to transmit and transmit resources are available.
 738  *  In case resources are not available stack is notified and
 739  *  the packet is requeued.
 740  **********************************************************************/
 741
 742 static void
 743 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 744 {
 745         struct mbuf    *m_head;
 746         struct adapter *adapter = txr->adapter;
 747
 748         IXGBE_TX_LOCK_ASSERT(txr);
 749
 750         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 751                 return;
 752         if (!adapter->link_active)
 753                 return;
 754
 755         while (!ifq_is_empty(&ifp->if_snd)) {
 756                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 757                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 758                         break;
 759                 }
 760
 761                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 762                 if (m_head == NULL)
 763                         break;
 764
 765                 if (ixgbe_xmit(txr, &m_head)) {
 766 #if 0 /* XXX: prepend to an ALTQ queue ? */
 767                         if (m_head != NULL)
 768                                 IF_PREPEND(&ifp->if_snd, m_head);
 769 #endif
 770                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 771                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 772                         break;
 773                 }
 774                 /* Send a copy of the frame to the BPF listener */
 775                 ETHER_BPF_MTAP(ifp, m_head);
 776
 777                 /* Set watchdog on */
 778                 txr->watchdog_time = ticks;
 779                 txr->queue_status = IXGBE_QUEUE_WORKING;
 780
 781         }
 782         return;
 783 }
 784
 785 /*
 786  * Legacy TX start - called by the stack, this
 787  * always uses the first tx ring, and should
 788  * not be used with multiqueue tx enabled.
 789  */
 790 static void
 791 ixgbe_start(struct ifnet *ifp)
 792 {
 793         struct adapter *adapter = ifp->if_softc;
 794         struct tx_ring  *txr = adapter->tx_rings;
 795
 796         if (ifp->if_flags & IFF_RUNNING) {
 797                 IXGBE_TX_LOCK(txr);
 798                 ixgbe_start_locked(txr, ifp);
 799                 IXGBE_TX_UNLOCK(txr);
 800         }
 801         return;
 802 }
 803
 804 #if 0 /* __FreeBSD_version >= 800000 */
 805 /*
 806 ** Multiqueue Transmit driver
 807 **
 808 */
 809 static int
 810 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 811 {
 812         struct adapter  *adapter = ifp->if_softc;
 813         struct ix_queue *que;
 814         struct tx_ring  *txr;
 815         int             i = 0, err = 0;
 816
 817         /* Which queue to use */
 818         if ((m->m_flags & M_FLOWID) != 0)
 819                 i = m->m_pkthdr.flowid % adapter->num_queues;
 820         else
 821                 i = curcpu % adapter->num_queues;
 822
 823         txr = &adapter->tx_rings[i];
 824         que = &adapter->queues[i];
 825
 826         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 827             IXGBE_TX_TRYLOCK(txr)) {
 828                 err = ixgbe_mq_start_locked(ifp, txr, m);
 829                 IXGBE_TX_UNLOCK(txr);
 830         } else {
 831                 err = drbr_enqueue(ifp, txr->br, m);
 832                 taskqueue_enqueue(que->tq, &que->que_task);
 833         }
 834
 835         return (err);
 836 }
 837
 838 static int
 839 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 840 {
 841         struct adapter  *adapter = txr->adapter;
 842         struct mbuf     *next;
 843         int             enqueued, err = 0;
 844
 845         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 846             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 847             adapter->link_active == 0) {
 848                 if (m != NULL)
 849                         err = drbr_enqueue(ifp, txr->br, m);
 850                 return (err);
 851         }
 852
 853         enqueued = 0;
 854         if (m == NULL) {
 855                 next = drbr_dequeue(ifp, txr->br);
 856         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 857                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 858                         return (err);
 859                 next = drbr_dequeue(ifp, txr->br);
 860         } else
 861                 next = m;
 862
 863         /* Process the queue */
 864         while (next != NULL) {
 865                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 866                         if (next != NULL)
 867                                 err = drbr_enqueue(ifp, txr->br, next);
 868                         break;
 869                 }
 870                 enqueued++;
 871                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 872                 /* Send a copy of the frame to the BPF listener */
 873                 ETHER_BPF_MTAP(ifp, next);
 874                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 875                         break;
 876                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 877                         ixgbe_txeof(txr);
 878                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 879                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 880                         break;
 881                 }
 882                 next = drbr_dequeue(ifp, txr->br);
 883         }
 884
 885         if (enqueued > 0) {
 886                 /* Set watchdog on */
 887                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 888                 txr->watchdog_time = ticks;
 889         }
 890
 891         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 892                 ixgbe_txeof(txr);
 893
 894         return (err);
 895 }
 896
 897 /*
 898 ** Flush all ring buffers
 899 */
 900 static void
 901 ixgbe_qflush(struct ifnet *ifp)
 902 {
 903         struct adapter  *adapter = ifp->if_softc;
 904         struct tx_ring  *txr = adapter->tx_rings;
 905         struct mbuf     *m;
 906
 907         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 908                 IXGBE_TX_LOCK(txr);
 909                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 910                         m_freem(m);
 911                 IXGBE_TX_UNLOCK(txr);
 912         }
 913         if_qflush(ifp);
 914 }
 915 #endif /* __FreeBSD_version >= 800000 */
 916
 917 /*********************************************************************
 918  *  Ioctl entry point
 919  *
 920  *  ixgbe_ioctl is called when the user wants to configure the
 921  *  interface.
 922  *
 923  *  return 0 on success, positive on failure
 924  **********************************************************************/
 925
 926 static int
 927 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 928 {
 929         struct adapter  *adapter = ifp->if_softc;
 930         struct ifreq    *ifr = (struct ifreq *) data;
 931 #if defined(INET) || defined(INET6)
 932         struct ifaddr *ifa = (struct ifaddr *)data;
 933         bool            avoid_reset = FALSE;
 934 #endif
 935         int             error = 0;
 936
 937         switch (command) {
 938
 939         case SIOCSIFADDR:
 940 #ifdef INET
 941                 if (ifa->ifa_addr->sa_family == AF_INET)
 942                         avoid_reset = TRUE;
 943 #endif
 944 #ifdef INET6
 945                 if (ifa->ifa_addr->sa_family == AF_INET6)
 946                         avoid_reset = TRUE;
 947 #endif
 948 #if defined(INET) || defined(INET6)
 949                 /*
 950                 ** Calling init results in link renegotiation,
 951                 ** so we avoid doing it when possible.
 952                 */
 953                 if (avoid_reset) {
 954                         ifp->if_flags |= IFF_UP;
 955                         if (!(ifp->if_flags & IFF_RUNNING))
 956                                 ixgbe_init(adapter);
 957                         if (!(ifp->if_flags & IFF_NOARP))
 958                                 arp_ifinit(ifp, ifa);
 959                 } else
 960                         error = ether_ioctl(ifp, command, data);
 961 #endif
 962                 break;
 963         case SIOCSIFMTU:
 964                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 965                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 966                         error = EINVAL;
 967                 } else {
 968                         IXGBE_CORE_LOCK(adapter);
 969                         ifp->if_mtu = ifr->ifr_mtu;
 970                         adapter->max_frame_size =
 971                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 972                         ixgbe_init_locked(adapter);
 973                         IXGBE_CORE_UNLOCK(adapter);
 974                 }
 975                 break;
 976         case SIOCSIFFLAGS:
 977                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 978                 IXGBE_CORE_LOCK(adapter);
 979                 if (ifp->if_flags & IFF_UP) {
 980                         if ((ifp->if_flags & IFF_RUNNING)) {
 981                                 if ((ifp->if_flags ^ adapter->if_flags) &
 982                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 983                                         ixgbe_set_promisc(adapter);
 984                                 }
 985                         } else
 986                                 ixgbe_init_locked(adapter);
 987                 } else
 988                         if (ifp->if_flags & IFF_RUNNING)
 989                                 ixgbe_stop(adapter);
 990                 adapter->if_flags = ifp->if_flags;
 991                 IXGBE_CORE_UNLOCK(adapter);
 992                 break;
 993         case SIOCADDMULTI:
 994         case SIOCDELMULTI:
 995                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 996                 if (ifp->if_flags & IFF_RUNNING) {
 997                         IXGBE_CORE_LOCK(adapter);
 998                         ixgbe_disable_intr(adapter);
 999                         ixgbe_set_multi(adapter);
1000                         ixgbe_enable_intr(adapter);
1001                         IXGBE_CORE_UNLOCK(adapter);
1002                 }
1003                 break;
1004         case SIOCSIFMEDIA:
1005         case SIOCGIFMEDIA:
1006                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1007                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1008                 break;
1009         case SIOCSIFCAP:
1010         {
1011                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1012                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1013                 if (mask & IFCAP_HWCSUM)
1014                         ifp->if_capenable ^= IFCAP_HWCSUM;
1015 #if 0 /* NET_TSO */
1016                 if (mask & IFCAP_TSO4)
1017                         ifp->if_capenable ^= IFCAP_TSO4;
1018                 if (mask & IFCAP_TSO6)
1019                         ifp->if_capenable ^= IFCAP_TSO6;
1020 #endif
1021 #if 0 /* NET_LRO */
1022                 if (mask & IFCAP_LRO)
1023                         ifp->if_capenable ^= IFCAP_LRO;
1024 #endif
1025                 if (mask & IFCAP_VLAN_HWTAGGING)
1026                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1027                 if (mask & IFCAP_VLAN_HWFILTER)
1028                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1029 #if 0 /* NET_TSO */
1030                 if (mask & IFCAP_VLAN_HWTSO)
1031                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1032 #endif
1033                 if (ifp->if_flags & IFF_RUNNING) {
1034                         IXGBE_CORE_LOCK(adapter);
1035                         ixgbe_init_locked(adapter);
1036                         IXGBE_CORE_UNLOCK(adapter);
1037                 }
1038 #if 0
1039                 VLAN_CAPABILITIES(ifp);
1040 #endif
1041                 break;
1042         }
1043
1044         default:
1045                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1046                 error = ether_ioctl(ifp, command, data);
1047                 break;
1048         }
1049
1050         return (error);
1051 }
1052
1053 /*********************************************************************
1054  *  Init entry point
1055  *
1056  *  This routine is used in two ways. It is used by the stack as
1057  *  init entry point in network interface structure. It is also used
1058  *  by the driver as a hw/sw initialization routine to get to a
1059  *  consistent state.
1060  *
1061  *  return 0 on success, positive on failure
1062  **********************************************************************/
1063 #define IXGBE_MHADD_MFS_SHIFT 16
1064
1065 static void
1066 ixgbe_init_locked(struct adapter *adapter)
1067 {
1068         struct ifnet   *ifp = adapter->ifp;
1069         device_t        dev = adapter->dev;
1070         struct ixgbe_hw *hw = &adapter->hw;
1071         u32             k, txdctl, mhadd, gpie;
1072         u32             rxdctl, rxctrl;
1073
1074         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1075         INIT_DEBUGOUT("ixgbe_init: begin");
1076         hw->adapter_stopped = FALSE;
1077         ixgbe_stop_adapter(hw);
1078         callout_stop(&adapter->timer);
1079
1080         /* reprogram the RAR[0] in case user changed it. */
1081         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1082
1083         /* Get the latest mac address, User can use a LAA */
1084         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1085               IXGBE_ETH_LENGTH_OF_ADDRESS);
1086         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1087         hw->addr_ctrl.rar_used_count = 1;
1088
1089         /* Set the various hardware offload abilities */
1090         ifp->if_hwassist = 0;
1091 #if 0 /* NET_TSO */
1092         if (ifp->if_capenable & IFCAP_TSO)
1093                 ifp->if_hwassist |= CSUM_TSO;
1094 #endif
1095         if (ifp->if_capenable & IFCAP_TXCSUM) {
1096                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1097 #if 0
1098                 if (hw->mac.type != ixgbe_mac_82598EB)
1099                         ifp->if_hwassist |= CSUM_SCTP;
1100 #endif
1101         }
1102
1103         /* Prepare transmit descriptors and buffers */
1104         if (ixgbe_setup_transmit_structures(adapter)) {
1105                 device_printf(dev,"Could not setup transmit structures\n");
1106                 ixgbe_stop(adapter);
1107                 return;
1108         }
1109
1110         ixgbe_init_hw(hw);
1111         ixgbe_initialize_transmit_units(adapter);
1112
1113         /* Setup Multicast table */
1114         ixgbe_set_multi(adapter);
1115
1116         /*
1117         ** Determine the correct mbuf pool
1118         ** for doing jumbo/headersplit
1119         */
1120         if (adapter->max_frame_size <= 2048)
1121                 adapter->rx_mbuf_sz = MCLBYTES;
1122         else if (adapter->max_frame_size <= 4096)
1123                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1124         else if (adapter->max_frame_size <= 9216)
1125                 adapter->rx_mbuf_sz = MJUM9BYTES;
1126         else
1127                 adapter->rx_mbuf_sz = MJUM16BYTES;
1128
1129         /* Prepare receive descriptors and buffers */
1130         if (ixgbe_setup_receive_structures(adapter)) {
1131                 device_printf(dev,"Could not setup receive structures\n");
1132                 ixgbe_stop(adapter);
1133                 return;
1134         }
1135
1136         /* Configure RX settings */
1137         ixgbe_initialize_receive_units(adapter);
1138
1139         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1140
1141         /* Enable Fan Failure Interrupt */
1142         gpie |= IXGBE_SDP1_GPIEN;
1143
1144         /* Add for Module detection */
1145         if (hw->mac.type == ixgbe_mac_82599EB)
1146                 gpie |= IXGBE_SDP2_GPIEN;
1147
1148         /* Thermal Failure Detection */
1149         if (hw->mac.type == ixgbe_mac_X540)
1150                 gpie |= IXGBE_SDP0_GPIEN;
1151
1152         if (adapter->msix > 1) {
1153                 /* Enable Enhanced MSIX mode */
1154                 gpie |= IXGBE_GPIE_MSIX_MODE;
1155                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1156                     IXGBE_GPIE_OCD;
1157         }
1158         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1159
1160         /* Set MTU size */
1161         if (ifp->if_mtu > ETHERMTU) {
1162                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1163                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1164                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1165                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1166         }
1167
1168         /* Now enable all the queues */
1169
1170         for (int i = 0; i < adapter->num_queues; i++) {
1171                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1172                 txdctl |= IXGBE_TXDCTL_ENABLE;
1173                 /* Set WTHRESH to 8, burst writeback */
1174                 txdctl |= (8 << 16);
1175                 /*
1176                  * When the internal queue falls below PTHRESH (32),
1177                  * start prefetching as long as there are at least
1178                  * HTHRESH (1) buffers ready. The values are taken
1179                  * from the Intel linux driver 3.8.21.
1180                  * Prefetching enables tx line rate even with 1 queue.
1181                  */
1182                 txdctl |= (32 << 0) | (1 << 8);
1183                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1184         }
1185
1186         for (int i = 0; i < adapter->num_queues; i++) {
1187                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1188                 if (hw->mac.type == ixgbe_mac_82598EB) {
1189                         /*
1190                         ** PTHRESH = 21
1191                         ** HTHRESH = 4
1192                         ** WTHRESH = 8
1193                         */
1194                         rxdctl &= ~0x3FFFFF;
1195                         rxdctl |= 0x080420;
1196                 }
1197                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1198                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1199                 for (k = 0; k < 10; k++) {
1200                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1201                             IXGBE_RXDCTL_ENABLE)
1202                                 break;
1203                         else
1204                                 msec_delay(1);
1205                 }
1206                 wmb();
1207 #ifdef DEV_NETMAP
1208                 /*
1209                  * In netmap mode, we must preserve the buffers made
1210                  * available to userspace before the if_init()
1211                  * (this is true by default on the TX side, because
1212                  * init makes all buffers available to userspace).
1213                  *
1214                  * netmap_reset() and the device specific routines
1215                  * (e.g. ixgbe_setup_receive_rings()) map these
1216                  * buffers at the end of the NIC ring, so here we
1217                  * must set the RDT (tail) register to make sure
1218                  * they are not overwritten.
1219                  *
1220                  * In this driver the NIC ring starts at RDH = 0,
1221                  * RDT points to the last slot available for reception (?),
1222                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1223                  */
1224                 if (ifp->if_capenable & IFCAP_NETMAP) {
1225                         struct netmap_adapter *na = NA(adapter->ifp);
1226                         struct netmap_kring *kring = &na->rx_rings[i];
1227                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1228
1229                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1230                 } else
1231 #endif /* DEV_NETMAP */
1232                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1233         }
1234
1235         /* Set up VLAN support and filter */
1236         ixgbe_setup_vlan_hw_support(adapter);
1237
1238         /* Enable Receive engine */
1239         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1240         if (hw->mac.type == ixgbe_mac_82598EB)
1241                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1242         rxctrl |= IXGBE_RXCTRL_RXEN;
1243         ixgbe_enable_rx_dma(hw, rxctrl);
1244
1245         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1246
1247         /* Set up MSI/X routing */
1248         if (ixgbe_enable_msix)  {
1249                 ixgbe_configure_ivars(adapter);
1250                 /* Set up auto-mask */
1251                 if (hw->mac.type == ixgbe_mac_82598EB)
1252                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1253                 else {
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1255                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1256                 }
1257         } else {  /* Simple settings for Legacy/MSI */
1258                 ixgbe_set_ivar(adapter, 0, 0, 0);
1259                 ixgbe_set_ivar(adapter, 0, 0, 1);
1260                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1261         }
1262
1263 #ifdef IXGBE_FDIR
1264         /* Init Flow director */
1265         if (hw->mac.type != ixgbe_mac_82598EB) {
1266                 u32 hdrm = 32 << fdir_pballoc;
1267
1268                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1269                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1270         }
1271 #endif
1272
1273         /*
1274         ** Check on any SFP devices that
1275         ** need to be kick-started
1276         */
1277         if (hw->phy.type == ixgbe_phy_none) {
1278                 int err = hw->phy.ops.identify(hw);
1279                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1280                         device_printf(dev,
1281                             "Unsupported SFP+ module type was detected.\n");
1282                         return;
1283                 }
1284         }
1285
1286         /* Set moderation on the Link interrupt */
1287         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1288
1289         /* Config/Enable Link */
1290         ixgbe_config_link(adapter);
1291
1292         /* Hardware Packet Buffer & Flow Control setup */
1293         {
1294                 u32 rxpb, frame, size, tmp;
1295
1296                 frame = adapter->max_frame_size;
1297
1298                 /* Calculate High Water */
1299                 if (hw->mac.type == ixgbe_mac_X540)
1300                         tmp = IXGBE_DV_X540(frame, frame);
1301                 else
1302                         tmp = IXGBE_DV(frame, frame);
1303                 size = IXGBE_BT2KB(tmp);
1304                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1305                 hw->fc.high_water[0] = rxpb - size;
1306
1307                 /* Now calculate Low Water */
1308                 if (hw->mac.type == ixgbe_mac_X540)
1309                         tmp = IXGBE_LOW_DV_X540(frame);
1310                 else
1311                         tmp = IXGBE_LOW_DV(frame);
1312                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1313
1314                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1315                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1316                 hw->fc.send_xon = TRUE;
1317         }
1318         /* Initialize the FC settings */
1319         ixgbe_start_hw(hw);
1320
1321         /* And now turn on interrupts */
1322         ixgbe_enable_intr(adapter);
1323
1324         /* Now inform the stack we're ready */
1325         ifp->if_flags |= IFF_RUNNING;
1326         ifp->if_flags &= ~IFF_OACTIVE;
1327
1328         return;
1329 }
1330
1331 static void
1332 ixgbe_init(void *arg)
1333 {
1334         struct adapter *adapter = arg;
1335
1336         IXGBE_CORE_LOCK(adapter);
1337         ixgbe_init_locked(adapter);
1338         IXGBE_CORE_UNLOCK(adapter);
1339         return;
1340 }
1341
1342
1343 /*
1344 **
1345 ** MSIX Interrupt Handlers and Tasklets
1346 **
1347 */
1348
1349 static inline void
1350 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1351 {
1352         struct ixgbe_hw *hw = &adapter->hw;
1353         u64     queue = (u64)(1 << vector);
1354         u32     mask;
1355
1356         if (hw->mac.type == ixgbe_mac_82598EB) {
1357                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1358                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1359         } else {
1360                 mask = (queue & 0xFFFFFFFF);
1361                 if (mask)
1362                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1363                 mask = (queue >> 32);
1364                 if (mask)
1365                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1366         }
1367 }
1368
1369 static inline void
1370 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1371 {
1372         struct ixgbe_hw *hw = &adapter->hw;
1373         u64     queue = (u64)(1 << vector);
1374         u32     mask;
1375
1376         if (hw->mac.type == ixgbe_mac_82598EB) {
1377                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1378                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1379         } else {
1380                 mask = (queue & 0xFFFFFFFF);
1381                 if (mask)
1382                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1383                 mask = (queue >> 32);
1384                 if (mask)
1385                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1386         }
1387 }
1388
1389 static inline void
1390 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1391 {
1392         u32 mask;
1393
1394         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1395                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1396                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1397         } else {
1398                 mask = (queues & 0xFFFFFFFF);
1399                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1400                 mask = (queues >> 32);
1401                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1402         }
1403 }
1404
1405
1406 static void
1407 ixgbe_handle_que(void *context, int pending)
1408 {
1409         struct ix_queue *que = context;
1410         struct adapter  *adapter = que->adapter;
1411         struct tx_ring  *txr = que->txr;
1412         struct ifnet    *ifp = adapter->ifp;
1413         bool            more;
1414
1415         if (ifp->if_flags & IFF_RUNNING) {
1416                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1417                 IXGBE_TX_LOCK(txr);
1418                 ixgbe_txeof(txr);
1419 #if 0 /*__FreeBSD_version >= 800000*/
1420                 if (!drbr_empty(ifp, txr->br))
1421                         ixgbe_mq_start_locked(ifp, txr, NULL);
1422 #else
1423                 if (!ifq_is_empty(&ifp->if_snd))
1424                         ixgbe_start_locked(txr, ifp);
1425 #endif
1426                 IXGBE_TX_UNLOCK(txr);
1427                 if (more) {
1428                         taskqueue_enqueue(que->tq, &que->que_task);
1429                         return;
1430                 }
1431         }
1432
1433         /* Reenable this interrupt */
1434         ixgbe_enable_queue(adapter, que->msix);
1435         return;
1436 }
1437
1438
1439 /*********************************************************************
1440  *
1441  *  Legacy Interrupt Service routine
1442  *
1443  **********************************************************************/
1444
1445 static void
1446 ixgbe_legacy_irq(void *arg)
1447 {
1448         struct ix_queue *que = arg;
1449         struct adapter  *adapter = que->adapter;
1450         struct ixgbe_hw *hw = &adapter->hw;
1451         struct          tx_ring *txr = adapter->tx_rings;
1452         bool            more_tx, more_rx;
1453         u32             reg_eicr, loop = MAX_LOOP;
1454
1455
1456         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1457
1458         ++que->irqs;
1459         if (reg_eicr == 0) {
1460                 ixgbe_enable_intr(adapter);
1461                 return;
1462         }
1463
1464         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1465
1466         IXGBE_TX_LOCK(txr);
1467         do {
1468                 more_tx = ixgbe_txeof(txr);
1469         } while (loop-- && more_tx);
1470         IXGBE_TX_UNLOCK(txr);
1471
1472         if (more_rx || more_tx)
1473                 taskqueue_enqueue(que->tq, &que->que_task);
1474
1475         /* Check for fan failure */
1476         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1477             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1478                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1479                     "REPLACE IMMEDIATELY!!\n");
1480                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1481         }
1482
1483         /* Link status change */
1484         if (reg_eicr & IXGBE_EICR_LSC)
1485                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1486
1487         ixgbe_enable_intr(adapter);
1488         return;
1489 }
1490
1491
1492 /*********************************************************************
1493  *
1494  *  MSIX Queue Interrupt Service routine
1495  *
1496  **********************************************************************/
1497 void
1498 ixgbe_msix_que(void *arg)
1499 {
1500         struct ix_queue *que = arg;
1501         struct adapter  *adapter = que->adapter;
1502         struct tx_ring  *txr = que->txr;
1503         struct rx_ring  *rxr = que->rxr;
1504         bool            more_tx, more_rx;
1505         u32             newitr = 0;
1506
1507         ixgbe_disable_queue(adapter, que->msix);
1508         ++que->irqs;
1509
1510         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1511
1512         IXGBE_TX_LOCK(txr);
1513         more_tx = ixgbe_txeof(txr);
1514         /*
1515         ** Make certain that if the stack
1516         ** has anything queued the task gets
1517         ** scheduled to handle it.
1518         */
1519 #if 0
1520 #if __FreeBSD_version < 800000
1521         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1522 #else
1523         if (!drbr_empty(adapter->ifp, txr->br))
1524 #endif
1525 #endif
1526         if (!ifq_is_empty(&adapter->ifp->if_snd))
1527                 more_tx = 1;
1528         IXGBE_TX_UNLOCK(txr);
1529
1530         /* Do AIM now? */
1531
1532         if (ixgbe_enable_aim == FALSE)
1533                 goto no_calc;
1534         /*
1535         ** Do Adaptive Interrupt Moderation:
1536         **  - Write out last calculated setting
1537         **  - Calculate based on average size over
1538         **    the last interval.
1539         */
1540         if (que->eitr_setting)
1541                 IXGBE_WRITE_REG(&adapter->hw,
1542                     IXGBE_EITR(que->msix), que->eitr_setting);
1543
1544         que->eitr_setting = 0;
1545
1546         /* Idle, do nothing */
1547         if ((txr->bytes == 0) && (rxr->bytes == 0))
1548                 goto no_calc;
1549
1550         if ((txr->bytes) && (txr->packets))
1551                 newitr = txr->bytes/txr->packets;
1552         if ((rxr->bytes) && (rxr->packets))
1553                 newitr = max(newitr,
1554                     (rxr->bytes / rxr->packets));
1555         newitr += 24; /* account for hardware frame, crc */
1556
1557         /* set an upper boundary */
1558         newitr = min(newitr, 3000);
1559
1560         /* Be nice to the mid range */
1561         if ((newitr > 300) && (newitr < 1200))
1562                 newitr = (newitr / 3);
1563         else
1564                 newitr = (newitr / 2);
1565
1566         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1567                 newitr |= newitr << 16;
1568         else
1569                 newitr |= IXGBE_EITR_CNT_WDIS;
1570
1571         /* save for next interrupt */
1572         que->eitr_setting = newitr;
1573
1574         /* Reset state */
1575         txr->bytes = 0;
1576         txr->packets = 0;
1577         rxr->bytes = 0;
1578         rxr->packets = 0;
1579
1580 no_calc:
1581         if (more_tx || more_rx)
1582                 taskqueue_enqueue(que->tq, &que->que_task);
1583         else /* Reenable this interrupt */
1584                 ixgbe_enable_queue(adapter, que->msix);
1585         return;
1586 }
1587
1588
1589 static void
1590 ixgbe_msix_link(void *arg)
1591 {
1592         struct adapter  *adapter = arg;
1593         struct ixgbe_hw *hw = &adapter->hw;
1594         u32             reg_eicr;
1595
1596         ++adapter->link_irq;
1597
1598         /* First get the cause */
1599         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1600         /* Clear interrupt with write */
1601         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1602
1603         /* Link status change */
1604         if (reg_eicr & IXGBE_EICR_LSC)
1605                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1606
1607         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1608 #ifdef IXGBE_FDIR
1609                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1610                         /* This is probably overkill :) */
1611                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1612                                 return;
1613                         /* Disable the interrupt */
1614                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1615                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1616                 } else
1617 #endif
1618                 if (reg_eicr & IXGBE_EICR_ECC) {
1619                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1620                             "Please Reboot!!\n");
1621                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1622                 } else
1623
1624                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1625                         /* Clear the interrupt */
1626                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1627                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1628                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1629                         /* Clear the interrupt */
1630                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1631                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1632                 }
1633         }
1634
1635         /* Check for fan failure */
1636         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1637             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1638                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1639                     "REPLACE IMMEDIATELY!!\n");
1640                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1641         }
1642
1643         /* Check for over temp condition */
1644         if ((hw->mac.type == ixgbe_mac_X540) &&
1645             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1646                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1647                     "PHY IS SHUT DOWN!!\n");
1648                 device_printf(adapter->dev, "System shutdown required\n");
1649                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1650         }
1651
1652         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1653         return;
1654 }
1655
1656 /*********************************************************************
1657  *
1658  *  Media Ioctl callback
1659  *
1660  *  This routine is called whenever the user queries the status of
1661  *  the interface using ifconfig.
1662  *
1663  **********************************************************************/
1664 static void
1665 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1666 {
1667         struct adapter *adapter = ifp->if_softc;
1668
1669         INIT_DEBUGOUT("ixgbe_media_status: begin");
1670         IXGBE_CORE_LOCK(adapter);
1671         ixgbe_update_link_status(adapter);
1672
1673         ifmr->ifm_status = IFM_AVALID;
1674         ifmr->ifm_active = IFM_ETHER;
1675
1676         if (!adapter->link_active) {
1677                 IXGBE_CORE_UNLOCK(adapter);
1678                 return;
1679         }
1680
1681         ifmr->ifm_status |= IFM_ACTIVE;
1682
1683         switch (adapter->link_speed) {
1684                 case IXGBE_LINK_SPEED_100_FULL:
1685                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1686                         break;
1687                 case IXGBE_LINK_SPEED_1GB_FULL:
1688                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1689                         break;
1690                 case IXGBE_LINK_SPEED_10GB_FULL:
1691                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1692                         break;
1693         }
1694
1695         IXGBE_CORE_UNLOCK(adapter);
1696
1697         return;
1698 }
1699
1700 /*********************************************************************
1701  *
1702  *  Media Ioctl callback
1703  *
1704  *  This routine is called when the user changes speed/duplex using
1705  *  media/mediopt option with ifconfig.
1706  *
1707  **********************************************************************/
1708 static int
1709 ixgbe_media_change(struct ifnet * ifp)
1710 {
1711         struct adapter *adapter = ifp->if_softc;
1712         struct ifmedia *ifm = &adapter->media;
1713
1714         INIT_DEBUGOUT("ixgbe_media_change: begin");
1715
1716         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1717                 return (EINVAL);
1718
1719         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1720         case IFM_AUTO:
1721                 adapter->hw.phy.autoneg_advertised =
1722                     IXGBE_LINK_SPEED_100_FULL |
1723                     IXGBE_LINK_SPEED_1GB_FULL |
1724                     IXGBE_LINK_SPEED_10GB_FULL;
1725                 break;
1726         default:
1727                 device_printf(adapter->dev, "Only auto media type\n");
1728                 return (EINVAL);
1729         }
1730
1731         return (0);
1732 }
1733
1734 /*********************************************************************
1735  *
1736  *  This routine maps the mbufs to tx descriptors, allowing the
1737  *  TX engine to transmit the packets.
1738  *      - return 0 on success, positive on failure
1739  *
1740  **********************************************************************/
1741
1742 static int
1743 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1744 {
1745         struct adapter  *adapter = txr->adapter;
1746         u32             olinfo_status = 0, cmd_type_len;
1747         u32             paylen = 0;
1748         int             i, j, error, nsegs;
1749         int             first, last = 0;
1750         struct mbuf     *m_head;
1751         bus_dma_segment_t segs[1];
1752         bus_dmamap_t    map;
1753         struct ixgbe_tx_buf *txbuf;
1754         union ixgbe_adv_tx_desc *txd = NULL;
1755
1756         m_head = *m_headp;
1757
1758         /* Basic descriptor defines */
1759         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1760             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1761
1762         if (m_head->m_flags & M_VLANTAG)
1763                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1764
1765         /*
1766          * Important to capture the first descriptor
1767          * used because it will contain the index of
1768          * the one we tell the hardware to report back
1769          */
1770         first = txr->next_avail_desc;
1771         txbuf = &txr->tx_buffers[first];
1772         map = txbuf->map;
1773
1774         /*
1775          * Map the packet for DMA.
1776          */
1777         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1778             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1779
1780         if (error == EFBIG) {
1781                 struct mbuf *m;
1782
1783                 m = m_defrag(*m_headp, MB_DONTWAIT);
1784                 if (m == NULL) {
1785                         adapter->mbuf_defrag_failed++;
1786                         m_freem(*m_headp);
1787                         *m_headp = NULL;
1788                         return (ENOBUFS);
1789                 }
1790                 *m_headp = m;
1791
1792                 /* Try it again */
1793                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1794                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1795
1796                 if (error == ENOMEM) {
1797                         adapter->no_tx_dma_setup++;
1798                         return (error);
1799                 } else if (error != 0) {
1800                         adapter->no_tx_dma_setup++;
1801                         m_freem(*m_headp);
1802                         *m_headp = NULL;
1803                         return (error);
1804                 }
1805         } else if (error == ENOMEM) {
1806                 adapter->no_tx_dma_setup++;
1807                 return (error);
1808         } else if (error != 0) {
1809                 adapter->no_tx_dma_setup++;
1810                 m_freem(*m_headp);
1811                 *m_headp = NULL;
1812                 return (error);
1813         }
1814
1815         /* Make certain there are enough descriptors */
1816         if (nsegs > txr->tx_avail - 2) {
1817                 txr->no_desc_avail++;
1818                 error = ENOBUFS;
1819                 goto xmit_fail;
1820         }
1821         m_head = *m_headp;
1822
1823         /*
1824         ** Set up the appropriate offload context
1825         ** this becomes the first descriptor of
1826         ** a packet.
1827         */
1828 #if 0 /* NET_TSO */
1829         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1830                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1831                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1832                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1833                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1834                         ++adapter->tso_tx;
1835                 } else
1836                         return (ENXIO);
1837         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1838 #endif
1839         if (ixgbe_tx_ctx_setup(txr, m_head))
1840                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1841
1842 #ifdef IXGBE_IEEE1588
1843         /* This is changing soon to an mtag detection */
1844         if (we detect this mbuf has a TSTAMP mtag)
1845                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1846 #endif
1847
1848 #ifdef IXGBE_FDIR
1849         /* Do the flow director magic */
1850         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1851                 ++txr->atr_count;
1852                 if (txr->atr_count >= atr_sample_rate) {
1853                         ixgbe_atr(txr, m_head);
1854                         txr->atr_count = 0;
1855                 }
1856         }
1857 #endif
1858         /* Record payload length */
1859         if (paylen == 0)
1860                 olinfo_status |= m_head->m_pkthdr.len <<
1861                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1862
1863         i = txr->next_avail_desc;
1864         for (j = 0; j < nsegs; j++) {
1865                 bus_size_t seglen;
1866                 bus_addr_t segaddr;
1867
1868                 txbuf = &txr->tx_buffers[i];
1869                 txd = &txr->tx_base[i];
1870                 seglen = segs[j].ds_len;
1871                 segaddr = htole64(segs[j].ds_addr);
1872
1873                 txd->read.buffer_addr = segaddr;
1874                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1875                     cmd_type_len |seglen);
1876                 txd->read.olinfo_status = htole32(olinfo_status);
1877                 last = i; /* descriptor that will get completion IRQ */
1878
1879                 if (++i == adapter->num_tx_desc)
1880                         i = 0;
1881
1882                 txbuf->m_head = NULL;
1883                 txbuf->eop_index = -1;
1884         }
1885
1886         txd->read.cmd_type_len |=
1887             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1888         txr->tx_avail -= nsegs;
1889         txr->next_avail_desc = i;
1890
1891         txbuf->m_head = m_head;
1892         /* Swap the dma map between the first and last descriptor */
1893         txr->tx_buffers[first].map = txbuf->map;
1894         txbuf->map = map;
1895         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1896
1897         /* Set the index of the descriptor that will be marked done */
1898         txbuf = &txr->tx_buffers[first];
1899         txbuf->eop_index = last;
1900
1901         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1902             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1903         /*
1904          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1905          * hardware that this frame is available to transmit.
1906          */
1907         ++txr->total_packets;
1908         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1909
1910         return (0);
1911
1912 xmit_fail:
1913         bus_dmamap_unload(txr->txtag, txbuf->map);
1914         return (error);
1915
1916 }
1917
1918 static void
1919 ixgbe_set_promisc(struct adapter *adapter)
1920 {
1921         u_int32_t       reg_rctl;
1922         struct ifnet   *ifp = adapter->ifp;
1923
1924         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1925         reg_rctl &= (~IXGBE_FCTRL_UPE);
1926         reg_rctl &= (~IXGBE_FCTRL_MPE);
1927         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1928
1929         if (ifp->if_flags & IFF_PROMISC) {
1930                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1931                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1932         } else if (ifp->if_flags & IFF_ALLMULTI) {
1933                 reg_rctl |= IXGBE_FCTRL_MPE;
1934                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1935                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1936         }
1937         return;
1938 }
1939
1940
1941 /*********************************************************************
1942  *  Multicast Update
1943  *
1944  *  This routine is called whenever multicast address list is updated.
1945  *
1946  **********************************************************************/
1947 #define IXGBE_RAR_ENTRIES 16
1948
1949 static void
1950 ixgbe_set_multi(struct adapter *adapter)
1951 {
1952         u32     fctrl;
1953         u8      *mta;
1954         u8      *update_ptr;
1955         struct  ifmultiaddr *ifma;
1956         int     mcnt = 0;
1957         struct ifnet   *ifp = adapter->ifp;
1958
1959         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1960
1961         mta = adapter->mta;
1962         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1963             MAX_NUM_MULTICAST_ADDRESSES);
1964
1965         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1966         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1967         if (ifp->if_flags & IFF_PROMISC)
1968                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969         else if (ifp->if_flags & IFF_ALLMULTI) {
1970                 fctrl |= IXGBE_FCTRL_MPE;
1971                 fctrl &= ~IXGBE_FCTRL_UPE;
1972         } else
1973                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1974
1975         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1976
1977         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1978                 if (ifma->ifma_addr->sa_family != AF_LINK)
1979                         continue;
1980                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1981                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1982                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1983                 mcnt++;
1984         }
1985
1986         update_ptr = mta;
1987         ixgbe_update_mc_addr_list(&adapter->hw,
1988             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1989
1990         return;
1991 }
1992
1993 /*
1994  * This is an iterator function now needed by the multicast
1995  * shared code. It simply feeds the shared code routine the
1996  * addresses in the array of ixgbe_set_multi() one by one.
1997  */
1998 static u8 *
1999 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2000 {
2001         u8 *addr = *update_ptr;
2002         u8 *newptr;
2003         *vmdq = 0;
2004
2005         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2006         *update_ptr = newptr;
2007         return addr;
2008 }
2009
2010
2011 /*********************************************************************
2012  *  Timer routine
2013  *
2014  *  This routine checks for link status,updates statistics,
2015  *  and runs the watchdog check.
2016  *
2017  **********************************************************************/
2018
2019 static void
2020 ixgbe_local_timer(void *arg)
2021 {
2022         struct adapter  *adapter = arg;
2023         device_t        dev = adapter->dev;
2024         struct ifnet    *ifp = adapter->ifp;
2025         struct ix_queue *que = adapter->queues;
2026         struct tx_ring  *txr = adapter->tx_rings;
2027         int             hung, busy, paused;
2028
2029         IXGBE_CORE_LOCK(adapter);
2030         hung = busy = paused = 0;
2031
2032         /* Check for pluggable optics */
2033         if (adapter->sfp_probe)
2034                 if (!ixgbe_sfp_probe(adapter))
2035                         goto out; /* Nothing to do */
2036
2037         ixgbe_update_link_status(adapter);
2038         ixgbe_update_stats_counters(adapter);
2039
2040         /*
2041          * If the interface has been paused
2042          * then don't do the watchdog check
2043          */
2044         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2045                 paused = 1;
2046
2047         /*
2048         ** Check the TX queues status
2049         **      - central locked handling of OACTIVE
2050         **      - watchdog only if all queues show hung
2051         */
2052         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2053                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2054                     (paused == 0))
2055                         ++hung;
2056                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2057                         ++busy;
2058                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2059                         taskqueue_enqueue(que->tq, &que->que_task);
2060         }
2061         /* Only truely watchdog if all queues show hung */
2062         if (hung == adapter->num_queues)
2063                 goto watchdog;
2064         /* Only turn off the stack flow when ALL are depleted */
2065         if (busy == adapter->num_queues)
2066                 ifp->if_flags |= IFF_OACTIVE;
2067         else if ((ifp->if_flags & IFF_OACTIVE) &&
2068             (busy < adapter->num_queues))
2069                 ifp->if_flags &= ~IFF_OACTIVE;
2070
2071 out:
2072         ixgbe_rearm_queues(adapter, adapter->que_mask);
2073         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2074         IXGBE_CORE_UNLOCK(adapter);
2075         return;
2076
2077 watchdog:
2078         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2079         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2080             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2081             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2082         device_printf(dev,"TX(%d) desc avail = %d,"
2083             "Next TX to Clean = %d\n",
2084             txr->me, txr->tx_avail, txr->next_to_clean);
2085         adapter->ifp->if_flags &= ~IFF_RUNNING;
2086         adapter->watchdog_events++;
2087         ixgbe_init_locked(adapter);
2088
2089         IXGBE_CORE_UNLOCK(adapter);
2090 }
2091
2092 /*
2093 ** Note: this routine updates the OS on the link state
2094 **      the real check of the hardware only happens with
2095 **      a link interrupt.
2096 */
2097 static void
2098 ixgbe_update_link_status(struct adapter *adapter)
2099 {
2100         struct ifnet    *ifp = adapter->ifp;
2101         struct tx_ring *txr = adapter->tx_rings;
2102         device_t dev = adapter->dev;
2103
2104
2105         if (adapter->link_up){
2106                 if (adapter->link_active == FALSE) {
2107                         if (bootverbose)
2108                                 device_printf(dev,"Link is up %d Gbps %s \n",
2109                                     ((adapter->link_speed == 128)? 10:1),
2110                                     "Full Duplex");
2111                         adapter->link_active = TRUE;
2112                         /* Update any Flow Control changes */
2113                         ixgbe_fc_enable(&adapter->hw);
2114                         ifp->if_link_state = LINK_STATE_UP;
2115                         if_link_state_change(ifp);
2116                 }
2117         } else { /* Link down */
2118                 if (adapter->link_active == TRUE) {
2119                         if (bootverbose)
2120                                 device_printf(dev,"Link is Down\n");
2121                         ifp->if_link_state = LINK_STATE_DOWN;
2122                         if_link_state_change(ifp);
2123                         adapter->link_active = FALSE;
2124                         for (int i = 0; i < adapter->num_queues;
2125                             i++, txr++)
2126                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2127                 }
2128         }
2129
2130         return;
2131 }
2132
2133
2134 /*********************************************************************
2135  *
2136  *  This routine disables all traffic on the adapter by issuing a
2137  *  global reset on the MAC and deallocates TX/RX buffers.
2138  *
2139  **********************************************************************/
2140
2141 static void
2142 ixgbe_stop(void *arg)
2143 {
2144         struct ifnet   *ifp;
2145         struct adapter *adapter = arg;
2146         struct ixgbe_hw *hw = &adapter->hw;
2147         ifp = adapter->ifp;
2148
2149         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2150
2151         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2152         ixgbe_disable_intr(adapter);
2153         callout_stop(&adapter->timer);
2154
2155         /* Let the stack know...*/
2156         ifp->if_flags &= ~IFF_RUNNING;
2157         ifp->if_flags |= IFF_OACTIVE;
2158
2159         ixgbe_reset_hw(hw);
2160         hw->adapter_stopped = FALSE;
2161         ixgbe_stop_adapter(hw);
2162         /* Turn off the laser */
2163         if (hw->phy.multispeed_fiber)
2164                 ixgbe_disable_tx_laser(hw);
2165
2166         /* reprogram the RAR[0] in case user changed it. */
2167         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2168
2169         return;
2170 }
2171
2172
2173 /*********************************************************************
2174  *
2175  *  Determine hardware revision.
2176  *
2177  **********************************************************************/
2178 static void
2179 ixgbe_identify_hardware(struct adapter *adapter)
2180 {
2181         device_t        dev = adapter->dev;
2182         struct ixgbe_hw *hw = &adapter->hw;
2183
2184         /* Save off the information about this board */
2185         hw->vendor_id = pci_get_vendor(dev);
2186         hw->device_id = pci_get_device(dev);
2187         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2188         hw->subsystem_vendor_id =
2189             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2190         hw->subsystem_device_id =
2191             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2192
2193         /* We need this here to set the num_segs below */
2194         ixgbe_set_mac_type(hw);
2195
2196         /* Pick up the 82599 and VF settings */
2197         if (hw->mac.type != ixgbe_mac_82598EB) {
2198                 hw->phy.smart_speed = ixgbe_smart_speed;
2199                 adapter->num_segs = IXGBE_82599_SCATTER;
2200         } else
2201                 adapter->num_segs = IXGBE_82598_SCATTER;
2202
2203         return;
2204 }
2205
2206 /*********************************************************************
2207  *
2208  *  Determine optic type
2209  *
2210  **********************************************************************/
2211 static void
2212 ixgbe_setup_optics(struct adapter *adapter)
2213 {
2214         struct ixgbe_hw *hw = &adapter->hw;
2215         int             layer;
2216
2217         layer = ixgbe_get_supported_physical_layer(hw);
2218
2219         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2220                 adapter->optics = IFM_10G_T;
2221                 return;
2222         }
2223
2224         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2225                 adapter->optics = IFM_1000_T;
2226                 return;
2227         }
2228
2229         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2230             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2231                 adapter->optics = IFM_10G_LR;
2232                 return;
2233         }
2234
2235         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2236                 adapter->optics = IFM_10G_SR;
2237                 return;
2238         }
2239
2240         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2241                 adapter->optics = IFM_10G_TWINAX;
2242                 return;
2243         }
2244
2245         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2246             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2247                 adapter->optics = IFM_10G_CX4;
2248                 return;
2249         }
2250
2251         /* If we get here just set the default */
2252         adapter->optics = IFM_ETHER | IFM_AUTO;
2253         return;
2254 }
2255
2256 /*********************************************************************
2257  *
2258  *  Setup the Legacy or MSI Interrupt handler
2259  *
2260  **********************************************************************/
2261 static int
2262 ixgbe_allocate_legacy(struct adapter *adapter)
2263 {
2264         device_t dev = adapter->dev;
2265         struct          ix_queue *que = adapter->queues;
2266         int error, rid = 0;
2267         unsigned int intr_flags;
2268
2269         /* MSI RID at 1 */
2270         if (adapter->msix == 1)
2271                 rid = 1;
2272
2273         /* Try allocating a MSI interrupt first */
2274         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2275                 &rid, &intr_flags);
2276
2277         /* We allocate a single interrupt resource */
2278         adapter->res = bus_alloc_resource_any(dev,
2279             SYS_RES_IRQ, &rid, intr_flags);
2280         if (adapter->res == NULL) {
2281                 device_printf(dev, "Unable to allocate bus resource: "
2282                     "interrupt\n");
2283                 return (ENXIO);
2284         }
2285
2286         /*
2287          * Try allocating a fast interrupt and the associated deferred
2288          * processing contexts.
2289          */
2290         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2291         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2292             taskqueue_thread_enqueue, &que->tq);
2293         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2294             device_get_nameunit(adapter->dev));
2295
2296         /* Tasklets for Link, SFP and Multispeed Fiber */
2297         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2298         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2299         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2300 #ifdef IXGBE_FDIR
2301         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2302 #endif
2303         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2304             taskqueue_thread_enqueue, &adapter->tq);
2305         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2306             device_get_nameunit(adapter->dev));
2307
2308         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2309             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2310                 device_printf(dev, "Failed to register fast interrupt "
2311                     "handler: %d\n", error);
2312                 taskqueue_free(que->tq);
2313                 taskqueue_free(adapter->tq);
2314                 que->tq = NULL;
2315                 adapter->tq = NULL;
2316                 return (error);
2317         }
2318         /* For simplicity in the handlers */
2319         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2320
2321         return (0);
2322 }
2323
2324
2325 /*********************************************************************
2326  *
2327  *  Setup MSIX Interrupt resources and handlers
2328  *
2329  **********************************************************************/
2330 static int
2331 ixgbe_allocate_msix(struct adapter *adapter)
2332 {
2333         device_t        dev = adapter->dev;
2334         struct          ix_queue *que = adapter->queues;
2335         int             error, rid, vector = 0;
2336
2337         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2338                 rid = vector + 1;
2339                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2340                     RF_SHAREABLE | RF_ACTIVE);
2341                 if (que->res == NULL) {
2342                         device_printf(dev,"Unable to allocate"
2343                             " bus resource: que interrupt [%d]\n", vector);
2344                         return (ENXIO);
2345                 }
2346                 /* Set the handler function */
2347                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2348                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2349                 if (error) {
2350                         que->res = NULL;
2351                         device_printf(dev, "Failed to register QUE handler");
2352                         return (error);
2353                 }
2354 #if 0 /* __FreeBSD_version >= 800504 */
2355                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2356 #endif
2357                 que->msix = vector;
2358                 adapter->que_mask |= (u64)(1 << que->msix);
2359                 /*
2360                 ** Bind the msix vector, and thus the
2361                 ** ring to the corresponding cpu.
2362                 */
2363 #if 0 /* XXX */
2364                 if (adapter->num_queues > 1)
2365                         bus_bind_intr(dev, que->res, i);
2366 #endif
2367
2368                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2369                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2370                     taskqueue_thread_enqueue, &que->tq);
2371                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2372                     device_get_nameunit(adapter->dev));
2373         }
2374
2375         /* and Link */
2376         rid = vector + 1;
2377         adapter->res = bus_alloc_resource_any(dev,
2378             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2379         if (!adapter->res) {
2380                 device_printf(dev,"Unable to allocate"
2381             " bus resource: Link interrupt [%d]\n", rid);
2382                 return (ENXIO);
2383         }
2384         /* Set the link handler function */
2385         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2386             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2387         if (error) {
2388                 adapter->res = NULL;
2389                 device_printf(dev, "Failed to register LINK handler");
2390                 return (error);
2391         }
2392 #if 0 /* __FreeBSD_version >= 800504 */
2393         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2394 #endif
2395         adapter->linkvec = vector;
2396         /* Tasklets for Link, SFP and Multispeed Fiber */
2397         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2398         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2399         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2400 #ifdef IXGBE_FDIR
2401         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2402 #endif
2403         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2404             taskqueue_thread_enqueue, &adapter->tq);
2405         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2406             device_get_nameunit(adapter->dev));
2407
2408         return (0);
2409 }
2410
2411 #if 0   /* HAVE_MSIX */
2412 /*
2413  * Setup Either MSI/X or MSI
2414  */
2415 static int
2416 ixgbe_setup_msix(struct adapter *adapter)
2417 {
2418         device_t dev = adapter->dev;
2419         int rid, want, queues, msgs;
2420
2421         /* Override by tuneable */
2422         if (ixgbe_enable_msix == 0)
2423                 goto msi;
2424
2425         /* First try MSI/X */
2426         rid = PCIR_BAR(MSIX_82598_BAR);
2427         adapter->msix_mem = bus_alloc_resource_any(dev,
2428             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2429         if (!adapter->msix_mem) {
2430                 rid += 4;       /* 82599 maps in higher BAR */
2431                 adapter->msix_mem = bus_alloc_resource_any(dev,
2432                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2433         }
2434         if (!adapter->msix_mem) {
2435                 /* May not be enabled */
2436                 device_printf(adapter->dev,
2437                     "Unable to map MSIX table \n");
2438                 goto msi;
2439         }
2440
2441         msgs = pci_msix_count(dev);
2442         if (msgs == 0) { /* system has msix disabled */
2443                 bus_release_resource(dev, SYS_RES_MEMORY,
2444                     rid, adapter->msix_mem);
2445                 adapter->msix_mem = NULL;
2446                 goto msi;
2447         }
2448
2449         /* Figure out a reasonable auto config value */
2450         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2451
2452         if (ixgbe_num_queues != 0)
2453                 queues = ixgbe_num_queues;
2454         /* Set max queues to 8 when autoconfiguring */
2455         else if ((ixgbe_num_queues == 0) && (queues > 8))
2456                 queues = 8;
2457
2458         /*
2459         ** Want one vector (RX/TX pair) per queue
2460         ** plus an additional for Link.
2461         */
2462         want = queues + 1;
2463         if (msgs >= want)
2464                 msgs = want;
2465         else {
2466                 device_printf(adapter->dev,
2467                     "MSIX Configuration Problem, "
2468                     "%d vectors but %d queues wanted!\n",
2469                     msgs, want);
2470                 return (0); /* Will go to Legacy setup */
2471         }
2472         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2473                 device_printf(adapter->dev,
2474                     "Using MSIX interrupts with %d vectors\n", msgs);
2475                 adapter->num_queues = queues;
2476                 return (msgs);
2477         }
2478 msi:
2479         msgs = pci_msi_count(dev);
2480         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2481                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2482         else
2483                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2484         return (msgs);
2485 }
2486 #endif
2487
2488
2489 static int
2490 ixgbe_allocate_pci_resources(struct adapter *adapter)
2491 {
2492         int             rid;
2493         device_t        dev = adapter->dev;
2494
2495         rid = PCIR_BAR(0);
2496         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2497             &rid, RF_ACTIVE);
2498
2499         if (!(adapter->pci_mem)) {
2500                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2501                 return (ENXIO);
2502         }
2503
2504         adapter->osdep.mem_bus_space_tag =
2505                 rman_get_bustag(adapter->pci_mem);
2506         adapter->osdep.mem_bus_space_handle =
2507                 rman_get_bushandle(adapter->pci_mem);
2508         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2509
2510         /* Legacy defaults */
2511         adapter->num_queues = 1;
2512         adapter->hw.back = &adapter->osdep;
2513
2514         /*
2515         ** Now setup MSI or MSI/X, should
2516         ** return us the number of supported
2517         ** vectors. (Will be 1 for MSI)
2518         */
2519 #if 0   /* HAVE_MSIX */
2520         adapter->msix = ixgbe_setup_msix(adapter);
2521 #endif
2522         return (0);
2523 }
2524
2525 static void
2526 ixgbe_free_pci_resources(struct adapter * adapter)
2527 {
2528         struct          ix_queue *que = adapter->queues;
2529         device_t        dev = adapter->dev;
2530         int             rid, memrid;
2531
2532         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2533                 memrid = PCIR_BAR(MSIX_82598_BAR);
2534         else
2535                 memrid = PCIR_BAR(MSIX_82599_BAR);
2536
2537         /*
2538         ** There is a slight possibility of a failure mode
2539         ** in attach that will result in entering this function
2540         ** before interrupt resources have been initialized, and
2541         ** in that case we do not want to execute the loops below
2542         ** We can detect this reliably by the state of the adapter
2543         ** res pointer.
2544         */
2545         if (adapter->res == NULL)
2546                 goto mem;
2547
2548         /*
2549         **  Release all msix queue resources:
2550         */
2551         for (int i = 0; i < adapter->num_queues; i++, que++) {
2552                 rid = que->msix + 1;
2553                 if (que->tag != NULL) {
2554                         bus_teardown_intr(dev, que->res, que->tag);
2555                         que->tag = NULL;
2556                 }
2557                 if (que->res != NULL)
2558                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2559         }
2560
2561
2562         /* Clean the Legacy or Link interrupt last */
2563         if (adapter->linkvec) /* we are doing MSIX */
2564                 rid = adapter->linkvec + 1;
2565         else
2566                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2567
2568         if (adapter->tag != NULL) {
2569                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2570                 adapter->tag = NULL;
2571         }
2572         if (adapter->res != NULL)
2573                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2574         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2575                 pci_release_msi(adapter->dev);
2576
2577 mem:
2578         if (adapter->msix)
2579                 pci_release_msi(dev);
2580
2581         if (adapter->msix_mem != NULL)
2582                 bus_release_resource(dev, SYS_RES_MEMORY,
2583                     memrid, adapter->msix_mem);
2584
2585         if (adapter->pci_mem != NULL)
2586                 bus_release_resource(dev, SYS_RES_MEMORY,
2587                     PCIR_BAR(0), adapter->pci_mem);
2588
2589         return;
2590 }
2591
2592 /*********************************************************************
2593  *
2594  *  Setup networking device structure and register an interface.
2595  *
2596  **********************************************************************/
2597 static int
2598 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2599 {
2600         struct ixgbe_hw *hw = &adapter->hw;
2601         struct ifnet   *ifp;
2602
2603         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2604
2605         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2606         if (ifp == NULL) {
2607                 device_printf(dev, "can not allocate ifnet structure\n");
2608                 return (-1);
2609         }
2610         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2611         ifp->if_baudrate = 1000000000;
2612         ifp->if_init = ixgbe_init;
2613         ifp->if_softc = adapter;
2614         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2615         ifp->if_ioctl = ixgbe_ioctl;
2616         ifp->if_start = ixgbe_start;
2617 #if 0 /* __FreeBSD_version >= 800000 */
2618         ifp->if_transmit = ixgbe_mq_start;
2619         ifp->if_qflush = ixgbe_qflush;
2620 #endif
2621         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2622
2623         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2624
2625         adapter->max_frame_size =
2626             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2627
2628         /*
2629          * Tell the upper layer(s) we support long frames.
2630          */
2631         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2632
2633 #if 0 /* NET_TSO */
2634         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2635 #endif
2636         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2637         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2638         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2639 #if 0 /* NET_TSO */
2640                              |  IFCAP_VLAN_HWTSO
2641 #endif
2642                              |  IFCAP_VLAN_MTU;
2643         ifp->if_capenable = ifp->if_capabilities;
2644
2645         /* Don't enable LRO by default */
2646 #if 0 /* NET_LRO */
2647         ifp->if_capabilities |= IFCAP_LRO;
2648 #endif
2649
2650         /*
2651         ** Don't turn this on by default, if vlans are
2652         ** created on another pseudo device (eg. lagg)
2653         ** then vlan events are not passed thru, breaking
2654         ** operation, but with HW FILTER off it works. If
2655         ** using vlans directly on the ixgbe driver you can
2656         ** enable this and get full hardware tag filtering.
2657         */
2658         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2659
2660         /*
2661          * Specify the media types supported by this adapter and register
2662          * callbacks to update media and link information
2663          */
2664         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2665                      ixgbe_media_status);
2666         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2667         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2668         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2669                 ifmedia_add(&adapter->media,
2670                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2671                 ifmedia_add(&adapter->media,
2672                     IFM_ETHER | IFM_1000_T, 0, NULL);
2673         }
2674         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2675         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2676
2677         return (0);
2678 }
2679
2680 static void
2681 ixgbe_config_link(struct adapter *adapter)
2682 {
2683         struct ixgbe_hw *hw = &adapter->hw;
2684         u32     autoneg, err = 0;
2685         bool    sfp, negotiate;
2686
2687         sfp = ixgbe_is_sfp(hw);
2688
2689         if (sfp) {
2690                 if (hw->phy.multispeed_fiber) {
2691                         hw->mac.ops.setup_sfp(hw);
2692                         ixgbe_enable_tx_laser(hw);
2693                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2694                 } else
2695                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2696         } else {
2697                 if (hw->mac.ops.check_link)
2698                         err = ixgbe_check_link(hw, &autoneg,
2699                             &adapter->link_up, FALSE);
2700                 if (err)
2701                         goto out;
2702                 autoneg = hw->phy.autoneg_advertised;
2703                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2704                         err  = hw->mac.ops.get_link_capabilities(hw,
2705                             &autoneg, &negotiate);
2706                 if (err)
2707                         goto out;
2708                 if (hw->mac.ops.setup_link)
2709                         err = hw->mac.ops.setup_link(hw, autoneg,
2710                             negotiate, adapter->link_up);
2711         }
2712 out:
2713         return;
2714 }
2715
2716 /********************************************************************
2717  * Manage DMA'able memory.
2718  *******************************************************************/
2719 static void
2720 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2721 {
2722         if (error)
2723                 return;
2724         *(bus_addr_t *) arg = segs->ds_addr;
2725         return;
2726 }
2727
2728 static int
2729 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2730                 struct ixgbe_dma_alloc *dma, int mapflags)
2731 {
2732         device_t dev = adapter->dev;
2733         int             r;
2734
2735         r = bus_dma_tag_create(NULL,    /* parent */
2736                                DBA_ALIGN, 0,    /* alignment, bounds */
2737                                BUS_SPACE_MAXADDR,       /* lowaddr */
2738                                BUS_SPACE_MAXADDR,       /* highaddr */
2739                                NULL, NULL,      /* filter, filterarg */
2740                                size,    /* maxsize */
2741                                1,       /* nsegments */
2742                                size,    /* maxsegsize */
2743                                BUS_DMA_ALLOCNOW,        /* flags */
2744                                &dma->dma_tag);
2745         if (r != 0) {
2746                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2747                        "error %u\n", r);
2748                 goto fail_0;
2749         }
2750         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2751                              BUS_DMA_NOWAIT, &dma->dma_map);
2752         if (r != 0) {
2753                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2754                        "error %u\n", r);
2755                 goto fail_1;
2756         }
2757         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2758                             size,
2759                             ixgbe_dmamap_cb,
2760                             &dma->dma_paddr,
2761                             mapflags | BUS_DMA_NOWAIT);
2762         if (r != 0) {
2763                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2764                        "error %u\n", r);
2765                 goto fail_2;
2766         }
2767         dma->dma_size = size;
2768         return (0);
2769 fail_2:
2770         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2771 fail_1:
2772         bus_dma_tag_destroy(dma->dma_tag);
2773 fail_0:
2774         dma->dma_map = NULL;
2775         dma->dma_tag = NULL;
2776         return (r);
2777 }
2778
2779 static void
2780 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2781 {
2782         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2783             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2784         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2785         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2786         bus_dma_tag_destroy(dma->dma_tag);
2787 }
2788
2789
2790 /*********************************************************************
2791  *
2792  *  Allocate memory for the transmit and receive rings, and then
2793  *  the descriptors associated with each, called only once at attach.
2794  *
2795  **********************************************************************/
2796 static int
2797 ixgbe_allocate_queues(struct adapter *adapter)
2798 {
2799         device_t        dev = adapter->dev;
2800         struct ix_queue *que;
2801         struct tx_ring  *txr;
2802         struct rx_ring  *rxr;
2803         int rsize, tsize, error = IXGBE_SUCCESS;
2804         int txconf = 0, rxconf = 0;
2805
2806         /* First allocate the top level queue structs */
2807         if (!(adapter->queues =
2808             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2809             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2810                 device_printf(dev, "Unable to allocate queue memory\n");
2811                 error = ENOMEM;
2812                 goto fail;
2813         }
2814
2815         /* First allocate the TX ring struct memory */
2816         if (!(adapter->tx_rings =
2817             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2818             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2819                 device_printf(dev, "Unable to allocate TX ring memory\n");
2820                 error = ENOMEM;
2821                 goto tx_fail;
2822         }
2823
2824         /* Next allocate the RX */
2825         if (!(adapter->rx_rings =
2826             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2827             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2828                 device_printf(dev, "Unable to allocate RX ring memory\n");
2829                 error = ENOMEM;
2830                 goto rx_fail;
2831         }
2832
2833         /* For the ring itself */
2834         tsize = roundup2(adapter->num_tx_desc *
2835             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2836
2837         /*
2838          * Now set up the TX queues, txconf is needed to handle the
2839          * possibility that things fail midcourse and we need to
2840          * undo memory gracefully
2841          */
2842         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2843                 /* Set up some basics */
2844                 txr = &adapter->tx_rings[i];
2845                 txr->adapter = adapter;
2846                 txr->me = i;
2847
2848                 /* Initialize the TX side lock */
2849                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2850                     device_get_nameunit(dev), txr->me);
2851                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2852
2853                 if (ixgbe_dma_malloc(adapter, tsize,
2854                         &txr->txdma, BUS_DMA_NOWAIT)) {
2855                         device_printf(dev,
2856                             "Unable to allocate TX Descriptor memory\n");
2857                         error = ENOMEM;
2858                         goto err_tx_desc;
2859                 }
2860                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2861                 bzero((void *)txr->tx_base, tsize);
2862
2863                 /* Now allocate transmit buffers for the ring */
2864                 if (ixgbe_allocate_transmit_buffers(txr)) {
2865                         device_printf(dev,
2866                             "Critical Failure setting up transmit buffers\n");
2867                         error = ENOMEM;
2868                         goto err_tx_desc;
2869                 }
2870 #if 0 /* __FreeBSD_version >= 800000 */
2871                 /* Allocate a buf ring */
2872                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2873                     M_WAITOK, &txr->tx_mtx);
2874                 if (txr->br == NULL) {
2875                         device_printf(dev,
2876                             "Critical Failure setting up buf ring\n");
2877                         error = ENOMEM;
2878                         goto err_tx_desc;
2879                 }
2880 #endif
2881         }
2882
2883         /*
2884          * Next the RX queues...
2885          */
2886         rsize = roundup2(adapter->num_rx_desc *
2887             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2888         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2889                 rxr = &adapter->rx_rings[i];
2890                 /* Set up some basics */
2891                 rxr->adapter = adapter;
2892                 rxr->me = i;
2893
2894                 /* Initialize the RX side lock */
2895                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2896                     device_get_nameunit(dev), rxr->me);
2897                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2898
2899                 if (ixgbe_dma_malloc(adapter, rsize,
2900                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2901                         device_printf(dev,
2902                             "Unable to allocate RxDescriptor memory\n");
2903                         error = ENOMEM;
2904                         goto err_rx_desc;
2905                 }
2906                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2907                 bzero((void *)rxr->rx_base, rsize);
2908
2909                 /* Allocate receive buffers for the ring*/
2910                 if (ixgbe_allocate_receive_buffers(rxr)) {
2911                         device_printf(dev,
2912                             "Critical Failure setting up receive buffers\n");
2913                         error = ENOMEM;
2914                         goto err_rx_desc;
2915                 }
2916         }
2917
2918         /*
2919         ** Finally set up the queue holding structs
2920         */
2921         for (int i = 0; i < adapter->num_queues; i++) {
2922                 que = &adapter->queues[i];
2923                 que->adapter = adapter;
2924                 que->txr = &adapter->tx_rings[i];
2925                 que->rxr = &adapter->rx_rings[i];
2926         }
2927
2928         return (0);
2929
2930 err_rx_desc:
2931         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2932                 ixgbe_dma_free(adapter, &rxr->rxdma);
2933 err_tx_desc:
2934         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2935                 ixgbe_dma_free(adapter, &txr->txdma);
2936         kfree(adapter->rx_rings, M_DEVBUF);
2937 rx_fail:
2938         kfree(adapter->tx_rings, M_DEVBUF);
2939 tx_fail:
2940         kfree(adapter->queues, M_DEVBUF);
2941 fail:
2942         return (error);
2943 }
2944
2945 /*********************************************************************
2946  *
2947  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2948  *  the information needed to transmit a packet on the wire. This is
2949  *  called only once at attach, setup is done every reset.
2950  *
2951  **********************************************************************/
2952 static int
2953 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2954 {
2955         struct adapter *adapter = txr->adapter;
2956         device_t dev = adapter->dev;
2957         struct ixgbe_tx_buf *txbuf;
2958         int error, i;
2959
2960         /*
2961          * Setup DMA descriptor areas.
2962          */
2963         if ((error = bus_dma_tag_create(
2964                                NULL,    /* parent */
2965                                1, 0,            /* alignment, bounds */
2966                                BUS_SPACE_MAXADDR,       /* lowaddr */
2967                                BUS_SPACE_MAXADDR,       /* highaddr */
2968                                NULL, NULL,              /* filter, filterarg */
2969                                IXGBE_TSO_SIZE,          /* maxsize */
2970                                adapter->num_segs,       /* nsegments */
2971                                PAGE_SIZE,               /* maxsegsize */
2972                                0,                       /* flags */
2973                                &txr->txtag))) {
2974                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2975                 goto fail;
2976         }
2977
2978         if (!(txr->tx_buffers =
2979             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2980             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2981                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2982                 error = ENOMEM;
2983                 goto fail;
2984         }
2985
2986         /* Create the descriptor buffer dma maps */
2987         txbuf = txr->tx_buffers;
2988         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2989                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2990                 if (error != 0) {
2991                         device_printf(dev, "Unable to create TX DMA map\n");
2992                         goto fail;
2993                 }
2994         }
2995
2996         return 0;
2997 fail:
2998         /* We free all, it handles case where we are in the middle */
2999         ixgbe_free_transmit_structures(adapter);
3000         return (error);
3001 }
3002
3003 /*********************************************************************
3004  *
3005  *  Initialize a transmit ring.
3006  *
3007  **********************************************************************/
3008 static void
3009 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3010 {
3011         struct adapter *adapter = txr->adapter;
3012         struct ixgbe_tx_buf *txbuf;
3013         int i;
3014 #ifdef DEV_NETMAP
3015         struct netmap_adapter *na = NA(adapter->ifp);
3016         struct netmap_slot *slot;
3017 #endif /* DEV_NETMAP */
3018
3019         /* Clear the old ring contents */
3020         IXGBE_TX_LOCK(txr);
3021 #ifdef DEV_NETMAP
3022         /*
3023          * (under lock): if in netmap mode, do some consistency
3024          * checks and set slot to entry 0 of the netmap ring.
3025          */
3026         slot = netmap_reset(na, NR_TX, txr->me, 0);
3027 #endif /* DEV_NETMAP */
3028         bzero((void *)txr->tx_base,
3029               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3030         /* Reset indices */
3031         txr->next_avail_desc = 0;
3032         txr->next_to_clean = 0;
3033
3034         /* Free any existing tx buffers. */
3035         txbuf = txr->tx_buffers;
3036         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3037                 if (txbuf->m_head != NULL) {
3038                         bus_dmamap_sync(txr->txtag, txbuf->map,
3039                             BUS_DMASYNC_POSTWRITE);
3040                         bus_dmamap_unload(txr->txtag, txbuf->map);
3041                         m_freem(txbuf->m_head);
3042                         txbuf->m_head = NULL;
3043                 }
3044 #ifdef DEV_NETMAP
3045                 /*
3046                  * In netmap mode, set the map for the packet buffer.
3047                  * NOTE: Some drivers (not this one) also need to set
3048                  * the physical buffer address in the NIC ring.
3049                  * Slots in the netmap ring (indexed by "si") are
3050                  * kring->nkr_hwofs positions "ahead" wrt the
3051                  * corresponding slot in the NIC ring. In some drivers
3052                  * (not here) nkr_hwofs can be negative. Function
3053                  * netmap_idx_n2k() handles wraparounds properly.
3054                  */
3055                 if (slot) {
3056                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3057                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3058                 }
3059 #endif /* DEV_NETMAP */
3060                 /* Clear the EOP index */
3061                 txbuf->eop_index = -1;
3062         }
3063
3064 #ifdef IXGBE_FDIR
3065         /* Set the rate at which we sample packets */
3066         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3067                 txr->atr_sample = atr_sample_rate;
3068 #endif
3069
3070         /* Set number of descriptors available */
3071         txr->tx_avail = adapter->num_tx_desc;
3072
3073         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3074             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3075         IXGBE_TX_UNLOCK(txr);
3076 }
3077
3078 /*********************************************************************
3079  *
3080  *  Initialize all transmit rings.
3081  *
3082  **********************************************************************/
3083 static int
3084 ixgbe_setup_transmit_structures(struct adapter *adapter)
3085 {
3086         struct tx_ring *txr = adapter->tx_rings;
3087
3088         for (int i = 0; i < adapter->num_queues; i++, txr++)
3089                 ixgbe_setup_transmit_ring(txr);
3090
3091         return (0);
3092 }
3093
3094 /*********************************************************************
3095  *
3096  *  Enable transmit unit.
3097  *
3098  **********************************************************************/
3099 static void
3100 ixgbe_initialize_transmit_units(struct adapter *adapter)
3101 {
3102         struct tx_ring  *txr = adapter->tx_rings;
3103         struct ixgbe_hw *hw = &adapter->hw;
3104
3105         /* Setup the Base and Length of the Tx Descriptor Ring */
3106
3107         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3108                 u64     tdba = txr->txdma.dma_paddr;
3109                 u32     txctrl;
3110
3111                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3112                        (tdba & 0x00000000ffffffffULL));
3113                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3114                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3115                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3116
3117                 /* Setup the HW Tx Head and Tail descriptor pointers */
3118                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3119                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3120
3121                 /* Setup Transmit Descriptor Cmd Settings */
3122                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3123                 txr->queue_status = IXGBE_QUEUE_IDLE;
3124
3125                 /* Disable Head Writeback */
3126                 switch (hw->mac.type) {
3127                 case ixgbe_mac_82598EB:
3128                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3129                         break;
3130                 case ixgbe_mac_82599EB:
3131                 case ixgbe_mac_X540:
3132                 default:
3133                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3134                         break;
3135                 }
3136                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3137                 switch (hw->mac.type) {
3138                 case ixgbe_mac_82598EB:
3139                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3140                         break;
3141                 case ixgbe_mac_82599EB:
3142                 case ixgbe_mac_X540:
3143                 default:
3144                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3145                         break;
3146                 }
3147
3148         }
3149
3150         if (hw->mac.type != ixgbe_mac_82598EB) {
3151                 u32 dmatxctl, rttdcs;
3152                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3153                 dmatxctl |= IXGBE_DMATXCTL_TE;
3154                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3155                 /* Disable arbiter to set MTQC */
3156                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3157                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3158                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3159                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3160                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3161                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3162         }
3163
3164         return;
3165 }
3166
3167 /*********************************************************************
3168  *
3169  *  Free all transmit rings.
3170  *
3171  **********************************************************************/
3172 static void
3173 ixgbe_free_transmit_structures(struct adapter *adapter)
3174 {
3175         struct tx_ring *txr = adapter->tx_rings;
3176
3177         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3178                 IXGBE_TX_LOCK(txr);
3179                 ixgbe_free_transmit_buffers(txr);
3180                 ixgbe_dma_free(adapter, &txr->txdma);
3181                 IXGBE_TX_UNLOCK(txr);
3182                 IXGBE_TX_LOCK_DESTROY(txr);
3183         }
3184         kfree(adapter->tx_rings, M_DEVBUF);
3185 }
3186
3187 /*********************************************************************
3188  *
3189  *  Free transmit ring related data structures.
3190  *
3191  **********************************************************************/
3192 static void
3193 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3194 {
3195         struct adapter *adapter = txr->adapter;
3196         struct ixgbe_tx_buf *tx_buffer;
3197         int             i;
3198
3199         INIT_DEBUGOUT("free_transmit_ring: begin");
3200
3201         if (txr->tx_buffers == NULL)
3202                 return;
3203
3204         tx_buffer = txr->tx_buffers;
3205         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3206                 if (tx_buffer->m_head != NULL) {
3207                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3208                             BUS_DMASYNC_POSTWRITE);
3209                         bus_dmamap_unload(txr->txtag,
3210                             tx_buffer->map);
3211                         m_freem(tx_buffer->m_head);
3212                         tx_buffer->m_head = NULL;
3213                         if (tx_buffer->map != NULL) {
3214                                 bus_dmamap_destroy(txr->txtag,
3215                                     tx_buffer->map);
3216                                 tx_buffer->map = NULL;
3217                         }
3218                 } else if (tx_buffer->map != NULL) {
3219                         bus_dmamap_unload(txr->txtag,
3220                             tx_buffer->map);
3221                         bus_dmamap_destroy(txr->txtag,
3222                             tx_buffer->map);
3223                         tx_buffer->map = NULL;
3224                 }
3225         }
3226 #if 0 /* __FreeBSD_version >= 800000 */
3227         if (txr->br != NULL)
3228                 buf_ring_free(txr->br, M_DEVBUF);
3229 #endif
3230         if (txr->tx_buffers != NULL) {
3231                 kfree(txr->tx_buffers, M_DEVBUF);
3232                 txr->tx_buffers = NULL;
3233         }
3234         if (txr->txtag != NULL) {
3235                 bus_dma_tag_destroy(txr->txtag);
3236                 txr->txtag = NULL;
3237         }
3238         return;
3239 }
3240
3241 /*********************************************************************
3242  *
3243  *  Advanced Context Descriptor setup for VLAN or CSUM
3244  *
3245  **********************************************************************/
3246
3247 static bool
3248 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3249 {
3250         struct adapter *adapter = txr->adapter;
3251         struct ixgbe_adv_tx_context_desc *TXD;
3252         struct ixgbe_tx_buf        *tx_buffer;
3253         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3254         struct ether_vlan_header *eh;
3255         struct ip *ip;
3256         struct ip6_hdr *ip6;
3257         int  ehdrlen, ip_hlen = 0;
3258         u16     etype;
3259         u8      ipproto = 0;
3260         bool    offload = TRUE;
3261         int ctxd = txr->next_avail_desc;
3262 #ifdef NET_VLAN
3263         u16 vtag = 0;
3264 #endif
3265
3266
3267         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3268                 offload = FALSE;
3269
3270         tx_buffer = &txr->tx_buffers[ctxd];
3271         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3272
3273         /*
3274         ** In advanced descriptors the vlan tag must
3275         ** be placed into the descriptor itself.
3276         */
3277 #ifdef NET_VLAN
3278         if (mp->m_flags & M_VLANTAG) {
3279                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3280                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3281         } else if (offload == FALSE)
3282                 return FALSE;
3283 #endif
3284
3285         /*
3286          * Determine where frame payload starts.
3287          * Jump over vlan headers if already present,
3288          * helpful for QinQ too.
3289          */
3290         eh = mtod(mp, struct ether_vlan_header *);
3291         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3292                 etype = ntohs(eh->evl_proto);
3293                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3294         } else {
3295                 etype = ntohs(eh->evl_encap_proto);
3296                 ehdrlen = ETHER_HDR_LEN;
3297         }
3298
3299         /* Set the ether header length */
3300         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3301
3302         switch (etype) {
3303                 case ETHERTYPE_IP:
3304                         ip = (struct ip *)(mp->m_data + ehdrlen);
3305                         ip_hlen = ip->ip_hl << 2;
3306                         ipproto = ip->ip_p;
3307                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3308                         break;
3309                 case ETHERTYPE_IPV6:
3310                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3311                         ip_hlen = sizeof(struct ip6_hdr);
3312                         /* XXX-BZ this will go badly in case of ext hdrs. */
3313                         ipproto = ip6->ip6_nxt;
3314                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3315                         break;
3316                 default:
3317                         offload = FALSE;
3318                         break;
3319         }
3320
3321         vlan_macip_lens |= ip_hlen;
3322         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3323
3324         switch (ipproto) {
3325                 case IPPROTO_TCP:
3326                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3327                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3328                         break;
3329
3330                 case IPPROTO_UDP:
3331                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3332                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3333                         break;
3334
3335 #if 0
3336                 case IPPROTO_SCTP:
3337                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3338                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3339                         break;
3340 #endif
3341                 default:
3342                         offload = FALSE;
3343                         break;
3344         }
3345
3346         /* Now copy bits into descriptor */
3347         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3348         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3349         TXD->seqnum_seed = htole32(0);
3350         TXD->mss_l4len_idx = htole32(0);
3351
3352         tx_buffer->m_head = NULL;
3353         tx_buffer->eop_index = -1;
3354
3355         /* We've consumed the first desc, adjust counters */
3356         if (++ctxd == adapter->num_tx_desc)
3357                 ctxd = 0;
3358         txr->next_avail_desc = ctxd;
3359         --txr->tx_avail;
3360
3361         return (offload);
3362 }
3363
3364 /**********************************************************************
3365  *
3366  *  Setup work for hardware segmentation offload (TSO) on
3367  *  adapters using advanced tx descriptors
3368  *
3369  **********************************************************************/
3370 #if 0   /* NET_TSO */
3371 static bool
3372 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3373     u32 *olinfo_status)
3374 {
3375         struct adapter *adapter = txr->adapter;
3376         struct ixgbe_adv_tx_context_desc *TXD;
3377         struct ixgbe_tx_buf        *tx_buffer;
3378 #ifdef NET_VLAN
3379         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3380         u16 vtag = 0, eh_type;
3381 #else
3382         u16 eh_type;
3383         u32 type_tucmd_mlhl = 0;
3384 #endif
3385         u32 mss_l4len_idx = 0, len;
3386         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3387         struct ether_vlan_header *eh;
3388 #ifdef INET6
3389         struct ip6_hdr *ip6;
3390 #endif
3391 #ifdef INET
3392         struct ip *ip;
3393 #endif
3394         struct tcphdr *th;
3395
3396
3397         /*
3398          * Determine where frame payload starts.
3399          * Jump over vlan headers if already present
3400          */
3401         eh = mtod(mp, struct ether_vlan_header *);
3402         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3403                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3404                 eh_type = eh->evl_proto;
3405         } else {
3406                 ehdrlen = ETHER_HDR_LEN;
3407                 eh_type = eh->evl_encap_proto;
3408         }
3409
3410         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3411         len = ehdrlen + sizeof(struct tcphdr);
3412         switch (ntohs(eh_type)) {
3413 #ifdef INET6
3414         case ETHERTYPE_IPV6:
3415                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3416                         return FALSE;
3417                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3418                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3419                 if (ip6->ip6_nxt != IPPROTO_TCP)
3420                         return FALSE;
3421                 ip_hlen = sizeof(struct ip6_hdr);
3422                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3423                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3424                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3425                 break;
3426 #endif
3427 #ifdef INET
3428         case ETHERTYPE_IP:
3429                 if (mp->m_len < len + sizeof(struct ip))
3430                         return FALSE;
3431                 ip = (struct ip *)(mp->m_data + ehdrlen);
3432                 if (ip->ip_p != IPPROTO_TCP)
3433                         return FALSE;
3434                 ip->ip_sum = 0;
3435                 ip_hlen = ip->ip_hl << 2;
3436                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3437                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3438                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3439                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3440                 /* Tell transmit desc to also do IPv4 checksum. */
3441                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3442                 break;
3443 #endif
3444         default:
3445                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3446                     __func__, ntohs(eh_type));
3447                 break;
3448         }
3449
3450         ctxd = txr->next_avail_desc;
3451         tx_buffer = &txr->tx_buffers[ctxd];
3452         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3453
3454         tcp_hlen = th->th_off << 2;
3455
3456         /* This is used in the transmit desc in encap */
3457         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3458
3459         /* VLAN MACLEN IPLEN */
3460 #ifdef NET_VLAN
3461         if (mp->m_flags & M_VLANTAG) {
3462                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3463                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3464         }
3465
3466         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3467         vlan_macip_lens |= ip_hlen;
3468         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3469 #endif
3470
3471         /* ADV DTYPE TUCMD */
3472         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3473         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3474         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3475
3476         /* MSS L4LEN IDX */
3477         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3478         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3479         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3480
3481         TXD->seqnum_seed = htole32(0);
3482         tx_buffer->m_head = NULL;
3483         tx_buffer->eop_index = -1;
3484
3485         if (++ctxd == adapter->num_tx_desc)
3486                 ctxd = 0;
3487
3488         txr->tx_avail--;
3489         txr->next_avail_desc = ctxd;
3490         return TRUE;
3491 }
3492 #endif
3493
3494 #ifdef IXGBE_FDIR
3495 /*
3496 ** This routine parses packet headers so that Flow
3497 ** Director can make a hashed filter table entry
3498 ** allowing traffic flows to be identified and kept
3499 ** on the same cpu.  This would be a performance
3500 ** hit, but we only do it at IXGBE_FDIR_RATE of
3501 ** packets.
3502 */
3503 static void
3504 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3505 {
3506         struct adapter                  *adapter = txr->adapter;
3507         struct ix_queue                 *que;
3508         struct ip                       *ip;
3509         struct tcphdr                   *th;
3510         struct udphdr                   *uh;
3511         struct ether_vlan_header        *eh;
3512         union ixgbe_atr_hash_dword      input = {.dword = 0};
3513         union ixgbe_atr_hash_dword      common = {.dword = 0};
3514         int                             ehdrlen, ip_hlen;
3515         u16                             etype;
3516
3517         eh = mtod(mp, struct ether_vlan_header *);
3518         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3519                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3520                 etype = eh->evl_proto;
3521         } else {
3522                 ehdrlen = ETHER_HDR_LEN;
3523                 etype = eh->evl_encap_proto;
3524         }
3525
3526         /* Only handling IPv4 */
3527         if (etype != htons(ETHERTYPE_IP))
3528                 return;
3529
3530         ip = (struct ip *)(mp->m_data + ehdrlen);
3531         ip_hlen = ip->ip_hl << 2;
3532
3533         /* check if we're UDP or TCP */
3534         switch (ip->ip_p) {
3535         case IPPROTO_TCP:
3536                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3537                 /* src and dst are inverted */
3538                 common.port.dst ^= th->th_sport;
3539                 common.port.src ^= th->th_dport;
3540                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3541                 break;
3542         case IPPROTO_UDP:
3543                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3544                 /* src and dst are inverted */
3545                 common.port.dst ^= uh->uh_sport;
3546                 common.port.src ^= uh->uh_dport;
3547                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3548                 break;
3549         default:
3550                 return;
3551         }
3552
3553         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3554         if (mp->m_pkthdr.ether_vtag)
3555                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3556         else
3557                 common.flex_bytes ^= etype;
3558         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3559
3560         que = &adapter->queues[txr->me];
3561         /*
3562         ** This assumes the Rx queue and Tx
3563         ** queue are bound to the same CPU
3564         */
3565         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3566             input, common, que->msix);
3567 }
3568 #endif /* IXGBE_FDIR */
3569
3570 /**********************************************************************
3571  *
3572  *  Examine each tx_buffer in the used queue. If the hardware is done
3573  *  processing the packet then free associated resources. The
3574  *  tx_buffer is put back on the free queue.
3575  *
3576  **********************************************************************/
3577 static bool
3578 ixgbe_txeof(struct tx_ring *txr)
3579 {
3580         struct adapter  *adapter = txr->adapter;
3581         struct ifnet    *ifp = adapter->ifp;
3582         u32     first, last, done, processed;
3583         struct ixgbe_tx_buf *tx_buffer;
3584         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3585
3586         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3587
3588 #ifdef DEV_NETMAP
3589         if (ifp->if_capenable & IFCAP_NETMAP) {
3590                 struct netmap_adapter *na = NA(ifp);
3591                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3592
3593                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3594
3595                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3596                     BUS_DMASYNC_POSTREAD);
3597                 /*
3598                  * In netmap mode, all the work is done in the context
3599                  * of the client thread. Interrupt handlers only wake up
3600                  * clients, which may be sleeping on individual rings
3601                  * or on a global resource for all rings.
3602                  * To implement tx interrupt mitigation, we wake up the client
3603                  * thread roughly every half ring, even if the NIC interrupts
3604                  * more frequently. This is implemented as follows:
3605                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3606                  *   the slot that should wake up the thread (nkr_num_slots
3607                  *   means the user thread should not be woken up);
3608                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3609                  *   or the slot has the DD bit set.
3610                  *
3611                  * When the driver has separate locks, we need to
3612                  * release and re-acquire txlock to avoid deadlocks.
3613                  * XXX see if we can find a better way.
3614                  */
3615                 if (!netmap_mitigate ||
3616                     (kring->nr_kflags < kring->nkr_num_slots &&
3617                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3618                         kring->nr_kflags = kring->nkr_num_slots;
3619                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3620                         IXGBE_TX_UNLOCK(txr);
3621                         IXGBE_CORE_LOCK(adapter);
3622                         selwakeuppri(&na->tx_si, PI_NET);
3623                         IXGBE_CORE_UNLOCK(adapter);
3624                         IXGBE_TX_LOCK(txr);
3625                 }
3626                 return FALSE;
3627         }
3628 #endif /* DEV_NETMAP */
3629
3630         if (txr->tx_avail == adapter->num_tx_desc) {
3631                 txr->queue_status = IXGBE_QUEUE_IDLE;
3632                 return FALSE;
3633         }
3634
3635         processed = 0;
3636         first = txr->next_to_clean;
3637         tx_buffer = &txr->tx_buffers[first];
3638         /* For cleanup we just use legacy struct */
3639         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3640         last = tx_buffer->eop_index;
3641         if (last == -1)
3642                 return FALSE;
3643         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3644
3645         /*
3646         ** Get the index of the first descriptor
3647         ** BEYOND the EOP and call that 'done'.
3648         ** I do this so the comparison in the
3649         ** inner while loop below can be simple
3650         */
3651         if (++last == adapter->num_tx_desc) last = 0;
3652         done = last;
3653
3654         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3655             BUS_DMASYNC_POSTREAD);
3656         /*
3657         ** Only the EOP descriptor of a packet now has the DD
3658         ** bit set, this is what we look for...
3659         */
3660         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3661                 /* We clean the range of the packet */
3662                 while (first != done) {
3663                         tx_desc->upper.data = 0;
3664                         tx_desc->lower.data = 0;
3665                         tx_desc->buffer_addr = 0;
3666                         ++txr->tx_avail;
3667                         ++processed;
3668
3669                         if (tx_buffer->m_head) {
3670                                 txr->bytes +=
3671                                     tx_buffer->m_head->m_pkthdr.len;
3672                                 bus_dmamap_sync(txr->txtag,
3673                                     tx_buffer->map,
3674                                     BUS_DMASYNC_POSTWRITE);
3675                                 bus_dmamap_unload(txr->txtag,
3676                                     tx_buffer->map);
3677                                 m_freem(tx_buffer->m_head);
3678                                 tx_buffer->m_head = NULL;
3679                                 tx_buffer->map = NULL;
3680                         }
3681                         tx_buffer->eop_index = -1;
3682                         txr->watchdog_time = ticks;
3683
3684                         if (++first == adapter->num_tx_desc)
3685                                 first = 0;
3686
3687                         tx_buffer = &txr->tx_buffers[first];
3688                         tx_desc =
3689                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3690                 }
3691                 ++txr->packets;
3692                 ++ifp->if_opackets;
3693                 /* See if there is more work now */
3694                 last = tx_buffer->eop_index;
3695                 if (last != -1) {
3696                         eop_desc =
3697                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3698                         /* Get next done point */
3699                         if (++last == adapter->num_tx_desc) last = 0;
3700                         done = last;
3701                 } else
3702                         break;
3703         }
3704         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3705             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3706
3707         txr->next_to_clean = first;
3708
3709         /*
3710         ** Watchdog calculation, we know there's
3711         ** work outstanding or the first return
3712         ** would have been taken, so none processed
3713         ** for too long indicates a hang.
3714         */
3715         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3716                 txr->queue_status = IXGBE_QUEUE_HUNG;
3717
3718         /* With a minimum free clear the depleted state bit.  */
3719         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3720                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3721
3722         if (txr->tx_avail == adapter->num_tx_desc) {
3723                 txr->queue_status = IXGBE_QUEUE_IDLE;
3724                 return (FALSE);
3725         }
3726
3727         return TRUE;
3728 }
3729
3730 /*********************************************************************
3731  *
3732  *  Refresh mbuf buffers for RX descriptor rings
3733  *   - now keeps its own state so discards due to resource
3734  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3735  *     it just returns, keeping its placeholder, thus it can simply
3736  *     be recalled to try again.
3737  *
3738  **********************************************************************/
3739 static void
3740 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3741 {
3742         struct adapter          *adapter = rxr->adapter;
3743         bus_dma_segment_t       hseg[1];
3744         bus_dma_segment_t       pseg[1];
3745         struct ixgbe_rx_buf     *rxbuf;
3746         struct mbuf             *mh, *mp;
3747         int                     i, j, nsegs, error;
3748         bool                    refreshed = FALSE;
3749
3750         i = j = rxr->next_to_refresh;
3751         /* Control the loop with one beyond */
3752         if (++j == adapter->num_rx_desc)
3753                 j = 0;
3754
3755         while (j != limit) {
3756                 rxbuf = &rxr->rx_buffers[i];
3757                 if (rxr->hdr_split == FALSE)
3758                         goto no_split;
3759
3760                 if (rxbuf->m_head == NULL) {
3761                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3762                         if (mh == NULL)
3763                                 goto update;
3764                 } else
3765                         mh = rxbuf->m_head;
3766
3767                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3768                 mh->m_len = MHLEN;
3769                 mh->m_flags |= M_PKTHDR;
3770                 /* Get the memory mapping */
3771                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3772                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3773                 if (error != 0) {
3774                         kprintf("Refresh mbufs: hdr dmamap load"
3775                             " failure - %d\n", error);
3776                         m_free(mh);
3777                         rxbuf->m_head = NULL;
3778                         goto update;
3779                 }
3780                 rxbuf->m_head = mh;
3781                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3782                     BUS_DMASYNC_PREREAD);
3783                 rxr->rx_base[i].read.hdr_addr =
3784                     htole64(hseg[0].ds_addr);
3785
3786 no_split:
3787                 if (rxbuf->m_pack == NULL) {
3788                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3789                             M_PKTHDR, adapter->rx_mbuf_sz);
3790                         if (mp == NULL)
3791                                 goto update;
3792                 } else
3793                         mp = rxbuf->m_pack;
3794
3795                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3796                 /* Get the memory mapping */
3797                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3798                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3799                 if (error != 0) {
3800                         kprintf("Refresh mbufs: payload dmamap load"
3801                             " failure - %d\n", error);
3802                         m_free(mp);
3803                         rxbuf->m_pack = NULL;
3804                         goto update;
3805                 }
3806                 rxbuf->m_pack = mp;
3807                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3808                     BUS_DMASYNC_PREREAD);
3809                 rxr->rx_base[i].read.pkt_addr =
3810                     htole64(pseg[0].ds_addr);
3811
3812                 refreshed = TRUE;
3813                 /* Next is precalculated */
3814                 i = j;
3815                 rxr->next_to_refresh = i;
3816                 if (++j == adapter->num_rx_desc)
3817                         j = 0;
3818         }
3819 update:
3820         if (refreshed) /* Update hardware tail index */
3821                 IXGBE_WRITE_REG(&adapter->hw,
3822                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3823         return;
3824 }
3825
3826 /*********************************************************************
3827  *
3828  *  Allocate memory for rx_buffer structures. Since we use one
3829  *  rx_buffer per received packet, the maximum number of rx_buffer's
3830  *  that we'll need is equal to the number of receive descriptors
3831  *  that we've allocated.
3832  *
3833  **********************************************************************/
3834 static int
3835 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3836 {
3837         struct  adapter         *adapter = rxr->adapter;
3838         device_t                dev = adapter->dev;
3839         struct ixgbe_rx_buf     *rxbuf;
3840         int                     i, bsize, error;
3841
3842         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3843         if (!(rxr->rx_buffers =
3844             (struct ixgbe_rx_buf *) kmalloc(bsize,
3845             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3846                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3847                 error = ENOMEM;
3848                 goto fail;
3849         }
3850
3851         if ((error = bus_dma_tag_create(NULL,   /* parent */
3852                                    1, 0,        /* alignment, bounds */
3853                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3854                                    BUS_SPACE_MAXADDR,   /* highaddr */
3855                                    NULL, NULL,          /* filter, filterarg */
3856                                    MSIZE,               /* maxsize */
3857                                    1,                   /* nsegments */
3858                                    MSIZE,               /* maxsegsize */
3859                                    0,                   /* flags */
3860                                    &rxr->htag))) {
3861                 device_printf(dev, "Unable to create RX DMA tag\n");
3862                 goto fail;
3863         }
3864
3865         if ((error = bus_dma_tag_create(NULL,   /* parent */
3866                                    1, 0,        /* alignment, bounds */
3867                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3868                                    BUS_SPACE_MAXADDR,   /* highaddr */
3869                                    NULL, NULL,          /* filter, filterarg */
3870                                    MJUM16BYTES,         /* maxsize */
3871                                    1,                   /* nsegments */
3872                                    MJUM16BYTES,         /* maxsegsize */
3873                                    0,                   /* flags */
3874                                    &rxr->ptag))) {
3875                 device_printf(dev, "Unable to create RX DMA tag\n");
3876                 goto fail;
3877         }
3878
3879         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3880                 rxbuf = &rxr->rx_buffers[i];
3881                 error = bus_dmamap_create(rxr->htag,
3882                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3883                 if (error) {
3884                         device_printf(dev, "Unable to create RX head map\n");
3885                         goto fail;
3886                 }
3887                 error = bus_dmamap_create(rxr->ptag,
3888                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3889                 if (error) {
3890                         device_printf(dev, "Unable to create RX pkt map\n");
3891                         goto fail;
3892                 }
3893         }
3894
3895         return (0);
3896
3897 fail:
3898         /* Frees all, but can handle partial completion */
3899         ixgbe_free_receive_structures(adapter);
3900         return (error);
3901 }
3902
3903 /*
3904 ** Used to detect a descriptor that has
3905 ** been merged by Hardware RSC.
3906 */
3907 static inline u32
3908 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3909 {
3910         return (le32toh(rx->wb.lower.lo_dword.data) &
3911             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3912 }
3913
3914 /*********************************************************************
3915  *
3916  *  Initialize Hardware RSC (LRO) feature on 82599
3917  *  for an RX ring, this is toggled by the LRO capability
3918  *  even though it is transparent to the stack.
3919  *
3920  **********************************************************************/
3921 #if 0   /* NET_LRO */
3922 static void
3923 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3924 {
3925         struct  adapter         *adapter = rxr->adapter;
3926         struct  ixgbe_hw        *hw = &adapter->hw;
3927         u32                     rscctrl, rdrxctl;
3928
3929         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3930         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3931 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3932         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3933 #endif /* DEV_NETMAP */
3934         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3935         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3936         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3937
3938         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3939         rscctrl |= IXGBE_RSCCTL_RSCEN;
3940         /*
3941         ** Limit the total number of descriptors that
3942         ** can be combined, so it does not exceed 64K
3943         */
3944         if (adapter->rx_mbuf_sz == MCLBYTES)
3945                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3946         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3947                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3948         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3949                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3950         else  /* Using 16K cluster */
3951                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3952
3953         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3954
3955         /* Enable TCP header recognition */
3956         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3957             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3958             IXGBE_PSRTYPE_TCPHDR));
3959
3960         /* Disable RSC for ACK packets */
3961         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3962             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3963
3964         rxr->hw_rsc = TRUE;
3965 }
3966 #endif
3967
3968 static void
3969 ixgbe_free_receive_ring(struct rx_ring *rxr)
3970 {
3971         struct  adapter         *adapter;
3972         struct ixgbe_rx_buf       *rxbuf;
3973         int i;
3974
3975         adapter = rxr->adapter;
3976         for (i = 0; i < adapter->num_rx_desc; i++) {
3977                 rxbuf = &rxr->rx_buffers[i];
3978                 if (rxbuf->m_head != NULL) {
3979                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3980                             BUS_DMASYNC_POSTREAD);
3981                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3982                         rxbuf->m_head->m_flags |= M_PKTHDR;
3983                         m_freem(rxbuf->m_head);
3984                 }
3985                 if (rxbuf->m_pack != NULL) {
3986                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3987                             BUS_DMASYNC_POSTREAD);
3988                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3989                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3990                         m_freem(rxbuf->m_pack);
3991                 }
3992                 rxbuf->m_head = NULL;
3993                 rxbuf->m_pack = NULL;
3994         }
3995 }
3996
3997
3998 /*********************************************************************
3999  *
4000  *  Initialize a receive ring and its buffers.
4001  *
4002  **********************************************************************/
4003 static int
4004 ixgbe_setup_receive_ring(struct rx_ring *rxr)
4005 {
4006         struct  adapter         *adapter;
4007         struct ifnet            *ifp;
4008         device_t                dev;
4009         struct ixgbe_rx_buf     *rxbuf;
4010         bus_dma_segment_t       pseg[1], hseg[1];
4011 #if 0   /* NET_LRO */
4012         struct lro_ctrl         *lro = &rxr->lro;
4013 #endif
4014         int                     rsize, nsegs, error = 0;
4015 #ifdef DEV_NETMAP
4016         struct netmap_adapter *na = NA(rxr->adapter->ifp);
4017         struct netmap_slot *slot;
4018 #endif /* DEV_NETMAP */
4019
4020         adapter = rxr->adapter;
4021         ifp = adapter->ifp;
4022         dev = adapter->dev;
4023
4024         /* Clear the ring contents */
4025         IXGBE_RX_LOCK(rxr);
4026 #ifdef DEV_NETMAP
4027         /* same as in ixgbe_setup_transmit_ring() */
4028         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4029 #endif /* DEV_NETMAP */
4030         rsize = roundup2(adapter->num_rx_desc *
4031             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4032         bzero((void *)rxr->rx_base, rsize);
4033
4034         /* Free current RX buffer structs and their mbufs */
4035         ixgbe_free_receive_ring(rxr);
4036
4037         /* Configure header split? */
4038         if (ixgbe_header_split)
4039                 rxr->hdr_split = TRUE;
4040
4041         /* Now replenish the mbufs */
4042         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4043                 struct mbuf     *mh, *mp;
4044
4045                 rxbuf = &rxr->rx_buffers[j];
4046 #ifdef DEV_NETMAP
4047                 /*
4048                  * In netmap mode, fill the map and set the buffer
4049                  * address in the NIC ring, considering the offset
4050                  * between the netmap and NIC rings (see comment in
4051                  * ixgbe_setup_transmit_ring() ). No need to allocate
4052                  * an mbuf, so end the block with a continue;
4053                  */
4054                 if (slot) {
4055                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4056                         uint64_t paddr;
4057                         void *addr;
4058
4059                         addr = PNMB(slot + sj, &paddr);
4060                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4061                         /* Update descriptor */
4062                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4063                         continue;
4064                 }
4065 #endif /* DEV_NETMAP */
4066                 /*
4067                 ** Don't allocate mbufs if not
4068                 ** doing header split, its wasteful
4069                 */
4070                 if (rxr->hdr_split == FALSE)
4071                         goto skip_head;
4072
4073                 /* First the header */
4074                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4075                 if (rxbuf->m_head == NULL) {
4076                         error = ENOBUFS;
4077                         goto fail;
4078                 }
4079                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4080                 mh = rxbuf->m_head;
4081                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4082                 mh->m_flags |= M_PKTHDR;
4083                 /* Get the memory mapping */
4084                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4085                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4086                     &nsegs, BUS_DMA_NOWAIT);
4087
4088                 if (error != 0) /* Nothing elegant to do here */
4089                         goto fail;
4090                 bus_dmamap_sync(rxr->htag,
4091                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4092                 /* Update descriptor */
4093                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4094
4095 skip_head:
4096                 /* Now the payload cluster */
4097                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4098                     M_PKTHDR, adapter->rx_mbuf_sz);
4099                 if (rxbuf->m_pack == NULL) {
4100                         error = ENOBUFS;
4101                         goto fail;
4102                 }
4103                 mp = rxbuf->m_pack;
4104                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4105                 /* Get the memory mapping */
4106                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4107                     rxbuf->pmap, mp, hseg, 1,
4108                     &nsegs, BUS_DMA_NOWAIT);
4109                 if (error != 0)
4110                         goto fail;
4111                 bus_dmamap_sync(rxr->ptag,
4112                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4113                 /* Update descriptor */
4114                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4115         }
4116
4117
4118         /* Setup our descriptor indices */
4119         rxr->next_to_check = 0;
4120         rxr->next_to_refresh = 0;
4121         rxr->lro_enabled = FALSE;
4122         rxr->rx_split_packets = 0;
4123         rxr->rx_bytes = 0;
4124         rxr->discard = FALSE;
4125         rxr->vtag_strip = FALSE;
4126
4127         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4128             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4129
4130         /*
4131         ** Now set up the LRO interface:
4132         ** 82598 uses software LRO, the
4133         ** 82599 and X540 use a hardware assist.
4134         */
4135 #if 0 /* NET_LRO */
4136         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4137             (ifp->if_capenable & IFCAP_RXCSUM) &&
4138             (ifp->if_capenable & IFCAP_LRO))
4139                 ixgbe_setup_hw_rsc(rxr);
4140         else if (ifp->if_capenable & IFCAP_LRO) {
4141                 int err = tcp_lro_init(lro);
4142                 if (err) {
4143                         device_printf(dev, "LRO Initialization failed!\n");
4144                         goto fail;
4145                 }
4146                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4147                 rxr->lro_enabled = TRUE;
4148                 lro->ifp = adapter->ifp;
4149         }
4150 #endif
4151
4152         IXGBE_RX_UNLOCK(rxr);
4153         return (0);
4154
4155 fail:
4156         ixgbe_free_receive_ring(rxr);
4157         IXGBE_RX_UNLOCK(rxr);
4158         return (error);
4159 }
4160
4161 /*********************************************************************
4162  *
4163  *  Initialize all receive rings.
4164  *
4165  **********************************************************************/
4166 static int
4167 ixgbe_setup_receive_structures(struct adapter *adapter)
4168 {
4169         struct rx_ring *rxr = adapter->rx_rings;
4170         int j;
4171
4172         for (j = 0; j < adapter->num_queues; j++, rxr++)
4173                 if (ixgbe_setup_receive_ring(rxr))
4174                         goto fail;
4175
4176         return (0);
4177 fail:
4178         /*
4179          * Free RX buffers allocated so far, we will only handle
4180          * the rings that completed, the failing case will have
4181          * cleaned up for itself. 'j' failed, so its the terminus.
4182          */
4183         for (int i = 0; i < j; ++i) {
4184                 rxr = &adapter->rx_rings[i];
4185                 ixgbe_free_receive_ring(rxr);
4186         }
4187
4188         return (ENOBUFS);
4189 }
4190
4191 /*********************************************************************
4192  *
4193  *  Setup receive registers and features.
4194  *
4195  **********************************************************************/
4196 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4197
4198 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4199
4200 static void
4201 ixgbe_initialize_receive_units(struct adapter *adapter)
4202 {
4203         struct  rx_ring *rxr = adapter->rx_rings;
4204         struct ixgbe_hw *hw = &adapter->hw;
4205         struct ifnet   *ifp = adapter->ifp;
4206         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4207         u32             reta, mrqc = 0, hlreg, random[10];
4208
4209
4210         /*
4211          * Make sure receives are disabled while
4212          * setting up the descriptor ring
4213          */
4214         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4215         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4216             rxctrl & ~IXGBE_RXCTRL_RXEN);
4217
4218         /* Enable broadcasts */
4219         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4220         fctrl |= IXGBE_FCTRL_BAM;
4221         fctrl |= IXGBE_FCTRL_DPF;
4222         fctrl |= IXGBE_FCTRL_PMCF;
4223         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4224
4225         /* Set for Jumbo Frames? */
4226         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4227         if (ifp->if_mtu > ETHERMTU)
4228                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4229         else
4230                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4231 #ifdef DEV_NETMAP
4232         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4233         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4234                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4235         else
4236                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4237 #endif /* DEV_NETMAP */
4238         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4239
4240         bufsz = (adapter->rx_mbuf_sz +
4241             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4242
4243         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4244                 u64 rdba = rxr->rxdma.dma_paddr;
4245
4246                 /* Setup the Base and Length of the Rx Descriptor Ring */
4247                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4248                                (rdba & 0x00000000ffffffffULL));
4249                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4250                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4251                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4252
4253                 /* Set up the SRRCTL register */
4254                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4255                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4256                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4257                 srrctl |= bufsz;
4258                 if (rxr->hdr_split) {
4259                         /* Use a standard mbuf for the header */
4260                         srrctl |= ((IXGBE_RX_HDR <<
4261                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4262                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4263                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4264                 } else
4265                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4266                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4267
4268                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4269                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4270                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4271         }
4272
4273         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4274                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4275                               IXGBE_PSRTYPE_UDPHDR |
4276                               IXGBE_PSRTYPE_IPV4HDR |
4277                               IXGBE_PSRTYPE_IPV6HDR;
4278                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4279         }
4280
4281         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4282
4283         /* Setup RSS */
4284         if (adapter->num_queues > 1) {
4285                 int i, j;
4286                 reta = 0;
4287
4288                 /* set up random bits */
4289                 karc4rand(&random, sizeof(random));
4290
4291                 /* Set up the redirection table */
4292                 for (i = 0, j = 0; i < 128; i++, j++) {
4293                         if (j == adapter->num_queues) j = 0;
4294                         reta = (reta << 8) | (j * 0x11);
4295                         if ((i & 3) == 3)
4296                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4297                 }
4298
4299                 /* Now fill our hash function seeds */
4300                 for (int i = 0; i < 10; i++)
4301                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4302
4303                 /* Perform hash on these packet types */
4304                 mrqc = IXGBE_MRQC_RSSEN
4305                      | IXGBE_MRQC_RSS_FIELD_IPV4
4306                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4307                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4308                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4309                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4310                      | IXGBE_MRQC_RSS_FIELD_IPV6
4311                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4312                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4313                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4314                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4315
4316                 /* RSS and RX IPP Checksum are mutually exclusive */
4317                 rxcsum |= IXGBE_RXCSUM_PCSD;
4318         }
4319
4320         if (ifp->if_capenable & IFCAP_RXCSUM)
4321                 rxcsum |= IXGBE_RXCSUM_PCSD;
4322
4323         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4324                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4325
4326         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4327
4328         return;
4329 }
4330
4331 /*********************************************************************
4332  *
4333  *  Free all receive rings.
4334  *
4335  **********************************************************************/
4336 static void
4337 ixgbe_free_receive_structures(struct adapter *adapter)
4338 {
4339         struct rx_ring *rxr = adapter->rx_rings;
4340
4341         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4342 #if 0   /* NET_LRO */
4343                 struct lro_ctrl         *lro = &rxr->lro;
4344 #endif
4345                 ixgbe_free_receive_buffers(rxr);
4346                 /* Free LRO memory */
4347 #if 0   /* NET_LRO */
4348                 tcp_lro_free(lro);
4349 #endif
4350                 /* Free the ring memory as well */
4351                 ixgbe_dma_free(adapter, &rxr->rxdma);
4352         }
4353
4354         kfree(adapter->rx_rings, M_DEVBUF);
4355 }
4356
4357
4358 /*********************************************************************
4359  *
4360  *  Free receive ring data structures
4361  *
4362  **********************************************************************/
4363 static void
4364 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4365 {
4366         struct adapter          *adapter = rxr->adapter;
4367         struct ixgbe_rx_buf     *rxbuf;
4368
4369         INIT_DEBUGOUT("free_receive_structures: begin");
4370
4371         /* Cleanup any existing buffers */
4372         if (rxr->rx_buffers != NULL) {
4373                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4374                         rxbuf = &rxr->rx_buffers[i];
4375                         if (rxbuf->m_head != NULL) {
4376                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4377                                     BUS_DMASYNC_POSTREAD);
4378                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4379                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4380                                 m_freem(rxbuf->m_head);
4381                         }
4382                         if (rxbuf->m_pack != NULL) {
4383                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4384                                     BUS_DMASYNC_POSTREAD);
4385                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4386                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4387                                 m_freem(rxbuf->m_pack);
4388                         }
4389                         rxbuf->m_head = NULL;
4390                         rxbuf->m_pack = NULL;
4391                         if (rxbuf->hmap != NULL) {
4392                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4393                                 rxbuf->hmap = NULL;
4394                         }
4395                         if (rxbuf->pmap != NULL) {
4396                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4397                                 rxbuf->pmap = NULL;
4398                         }
4399                 }
4400                 if (rxr->rx_buffers != NULL) {
4401                         kfree(rxr->rx_buffers, M_DEVBUF);
4402                         rxr->rx_buffers = NULL;
4403                 }
4404         }
4405
4406         if (rxr->htag != NULL) {
4407                 bus_dma_tag_destroy(rxr->htag);
4408                 rxr->htag = NULL;
4409         }
4410         if (rxr->ptag != NULL) {
4411                 bus_dma_tag_destroy(rxr->ptag);
4412                 rxr->ptag = NULL;
4413         }
4414
4415         return;
4416 }
4417
4418 static __inline void
4419 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4420 {
4421
4422         /*
4423          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4424          * should be computed by hardware. Also it should not have VLAN tag in
4425          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4426          */
4427 #if 0   /* NET_LRO */
4428         if (rxr->lro_enabled &&
4429             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4430             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4431             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4432             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4433             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4434             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4435             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4436             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4437                 /*
4438                  * Send to the stack if:
4439                  **  - LRO not enabled, or
4440                  **  - no LRO resources, or
4441                  **  - lro enqueue fails
4442                  */
4443                 if (rxr->lro.lro_cnt != 0)
4444                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4445                                 return;
4446         }
4447 #endif
4448         IXGBE_RX_UNLOCK(rxr);
4449         (*ifp->if_input)(ifp, m);
4450         IXGBE_RX_LOCK(rxr);
4451 }
4452
4453 static __inline void
4454 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4455 {
4456         struct ixgbe_rx_buf     *rbuf;
4457
4458         rbuf = &rxr->rx_buffers[i];
4459
4460         if (rbuf->fmp != NULL) {/* Partial chain ? */
4461                 rbuf->fmp->m_flags |= M_PKTHDR;
4462                 m_freem(rbuf->fmp);
4463                 rbuf->fmp = NULL;
4464         }
4465
4466         /*
4467         ** With advanced descriptors the writeback
4468         ** clobbers the buffer addrs, so its easier
4469         ** to just free the existing mbufs and take
4470         ** the normal refresh path to get new buffers
4471         ** and mapping.
4472         */
4473         if (rbuf->m_head) {
4474                 m_free(rbuf->m_head);
4475                 rbuf->m_head = NULL;
4476         }
4477
4478         if (rbuf->m_pack) {
4479                 m_free(rbuf->m_pack);
4480                 rbuf->m_pack = NULL;
4481         }
4482
4483         return;
4484 }
4485
4486
4487 /*********************************************************************
4488  *
4489  *  This routine executes in interrupt context. It replenishes
4490  *  the mbufs in the descriptor and sends data which has been
4491  *  dma'ed into host memory to upper layer.
4492  *
4493  *  We loop at most count times if count is > 0, or until done if
4494  *  count < 0.
4495  *
4496  *  Return TRUE for more work, FALSE for all clean.
4497  *********************************************************************/
4498 static bool
4499 ixgbe_rxeof(struct ix_queue *que, int count)
4500 {
4501         struct adapter          *adapter = que->adapter;
4502         struct rx_ring          *rxr = que->rxr;
4503         struct ifnet            *ifp = adapter->ifp;
4504 #if 0   /* NET_LRO */
4505         struct lro_ctrl         *lro = &rxr->lro;
4506         struct lro_entry        *queued;
4507 #endif
4508         int                     i, nextp, processed = 0;
4509         u32                     staterr = 0;
4510         union ixgbe_adv_rx_desc *cur;
4511         struct ixgbe_rx_buf     *rbuf, *nbuf;
4512
4513         IXGBE_RX_LOCK(rxr);
4514
4515 #ifdef DEV_NETMAP
4516         if (ifp->if_capenable & IFCAP_NETMAP) {
4517                 /*
4518                  * Same as the txeof routine: only wakeup clients on intr.
4519                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4520                  * mitigation (ixgbe_rxsync() will not look for new packets
4521                  * unless NKR_PENDINTR is set).
4522                  */
4523                 struct netmap_adapter *na = NA(ifp);
4524
4525                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4526                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4527                 IXGBE_RX_UNLOCK(rxr);
4528                 IXGBE_CORE_LOCK(adapter);
4529                 selwakeuppri(&na->rx_si, PI_NET);
4530                 IXGBE_CORE_UNLOCK(adapter);
4531                 return (FALSE);
4532         }
4533 #endif /* DEV_NETMAP */
4534         for (i = rxr->next_to_check; count != 0;) {
4535                 struct mbuf     *sendmp, *mh, *mp;
4536                 u32             rsc, ptype;
4537                 u16             hlen, plen, hdr;
4538 #ifdef NET_VLAN
4539                 u16             vtag = 0;
4540 #endif
4541                 bool            eop;
4542
4543                 /* Sync the ring. */
4544                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4545                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4546
4547                 cur = &rxr->rx_base[i];
4548                 staterr = le32toh(cur->wb.upper.status_error);
4549
4550                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4551                         break;
4552                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4553                         break;
4554
4555                 count--;
4556                 sendmp = NULL;
4557                 nbuf = NULL;
4558                 rsc = 0;
4559                 cur->wb.upper.status_error = 0;
4560                 rbuf = &rxr->rx_buffers[i];
4561                 mh = rbuf->m_head;
4562                 mp = rbuf->m_pack;
4563
4564                 plen = le16toh(cur->wb.upper.length);
4565                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4566                     IXGBE_RXDADV_PKTTYPE_MASK;
4567                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4568                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4569
4570                 /* Process vlan info */
4571 #ifdef NET_VLAN
4572                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4573                         vtag = le16toh(cur->wb.upper.vlan);
4574 #endif
4575
4576                 /* Make sure bad packets are discarded */
4577                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4578                     (rxr->discard)) {
4579                         ifp->if_ierrors++;
4580                         rxr->rx_discarded++;
4581                         if (eop)
4582                                 rxr->discard = FALSE;
4583                         else
4584                                 rxr->discard = TRUE;
4585                         ixgbe_rx_discard(rxr, i);
4586                         goto next_desc;
4587                 }
4588
4589                 /*
4590                 ** On 82599 which supports a hardware
4591                 ** LRO (called HW RSC), packets need
4592                 ** not be fragmented across sequential
4593                 ** descriptors, rather the next descriptor
4594                 ** is indicated in bits of the descriptor.
4595                 ** This also means that we might proceses
4596                 ** more than one packet at a time, something
4597                 ** that has never been true before, it
4598                 ** required eliminating global chain pointers
4599                 ** in favor of what we are doing here.  -jfv
4600                 */
4601                 if (!eop) {
4602                         /*
4603                         ** Figure out the next descriptor
4604                         ** of this frame.
4605                         */
4606                         if (rxr->hw_rsc == TRUE) {
4607                                 rsc = ixgbe_rsc_count(cur);
4608                                 rxr->rsc_num += (rsc - 1);
4609                         }
4610                         if (rsc) { /* Get hardware index */
4611                                 nextp = ((staterr &
4612                                     IXGBE_RXDADV_NEXTP_MASK) >>
4613                                     IXGBE_RXDADV_NEXTP_SHIFT);
4614                         } else { /* Just sequential */
4615                                 nextp = i + 1;
4616                                 if (nextp == adapter->num_rx_desc)
4617                                         nextp = 0;
4618                         }
4619                         nbuf = &rxr->rx_buffers[nextp];
4620                         prefetch(nbuf);
4621                 }
4622                 /*
4623                 ** The header mbuf is ONLY used when header
4624                 ** split is enabled, otherwise we get normal
4625                 ** behavior, ie, both header and payload
4626                 ** are DMA'd into the payload buffer.
4627                 **
4628                 ** Rather than using the fmp/lmp global pointers
4629                 ** we now keep the head of a packet chain in the
4630                 ** buffer struct and pass this along from one
4631                 ** descriptor to the next, until we get EOP.
4632                 */
4633                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4634                         /* This must be an initial descriptor */
4635                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4636                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4637                         if (hlen > IXGBE_RX_HDR)
4638                                 hlen = IXGBE_RX_HDR;
4639                         mh->m_len = hlen;
4640                         mh->m_flags |= M_PKTHDR;
4641                         mh->m_next = NULL;
4642                         mh->m_pkthdr.len = mh->m_len;
4643                         /* Null buf pointer so it is refreshed */
4644                         rbuf->m_head = NULL;
4645                         /*
4646                         ** Check the payload length, this
4647                         ** could be zero if its a small
4648                         ** packet.
4649                         */
4650                         if (plen > 0) {
4651                                 mp->m_len = plen;
4652                                 mp->m_next = NULL;
4653                                 mp->m_flags &= ~M_PKTHDR;
4654                                 mh->m_next = mp;
4655                                 mh->m_pkthdr.len += mp->m_len;
4656                                 /* Null buf pointer so it is refreshed */
4657                                 rbuf->m_pack = NULL;
4658                                 rxr->rx_split_packets++;
4659                         }
4660                         /*
4661                         ** Now create the forward
4662                         ** chain so when complete
4663                         ** we wont have to.
4664                         */
4665                         if (eop == 0) {
4666                                 /* stash the chain head */
4667                                 nbuf->fmp = mh;
4668                                 /* Make forward chain */
4669                                 if (plen)
4670                                         mp->m_next = nbuf->m_pack;
4671                                 else
4672                                         mh->m_next = nbuf->m_pack;
4673                         } else {
4674                                 /* Singlet, prepare to send */
4675                                 sendmp = mh;
4676                                 /* If hardware handled vtag */
4677 #ifdef NET_VLAN
4678                                 if (vtag) {
4679                                         sendmp->m_pkthdr.ether_vtag = vtag;
4680                                         sendmp->m_flags |= M_VLANTAG;
4681                                 }
4682 #endif
4683                         }
4684                 } else {
4685                         /*
4686                         ** Either no header split, or a
4687                         ** secondary piece of a fragmented
4688                         ** split packet.
4689                         */
4690                         mp->m_len = plen;
4691                         /*
4692                         ** See if there is a stored head
4693                         ** that determines what we are
4694                         */
4695                         sendmp = rbuf->fmp;
4696                         rbuf->m_pack = rbuf->fmp = NULL;
4697
4698                         if (sendmp != NULL) {  /* secondary frag */
4699                                 mp->m_flags &= ~M_PKTHDR;
4700                                 sendmp->m_pkthdr.len += mp->m_len;
4701                         } else {
4702                                 /* first desc of a non-ps chain */
4703                                 sendmp = mp;
4704                                 sendmp->m_flags |= M_PKTHDR;
4705                                 sendmp->m_pkthdr.len = mp->m_len;
4706 #ifdef NET_VLAN
4707                                 if (staterr & IXGBE_RXD_STAT_VP) {
4708                                         sendmp->m_pkthdr.ether_vtag = vtag;
4709                                         sendmp->m_flags |= M_VLANTAG;
4710                                 }
4711 #endif
4712                         }
4713                         /* Pass the head pointer on */
4714                         if (eop == 0) {
4715                                 nbuf->fmp = sendmp;
4716                                 sendmp = NULL;
4717                                 mp->m_next = nbuf->m_pack;
4718                         }
4719                 }
4720                 ++processed;
4721                 /* Sending this frame? */
4722                 if (eop) {
4723                         sendmp->m_pkthdr.rcvif = ifp;
4724                         ifp->if_ipackets++;
4725                         rxr->rx_packets++;
4726                         /* capture data for AIM */
4727                         rxr->bytes += sendmp->m_pkthdr.len;
4728                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4729                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4730                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4731 #if 0 /* __FreeBSD_version >= 800000 */
4732                         sendmp->m_pkthdr.flowid = que->msix;
4733                         sendmp->m_flags |= M_FLOWID;
4734 #endif
4735                 }
4736 next_desc:
4737                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4738                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4739
4740                 /* Advance our pointers to the next descriptor. */
4741                 if (++i == adapter->num_rx_desc)
4742                         i = 0;
4743
4744                 /* Now send to the stack or do LRO */
4745                 if (sendmp != NULL) {
4746                         rxr->next_to_check = i;
4747                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4748                         i = rxr->next_to_check;
4749                 }
4750
4751                /* Every 8 descriptors we go to refresh mbufs */
4752                 if (processed == 8) {
4753                         ixgbe_refresh_mbufs(rxr, i);
4754                         processed = 0;
4755                 }
4756         }
4757
4758         /* Refresh any remaining buf structs */
4759         if (ixgbe_rx_unrefreshed(rxr))
4760                 ixgbe_refresh_mbufs(rxr, i);
4761
4762         rxr->next_to_check = i;
4763
4764         /*
4765          * Flush any outstanding LRO work
4766          */
4767 #if 0   /* NET_LRO */
4768         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4769                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4770                 tcp_lro_flush(lro, queued);
4771         }
4772 #endif
4773
4774         IXGBE_RX_UNLOCK(rxr);
4775
4776         /*
4777         ** We still have cleaning to do?
4778         ** Schedule another interrupt if so.
4779         */
4780         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4781                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4782                 return (TRUE);
4783         }
4784
4785         return (FALSE);
4786 }
4787
4788
4789 /*********************************************************************
4790  *
4791  *  Verify that the hardware indicated that the checksum is valid.
4792  *  Inform the stack about the status of checksum so that stack
4793  *  doesn't spend time verifying the checksum.
4794  *
4795  *********************************************************************/
4796 static void
4797 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4798 {
4799         u16     status = (u16) staterr;
4800         u8      errors = (u8) (staterr >> 24);
4801         bool    sctp = FALSE;
4802
4803         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4804             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4805                 sctp = TRUE;
4806
4807         if (status & IXGBE_RXD_STAT_IPCS) {
4808                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4809                         /* IP Checksum Good */
4810                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4811                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4812
4813                 } else
4814                         mp->m_pkthdr.csum_flags = 0;
4815         }
4816         if (status & IXGBE_RXD_STAT_L4CS) {
4817                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4818 #if 0
4819                 if (sctp)
4820                         type = CSUM_SCTP_VALID;
4821 #endif
4822                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4823                         mp->m_pkthdr.csum_flags |= type;
4824                         if (!sctp)
4825                                 mp->m_pkthdr.csum_data = htons(0xffff);
4826                 }
4827         }
4828         return;
4829 }
4830
4831
4832 /*
4833 ** This routine is run via an vlan config EVENT,
4834 ** it enables us to use the HW Filter table since
4835 ** we can get the vlan id. This just creates the
4836 ** entry in the soft version of the VFTA, init will
4837 ** repopulate the real table.
4838 */
4839 static void
4840 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4841 {
4842         struct adapter  *adapter = ifp->if_softc;
4843         u16             index, bit;
4844
4845         if (ifp->if_softc !=  arg)   /* Not our event */
4846                 return;
4847
4848         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4849                 return;
4850
4851         IXGBE_CORE_LOCK(adapter);
4852         index = (vtag >> 5) & 0x7F;
4853         bit = vtag & 0x1F;
4854         adapter->shadow_vfta[index] |= (1 << bit);
4855         ++adapter->num_vlans;
4856         ixgbe_init_locked(adapter);
4857         IXGBE_CORE_UNLOCK(adapter);
4858 }
4859
4860 /*
4861 ** This routine is run via an vlan
4862 ** unconfig EVENT, remove our entry
4863 ** in the soft vfta.
4864 */
4865 static void
4866 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4867 {
4868         struct adapter  *adapter = ifp->if_softc;
4869         u16             index, bit;
4870
4871         if (ifp->if_softc !=  arg)
4872                 return;
4873
4874         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4875                 return;
4876
4877         IXGBE_CORE_LOCK(adapter);
4878         index = (vtag >> 5) & 0x7F;
4879         bit = vtag & 0x1F;
4880         adapter->shadow_vfta[index] &= ~(1 << bit);
4881         --adapter->num_vlans;
4882         /* Re-init to load the changes */
4883         ixgbe_init_locked(adapter);
4884         IXGBE_CORE_UNLOCK(adapter);
4885 }
4886
4887 static void
4888 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4889 {
4890 #ifdef NET_VLAN
4891         struct ifnet    *ifp = adapter->ifp;
4892         struct ixgbe_hw *hw = &adapter->hw;
4893         struct rx_ring  *rxr;
4894         u32             ctrl;
4895
4896         /*
4897         ** We get here thru init_locked, meaning
4898         ** a soft reset, this has already cleared
4899         ** the VFTA and other state, so if there
4900         ** have been no vlan's registered do nothing.
4901         */
4902         if (adapter->num_vlans == 0)
4903                 return;
4904
4905         /*
4906         ** A soft reset zero's out the VFTA, so
4907         ** we need to repopulate it now.
4908         */
4909         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4910                 if (adapter->shadow_vfta[i] != 0)
4911                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4912                             adapter->shadow_vfta[i]);
4913
4914         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4915         /* Enable the Filter Table if enabled */
4916         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4917                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4918                 ctrl |= IXGBE_VLNCTRL_VFE;
4919         }
4920         if (hw->mac.type == ixgbe_mac_82598EB)
4921                 ctrl |= IXGBE_VLNCTRL_VME;
4922         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4923
4924         /* Setup the queues for vlans */
4925         for (int i = 0; i < adapter->num_queues; i++) {
4926                 rxr = &adapter->rx_rings[i];
4927                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4928                 if (hw->mac.type != ixgbe_mac_82598EB) {
4929                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4930                         ctrl |= IXGBE_RXDCTL_VME;
4931                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4932                 }
4933                 rxr->vtag_strip = TRUE;
4934         }
4935 #endif
4936 }
4937
4938 static void
4939 ixgbe_enable_intr(struct adapter *adapter)
4940 {
4941         struct ixgbe_hw *hw = &adapter->hw;
4942         struct ix_queue *que = adapter->queues;
4943         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4944
4945
4946         /* Enable Fan Failure detection */
4947         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4948                     mask |= IXGBE_EIMS_GPI_SDP1;
4949         else {
4950                     mask |= IXGBE_EIMS_ECC;
4951                     mask |= IXGBE_EIMS_GPI_SDP0;
4952                     mask |= IXGBE_EIMS_GPI_SDP1;
4953                     mask |= IXGBE_EIMS_GPI_SDP2;
4954 #ifdef IXGBE_FDIR
4955                     mask |= IXGBE_EIMS_FLOW_DIR;
4956 #endif
4957         }
4958
4959         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4960
4961         /* With RSS we use auto clear */
4962         if (adapter->msix_mem) {
4963                 mask = IXGBE_EIMS_ENABLE_MASK;
4964                 /* Don't autoclear Link */
4965                 mask &= ~IXGBE_EIMS_OTHER;
4966                 mask &= ~IXGBE_EIMS_LSC;
4967                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4968         }
4969
4970         /*
4971         ** Now enable all queues, this is done separately to
4972         ** allow for handling the extended (beyond 32) MSIX
4973         ** vectors that can be used by 82599
4974         */
4975         for (int i = 0; i < adapter->num_queues; i++, que++)
4976                 ixgbe_enable_queue(adapter, que->msix);
4977
4978         IXGBE_WRITE_FLUSH(hw);
4979
4980         return;
4981 }
4982
4983 static void
4984 ixgbe_disable_intr(struct adapter *adapter)
4985 {
4986         if (adapter->msix_mem)
4987                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4988         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4989                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4990         } else {
4991                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4992                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4993                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4994         }
4995         IXGBE_WRITE_FLUSH(&adapter->hw);
4996         return;
4997 }
4998
4999 u16
5000 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5001 {
5002         u16 value;
5003
5004         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
5005             reg, 2);
5006
5007         return (value);
5008 }
5009
5010 void
5011 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5012 {
5013         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
5014             reg, value, 2);
5015
5016         return;
5017 }
5018
5019 /*
5020 ** Setup the correct IVAR register for a particular MSIX interrupt
5021 **   (yes this is all very magic and confusing :)
5022 **  - entry is the register array entry
5023 **  - vector is the MSIX vector for this queue
5024 **  - type is RX/TX/MISC
5025 */
5026 static void
5027 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5028 {
5029         struct ixgbe_hw *hw = &adapter->hw;
5030         u32 ivar, index;
5031
5032         vector |= IXGBE_IVAR_ALLOC_VAL;
5033
5034         switch (hw->mac.type) {
5035
5036         case ixgbe_mac_82598EB:
5037                 if (type == -1)
5038                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5039                 else
5040                         entry += (type * 64);
5041                 index = (entry >> 2) & 0x1F;
5042                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5043                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5044                 ivar |= (vector << (8 * (entry & 0x3)));
5045                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5046                 break;
5047
5048         case ixgbe_mac_82599EB:
5049         case ixgbe_mac_X540:
5050                 if (type == -1) { /* MISC IVAR */
5051                         index = (entry & 1) * 8;
5052                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5053                         ivar &= ~(0xFF << index);
5054                         ivar |= (vector << index);
5055                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5056                 } else {        /* RX/TX IVARS */
5057                         index = (16 * (entry & 1)) + (8 * type);
5058                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5059                         ivar &= ~(0xFF << index);
5060                         ivar |= (vector << index);
5061                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5062                 }
5063
5064         default:
5065                 break;
5066         }
5067 }
5068
5069 static void
5070 ixgbe_configure_ivars(struct adapter *adapter)
5071 {
5072         struct  ix_queue *que = adapter->queues;
5073         u32 newitr;
5074
5075         if (ixgbe_max_interrupt_rate > 0)
5076                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5077         else
5078                 newitr = 0;
5079
5080         for (int i = 0; i < adapter->num_queues; i++, que++) {
5081                 /* First the RX queue entry */
5082                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5083                 /* ... and the TX */
5084                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5085                 /* Set an Initial EITR value */
5086                 IXGBE_WRITE_REG(&adapter->hw,
5087                     IXGBE_EITR(que->msix), newitr);
5088         }
5089
5090         /* For the Link interrupt */
5091         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5092 }
5093
5094 /*
5095 ** ixgbe_sfp_probe - called in the local timer to
5096 ** determine if a port had optics inserted.
5097 */
5098 static bool ixgbe_sfp_probe(struct adapter *adapter)
5099 {
5100         struct ixgbe_hw *hw = &adapter->hw;
5101         device_t        dev = adapter->dev;
5102         bool            result = FALSE;
5103
5104         if ((hw->phy.type == ixgbe_phy_nl) &&
5105             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5106                 s32 ret = hw->phy.ops.identify_sfp(hw);
5107                 if (ret)
5108                         goto out;
5109                 ret = hw->phy.ops.reset(hw);
5110                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5111                         device_printf(dev,"Unsupported SFP+ module detected!");
5112                         kprintf(" Reload driver with supported module.\n");
5113                         adapter->sfp_probe = FALSE;
5114                         goto out;
5115                 } else
5116                         device_printf(dev,"SFP+ module detected!\n");
5117                 /* We now have supported optics */
5118                 adapter->sfp_probe = FALSE;
5119                 /* Set the optics type so system reports correctly */
5120                 ixgbe_setup_optics(adapter);
5121                 result = TRUE;
5122         }
5123 out:
5124         return (result);
5125 }
5126
5127 /*
5128 ** Tasklet handler for MSIX Link interrupts
5129 **  - do outside interrupt since it might sleep
5130 */
5131 static void
5132 ixgbe_handle_link(void *context, int pending)
5133 {
5134         struct adapter  *adapter = context;
5135
5136         ixgbe_check_link(&adapter->hw,
5137             &adapter->link_speed, &adapter->link_up, 0);
5138         ixgbe_update_link_status(adapter);
5139 }
5140
5141 /*
5142 ** Tasklet for handling SFP module interrupts
5143 */
5144 static void
5145 ixgbe_handle_mod(void *context, int pending)
5146 {
5147         struct adapter  *adapter = context;
5148         struct ixgbe_hw *hw = &adapter->hw;
5149         device_t        dev = adapter->dev;
5150         u32 err;
5151
5152         err = hw->phy.ops.identify_sfp(hw);
5153         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5154                 device_printf(dev,
5155                     "Unsupported SFP+ module type was detected.\n");
5156                 return;
5157         }
5158         err = hw->mac.ops.setup_sfp(hw);
5159         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5160                 device_printf(dev,
5161                     "Setup failure - unsupported SFP+ module type.\n");
5162                 return;
5163         }
5164         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5165         return;
5166 }
5167
5168
5169 /*
5170 ** Tasklet for handling MSF (multispeed fiber) interrupts
5171 */
5172 static void
5173 ixgbe_handle_msf(void *context, int pending)
5174 {
5175         struct adapter  *adapter = context;
5176         struct ixgbe_hw *hw = &adapter->hw;
5177         u32 autoneg;
5178         bool negotiate;
5179
5180         autoneg = hw->phy.autoneg_advertised;
5181         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5182                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5183         if (hw->mac.ops.setup_link)
5184                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5185         return;
5186 }
5187
5188 #ifdef IXGBE_FDIR
5189 /*
5190 ** Tasklet for reinitializing the Flow Director filter table
5191 */
5192 static void
5193 ixgbe_reinit_fdir(void *context, int pending)
5194 {
5195         struct adapter  *adapter = context;
5196         struct ifnet   *ifp = adapter->ifp;
5197
5198         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5199                 return;
5200         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5201         adapter->fdir_reinit = 0;
5202         /* re-enable flow director interrupts */
5203         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5204         /* Restart the interface */
5205         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5206         return;
5207 }
5208 #endif
5209
5210 /**********************************************************************
5211  *
5212  *  Update the board statistics counters.
5213  *
5214  **********************************************************************/
5215 static void
5216 ixgbe_update_stats_counters(struct adapter *adapter)
5217 {
5218         struct ifnet   *ifp = adapter->ifp;
5219         struct ixgbe_hw *hw = &adapter->hw;
5220         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5221         u64  total_missed_rx = 0;
5222
5223         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5224         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5225         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5226         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5227
5228         for (int i = 0; i < 8; i++) {
5229                 u32 mp;
5230                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5231                 /* missed_rx tallies misses for the gprc workaround */
5232                 missed_rx += mp;
5233                 /* global total per queue */
5234                 adapter->stats.mpc[i] += mp;
5235                 /* Running comprehensive total for stats display */
5236                 total_missed_rx += adapter->stats.mpc[i];
5237                 if (hw->mac.type == ixgbe_mac_82598EB)
5238                         adapter->stats.rnbc[i] +=
5239                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5240                 adapter->stats.pxontxc[i] +=
5241                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5242                 adapter->stats.pxonrxc[i] +=
5243                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5244                 adapter->stats.pxofftxc[i] +=
5245                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5246                 adapter->stats.pxoffrxc[i] +=
5247                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5248                 adapter->stats.pxon2offc[i] +=
5249                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5250         }
5251         for (int i = 0; i < 16; i++) {
5252                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5253                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5254                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5255                 adapter->stats.qbrc[i] +=
5256                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5257                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5258                 adapter->stats.qbtc[i] +=
5259                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5260                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5261         }
5262         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5263         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5264         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5265
5266         /* Hardware workaround, gprc counts missed packets */
5267         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5268         adapter->stats.gprc -= missed_rx;
5269
5270         if (hw->mac.type != ixgbe_mac_82598EB) {
5271                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5272                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5273                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5274                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5275                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5276                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5277                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5278                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5279         } else {
5280                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5281                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5282                 /* 82598 only has a counter in the high register */
5283                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5284                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5285                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5286         }
5287
5288         /*
5289          * Workaround: mprc hardware is incorrectly counting
5290          * broadcasts, so for now we subtract those.
5291          */
5292         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5293         adapter->stats.bprc += bprc;
5294         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5295         if (hw->mac.type == ixgbe_mac_82598EB)
5296                 adapter->stats.mprc -= bprc;
5297
5298         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5299         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5300         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5301         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5302         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5303         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5304
5305         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5306         adapter->stats.lxontxc += lxon;
5307         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5308         adapter->stats.lxofftxc += lxoff;
5309         total = lxon + lxoff;
5310
5311         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5312         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5313         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5314         adapter->stats.gptc -= total;
5315         adapter->stats.mptc -= total;
5316         adapter->stats.ptc64 -= total;
5317         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5318
5319         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5320         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5321         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5322         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5323         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5324         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5325         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5326         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5327         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5328         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5329         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5330         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5331         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5332         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5333         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5334         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5335         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5336         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5337         /* Only read FCOE on 82599 */
5338         if (hw->mac.type != ixgbe_mac_82598EB) {
5339                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5340                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5341                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5342                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5343                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5344         }
5345
5346         /* Fill out the OS statistics structure */
5347         ifp->if_ipackets = adapter->stats.gprc;
5348         ifp->if_opackets = adapter->stats.gptc;
5349         ifp->if_ibytes = adapter->stats.gorc;
5350         ifp->if_obytes = adapter->stats.gotc;
5351         ifp->if_imcasts = adapter->stats.mprc;
5352         ifp->if_collisions = 0;
5353
5354         /* Rx Errors */
5355         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5356                 adapter->stats.rlec;
5357 }
5358
5359 /** ixgbe_sysctl_tdh_handler - Handler function
5360  *  Retrieves the TDH value from the hardware
5361  */
5362 static int
5363 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5364 {
5365         int error;
5366
5367         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5368         if (!txr) return 0;
5369
5370         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5371         error = sysctl_handle_int(oidp, &val, 0, req);
5372         if (error || !req->newptr)
5373                 return error;
5374         return 0;
5375 }
5376
5377 /** ixgbe_sysctl_tdt_handler - Handler function
5378  *  Retrieves the TDT value from the hardware
5379  */
5380 static int
5381 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5382 {
5383         int error;
5384
5385         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5386         if (!txr) return 0;
5387
5388         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5389         error = sysctl_handle_int(oidp, &val, 0, req);
5390         if (error || !req->newptr)
5391                 return error;
5392         return 0;
5393 }
5394
5395 /** ixgbe_sysctl_rdh_handler - Handler function
5396  *  Retrieves the RDH value from the hardware
5397  */
5398 static int
5399 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5400 {
5401         int error;
5402
5403         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5404         if (!rxr) return 0;
5405
5406         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5407         error = sysctl_handle_int(oidp, &val, 0, req);
5408         if (error || !req->newptr)
5409                 return error;
5410         return 0;
5411 }
5412
5413 /** ixgbe_sysctl_rdt_handler - Handler function
5414  *  Retrieves the RDT value from the hardware
5415  */
5416 static int
5417 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5418 {
5419         int error;
5420
5421         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5422         if (!rxr) return 0;
5423
5424         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5425         error = sysctl_handle_int(oidp, &val, 0, req);
5426         if (error || !req->newptr)
5427                 return error;
5428         return 0;
5429 }
5430
5431 static int
5432 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5433 {
5434         int error;
5435         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5436         unsigned int reg, usec, rate;
5437
5438         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5439         usec = ((reg & 0x0FF8) >> 3);
5440         if (usec > 0)
5441                 rate = 500000 / usec;
5442         else
5443                 rate = 0;
5444         error = sysctl_handle_int(oidp, &rate, 0, req);
5445         if (error || !req->newptr)
5446                 return error;
5447         reg &= ~0xfff; /* default, no limitation */
5448         ixgbe_max_interrupt_rate = 0;
5449         if (rate > 0 && rate < 500000) {
5450                 if (rate < 1000)
5451                         rate = 1000;
5452                 ixgbe_max_interrupt_rate = rate;
5453                 reg |= ((4000000/rate) & 0xff8 );
5454         }
5455         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5456         return 0;
5457 }
5458
5459 /*
5460  * Add sysctl variables, one per statistic, to the system.
5461  */
5462 static void
5463 ixgbe_add_hw_stats(struct adapter *adapter)
5464 {
5465         struct tx_ring *txr = adapter->tx_rings;
5466         struct rx_ring *rxr = adapter->rx_rings;
5467
5468         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5469         struct sysctl_oid *tree = adapter->sysctl_tree;
5470         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5471         struct ixgbe_hw_stats *stats = &adapter->stats;
5472
5473         struct sysctl_oid *stat_node, *queue_node;
5474         struct sysctl_oid_list *stat_list, *queue_list;
5475
5476 #define QUEUE_NAME_LEN 32
5477         char namebuf[QUEUE_NAME_LEN];
5478
5479         /* Driver Statistics */
5480         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5481                         CTLFLAG_RD, &adapter->dropped_pkts,
5482                         "Driver dropped packets");
5483         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5484                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5485                         "m_defrag() failed");
5486         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5487                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5488                         "Driver tx dma failure in xmit");
5489         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5490                         CTLFLAG_RD, &adapter->watchdog_events,
5491                         "Watchdog timeouts");
5492 #if 0   /* NET_TSO */
5493         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5494                         CTLFLAG_RD, &adapter->tso_tx,
5495                         "TSO");
5496 #endif
5497         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5498                         CTLFLAG_RD, &adapter->link_irq,
5499                         "Link MSIX IRQ Handled");
5500
5501         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5502         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5503                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5504                                             CTLFLAG_RD, NULL, "Queue Name");
5505                 queue_list = SYSCTL_CHILDREN(queue_node);
5506
5507                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5508                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5509                                 sizeof(&adapter->queues[i]),
5510                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5511                                 "Interrupt Rate");
5512                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5513                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5514                                 "irqs on this queue");
5515                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5516                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5517                                 ixgbe_sysctl_tdh_handler, "IU",
5518                                 "Transmit Descriptor Head");
5519                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5520                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5521                                 ixgbe_sysctl_tdt_handler, "IU",
5522                                 "Transmit Descriptor Tail");
5523                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5524                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5525                                 "Queue No Descriptor Available");
5526                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5527                                 CTLFLAG_RD, &txr->total_packets, 0,
5528                                 "Queue Packets Transmitted");
5529         }
5530
5531         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5532         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5533                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5534                                             CTLFLAG_RD, NULL, "Queue Name");
5535                 queue_list = SYSCTL_CHILDREN(queue_node);
5536
5537 #if 0   /* NET_LRO */
5538                 struct lro_ctrl *lro = &rxr->lro;
5539 #endif
5540
5541         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5542                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5543                                             CTLFLAG_RD, NULL, "Queue Name");
5544                 queue_list = SYSCTL_CHILDREN(queue_node);
5545
5546                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5547                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5548                                 ixgbe_sysctl_rdh_handler, "IU",
5549                                 "Receive Descriptor Head");
5550                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5551                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5552                                 ixgbe_sysctl_rdt_handler, "IU",
5553                                 "Receive Descriptor Tail");
5554                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5555                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5556                                 "Queue Packets Received");
5557                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5558                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5559                                 "Queue Bytes Received");
5560 #if 0   /* NET_LRO */
5561                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5562                                 CTLFLAG_RD, &lro->lro_queued, 0,
5563                                 "LRO Queued");
5564                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5565                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5566                                 "LRO Flushed");
5567 #endif
5568         }
5569
5570         /* MAC stats get the own sub node */
5571
5572         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5573                                     CTLFLAG_RD, NULL, "MAC Statistics");
5574         stat_list = SYSCTL_CHILDREN(stat_node);
5575
5576         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5577                         CTLFLAG_RD, &stats->crcerrs, 0,
5578                         "CRC Errors");
5579         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5580                         CTLFLAG_RD, &stats->illerrc, 0,
5581                         "Illegal Byte Errors");
5582         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5583                         CTLFLAG_RD, &stats->errbc, 0,
5584                         "Byte Errors");
5585         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5586                         CTLFLAG_RD, &stats->mspdc, 0,
5587                         "MAC Short Packets Discarded");
5588         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5589                         CTLFLAG_RD, &stats->mlfc, 0,
5590                         "MAC Local Faults");
5591         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5592                         CTLFLAG_RD, &stats->mrfc, 0,
5593                         "MAC Remote Faults");
5594         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5595                         CTLFLAG_RD, &stats->rlec, 0,
5596                         "Receive Length Errors");
5597         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5598                         CTLFLAG_RD, &stats->lxontxc, 0,
5599                         "Link XON Transmitted");
5600         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5601                         CTLFLAG_RD, &stats->lxonrxc, 0,
5602                         "Link XON Received");
5603         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5604                         CTLFLAG_RD, &stats->lxofftxc, 0,
5605                         "Link XOFF Transmitted");
5606         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5607                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5608                         "Link XOFF Received");
5609
5610         /* Packet Reception Stats */
5611         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5612                         CTLFLAG_RD, &stats->tor, 0,
5613                         "Total Octets Received");
5614         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5615                         CTLFLAG_RD, &stats->gorc, 0,
5616                         "Good Octets Received");
5617         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5618                         CTLFLAG_RD, &stats->tpr, 0,
5619                         "Total Packets Received");
5620         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5621                         CTLFLAG_RD, &stats->gprc, 0,
5622                         "Good Packets Received");
5623         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5624                         CTLFLAG_RD, &stats->mprc, 0,
5625                         "Multicast Packets Received");
5626         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5627                         CTLFLAG_RD, &stats->bprc, 0,
5628                         "Broadcast Packets Received");
5629         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5630                         CTLFLAG_RD, &stats->prc64, 0,
5631                         "64 byte frames received ");
5632         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5633                         CTLFLAG_RD, &stats->prc127, 0,
5634                         "65-127 byte frames received");
5635         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5636                         CTLFLAG_RD, &stats->prc255, 0,
5637                         "128-255 byte frames received");
5638         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5639                         CTLFLAG_RD, &stats->prc511, 0,
5640                         "256-511 byte frames received");
5641         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5642                         CTLFLAG_RD, &stats->prc1023, 0,
5643                         "512-1023 byte frames received");
5644         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5645                         CTLFLAG_RD, &stats->prc1522, 0,
5646                         "1023-1522 byte frames received");
5647         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5648                         CTLFLAG_RD, &stats->ruc, 0,
5649                         "Receive Undersized");
5650         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5651                         CTLFLAG_RD, &stats->rfc, 0,
5652                         "Fragmented Packets Received ");
5653         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5654                         CTLFLAG_RD, &stats->roc, 0,
5655                         "Oversized Packets Received");
5656         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5657                         CTLFLAG_RD, &stats->rjc, 0,
5658                         "Received Jabber");
5659         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5660                         CTLFLAG_RD, &stats->mngprc, 0,
5661                         "Management Packets Received");
5662         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5663                         CTLFLAG_RD, &stats->mngptc, 0,
5664                         "Management Packets Dropped");
5665         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5666                         CTLFLAG_RD, &stats->xec, 0,
5667                         "Checksum Errors");
5668
5669         /* Packet Transmission Stats */
5670         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5671                         CTLFLAG_RD, &stats->gotc, 0,
5672                         "Good Octets Transmitted");
5673         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5674                         CTLFLAG_RD, &stats->tpt, 0,
5675                         "Total Packets Transmitted");
5676         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5677                         CTLFLAG_RD, &stats->gptc, 0,
5678                         "Good Packets Transmitted");
5679         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5680                         CTLFLAG_RD, &stats->bptc, 0,
5681                         "Broadcast Packets Transmitted");
5682         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5683                         CTLFLAG_RD, &stats->mptc, 0,
5684                         "Multicast Packets Transmitted");
5685         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5686                         CTLFLAG_RD, &stats->mngptc, 0,
5687                         "Management Packets Transmitted");
5688         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5689                         CTLFLAG_RD, &stats->ptc64, 0,
5690                         "64 byte frames transmitted ");
5691         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5692                         CTLFLAG_RD, &stats->ptc127, 0,
5693                         "65-127 byte frames transmitted");
5694         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5695                         CTLFLAG_RD, &stats->ptc255, 0,
5696                         "128-255 byte frames transmitted");
5697         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5698                         CTLFLAG_RD, &stats->ptc511, 0,
5699                         "256-511 byte frames transmitted");
5700         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5701                         CTLFLAG_RD, &stats->ptc1023, 0,
5702                         "512-1023 byte frames transmitted");
5703         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5704                         CTLFLAG_RD, &stats->ptc1522, 0,
5705                         "1024-1522 byte frames transmitted");
5706
5707         /* FC Stats */
5708         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5709                 CTLFLAG_RD, &stats->fccrc, 0,
5710                 "FC CRC Errors");
5711         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5712                 CTLFLAG_RD, &stats->fclast, 0,
5713                 "FC Last Error");
5714         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5715                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5716                 "FCoE Packets Dropped");
5717         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5718                 CTLFLAG_RD, &stats->fcoeprc, 0,
5719                 "FCoE Packets Received");
5720         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5721                 CTLFLAG_RD, &stats->fcoeptc, 0,
5722                 "FCoE Packets Transmitted");
5723         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5724                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5725                 "FCoE DWords Received");
5726         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5727                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5728                 "FCoE DWords Transmitted");
5729 }
5730
5731 /*
5732 ** Set flow control using sysctl:
5733 ** Flow control values:
5734 **      0 - off
5735 **      1 - rx pause
5736 **      2 - tx pause
5737 **      3 - full
5738 */
5739 static int
5740 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5741 {
5742         int error, last;
5743         struct adapter *adapter = (struct adapter *) arg1;
5744
5745         last = adapter->fc;
5746         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5747         if ((error) || (req->newptr == NULL))
5748                 return (error);
5749
5750         /* Don't bother if it's not changed */
5751         if (adapter->fc == last)
5752                 return (0);
5753
5754         switch (adapter->fc) {
5755                 case ixgbe_fc_rx_pause:
5756                 case ixgbe_fc_tx_pause:
5757                 case ixgbe_fc_full:
5758                         adapter->hw.fc.requested_mode = adapter->fc;
5759                         break;
5760                 case ixgbe_fc_none:
5761                 default:
5762                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5763         }
5764         /* Don't autoneg if forcing a value */
5765         adapter->hw.fc.disable_fc_autoneg = TRUE;
5766         ixgbe_fc_enable(&adapter->hw);
5767         return error;
5768 }
5769
5770 static void
5771 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5772         const char *description, int *limit, int value)
5773 {
5774         *limit = value;
5775         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5776             SYSCTL_CHILDREN(adapter->sysctl_tree),
5777             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5778 }
5779
5780 /*
5781 ** Control link advertise speed:
5782 **      1 - advertise only 1G
5783 **      2 - advertise 100Mb
5784 **      3 - advertise normal
5785 */
5786 static int
5787 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5788 {
5789         int                     error = 0;
5790         struct adapter          *adapter;
5791         device_t                dev;
5792         struct ixgbe_hw         *hw;
5793         ixgbe_link_speed        speed, last;
5794
5795         adapter = (struct adapter *) arg1;
5796         dev = adapter->dev;
5797         hw = &adapter->hw;
5798         last = adapter->advertise;
5799
5800         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5801         if ((error) || (adapter->advertise == -1))
5802                 return (error);
5803
5804         if (adapter->advertise == last) /* no change */
5805                 return (0);
5806
5807         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5808             (hw->phy.multispeed_fiber)))
5809                 return (error);
5810
5811         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5812                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5813                 return (error);
5814         }
5815
5816         if (adapter->advertise == 1)
5817                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5818         else if (adapter->advertise == 2)
5819                 speed = IXGBE_LINK_SPEED_100_FULL;
5820         else if (adapter->advertise == 3)
5821                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5822                         IXGBE_LINK_SPEED_10GB_FULL;
5823         else /* bogus value */
5824                 return (error);
5825
5826         hw->mac.autotry_restart = TRUE;
5827         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5828
5829         return (error);
5830 }
5831
5832 /*
5833 ** Thermal Shutdown Trigger
5834 **   - cause a Thermal Overtemp IRQ
5835 */
5836 static int
5837 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5838 {
5839         int             error, fire = 0;
5840         struct adapter  *adapter = (struct adapter *) arg1;
5841         struct ixgbe_hw *hw = &adapter->hw;
5842
5843
5844         if (hw->mac.type != ixgbe_mac_X540)
5845                 return (0);
5846
5847         error = sysctl_handle_int(oidp, &fire, 0, req);
5848         if ((error) || (req->newptr == NULL))
5849                 return (error);
5850
5851         if (fire) {
5852                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5853                 reg |= IXGBE_EICR_TS;
5854                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5855         }
5856
5857         return (0);
5858 }