sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 #if 0   /* HAVE_MSIX */
 125 static int      ixgbe_setup_msix(struct adapter *);
 126 #endif
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 #if 0   /* NET_LRO */
 146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 147 #endif
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *, int);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 167                     const char *, int *, int);
 168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 169 #if 0   /* NET_TSO */
 170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 171 #endif
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_que(void *, int);
 199 static void     ixgbe_handle_link(void *, int);
 200 static void     ixgbe_handle_msf(void *, int);
 201 static void     ixgbe_handle_mod(void *, int);
 202
 203 #ifdef IXGBE_FDIR
 204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 205 static void     ixgbe_reinit_fdir(void *, int);
 206 #endif
 207
 208 /*********************************************************************
 209  *  FreeBSD Device Interface Entry Points
 210  *********************************************************************/
 211
 212 static device_method_t ixgbe_methods[] = {
 213         /* Device interface */
 214         DEVMETHOD(device_probe, ixgbe_probe),
 215         DEVMETHOD(device_attach, ixgbe_attach),
 216         DEVMETHOD(device_detach, ixgbe_detach),
 217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 218         {0, 0}
 219 };
 220
 221 static driver_t ixgbe_driver = {
 222         "ix", ixgbe_methods, sizeof(struct adapter),
 223 };
 224
 225 devclass_t ixgbe_devclass;
 226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 227
 228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 230
 231 /*
 232 ** TUNEABLE PARAMETERS:
 233 */
 234
 235 /*
 236 ** AIM: Adaptive Interrupt Moderation
 237 ** which means that the interrupt rate
 238 ** is varied over time based on the
 239 ** traffic for that interrupt vector
 240 */
 241 static int ixgbe_enable_aim = TRUE;
 242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 243
 244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 246
 247 /* How many packets rxeof tries to clean at a time */
 248 static int ixgbe_rx_process_limit = 128;
 249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 static int ixgbe_msi_enable = 1;
 261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 262
 263 /*
 264  * MSIX should be the default for best performance,
 265  * but this allows it to be forced off for testing.
 266  */
 267 static int ixgbe_enable_msix = 1;
 268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 269
 270 /*
 271  * Header split: this causes the hardware to DMA
 272  * the header into a separate mbuf from the payload,
 273  * it can be a performance win in some workloads, but
 274  * in others it actually hurts, its off by default.
 275  */
 276 static int ixgbe_header_split = FALSE;
 277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 278
 279 /*
 280  * Number of Queues, can be set to 0,
 281  * it then autoconfigures based on the
 282  * number of cpus with a max of 8. This
 283  * can be overriden manually here.
 284  */
 285 static int ixgbe_num_queues = 0;
 286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 287
 288 /*
 289 ** Number of TX descriptors per ring,
 290 ** setting higher than RX as this seems
 291 ** the better performing choice.
 292 */
 293 static int ixgbe_txd = PERFORM_TXD;
 294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 295
 296 /* Number of RX descriptors per ring */
 297 static int ixgbe_rxd = PERFORM_RXD;
 298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** For Flow Director: this is the
 306 ** number of TX packets we sample
 307 ** for the filter pool, this means
 308 ** every 20th packet will be probed.
 309 **
 310 ** This feature can be disabled by
 311 ** setting this to 0.
 312 */
 313 static int atr_sample_rate = 20;
 314 /*
 315 ** Flow Director actually 'steals'
 316 ** part of the packet buffer as its
 317 ** filter pool, this variable controls
 318 ** how much it uses:
 319 **  0 = 64K, 1 = 128K, 2 = 256K
 320 */
 321 static int fdir_pballoc = 1;
 322 #endif
 323
 324 #ifdef DEV_NETMAP
 325 /*
 326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 327  * be a reference on how to implement netmap support in a driver.
 328  * Additional comments are in ixgbe_netmap.h .
 329  *
 330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 331  * that extend the standard driver.
 332  */
 333 #include <dev/netmap/ixgbe_netmap.h>
 334 #endif /* DEV_NETMAP */
 335
 336 /*********************************************************************
 337  *  Device identification routine
 338  *
 339  *  ixgbe_probe determines if the driver should be loaded on
 340  *  adapter based on PCI vendor/device id of the adapter.
 341  *
 342  *  return BUS_PROBE_DEFAULT on success, positive on failure
 343  *********************************************************************/
 344
 345 static int
 346 ixgbe_probe(device_t dev)
 347 {
 348         ixgbe_vendor_info_t *ent;
 349
 350         u16     pci_vendor_id = 0;
 351         u16     pci_device_id = 0;
 352         u16     pci_subvendor_id = 0;
 353         u16     pci_subdevice_id = 0;
 354         char    adapter_name[256];
 355
 356         INIT_DEBUGOUT("ixgbe_probe: begin");
 357
 358         pci_vendor_id = pci_get_vendor(dev);
 359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 360                 return (ENXIO);
 361
 362         pci_device_id = pci_get_device(dev);
 363         pci_subvendor_id = pci_get_subvendor(dev);
 364         pci_subdevice_id = pci_get_subdevice(dev);
 365
 366         ent = ixgbe_vendor_info_array;
 367         while (ent->vendor_id != 0) {
 368                 if ((pci_vendor_id == ent->vendor_id) &&
 369                     (pci_device_id == ent->device_id) &&
 370
 371                     ((pci_subvendor_id == ent->subvendor_id) ||
 372                      (ent->subvendor_id == 0)) &&
 373
 374                     ((pci_subdevice_id == ent->subdevice_id) ||
 375                      (ent->subdevice_id == 0))) {
 376                         ksprintf(adapter_name, "%s, Version - %s",
 377                                 ixgbe_strings[ent->index],
 378                                 ixgbe_driver_version);
 379                         device_set_desc_copy(dev, adapter_name);
 380                         ++ixgbe_total_ports;
 381                         return (BUS_PROBE_DEFAULT);
 382                 }
 383                 ent++;
 384         }
 385         return (ENXIO);
 386 }
 387
 388 /*********************************************************************
 389  *  Device initialization routine
 390  *
 391  *  The attach entry point is called when the driver is being loaded.
 392  *  This routine identifies the type of hardware, allocates all resources
 393  *  and initializes the hardware.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 ixgbe_attach(device_t dev)
 400 {
 401         struct adapter *adapter;
 402         struct ixgbe_hw *hw;
 403         int             error = 0;
 404         u16             csum;
 405         u32             ctrl_ext;
 406
 407         INIT_DEBUGOUT("ixgbe_attach: begin");
 408
 409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 410                 device_printf(dev, "Disabled by device hint\n");
 411                 return (ENXIO);
 412         }
 413
 414         /* Allocate, clear, and link in our adapter structure */
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         hw = &adapter->hw;
 418
 419         /* Core Lock Init*/
 420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 421
 422         /* SYSCTL APIs */
 423
 424         sysctl_ctx_init(&adapter->sysctl_ctx);
 425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 428         if (adapter->sysctl_tree == NULL) {
 429                 device_printf(adapter->dev, "can't add sysctl node\n");
 430                 return (EINVAL);
 431         }
 432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 436
 437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 441
 442         /*
 443         ** Allow a kind of speed control by forcing the autoneg
 444         ** advertised speed list to only a certain value, this
 445         ** supports 1G on 82599 devices, and 100Mb on x540.
 446         */
 447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 451
 452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 456
 457         /* Set up the timer callout */
 458         callout_init_mp(&adapter->timer);
 459
 460         /* Determine hardware revision */
 461         ixgbe_identify_hardware(adapter);
 462
 463         /* Do base PCI setup - map BAR0 */
 464         if (ixgbe_allocate_pci_resources(adapter)) {
 465                 device_printf(dev, "Allocation of PCI resources failed\n");
 466                 error = ENXIO;
 467                 goto err_out;
 468         }
 469
 470         /* Do descriptor calc and sanity checks */
 471         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 472             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 473                 device_printf(dev, "TXD config issue, using default!\n");
 474                 adapter->num_tx_desc = DEFAULT_TXD;
 475         } else
 476                 adapter->num_tx_desc = ixgbe_txd;
 477
 478         /*
 479         ** With many RX rings it is easy to exceed the
 480         ** system mbuf allocation. Tuning nmbclusters
 481         ** can alleviate this.
 482         */
 483         if (nmbclusters > 0 ) {
 484                 int s;
 485                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 486                 if (s > nmbclusters) {
 487                         device_printf(dev, "RX Descriptors exceed "
 488                             "system mbuf max, using default instead!\n");
 489                         ixgbe_rxd = DEFAULT_RXD;
 490                 }
 491         }
 492
 493         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 494             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 495                 device_printf(dev, "RXD config issue, using default!\n");
 496                 adapter->num_rx_desc = DEFAULT_RXD;
 497         } else
 498                 adapter->num_rx_desc = ixgbe_rxd;
 499
 500         /* Allocate our TX/RX Queues */
 501         if (ixgbe_allocate_queues(adapter)) {
 502                 error = ENOMEM;
 503                 goto err_out;
 504         }
 505
 506         /* Allocate multicast array memory. */
 507         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 508             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 509         if (adapter->mta == NULL) {
 510                 device_printf(dev, "Can not allocate multicast setup array\n");
 511                 error = ENOMEM;
 512                 goto err_late;
 513         }
 514
 515         /* Initialize the shared code */
 516         error = ixgbe_init_shared_code(hw);
 517         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 518                 /*
 519                 ** No optics in this port, set up
 520                 ** so the timer routine will probe
 521                 ** for later insertion.
 522                 */
 523                 adapter->sfp_probe = TRUE;
 524                 error = 0;
 525         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 526                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 527                 error = EIO;
 528                 goto err_late;
 529         } else if (error) {
 530                 device_printf(dev,"Unable to initialize the shared code\n");
 531                 error = EIO;
 532                 goto err_late;
 533         }
 534
 535         /* Make sure we have a good EEPROM before we read from it */
 536         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 537                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 538                 error = EIO;
 539                 goto err_late;
 540         }
 541
 542         error = ixgbe_init_hw(hw);
 543         switch (error) {
 544         case IXGBE_ERR_EEPROM_VERSION:
 545                 device_printf(dev, "This device is a pre-production adapter/"
 546                     "LOM.  Please be aware there may be issues associated "
 547                     "with your hardware.\n If you are experiencing problems "
 548                     "please contact your Intel or hardware representative "
 549                     "who provided you with this hardware.\n");
 550                 break;
 551         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 552                 device_printf(dev,"Unsupported SFP+ Module\n");
 553                 error = EIO;
 554                 device_printf(dev,"Hardware Initialization Failure\n");
 555                 goto err_late;
 556         case IXGBE_ERR_SFP_NOT_PRESENT:
 557                 device_printf(dev,"No SFP+ Module found\n");
 558                 /* falls thru */
 559         default:
 560                 break;
 561         }
 562
 563         /* Detect and set physical type */
 564         ixgbe_setup_optics(adapter);
 565
 566         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 567                 error = ixgbe_allocate_msix(adapter);
 568         else
 569                 error = ixgbe_allocate_legacy(adapter);
 570         if (error)
 571                 goto err_late;
 572
 573         /* Setup OS specific network interface */
 574         if (ixgbe_setup_interface(dev, adapter) != 0)
 575                 goto err_late;
 576
 577         /* Sysctl for limiting the amount of work done in the taskqueue */
 578         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 579             "max number of rx packets to process", &adapter->rx_process_limit,
 580             ixgbe_rx_process_limit);
 581
 582         /* Initialize statistics */
 583         ixgbe_update_stats_counters(adapter);
 584
 585         /* Register for VLAN events */
 586         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 587             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 588         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 589             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 590
 591         /* Print PCIE bus type/speed/width info */
 592         ixgbe_get_bus_info(hw);
 593         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 594             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 595             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 596             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 597             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 598             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 599             ("Unknown"));
 600
 601         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 602             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 603                 device_printf(dev, "PCI-Express bandwidth available"
 604                     " for this card\n     is not sufficient for"
 605                     " optimal performance.\n");
 606                 device_printf(dev, "For optimal performance a x8 "
 607                     "PCIE, or x4 PCIE 2 slot is required.\n");
 608         }
 609
 610         /* let hardware know driver is loaded */
 611         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 612         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 613         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 614
 615         ixgbe_add_hw_stats(adapter);
 616
 617 #ifdef DEV_NETMAP
 618         ixgbe_netmap_attach(adapter);
 619 #endif /* DEV_NETMAP */
 620         INIT_DEBUGOUT("ixgbe_attach: end");
 621         return (0);
 622 err_late:
 623         ixgbe_free_transmit_structures(adapter);
 624         ixgbe_free_receive_structures(adapter);
 625 err_out:
 626         if (adapter->ifp != NULL)
 627                 if_free(adapter->ifp);
 628         ixgbe_free_pci_resources(adapter);
 629         kfree(adapter->mta, M_DEVBUF);
 630         return (error);
 631
 632 }
 633
 634 /*********************************************************************
 635  *  Device removal routine
 636  *
 637  *  The detach entry point is called when the driver is being removed.
 638  *  This routine stops the adapter and deallocates all the resources
 639  *  that were allocated for driver operation.
 640  *
 641  *  return 0 on success, positive on failure
 642  *********************************************************************/
 643
 644 static int
 645 ixgbe_detach(device_t dev)
 646 {
 647         struct adapter *adapter = device_get_softc(dev);
 648         struct ix_queue *que = adapter->queues;
 649         u32     ctrl_ext;
 650
 651         INIT_DEBUGOUT("ixgbe_detach: begin");
 652
 653 #ifdef NET_VLAN
 654         /* Make sure VLANS are not using driver */
 655         if (adapter->ifp->if_vlantrunk != NULL) {
 656                 device_printf(dev,"Vlan in use, detach first\n");
 657                 return (EBUSY);
 658         }
 659 #endif
 660
 661         IXGBE_CORE_LOCK(adapter);
 662         ixgbe_stop(adapter);
 663         IXGBE_CORE_UNLOCK(adapter);
 664
 665         for (int i = 0; i < adapter->num_queues; i++, que++) {
 666                 if (que->tq) {
 667                         taskqueue_drain(que->tq, &que->que_task);
 668                         taskqueue_free(que->tq);
 669                 }
 670         }
 671
 672         /* Drain the Link queue */
 673         if (adapter->tq) {
 674                 taskqueue_drain(adapter->tq, &adapter->link_task);
 675                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 676                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 677 #ifdef IXGBE_FDIR
 678                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 679 #endif
 680                 taskqueue_free(adapter->tq);
 681         }
 682
 683         /* let hardware know driver is unloading */
 684         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 685         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 686         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 687
 688         /* Unregister VLAN events */
 689 #ifdef NET_VLAN
 690         if (adapter->vlan_attach != NULL)
 691                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 692         if (adapter->vlan_detach != NULL)
 693                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 694 #endif
 695
 696         ether_ifdetach(adapter->ifp);
 697         callout_stop(&adapter->timer);
 698 #ifdef DEV_NETMAP
 699         netmap_detach(adapter->ifp);
 700 #endif /* DEV_NETMAP */
 701         ixgbe_free_pci_resources(adapter);
 702         bus_generic_detach(dev);
 703         if_free(adapter->ifp);
 704
 705         ixgbe_free_transmit_structures(adapter);
 706         ixgbe_free_receive_structures(adapter);
 707         kfree(adapter->mta, M_DEVBUF);
 708         sysctl_ctx_free(&adapter->sysctl_ctx);
 709
 710         IXGBE_CORE_LOCK_DESTROY(adapter);
 711         return (0);
 712 }
 713
 714 /*********************************************************************
 715  *
 716  *  Shutdown entry point
 717  *
 718  **********************************************************************/
 719
 720 static int
 721 ixgbe_shutdown(device_t dev)
 722 {
 723         struct adapter *adapter = device_get_softc(dev);
 724         IXGBE_CORE_LOCK(adapter);
 725         ixgbe_stop(adapter);
 726         IXGBE_CORE_UNLOCK(adapter);
 727         return (0);
 728 }
 729
 730
 731 /*********************************************************************
 732  *  Transmit entry point
 733  *
 734  *  ixgbe_start is called by the stack to initiate a transmit.
 735  *  The driver will remain in this routine as long as there are
 736  *  packets to transmit and transmit resources are available.
 737  *  In case resources are not available stack is notified and
 738  *  the packet is requeued.
 739  **********************************************************************/
 740
 741 static void
 742 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 743 {
 744         struct mbuf    *m_head;
 745         struct adapter *adapter = txr->adapter;
 746
 747         IXGBE_TX_LOCK_ASSERT(txr);
 748
 749         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 750                 return;
 751         if (!adapter->link_active)
 752                 return;
 753
 754         while (!ifq_is_empty(&ifp->if_snd)) {
 755                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 756                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 757                         break;
 758                 }
 759
 760                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 761                 if (m_head == NULL)
 762                         break;
 763
 764                 if (ixgbe_xmit(txr, &m_head)) {
 765 #if 0 /* XXX: prepend to an ALTQ queue ? */
 766                         if (m_head != NULL)
 767                                 IF_PREPEND(&ifp->if_snd, m_head);
 768 #endif
 769                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 770                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 771                         break;
 772                 }
 773                 /* Send a copy of the frame to the BPF listener */
 774                 ETHER_BPF_MTAP(ifp, m_head);
 775
 776                 /* Set watchdog on */
 777                 txr->watchdog_time = ticks;
 778                 txr->queue_status = IXGBE_QUEUE_WORKING;
 779
 780         }
 781         return;
 782 }
 783
 784 /*
 785  * Legacy TX start - called by the stack, this
 786  * always uses the first tx ring, and should
 787  * not be used with multiqueue tx enabled.
 788  */
 789 static void
 790 ixgbe_start(struct ifnet *ifp)
 791 {
 792         struct adapter *adapter = ifp->if_softc;
 793         struct tx_ring  *txr = adapter->tx_rings;
 794
 795         if (ifp->if_flags & IFF_RUNNING) {
 796                 IXGBE_TX_LOCK(txr);
 797                 ixgbe_start_locked(txr, ifp);
 798                 IXGBE_TX_UNLOCK(txr);
 799         }
 800         return;
 801 }
 802
 803 #if 0 /* __FreeBSD_version >= 800000 */
 804 /*
 805 ** Multiqueue Transmit driver
 806 **
 807 */
 808 static int
 809 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 810 {
 811         struct adapter  *adapter = ifp->if_softc;
 812         struct ix_queue *que;
 813         struct tx_ring  *txr;
 814         int             i = 0, err = 0;
 815
 816         /* Which queue to use */
 817         if ((m->m_flags & M_FLOWID) != 0)
 818                 i = m->m_pkthdr.flowid % adapter->num_queues;
 819         else
 820                 i = curcpu % adapter->num_queues;
 821
 822         txr = &adapter->tx_rings[i];
 823         que = &adapter->queues[i];
 824
 825         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 826             IXGBE_TX_TRYLOCK(txr)) {
 827                 err = ixgbe_mq_start_locked(ifp, txr, m);
 828                 IXGBE_TX_UNLOCK(txr);
 829         } else {
 830                 err = drbr_enqueue(ifp, txr->br, m);
 831                 taskqueue_enqueue(que->tq, &que->que_task);
 832         }
 833
 834         return (err);
 835 }
 836
 837 static int
 838 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 839 {
 840         struct adapter  *adapter = txr->adapter;
 841         struct mbuf     *next;
 842         int             enqueued, err = 0;
 843
 844         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 845             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 846             adapter->link_active == 0) {
 847                 if (m != NULL)
 848                         err = drbr_enqueue(ifp, txr->br, m);
 849                 return (err);
 850         }
 851
 852         enqueued = 0;
 853         if (m == NULL) {
 854                 next = drbr_dequeue(ifp, txr->br);
 855         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 856                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 857                         return (err);
 858                 next = drbr_dequeue(ifp, txr->br);
 859         } else
 860                 next = m;
 861
 862         /* Process the queue */
 863         while (next != NULL) {
 864                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 865                         if (next != NULL)
 866                                 err = drbr_enqueue(ifp, txr->br, next);
 867                         break;
 868                 }
 869                 enqueued++;
 870                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 871                 /* Send a copy of the frame to the BPF listener */
 872                 ETHER_BPF_MTAP(ifp, next);
 873                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 874                         break;
 875                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 876                         ixgbe_txeof(txr);
 877                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 878                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 879                         break;
 880                 }
 881                 next = drbr_dequeue(ifp, txr->br);
 882         }
 883
 884         if (enqueued > 0) {
 885                 /* Set watchdog on */
 886                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 887                 txr->watchdog_time = ticks;
 888         }
 889
 890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 891                 ixgbe_txeof(txr);
 892
 893         return (err);
 894 }
 895
 896 /*
 897 ** Flush all ring buffers
 898 */
 899 static void
 900 ixgbe_qflush(struct ifnet *ifp)
 901 {
 902         struct adapter  *adapter = ifp->if_softc;
 903         struct tx_ring  *txr = adapter->tx_rings;
 904         struct mbuf     *m;
 905
 906         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 907                 IXGBE_TX_LOCK(txr);
 908                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 909                         m_freem(m);
 910                 IXGBE_TX_UNLOCK(txr);
 911         }
 912         if_qflush(ifp);
 913 }
 914 #endif /* __FreeBSD_version >= 800000 */
 915
 916 /*********************************************************************
 917  *  Ioctl entry point
 918  *
 919  *  ixgbe_ioctl is called when the user wants to configure the
 920  *  interface.
 921  *
 922  *  return 0 on success, positive on failure
 923  **********************************************************************/
 924
 925 static int
 926 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 927 {
 928         struct adapter  *adapter = ifp->if_softc;
 929         struct ifreq    *ifr = (struct ifreq *) data;
 930 #if defined(INET) || defined(INET6)
 931         struct ifaddr *ifa = (struct ifaddr *)data;
 932         bool            avoid_reset = FALSE;
 933 #endif
 934         int             error = 0;
 935
 936         switch (command) {
 937
 938         case SIOCSIFADDR:
 939 #ifdef INET
 940                 if (ifa->ifa_addr->sa_family == AF_INET)
 941                         avoid_reset = TRUE;
 942 #endif
 943 #ifdef INET6
 944                 if (ifa->ifa_addr->sa_family == AF_INET6)
 945                         avoid_reset = TRUE;
 946 #endif
 947 #if defined(INET) || defined(INET6)
 948                 /*
 949                 ** Calling init results in link renegotiation,
 950                 ** so we avoid doing it when possible.
 951                 */
 952                 if (avoid_reset) {
 953                         ifp->if_flags |= IFF_UP;
 954                         if (!(ifp->if_flags & IFF_RUNNING))
 955                                 ixgbe_init(adapter);
 956                         if (!(ifp->if_flags & IFF_NOARP))
 957                                 arp_ifinit(ifp, ifa);
 958                 } else
 959                         error = ether_ioctl(ifp, command, data);
 960 #endif
 961                 break;
 962         case SIOCSIFMTU:
 963                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 964                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 965                         error = EINVAL;
 966                 } else {
 967                         IXGBE_CORE_LOCK(adapter);
 968                         ifp->if_mtu = ifr->ifr_mtu;
 969                         adapter->max_frame_size =
 970                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 971                         ixgbe_init_locked(adapter);
 972                         IXGBE_CORE_UNLOCK(adapter);
 973                 }
 974                 break;
 975         case SIOCSIFFLAGS:
 976                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 977                 IXGBE_CORE_LOCK(adapter);
 978                 if (ifp->if_flags & IFF_UP) {
 979                         if ((ifp->if_flags & IFF_RUNNING)) {
 980                                 if ((ifp->if_flags ^ adapter->if_flags) &
 981                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 982                                         ixgbe_set_promisc(adapter);
 983                                 }
 984                         } else
 985                                 ixgbe_init_locked(adapter);
 986                 } else
 987                         if (ifp->if_flags & IFF_RUNNING)
 988                                 ixgbe_stop(adapter);
 989                 adapter->if_flags = ifp->if_flags;
 990                 IXGBE_CORE_UNLOCK(adapter);
 991                 break;
 992         case SIOCADDMULTI:
 993         case SIOCDELMULTI:
 994                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 995                 if (ifp->if_flags & IFF_RUNNING) {
 996                         IXGBE_CORE_LOCK(adapter);
 997                         ixgbe_disable_intr(adapter);
 998                         ixgbe_set_multi(adapter);
 999                         ixgbe_enable_intr(adapter);
1000                         IXGBE_CORE_UNLOCK(adapter);
1001                 }
1002                 break;
1003         case SIOCSIFMEDIA:
1004         case SIOCGIFMEDIA:
1005                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1006                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1007                 break;
1008         case SIOCSIFCAP:
1009         {
1010                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1011                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1012                 if (mask & IFCAP_HWCSUM)
1013                         ifp->if_capenable ^= IFCAP_HWCSUM;
1014 #if 0 /* NET_TSO */
1015                 if (mask & IFCAP_TSO4)
1016                         ifp->if_capenable ^= IFCAP_TSO4;
1017                 if (mask & IFCAP_TSO6)
1018                         ifp->if_capenable ^= IFCAP_TSO6;
1019 #endif
1020 #if 0 /* NET_LRO */
1021                 if (mask & IFCAP_LRO)
1022                         ifp->if_capenable ^= IFCAP_LRO;
1023 #endif
1024                 if (mask & IFCAP_VLAN_HWTAGGING)
1025                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1026                 if (mask & IFCAP_VLAN_HWFILTER)
1027                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1028 #if 0 /* NET_TSO */
1029                 if (mask & IFCAP_VLAN_HWTSO)
1030                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1031 #endif
1032                 if (ifp->if_flags & IFF_RUNNING) {
1033                         IXGBE_CORE_LOCK(adapter);
1034                         ixgbe_init_locked(adapter);
1035                         IXGBE_CORE_UNLOCK(adapter);
1036                 }
1037 #if 0
1038                 VLAN_CAPABILITIES(ifp);
1039 #endif
1040                 break;
1041         }
1042
1043         default:
1044                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045                 error = ether_ioctl(ifp, command, data);
1046                 break;
1047         }
1048
1049         return (error);
1050 }
1051
1052 /*********************************************************************
1053  *  Init entry point
1054  *
1055  *  This routine is used in two ways. It is used by the stack as
1056  *  init entry point in network interface structure. It is also used
1057  *  by the driver as a hw/sw initialization routine to get to a
1058  *  consistent state.
1059  *
1060  *  return 0 on success, positive on failure
1061  **********************************************************************/
1062 #define IXGBE_MHADD_MFS_SHIFT 16
1063
1064 static void
1065 ixgbe_init_locked(struct adapter *adapter)
1066 {
1067         struct ifnet   *ifp = adapter->ifp;
1068         device_t        dev = adapter->dev;
1069         struct ixgbe_hw *hw = &adapter->hw;
1070         u32             k, txdctl, mhadd, gpie;
1071         u32             rxdctl, rxctrl;
1072
1073         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1074         INIT_DEBUGOUT("ixgbe_init: begin");
1075         hw->adapter_stopped = FALSE;
1076         ixgbe_stop_adapter(hw);
1077         callout_stop(&adapter->timer);
1078
1079         /* reprogram the RAR[0] in case user changed it. */
1080         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082         /* Get the latest mac address, User can use a LAA */
1083         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084               IXGBE_ETH_LENGTH_OF_ADDRESS);
1085         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086         hw->addr_ctrl.rar_used_count = 1;
1087
1088         /* Set the various hardware offload abilities */
1089         ifp->if_hwassist = 0;
1090 #if 0 /* NET_TSO */
1091         if (ifp->if_capenable & IFCAP_TSO)
1092                 ifp->if_hwassist |= CSUM_TSO;
1093 #endif
1094         if (ifp->if_capenable & IFCAP_TXCSUM) {
1095                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1096 #if 0
1097                 if (hw->mac.type != ixgbe_mac_82598EB)
1098                         ifp->if_hwassist |= CSUM_SCTP;
1099 #endif
1100         }
1101
1102         /* Prepare transmit descriptors and buffers */
1103         if (ixgbe_setup_transmit_structures(adapter)) {
1104                 device_printf(dev,"Could not setup transmit structures\n");
1105                 ixgbe_stop(adapter);
1106                 return;
1107         }
1108
1109         ixgbe_init_hw(hw);
1110         ixgbe_initialize_transmit_units(adapter);
1111
1112         /* Setup Multicast table */
1113         ixgbe_set_multi(adapter);
1114
1115         /*
1116         ** Determine the correct mbuf pool
1117         ** for doing jumbo/headersplit
1118         */
1119         if (adapter->max_frame_size <= 2048)
1120                 adapter->rx_mbuf_sz = MCLBYTES;
1121         else if (adapter->max_frame_size <= 4096)
1122                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1123         else if (adapter->max_frame_size <= 9216)
1124                 adapter->rx_mbuf_sz = MJUM9BYTES;
1125         else
1126                 adapter->rx_mbuf_sz = MJUM16BYTES;
1127
1128         /* Prepare receive descriptors and buffers */
1129         if (ixgbe_setup_receive_structures(adapter)) {
1130                 device_printf(dev,"Could not setup receive structures\n");
1131                 ixgbe_stop(adapter);
1132                 return;
1133         }
1134
1135         /* Configure RX settings */
1136         ixgbe_initialize_receive_units(adapter);
1137
1138         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1139
1140         /* Enable Fan Failure Interrupt */
1141         gpie |= IXGBE_SDP1_GPIEN;
1142
1143         /* Add for Module detection */
1144         if (hw->mac.type == ixgbe_mac_82599EB)
1145                 gpie |= IXGBE_SDP2_GPIEN;
1146
1147         /* Thermal Failure Detection */
1148         if (hw->mac.type == ixgbe_mac_X540)
1149                 gpie |= IXGBE_SDP0_GPIEN;
1150
1151         if (adapter->msix > 1) {
1152                 /* Enable Enhanced MSIX mode */
1153                 gpie |= IXGBE_GPIE_MSIX_MODE;
1154                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155                     IXGBE_GPIE_OCD;
1156         }
1157         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158
1159         /* Set MTU size */
1160         if (ifp->if_mtu > ETHERMTU) {
1161                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1162                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1163                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1164                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165         }
1166
1167         /* Now enable all the queues */
1168
1169         for (int i = 0; i < adapter->num_queues; i++) {
1170                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1171                 txdctl |= IXGBE_TXDCTL_ENABLE;
1172                 /* Set WTHRESH to 8, burst writeback */
1173                 txdctl |= (8 << 16);
1174                 /*
1175                  * When the internal queue falls below PTHRESH (32),
1176                  * start prefetching as long as there are at least
1177                  * HTHRESH (1) buffers ready. The values are taken
1178                  * from the Intel linux driver 3.8.21.
1179                  * Prefetching enables tx line rate even with 1 queue.
1180                  */
1181                 txdctl |= (32 << 0) | (1 << 8);
1182                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183         }
1184
1185         for (int i = 0; i < adapter->num_queues; i++) {
1186                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1187                 if (hw->mac.type == ixgbe_mac_82598EB) {
1188                         /*
1189                         ** PTHRESH = 21
1190                         ** HTHRESH = 4
1191                         ** WTHRESH = 8
1192                         */
1193                         rxdctl &= ~0x3FFFFF;
1194                         rxdctl |= 0x080420;
1195                 }
1196                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1197                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1198                 for (k = 0; k < 10; k++) {
1199                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1200                             IXGBE_RXDCTL_ENABLE)
1201                                 break;
1202                         else
1203                                 msec_delay(1);
1204                 }
1205                 wmb();
1206 #ifdef DEV_NETMAP
1207                 /*
1208                  * In netmap mode, we must preserve the buffers made
1209                  * available to userspace before the if_init()
1210                  * (this is true by default on the TX side, because
1211                  * init makes all buffers available to userspace).
1212                  *
1213                  * netmap_reset() and the device specific routines
1214                  * (e.g. ixgbe_setup_receive_rings()) map these
1215                  * buffers at the end of the NIC ring, so here we
1216                  * must set the RDT (tail) register to make sure
1217                  * they are not overwritten.
1218                  *
1219                  * In this driver the NIC ring starts at RDH = 0,
1220                  * RDT points to the last slot available for reception (?),
1221                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1222                  */
1223                 if (ifp->if_capenable & IFCAP_NETMAP) {
1224                         struct netmap_adapter *na = NA(adapter->ifp);
1225                         struct netmap_kring *kring = &na->rx_rings[i];
1226                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1227
1228                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1229                 } else
1230 #endif /* DEV_NETMAP */
1231                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232         }
1233
1234         /* Set up VLAN support and filter */
1235         ixgbe_setup_vlan_hw_support(adapter);
1236
1237         /* Enable Receive engine */
1238         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1239         if (hw->mac.type == ixgbe_mac_82598EB)
1240                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1241         rxctrl |= IXGBE_RXCTRL_RXEN;
1242         ixgbe_enable_rx_dma(hw, rxctrl);
1243
1244         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1245
1246         /* Set up MSI/X routing */
1247         if (ixgbe_enable_msix)  {
1248                 ixgbe_configure_ivars(adapter);
1249                 /* Set up auto-mask */
1250                 if (hw->mac.type == ixgbe_mac_82598EB)
1251                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1252                 else {
1253                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1255                 }
1256         } else {  /* Simple settings for Legacy/MSI */
1257                 ixgbe_set_ivar(adapter, 0, 0, 0);
1258                 ixgbe_set_ivar(adapter, 0, 0, 1);
1259                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260         }
1261
1262 #ifdef IXGBE_FDIR
1263         /* Init Flow director */
1264         if (hw->mac.type != ixgbe_mac_82598EB) {
1265                 u32 hdrm = 32 << fdir_pballoc;
1266
1267                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1268                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269         }
1270 #endif
1271
1272         /*
1273         ** Check on any SFP devices that
1274         ** need to be kick-started
1275         */
1276         if (hw->phy.type == ixgbe_phy_none) {
1277                 int err = hw->phy.ops.identify(hw);
1278                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1279                         device_printf(dev,
1280                             "Unsupported SFP+ module type was detected.\n");
1281                         return;
1282                 }
1283         }
1284
1285         /* Set moderation on the Link interrupt */
1286         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1287
1288         /* Config/Enable Link */
1289         ixgbe_config_link(adapter);
1290
1291         /* Hardware Packet Buffer & Flow Control setup */
1292         {
1293                 u32 rxpb, frame, size, tmp;
1294
1295                 frame = adapter->max_frame_size;
1296
1297                 /* Calculate High Water */
1298                 if (hw->mac.type == ixgbe_mac_X540)
1299                         tmp = IXGBE_DV_X540(frame, frame);
1300                 else
1301                         tmp = IXGBE_DV(frame, frame);
1302                 size = IXGBE_BT2KB(tmp);
1303                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1304                 hw->fc.high_water[0] = rxpb - size;
1305
1306                 /* Now calculate Low Water */
1307                 if (hw->mac.type == ixgbe_mac_X540)
1308                         tmp = IXGBE_LOW_DV_X540(frame);
1309                 else
1310                         tmp = IXGBE_LOW_DV(frame);
1311                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1312
1313                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1314                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1315                 hw->fc.send_xon = TRUE;
1316         }
1317         /* Initialize the FC settings */
1318         ixgbe_start_hw(hw);
1319
1320         /* And now turn on interrupts */
1321         ixgbe_enable_intr(adapter);
1322
1323         /* Now inform the stack we're ready */
1324         ifp->if_flags |= IFF_RUNNING;
1325         ifp->if_flags &= ~IFF_OACTIVE;
1326
1327         return;
1328 }
1329
1330 static void
1331 ixgbe_init(void *arg)
1332 {
1333         struct adapter *adapter = arg;
1334
1335         IXGBE_CORE_LOCK(adapter);
1336         ixgbe_init_locked(adapter);
1337         IXGBE_CORE_UNLOCK(adapter);
1338         return;
1339 }
1340
1341
1342 /*
1343 **
1344 ** MSIX Interrupt Handlers and Tasklets
1345 **
1346 */
1347
1348 static inline void
1349 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1350 {
1351         struct ixgbe_hw *hw = &adapter->hw;
1352         u64     queue = (u64)(1 << vector);
1353         u32     mask;
1354
1355         if (hw->mac.type == ixgbe_mac_82598EB) {
1356                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1357                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1358         } else {
1359                 mask = (queue & 0xFFFFFFFF);
1360                 if (mask)
1361                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1362                 mask = (queue >> 32);
1363                 if (mask)
1364                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365         }
1366 }
1367
1368 static inline void
1369 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1370 {
1371         struct ixgbe_hw *hw = &adapter->hw;
1372         u64     queue = (u64)(1 << vector);
1373         u32     mask;
1374
1375         if (hw->mac.type == ixgbe_mac_82598EB) {
1376                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1377                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1378         } else {
1379                 mask = (queue & 0xFFFFFFFF);
1380                 if (mask)
1381                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1382                 mask = (queue >> 32);
1383                 if (mask)
1384                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385         }
1386 }
1387
1388 static inline void
1389 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390 {
1391         u32 mask;
1392
1393         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1394                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1395                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1396         } else {
1397                 mask = (queues & 0xFFFFFFFF);
1398                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1399                 mask = (queues >> 32);
1400                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1401         }
1402 }
1403
1404
1405 static void
1406 ixgbe_handle_que(void *context, int pending)
1407 {
1408         struct ix_queue *que = context;
1409         struct adapter  *adapter = que->adapter;
1410         struct tx_ring  *txr = que->txr;
1411         struct ifnet    *ifp = adapter->ifp;
1412         bool            more;
1413
1414         if (ifp->if_flags & IFF_RUNNING) {
1415                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1416                 IXGBE_TX_LOCK(txr);
1417                 ixgbe_txeof(txr);
1418 #if 0 /*__FreeBSD_version >= 800000*/
1419                 if (!drbr_empty(ifp, txr->br))
1420                         ixgbe_mq_start_locked(ifp, txr, NULL);
1421 #else
1422                 if (!ifq_is_empty(&ifp->if_snd))
1423                         ixgbe_start_locked(txr, ifp);
1424 #endif
1425                 IXGBE_TX_UNLOCK(txr);
1426                 if (more) {
1427                         taskqueue_enqueue(que->tq, &que->que_task);
1428                         return;
1429                 }
1430         }
1431
1432         /* Reenable this interrupt */
1433         ixgbe_enable_queue(adapter, que->msix);
1434         return;
1435 }
1436
1437
1438 /*********************************************************************
1439  *
1440  *  Legacy Interrupt Service routine
1441  *
1442  **********************************************************************/
1443
1444 static void
1445 ixgbe_legacy_irq(void *arg)
1446 {
1447         struct ix_queue *que = arg;
1448         struct adapter  *adapter = que->adapter;
1449         struct ixgbe_hw *hw = &adapter->hw;
1450         struct          tx_ring *txr = adapter->tx_rings;
1451         bool            more_tx, more_rx;
1452         u32             reg_eicr, loop = MAX_LOOP;
1453
1454
1455         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1456
1457         ++que->irqs;
1458         if (reg_eicr == 0) {
1459                 ixgbe_enable_intr(adapter);
1460                 return;
1461         }
1462
1463         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1464
1465         IXGBE_TX_LOCK(txr);
1466         do {
1467                 more_tx = ixgbe_txeof(txr);
1468         } while (loop-- && more_tx);
1469         IXGBE_TX_UNLOCK(txr);
1470
1471         if (more_rx || more_tx)
1472                 taskqueue_enqueue(que->tq, &que->que_task);
1473
1474         /* Check for fan failure */
1475         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1476             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1477                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1478                     "REPLACE IMMEDIATELY!!\n");
1479                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1480         }
1481
1482         /* Link status change */
1483         if (reg_eicr & IXGBE_EICR_LSC)
1484                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1485
1486         ixgbe_enable_intr(adapter);
1487         return;
1488 }
1489
1490
1491 /*********************************************************************
1492  *
1493  *  MSIX Queue Interrupt Service routine
1494  *
1495  **********************************************************************/
1496 void
1497 ixgbe_msix_que(void *arg)
1498 {
1499         struct ix_queue *que = arg;
1500         struct adapter  *adapter = que->adapter;
1501         struct tx_ring  *txr = que->txr;
1502         struct rx_ring  *rxr = que->rxr;
1503         bool            more_tx, more_rx;
1504         u32             newitr = 0;
1505
1506         ixgbe_disable_queue(adapter, que->msix);
1507         ++que->irqs;
1508
1509         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1510
1511         IXGBE_TX_LOCK(txr);
1512         more_tx = ixgbe_txeof(txr);
1513         /*
1514         ** Make certain that if the stack
1515         ** has anything queued the task gets
1516         ** scheduled to handle it.
1517         */
1518 #if 0
1519 #if __FreeBSD_version < 800000
1520         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521 #else
1522         if (!drbr_empty(adapter->ifp, txr->br))
1523 #endif
1524 #endif
1525         if (!ifq_is_empty(&adapter->ifp->if_snd))
1526                 more_tx = 1;
1527         IXGBE_TX_UNLOCK(txr);
1528
1529         /* Do AIM now? */
1530
1531         if (ixgbe_enable_aim == FALSE)
1532                 goto no_calc;
1533         /*
1534         ** Do Adaptive Interrupt Moderation:
1535         **  - Write out last calculated setting
1536         **  - Calculate based on average size over
1537         **    the last interval.
1538         */
1539         if (que->eitr_setting)
1540                 IXGBE_WRITE_REG(&adapter->hw,
1541                     IXGBE_EITR(que->msix), que->eitr_setting);
1542
1543         que->eitr_setting = 0;
1544
1545         /* Idle, do nothing */
1546         if ((txr->bytes == 0) && (rxr->bytes == 0))
1547                 goto no_calc;
1548
1549         if ((txr->bytes) && (txr->packets))
1550                 newitr = txr->bytes/txr->packets;
1551         if ((rxr->bytes) && (rxr->packets))
1552                 newitr = max(newitr,
1553                     (rxr->bytes / rxr->packets));
1554         newitr += 24; /* account for hardware frame, crc */
1555
1556         /* set an upper boundary */
1557         newitr = min(newitr, 3000);
1558
1559         /* Be nice to the mid range */
1560         if ((newitr > 300) && (newitr < 1200))
1561                 newitr = (newitr / 3);
1562         else
1563                 newitr = (newitr / 2);
1564
1565         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1566                 newitr |= newitr << 16;
1567         else
1568                 newitr |= IXGBE_EITR_CNT_WDIS;
1569
1570         /* save for next interrupt */
1571         que->eitr_setting = newitr;
1572
1573         /* Reset state */
1574         txr->bytes = 0;
1575         txr->packets = 0;
1576         rxr->bytes = 0;
1577         rxr->packets = 0;
1578
1579 no_calc:
1580         if (more_tx || more_rx)
1581                 taskqueue_enqueue(que->tq, &que->que_task);
1582         else /* Reenable this interrupt */
1583                 ixgbe_enable_queue(adapter, que->msix);
1584         return;
1585 }
1586
1587
1588 static void
1589 ixgbe_msix_link(void *arg)
1590 {
1591         struct adapter  *adapter = arg;
1592         struct ixgbe_hw *hw = &adapter->hw;
1593         u32             reg_eicr;
1594
1595         ++adapter->link_irq;
1596
1597         /* First get the cause */
1598         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1599         /* Clear interrupt with write */
1600         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1601
1602         /* Link status change */
1603         if (reg_eicr & IXGBE_EICR_LSC)
1604                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1605
1606         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1607 #ifdef IXGBE_FDIR
1608                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1609                         /* This is probably overkill :) */
1610                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1611                                 return;
1612                         /* Disable the interrupt */
1613                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1614                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1615                 } else
1616 #endif
1617                 if (reg_eicr & IXGBE_EICR_ECC) {
1618                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1619                             "Please Reboot!!\n");
1620                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1621                 } else
1622
1623                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1624                         /* Clear the interrupt */
1625                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1626                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1627                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1628                         /* Clear the interrupt */
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1630                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1631                 }
1632         }
1633
1634         /* Check for fan failure */
1635         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1636             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1637                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1638                     "REPLACE IMMEDIATELY!!\n");
1639                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1640         }
1641
1642         /* Check for over temp condition */
1643         if ((hw->mac.type == ixgbe_mac_X540) &&
1644             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1645                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1646                     "PHY IS SHUT DOWN!!\n");
1647                 device_printf(adapter->dev, "System shutdown required\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1649         }
1650
1651         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1652         return;
1653 }
1654
1655 /*********************************************************************
1656  *
1657  *  Media Ioctl callback
1658  *
1659  *  This routine is called whenever the user queries the status of
1660  *  the interface using ifconfig.
1661  *
1662  **********************************************************************/
1663 static void
1664 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1665 {
1666         struct adapter *adapter = ifp->if_softc;
1667
1668         INIT_DEBUGOUT("ixgbe_media_status: begin");
1669         IXGBE_CORE_LOCK(adapter);
1670         ixgbe_update_link_status(adapter);
1671
1672         ifmr->ifm_status = IFM_AVALID;
1673         ifmr->ifm_active = IFM_ETHER;
1674
1675         if (!adapter->link_active) {
1676                 IXGBE_CORE_UNLOCK(adapter);
1677                 return;
1678         }
1679
1680         ifmr->ifm_status |= IFM_ACTIVE;
1681
1682         switch (adapter->link_speed) {
1683                 case IXGBE_LINK_SPEED_100_FULL:
1684                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1685                         break;
1686                 case IXGBE_LINK_SPEED_1GB_FULL:
1687                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1688                         break;
1689                 case IXGBE_LINK_SPEED_10GB_FULL:
1690                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1691                         break;
1692         }
1693
1694         IXGBE_CORE_UNLOCK(adapter);
1695
1696         return;
1697 }
1698
1699 /*********************************************************************
1700  *
1701  *  Media Ioctl callback
1702  *
1703  *  This routine is called when the user changes speed/duplex using
1704  *  media/mediopt option with ifconfig.
1705  *
1706  **********************************************************************/
1707 static int
1708 ixgbe_media_change(struct ifnet * ifp)
1709 {
1710         struct adapter *adapter = ifp->if_softc;
1711         struct ifmedia *ifm = &adapter->media;
1712
1713         INIT_DEBUGOUT("ixgbe_media_change: begin");
1714
1715         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1716                 return (EINVAL);
1717
1718         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1719         case IFM_AUTO:
1720                 adapter->hw.phy.autoneg_advertised =
1721                     IXGBE_LINK_SPEED_100_FULL |
1722                     IXGBE_LINK_SPEED_1GB_FULL |
1723                     IXGBE_LINK_SPEED_10GB_FULL;
1724                 break;
1725         default:
1726                 device_printf(adapter->dev, "Only auto media type\n");
1727                 return (EINVAL);
1728         }
1729
1730         return (0);
1731 }
1732
1733 /*********************************************************************
1734  *
1735  *  This routine maps the mbufs to tx descriptors, allowing the
1736  *  TX engine to transmit the packets.
1737  *      - return 0 on success, positive on failure
1738  *
1739  **********************************************************************/
1740
1741 static int
1742 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1743 {
1744         struct adapter  *adapter = txr->adapter;
1745         u32             olinfo_status = 0, cmd_type_len;
1746         u32             paylen = 0;
1747         int             i, j, error, nsegs;
1748         int             first, last = 0;
1749         struct mbuf     *m_head;
1750         bus_dma_segment_t segs[1];
1751         bus_dmamap_t    map;
1752         struct ixgbe_tx_buf *txbuf;
1753         union ixgbe_adv_tx_desc *txd = NULL;
1754
1755         m_head = *m_headp;
1756
1757         /* Basic descriptor defines */
1758         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1759             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1760
1761         if (m_head->m_flags & M_VLANTAG)
1762                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1763
1764         /*
1765          * Important to capture the first descriptor
1766          * used because it will contain the index of
1767          * the one we tell the hardware to report back
1768          */
1769         first = txr->next_avail_desc;
1770         txbuf = &txr->tx_buffers[first];
1771         map = txbuf->map;
1772
1773         /*
1774          * Map the packet for DMA.
1775          */
1776         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1777             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1778
1779         if (error == EFBIG) {
1780                 struct mbuf *m;
1781
1782                 m = m_defrag(*m_headp, MB_DONTWAIT);
1783                 if (m == NULL) {
1784                         adapter->mbuf_defrag_failed++;
1785                         m_freem(*m_headp);
1786                         *m_headp = NULL;
1787                         return (ENOBUFS);
1788                 }
1789                 *m_headp = m;
1790
1791                 /* Try it again */
1792                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1793                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1794
1795                 if (error == ENOMEM) {
1796                         adapter->no_tx_dma_setup++;
1797                         return (error);
1798                 } else if (error != 0) {
1799                         adapter->no_tx_dma_setup++;
1800                         m_freem(*m_headp);
1801                         *m_headp = NULL;
1802                         return (error);
1803                 }
1804         } else if (error == ENOMEM) {
1805                 adapter->no_tx_dma_setup++;
1806                 return (error);
1807         } else if (error != 0) {
1808                 adapter->no_tx_dma_setup++;
1809                 m_freem(*m_headp);
1810                 *m_headp = NULL;
1811                 return (error);
1812         }
1813
1814         /* Make certain there are enough descriptors */
1815         if (nsegs > txr->tx_avail - 2) {
1816                 txr->no_desc_avail++;
1817                 error = ENOBUFS;
1818                 goto xmit_fail;
1819         }
1820         m_head = *m_headp;
1821
1822         /*
1823         ** Set up the appropriate offload context
1824         ** this becomes the first descriptor of
1825         ** a packet.
1826         */
1827 #if 0 /* NET_TSO */
1828         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1829                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1830                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1831                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1832                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1833                         ++adapter->tso_tx;
1834                 } else
1835                         return (ENXIO);
1836         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1837 #endif
1838         if (ixgbe_tx_ctx_setup(txr, m_head))
1839                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1840
1841 #ifdef IXGBE_IEEE1588
1842         /* This is changing soon to an mtag detection */
1843         if (we detect this mbuf has a TSTAMP mtag)
1844                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1845 #endif
1846
1847 #ifdef IXGBE_FDIR
1848         /* Do the flow director magic */
1849         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1850                 ++txr->atr_count;
1851                 if (txr->atr_count >= atr_sample_rate) {
1852                         ixgbe_atr(txr, m_head);
1853                         txr->atr_count = 0;
1854                 }
1855         }
1856 #endif
1857         /* Record payload length */
1858         if (paylen == 0)
1859                 olinfo_status |= m_head->m_pkthdr.len <<
1860                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1861
1862         i = txr->next_avail_desc;
1863         for (j = 0; j < nsegs; j++) {
1864                 bus_size_t seglen;
1865                 bus_addr_t segaddr;
1866
1867                 txbuf = &txr->tx_buffers[i];
1868                 txd = &txr->tx_base[i];
1869                 seglen = segs[j].ds_len;
1870                 segaddr = htole64(segs[j].ds_addr);
1871
1872                 txd->read.buffer_addr = segaddr;
1873                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1874                     cmd_type_len |seglen);
1875                 txd->read.olinfo_status = htole32(olinfo_status);
1876                 last = i; /* descriptor that will get completion IRQ */
1877
1878                 if (++i == adapter->num_tx_desc)
1879                         i = 0;
1880
1881                 txbuf->m_head = NULL;
1882                 txbuf->eop_index = -1;
1883         }
1884
1885         txd->read.cmd_type_len |=
1886             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1887         txr->tx_avail -= nsegs;
1888         txr->next_avail_desc = i;
1889
1890         txbuf->m_head = m_head;
1891         /* Swap the dma map between the first and last descriptor */
1892         txr->tx_buffers[first].map = txbuf->map;
1893         txbuf->map = map;
1894         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1895
1896         /* Set the index of the descriptor that will be marked done */
1897         txbuf = &txr->tx_buffers[first];
1898         txbuf->eop_index = last;
1899
1900         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1901             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1902         /*
1903          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1904          * hardware that this frame is available to transmit.
1905          */
1906         ++txr->total_packets;
1907         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1908
1909         return (0);
1910
1911 xmit_fail:
1912         bus_dmamap_unload(txr->txtag, txbuf->map);
1913         return (error);
1914
1915 }
1916
1917 static void
1918 ixgbe_set_promisc(struct adapter *adapter)
1919 {
1920         u_int32_t       reg_rctl;
1921         struct ifnet   *ifp = adapter->ifp;
1922
1923         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1924         reg_rctl &= (~IXGBE_FCTRL_UPE);
1925         reg_rctl &= (~IXGBE_FCTRL_MPE);
1926         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1927
1928         if (ifp->if_flags & IFF_PROMISC) {
1929                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1930                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1931         } else if (ifp->if_flags & IFF_ALLMULTI) {
1932                 reg_rctl |= IXGBE_FCTRL_MPE;
1933                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1934                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1935         }
1936         return;
1937 }
1938
1939
1940 /*********************************************************************
1941  *  Multicast Update
1942  *
1943  *  This routine is called whenever multicast address list is updated.
1944  *
1945  **********************************************************************/
1946 #define IXGBE_RAR_ENTRIES 16
1947
1948 static void
1949 ixgbe_set_multi(struct adapter *adapter)
1950 {
1951         u32     fctrl;
1952         u8      *mta;
1953         u8      *update_ptr;
1954         struct  ifmultiaddr *ifma;
1955         int     mcnt = 0;
1956         struct ifnet   *ifp = adapter->ifp;
1957
1958         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1959
1960         mta = adapter->mta;
1961         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1962             MAX_NUM_MULTICAST_ADDRESSES);
1963
1964         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1965         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1966         if (ifp->if_flags & IFF_PROMISC)
1967                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1968         else if (ifp->if_flags & IFF_ALLMULTI) {
1969                 fctrl |= IXGBE_FCTRL_MPE;
1970                 fctrl &= ~IXGBE_FCTRL_UPE;
1971         } else
1972                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1973
1974         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1975
1976         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1977                 if (ifma->ifma_addr->sa_family != AF_LINK)
1978                         continue;
1979                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1980                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1981                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1982                 mcnt++;
1983         }
1984
1985         update_ptr = mta;
1986         ixgbe_update_mc_addr_list(&adapter->hw,
1987             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1988
1989         return;
1990 }
1991
1992 /*
1993  * This is an iterator function now needed by the multicast
1994  * shared code. It simply feeds the shared code routine the
1995  * addresses in the array of ixgbe_set_multi() one by one.
1996  */
1997 static u8 *
1998 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1999 {
2000         u8 *addr = *update_ptr;
2001         u8 *newptr;
2002         *vmdq = 0;
2003
2004         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2005         *update_ptr = newptr;
2006         return addr;
2007 }
2008
2009
2010 /*********************************************************************
2011  *  Timer routine
2012  *
2013  *  This routine checks for link status,updates statistics,
2014  *  and runs the watchdog check.
2015  *
2016  **********************************************************************/
2017
2018 static void
2019 ixgbe_local_timer(void *arg)
2020 {
2021         struct adapter  *adapter = arg;
2022         device_t        dev = adapter->dev;
2023         struct ifnet    *ifp = adapter->ifp;
2024         struct ix_queue *que = adapter->queues;
2025         struct tx_ring  *txr = adapter->tx_rings;
2026         int             hung, busy, paused;
2027
2028         IXGBE_CORE_LOCK(adapter);
2029         hung = busy = paused = 0;
2030
2031         /* Check for pluggable optics */
2032         if (adapter->sfp_probe)
2033                 if (!ixgbe_sfp_probe(adapter))
2034                         goto out; /* Nothing to do */
2035
2036         ixgbe_update_link_status(adapter);
2037         ixgbe_update_stats_counters(adapter);
2038
2039         /*
2040          * If the interface has been paused
2041          * then don't do the watchdog check
2042          */
2043         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2044                 paused = 1;
2045
2046         /*
2047         ** Check the TX queues status
2048         **      - central locked handling of OACTIVE
2049         **      - watchdog only if all queues show hung
2050         */
2051         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2052                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2053                     (paused == 0))
2054                         ++hung;
2055                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2056                         ++busy;
2057                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2058                         taskqueue_enqueue(que->tq, &que->que_task);
2059         }
2060         /* Only truely watchdog if all queues show hung */
2061         if (hung == adapter->num_queues)
2062                 goto watchdog;
2063         /* Only turn off the stack flow when ALL are depleted */
2064         if (busy == adapter->num_queues)
2065                 ifp->if_flags |= IFF_OACTIVE;
2066         else if ((ifp->if_flags & IFF_OACTIVE) &&
2067             (busy < adapter->num_queues))
2068                 ifp->if_flags &= ~IFF_OACTIVE;
2069
2070 out:
2071         ixgbe_rearm_queues(adapter, adapter->que_mask);
2072         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2073         IXGBE_CORE_UNLOCK(adapter);
2074         return;
2075
2076 watchdog:
2077         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2078         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2079             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2080             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2081         device_printf(dev,"TX(%d) desc avail = %d,"
2082             "Next TX to Clean = %d\n",
2083             txr->me, txr->tx_avail, txr->next_to_clean);
2084         adapter->ifp->if_flags &= ~IFF_RUNNING;
2085         adapter->watchdog_events++;
2086         ixgbe_init_locked(adapter);
2087
2088         IXGBE_CORE_UNLOCK(adapter);
2089 }
2090
2091 /*
2092 ** Note: this routine updates the OS on the link state
2093 **      the real check of the hardware only happens with
2094 **      a link interrupt.
2095 */
2096 static void
2097 ixgbe_update_link_status(struct adapter *adapter)
2098 {
2099         struct ifnet    *ifp = adapter->ifp;
2100         struct tx_ring *txr = adapter->tx_rings;
2101         device_t dev = adapter->dev;
2102
2103
2104         if (adapter->link_up){
2105                 if (adapter->link_active == FALSE) {
2106                         if (bootverbose)
2107                                 device_printf(dev,"Link is up %d Gbps %s \n",
2108                                     ((adapter->link_speed == 128)? 10:1),
2109                                     "Full Duplex");
2110                         adapter->link_active = TRUE;
2111                         /* Update any Flow Control changes */
2112                         ixgbe_fc_enable(&adapter->hw);
2113                         ifp->if_link_state = LINK_STATE_UP;
2114                         if_link_state_change(ifp);
2115                 }
2116         } else { /* Link down */
2117                 if (adapter->link_active == TRUE) {
2118                         if (bootverbose)
2119                                 device_printf(dev,"Link is Down\n");
2120                         ifp->if_link_state = LINK_STATE_DOWN;
2121                         if_link_state_change(ifp);
2122                         adapter->link_active = FALSE;
2123                         for (int i = 0; i < adapter->num_queues;
2124                             i++, txr++)
2125                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2126                 }
2127         }
2128
2129         return;
2130 }
2131
2132
2133 /*********************************************************************
2134  *
2135  *  This routine disables all traffic on the adapter by issuing a
2136  *  global reset on the MAC and deallocates TX/RX buffers.
2137  *
2138  **********************************************************************/
2139
2140 static void
2141 ixgbe_stop(void *arg)
2142 {
2143         struct ifnet   *ifp;
2144         struct adapter *adapter = arg;
2145         struct ixgbe_hw *hw = &adapter->hw;
2146         ifp = adapter->ifp;
2147
2148         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2149
2150         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2151         ixgbe_disable_intr(adapter);
2152         callout_stop(&adapter->timer);
2153
2154         /* Let the stack know...*/
2155         ifp->if_flags &= ~IFF_RUNNING;
2156         ifp->if_flags |= IFF_OACTIVE;
2157
2158         ixgbe_reset_hw(hw);
2159         hw->adapter_stopped = FALSE;
2160         ixgbe_stop_adapter(hw);
2161         /* Turn off the laser */
2162         if (hw->phy.multispeed_fiber)
2163                 ixgbe_disable_tx_laser(hw);
2164
2165         /* reprogram the RAR[0] in case user changed it. */
2166         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2167
2168         return;
2169 }
2170
2171
2172 /*********************************************************************
2173  *
2174  *  Determine hardware revision.
2175  *
2176  **********************************************************************/
2177 static void
2178 ixgbe_identify_hardware(struct adapter *adapter)
2179 {
2180         device_t        dev = adapter->dev;
2181         struct ixgbe_hw *hw = &adapter->hw;
2182
2183         /* Save off the information about this board */
2184         hw->vendor_id = pci_get_vendor(dev);
2185         hw->device_id = pci_get_device(dev);
2186         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2187         hw->subsystem_vendor_id =
2188             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2189         hw->subsystem_device_id =
2190             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2191
2192         /* We need this here to set the num_segs below */
2193         ixgbe_set_mac_type(hw);
2194
2195         /* Pick up the 82599 and VF settings */
2196         if (hw->mac.type != ixgbe_mac_82598EB) {
2197                 hw->phy.smart_speed = ixgbe_smart_speed;
2198                 adapter->num_segs = IXGBE_82599_SCATTER;
2199         } else
2200                 adapter->num_segs = IXGBE_82598_SCATTER;
2201
2202         return;
2203 }
2204
2205 /*********************************************************************
2206  *
2207  *  Determine optic type
2208  *
2209  **********************************************************************/
2210 static void
2211 ixgbe_setup_optics(struct adapter *adapter)
2212 {
2213         struct ixgbe_hw *hw = &adapter->hw;
2214         int             layer;
2215
2216         layer = ixgbe_get_supported_physical_layer(hw);
2217
2218         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2219                 adapter->optics = IFM_10G_T;
2220                 return;
2221         }
2222
2223         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2224                 adapter->optics = IFM_1000_T;
2225                 return;
2226         }
2227
2228         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2229             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2230                 adapter->optics = IFM_10G_LR;
2231                 return;
2232         }
2233
2234         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2235                 adapter->optics = IFM_10G_SR;
2236                 return;
2237         }
2238
2239         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2240                 adapter->optics = IFM_10G_TWINAX;
2241                 return;
2242         }
2243
2244         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2245             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2246                 adapter->optics = IFM_10G_CX4;
2247                 return;
2248         }
2249
2250         /* If we get here just set the default */
2251         adapter->optics = IFM_ETHER | IFM_AUTO;
2252         return;
2253 }
2254
2255 /*********************************************************************
2256  *
2257  *  Setup the Legacy or MSI Interrupt handler
2258  *
2259  **********************************************************************/
2260 static int
2261 ixgbe_allocate_legacy(struct adapter *adapter)
2262 {
2263         device_t dev = adapter->dev;
2264         struct          ix_queue *que = adapter->queues;
2265         int error, rid = 0;
2266         unsigned int intr_flags;
2267
2268         /* MSI RID at 1 */
2269         if (adapter->msix == 1)
2270                 rid = 1;
2271
2272         /* Try allocating a MSI interrupt first */
2273         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2274                 &rid, &intr_flags);
2275
2276         /* We allocate a single interrupt resource */
2277         adapter->res = bus_alloc_resource_any(dev,
2278             SYS_RES_IRQ, &rid, intr_flags);
2279         if (adapter->res == NULL) {
2280                 device_printf(dev, "Unable to allocate bus resource: "
2281                     "interrupt\n");
2282                 return (ENXIO);
2283         }
2284
2285         /*
2286          * Try allocating a fast interrupt and the associated deferred
2287          * processing contexts.
2288          */
2289         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2290         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2291             taskqueue_thread_enqueue, &que->tq);
2292         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2293             device_get_nameunit(adapter->dev));
2294
2295         /* Tasklets for Link, SFP and Multispeed Fiber */
2296         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2297         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2298         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2299 #ifdef IXGBE_FDIR
2300         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2301 #endif
2302         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2303             taskqueue_thread_enqueue, &adapter->tq);
2304         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2305             device_get_nameunit(adapter->dev));
2306
2307         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2308             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2309                 device_printf(dev, "Failed to register fast interrupt "
2310                     "handler: %d\n", error);
2311                 taskqueue_free(que->tq);
2312                 taskqueue_free(adapter->tq);
2313                 que->tq = NULL;
2314                 adapter->tq = NULL;
2315                 return (error);
2316         }
2317         /* For simplicity in the handlers */
2318         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2319
2320         return (0);
2321 }
2322
2323
2324 /*********************************************************************
2325  *
2326  *  Setup MSIX Interrupt resources and handlers
2327  *
2328  **********************************************************************/
2329 static int
2330 ixgbe_allocate_msix(struct adapter *adapter)
2331 {
2332         device_t        dev = adapter->dev;
2333         struct          ix_queue *que = adapter->queues;
2334         int             error, rid, vector = 0;
2335
2336         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2337                 rid = vector + 1;
2338                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2339                     RF_SHAREABLE | RF_ACTIVE);
2340                 if (que->res == NULL) {
2341                         device_printf(dev,"Unable to allocate"
2342                             " bus resource: que interrupt [%d]\n", vector);
2343                         return (ENXIO);
2344                 }
2345                 /* Set the handler function */
2346                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2347                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2348                 if (error) {
2349                         que->res = NULL;
2350                         device_printf(dev, "Failed to register QUE handler");
2351                         return (error);
2352                 }
2353 #if 0 /* __FreeBSD_version >= 800504 */
2354                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2355 #endif
2356                 que->msix = vector;
2357                 adapter->que_mask |= (u64)(1 << que->msix);
2358                 /*
2359                 ** Bind the msix vector, and thus the
2360                 ** ring to the corresponding cpu.
2361                 */
2362 #if 0 /* XXX */
2363                 if (adapter->num_queues > 1)
2364                         bus_bind_intr(dev, que->res, i);
2365 #endif
2366
2367                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2368                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2369                     taskqueue_thread_enqueue, &que->tq);
2370                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2371                     device_get_nameunit(adapter->dev));
2372         }
2373
2374         /* and Link */
2375         rid = vector + 1;
2376         adapter->res = bus_alloc_resource_any(dev,
2377             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2378         if (!adapter->res) {
2379                 device_printf(dev,"Unable to allocate"
2380             " bus resource: Link interrupt [%d]\n", rid);
2381                 return (ENXIO);
2382         }
2383         /* Set the link handler function */
2384         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2385             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2386         if (error) {
2387                 adapter->res = NULL;
2388                 device_printf(dev, "Failed to register LINK handler");
2389                 return (error);
2390         }
2391 #if 0 /* __FreeBSD_version >= 800504 */
2392         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2393 #endif
2394         adapter->linkvec = vector;
2395         /* Tasklets for Link, SFP and Multispeed Fiber */
2396         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2397         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2398         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2399 #ifdef IXGBE_FDIR
2400         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2401 #endif
2402         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2403             taskqueue_thread_enqueue, &adapter->tq);
2404         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2405             device_get_nameunit(adapter->dev));
2406
2407         return (0);
2408 }
2409
2410 #if 0   /* HAVE_MSIX */
2411 /*
2412  * Setup Either MSI/X or MSI
2413  */
2414 static int
2415 ixgbe_setup_msix(struct adapter *adapter)
2416 {
2417         device_t dev = adapter->dev;
2418         int rid, want, queues, msgs;
2419
2420         /* Override by tuneable */
2421         if (ixgbe_enable_msix == 0)
2422                 goto msi;
2423
2424         /* First try MSI/X */
2425         rid = PCIR_BAR(MSIX_82598_BAR);
2426         adapter->msix_mem = bus_alloc_resource_any(dev,
2427             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2428         if (!adapter->msix_mem) {
2429                 rid += 4;       /* 82599 maps in higher BAR */
2430                 adapter->msix_mem = bus_alloc_resource_any(dev,
2431                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2432         }
2433         if (!adapter->msix_mem) {
2434                 /* May not be enabled */
2435                 device_printf(adapter->dev,
2436                     "Unable to map MSIX table \n");
2437                 goto msi;
2438         }
2439
2440         msgs = pci_msix_count(dev);
2441         if (msgs == 0) { /* system has msix disabled */
2442                 bus_release_resource(dev, SYS_RES_MEMORY,
2443                     rid, adapter->msix_mem);
2444                 adapter->msix_mem = NULL;
2445                 goto msi;
2446         }
2447
2448         /* Figure out a reasonable auto config value */
2449         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2450
2451         if (ixgbe_num_queues != 0)
2452                 queues = ixgbe_num_queues;
2453         /* Set max queues to 8 when autoconfiguring */
2454         else if ((ixgbe_num_queues == 0) && (queues > 8))
2455                 queues = 8;
2456
2457         /*
2458         ** Want one vector (RX/TX pair) per queue
2459         ** plus an additional for Link.
2460         */
2461         want = queues + 1;
2462         if (msgs >= want)
2463                 msgs = want;
2464         else {
2465                 device_printf(adapter->dev,
2466                     "MSIX Configuration Problem, "
2467                     "%d vectors but %d queues wanted!\n",
2468                     msgs, want);
2469                 return (0); /* Will go to Legacy setup */
2470         }
2471         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2472                 device_printf(adapter->dev,
2473                     "Using MSIX interrupts with %d vectors\n", msgs);
2474                 adapter->num_queues = queues;
2475                 return (msgs);
2476         }
2477 msi:
2478         msgs = pci_msi_count(dev);
2479         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2480                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2481         else
2482                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2483         return (msgs);
2484 }
2485 #endif
2486
2487
2488 static int
2489 ixgbe_allocate_pci_resources(struct adapter *adapter)
2490 {
2491         int             rid;
2492         device_t        dev = adapter->dev;
2493
2494         rid = PCIR_BAR(0);
2495         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2496             &rid, RF_ACTIVE);
2497
2498         if (!(adapter->pci_mem)) {
2499                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2500                 return (ENXIO);
2501         }
2502
2503         adapter->osdep.mem_bus_space_tag =
2504                 rman_get_bustag(adapter->pci_mem);
2505         adapter->osdep.mem_bus_space_handle =
2506                 rman_get_bushandle(adapter->pci_mem);
2507         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2508
2509         /* Legacy defaults */
2510         adapter->num_queues = 1;
2511         adapter->hw.back = &adapter->osdep;
2512
2513         /*
2514         ** Now setup MSI or MSI/X, should
2515         ** return us the number of supported
2516         ** vectors. (Will be 1 for MSI)
2517         */
2518 #if 0   /* HAVE_MSIX */
2519         adapter->msix = ixgbe_setup_msix(adapter);
2520 #endif
2521         return (0);
2522 }
2523
2524 static void
2525 ixgbe_free_pci_resources(struct adapter * adapter)
2526 {
2527         struct          ix_queue *que = adapter->queues;
2528         device_t        dev = adapter->dev;
2529         int             rid, memrid;
2530
2531         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2532                 memrid = PCIR_BAR(MSIX_82598_BAR);
2533         else
2534                 memrid = PCIR_BAR(MSIX_82599_BAR);
2535
2536         /*
2537         ** There is a slight possibility of a failure mode
2538         ** in attach that will result in entering this function
2539         ** before interrupt resources have been initialized, and
2540         ** in that case we do not want to execute the loops below
2541         ** We can detect this reliably by the state of the adapter
2542         ** res pointer.
2543         */
2544         if (adapter->res == NULL)
2545                 goto mem;
2546
2547         /*
2548         **  Release all msix queue resources:
2549         */
2550         for (int i = 0; i < adapter->num_queues; i++, que++) {
2551                 rid = que->msix + 1;
2552                 if (que->tag != NULL) {
2553                         bus_teardown_intr(dev, que->res, que->tag);
2554                         que->tag = NULL;
2555                 }
2556                 if (que->res != NULL)
2557                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2558         }
2559
2560
2561         /* Clean the Legacy or Link interrupt last */
2562         if (adapter->linkvec) /* we are doing MSIX */
2563                 rid = adapter->linkvec + 1;
2564         else
2565                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2566
2567         if (adapter->tag != NULL) {
2568                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2569                 adapter->tag = NULL;
2570         }
2571         if (adapter->res != NULL)
2572                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2573         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2574                 pci_release_msi(adapter->dev);
2575
2576 mem:
2577         if (adapter->msix)
2578                 pci_release_msi(dev);
2579
2580         if (adapter->msix_mem != NULL)
2581                 bus_release_resource(dev, SYS_RES_MEMORY,
2582                     memrid, adapter->msix_mem);
2583
2584         if (adapter->pci_mem != NULL)
2585                 bus_release_resource(dev, SYS_RES_MEMORY,
2586                     PCIR_BAR(0), adapter->pci_mem);
2587
2588         return;
2589 }
2590
2591 /*********************************************************************
2592  *
2593  *  Setup networking device structure and register an interface.
2594  *
2595  **********************************************************************/
2596 static int
2597 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2598 {
2599         struct ixgbe_hw *hw = &adapter->hw;
2600         struct ifnet   *ifp;
2601
2602         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2603
2604         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2605         if (ifp == NULL) {
2606                 device_printf(dev, "can not allocate ifnet structure\n");
2607                 return (-1);
2608         }
2609         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2610         ifp->if_baudrate = 1000000000;
2611         ifp->if_init = ixgbe_init;
2612         ifp->if_softc = adapter;
2613         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2614         ifp->if_ioctl = ixgbe_ioctl;
2615         ifp->if_start = ixgbe_start;
2616 #if 0 /* __FreeBSD_version >= 800000 */
2617         ifp->if_transmit = ixgbe_mq_start;
2618         ifp->if_qflush = ixgbe_qflush;
2619 #endif
2620         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2621
2622         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2623
2624         adapter->max_frame_size =
2625             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2626
2627         /*
2628          * Tell the upper layer(s) we support long frames.
2629          */
2630         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2631
2632 #if 0 /* NET_TSO */
2633         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2634 #endif
2635         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2636         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2637         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2638 #if 0 /* NET_TSO */
2639                              |  IFCAP_VLAN_HWTSO
2640 #endif
2641                              |  IFCAP_VLAN_MTU;
2642         ifp->if_capenable = ifp->if_capabilities;
2643
2644         /* Don't enable LRO by default */
2645 #if 0 /* NET_LRO */
2646         ifp->if_capabilities |= IFCAP_LRO;
2647 #endif
2648
2649         /*
2650         ** Don't turn this on by default, if vlans are
2651         ** created on another pseudo device (eg. lagg)
2652         ** then vlan events are not passed thru, breaking
2653         ** operation, but with HW FILTER off it works. If
2654         ** using vlans directly on the ixgbe driver you can
2655         ** enable this and get full hardware tag filtering.
2656         */
2657         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2658
2659         /*
2660          * Specify the media types supported by this adapter and register
2661          * callbacks to update media and link information
2662          */
2663         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2664                      ixgbe_media_status);
2665         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2666         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2667         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2668                 ifmedia_add(&adapter->media,
2669                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2670                 ifmedia_add(&adapter->media,
2671                     IFM_ETHER | IFM_1000_T, 0, NULL);
2672         }
2673         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2674         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2675
2676         return (0);
2677 }
2678
2679 static void
2680 ixgbe_config_link(struct adapter *adapter)
2681 {
2682         struct ixgbe_hw *hw = &adapter->hw;
2683         u32     autoneg, err = 0;
2684         bool    sfp, negotiate;
2685
2686         sfp = ixgbe_is_sfp(hw);
2687
2688         if (sfp) {
2689                 if (hw->phy.multispeed_fiber) {
2690                         hw->mac.ops.setup_sfp(hw);
2691                         ixgbe_enable_tx_laser(hw);
2692                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2693                 } else
2694                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2695         } else {
2696                 if (hw->mac.ops.check_link)
2697                         err = ixgbe_check_link(hw, &autoneg,
2698                             &adapter->link_up, FALSE);
2699                 if (err)
2700                         goto out;
2701                 autoneg = hw->phy.autoneg_advertised;
2702                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2703                         err  = hw->mac.ops.get_link_capabilities(hw,
2704                             &autoneg, &negotiate);
2705                 if (err)
2706                         goto out;
2707                 if (hw->mac.ops.setup_link)
2708                         err = hw->mac.ops.setup_link(hw, autoneg,
2709                             negotiate, adapter->link_up);
2710         }
2711 out:
2712         return;
2713 }
2714
2715 /********************************************************************
2716  * Manage DMA'able memory.
2717  *******************************************************************/
2718 static void
2719 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2720 {
2721         if (error)
2722                 return;
2723         *(bus_addr_t *) arg = segs->ds_addr;
2724         return;
2725 }
2726
2727 static int
2728 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2729                 struct ixgbe_dma_alloc *dma, int mapflags)
2730 {
2731         device_t dev = adapter->dev;
2732         int             r;
2733
2734         r = bus_dma_tag_create(NULL,    /* parent */
2735                                DBA_ALIGN, 0,    /* alignment, bounds */
2736                                BUS_SPACE_MAXADDR,       /* lowaddr */
2737                                BUS_SPACE_MAXADDR,       /* highaddr */
2738                                NULL, NULL,      /* filter, filterarg */
2739                                size,    /* maxsize */
2740                                1,       /* nsegments */
2741                                size,    /* maxsegsize */
2742                                BUS_DMA_ALLOCNOW,        /* flags */
2743                                &dma->dma_tag);
2744         if (r != 0) {
2745                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2746                        "error %u\n", r);
2747                 goto fail_0;
2748         }
2749         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2750                              BUS_DMA_NOWAIT, &dma->dma_map);
2751         if (r != 0) {
2752                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2753                        "error %u\n", r);
2754                 goto fail_1;
2755         }
2756         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2757                             size,
2758                             ixgbe_dmamap_cb,
2759                             &dma->dma_paddr,
2760                             mapflags | BUS_DMA_NOWAIT);
2761         if (r != 0) {
2762                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2763                        "error %u\n", r);
2764                 goto fail_2;
2765         }
2766         dma->dma_size = size;
2767         return (0);
2768 fail_2:
2769         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2770 fail_1:
2771         bus_dma_tag_destroy(dma->dma_tag);
2772 fail_0:
2773         dma->dma_map = NULL;
2774         dma->dma_tag = NULL;
2775         return (r);
2776 }
2777
2778 static void
2779 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2780 {
2781         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2782             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2783         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2784         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2785         bus_dma_tag_destroy(dma->dma_tag);
2786 }
2787
2788
2789 /*********************************************************************
2790  *
2791  *  Allocate memory for the transmit and receive rings, and then
2792  *  the descriptors associated with each, called only once at attach.
2793  *
2794  **********************************************************************/
2795 static int
2796 ixgbe_allocate_queues(struct adapter *adapter)
2797 {
2798         device_t        dev = adapter->dev;
2799         struct ix_queue *que;
2800         struct tx_ring  *txr;
2801         struct rx_ring  *rxr;
2802         int rsize, tsize, error = IXGBE_SUCCESS;
2803         int txconf = 0, rxconf = 0;
2804
2805         /* First allocate the top level queue structs */
2806         if (!(adapter->queues =
2807             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2808             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2809                 device_printf(dev, "Unable to allocate queue memory\n");
2810                 error = ENOMEM;
2811                 goto fail;
2812         }
2813
2814         /* First allocate the TX ring struct memory */
2815         if (!(adapter->tx_rings =
2816             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2817             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2818                 device_printf(dev, "Unable to allocate TX ring memory\n");
2819                 error = ENOMEM;
2820                 goto tx_fail;
2821         }
2822
2823         /* Next allocate the RX */
2824         if (!(adapter->rx_rings =
2825             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2826             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2827                 device_printf(dev, "Unable to allocate RX ring memory\n");
2828                 error = ENOMEM;
2829                 goto rx_fail;
2830         }
2831
2832         /* For the ring itself */
2833         tsize = roundup2(adapter->num_tx_desc *
2834             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2835
2836         /*
2837          * Now set up the TX queues, txconf is needed to handle the
2838          * possibility that things fail midcourse and we need to
2839          * undo memory gracefully
2840          */
2841         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2842                 /* Set up some basics */
2843                 txr = &adapter->tx_rings[i];
2844                 txr->adapter = adapter;
2845                 txr->me = i;
2846
2847                 /* Initialize the TX side lock */
2848                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2849                     device_get_nameunit(dev), txr->me);
2850                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2851
2852                 if (ixgbe_dma_malloc(adapter, tsize,
2853                         &txr->txdma, BUS_DMA_NOWAIT)) {
2854                         device_printf(dev,
2855                             "Unable to allocate TX Descriptor memory\n");
2856                         error = ENOMEM;
2857                         goto err_tx_desc;
2858                 }
2859                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2860                 bzero((void *)txr->tx_base, tsize);
2861
2862                 /* Now allocate transmit buffers for the ring */
2863                 if (ixgbe_allocate_transmit_buffers(txr)) {
2864                         device_printf(dev,
2865                             "Critical Failure setting up transmit buffers\n");
2866                         error = ENOMEM;
2867                         goto err_tx_desc;
2868                 }
2869 #if 0 /* __FreeBSD_version >= 800000 */
2870                 /* Allocate a buf ring */
2871                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2872                     M_WAITOK, &txr->tx_mtx);
2873                 if (txr->br == NULL) {
2874                         device_printf(dev,
2875                             "Critical Failure setting up buf ring\n");
2876                         error = ENOMEM;
2877                         goto err_tx_desc;
2878                 }
2879 #endif
2880         }
2881
2882         /*
2883          * Next the RX queues...
2884          */
2885         rsize = roundup2(adapter->num_rx_desc *
2886             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2887         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2888                 rxr = &adapter->rx_rings[i];
2889                 /* Set up some basics */
2890                 rxr->adapter = adapter;
2891                 rxr->me = i;
2892
2893                 /* Initialize the RX side lock */
2894                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2895                     device_get_nameunit(dev), rxr->me);
2896                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2897
2898                 if (ixgbe_dma_malloc(adapter, rsize,
2899                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2900                         device_printf(dev,
2901                             "Unable to allocate RxDescriptor memory\n");
2902                         error = ENOMEM;
2903                         goto err_rx_desc;
2904                 }
2905                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2906                 bzero((void *)rxr->rx_base, rsize);
2907
2908                 /* Allocate receive buffers for the ring*/
2909                 if (ixgbe_allocate_receive_buffers(rxr)) {
2910                         device_printf(dev,
2911                             "Critical Failure setting up receive buffers\n");
2912                         error = ENOMEM;
2913                         goto err_rx_desc;
2914                 }
2915         }
2916
2917         /*
2918         ** Finally set up the queue holding structs
2919         */
2920         for (int i = 0; i < adapter->num_queues; i++) {
2921                 que = &adapter->queues[i];
2922                 que->adapter = adapter;
2923                 que->txr = &adapter->tx_rings[i];
2924                 que->rxr = &adapter->rx_rings[i];
2925         }
2926
2927         return (0);
2928
2929 err_rx_desc:
2930         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2931                 ixgbe_dma_free(adapter, &rxr->rxdma);
2932 err_tx_desc:
2933         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2934                 ixgbe_dma_free(adapter, &txr->txdma);
2935         kfree(adapter->rx_rings, M_DEVBUF);
2936 rx_fail:
2937         kfree(adapter->tx_rings, M_DEVBUF);
2938 tx_fail:
2939         kfree(adapter->queues, M_DEVBUF);
2940 fail:
2941         return (error);
2942 }
2943
2944 /*********************************************************************
2945  *
2946  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2947  *  the information needed to transmit a packet on the wire. This is
2948  *  called only once at attach, setup is done every reset.
2949  *
2950  **********************************************************************/
2951 static int
2952 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2953 {
2954         struct adapter *adapter = txr->adapter;
2955         device_t dev = adapter->dev;
2956         struct ixgbe_tx_buf *txbuf;
2957         int error, i;
2958
2959         /*
2960          * Setup DMA descriptor areas.
2961          */
2962         if ((error = bus_dma_tag_create(
2963                                NULL,    /* parent */
2964                                1, 0,            /* alignment, bounds */
2965                                BUS_SPACE_MAXADDR,       /* lowaddr */
2966                                BUS_SPACE_MAXADDR,       /* highaddr */
2967                                NULL, NULL,              /* filter, filterarg */
2968                                IXGBE_TSO_SIZE,          /* maxsize */
2969                                adapter->num_segs,       /* nsegments */
2970                                PAGE_SIZE,               /* maxsegsize */
2971                                0,                       /* flags */
2972                                &txr->txtag))) {
2973                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2974                 goto fail;
2975         }
2976
2977         if (!(txr->tx_buffers =
2978             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2979             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2980                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2981                 error = ENOMEM;
2982                 goto fail;
2983         }
2984
2985         /* Create the descriptor buffer dma maps */
2986         txbuf = txr->tx_buffers;
2987         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2988                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2989                 if (error != 0) {
2990                         device_printf(dev, "Unable to create TX DMA map\n");
2991                         goto fail;
2992                 }
2993         }
2994
2995         return 0;
2996 fail:
2997         /* We free all, it handles case where we are in the middle */
2998         ixgbe_free_transmit_structures(adapter);
2999         return (error);
3000 }
3001
3002 /*********************************************************************
3003  *
3004  *  Initialize a transmit ring.
3005  *
3006  **********************************************************************/
3007 static void
3008 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3009 {
3010         struct adapter *adapter = txr->adapter;
3011         struct ixgbe_tx_buf *txbuf;
3012         int i;
3013 #ifdef DEV_NETMAP
3014         struct netmap_adapter *na = NA(adapter->ifp);
3015         struct netmap_slot *slot;
3016 #endif /* DEV_NETMAP */
3017
3018         /* Clear the old ring contents */
3019         IXGBE_TX_LOCK(txr);
3020 #ifdef DEV_NETMAP
3021         /*
3022          * (under lock): if in netmap mode, do some consistency
3023          * checks and set slot to entry 0 of the netmap ring.
3024          */
3025         slot = netmap_reset(na, NR_TX, txr->me, 0);
3026 #endif /* DEV_NETMAP */
3027         bzero((void *)txr->tx_base,
3028               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3029         /* Reset indices */
3030         txr->next_avail_desc = 0;
3031         txr->next_to_clean = 0;
3032
3033         /* Free any existing tx buffers. */
3034         txbuf = txr->tx_buffers;
3035         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3036                 if (txbuf->m_head != NULL) {
3037                         bus_dmamap_sync(txr->txtag, txbuf->map,
3038                             BUS_DMASYNC_POSTWRITE);
3039                         bus_dmamap_unload(txr->txtag, txbuf->map);
3040                         m_freem(txbuf->m_head);
3041                         txbuf->m_head = NULL;
3042                 }
3043 #ifdef DEV_NETMAP
3044                 /*
3045                  * In netmap mode, set the map for the packet buffer.
3046                  * NOTE: Some drivers (not this one) also need to set
3047                  * the physical buffer address in the NIC ring.
3048                  * Slots in the netmap ring (indexed by "si") are
3049                  * kring->nkr_hwofs positions "ahead" wrt the
3050                  * corresponding slot in the NIC ring. In some drivers
3051                  * (not here) nkr_hwofs can be negative. Function
3052                  * netmap_idx_n2k() handles wraparounds properly.
3053                  */
3054                 if (slot) {
3055                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3056                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3057                 }
3058 #endif /* DEV_NETMAP */
3059                 /* Clear the EOP index */
3060                 txbuf->eop_index = -1;
3061         }
3062
3063 #ifdef IXGBE_FDIR
3064         /* Set the rate at which we sample packets */
3065         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3066                 txr->atr_sample = atr_sample_rate;
3067 #endif
3068
3069         /* Set number of descriptors available */
3070         txr->tx_avail = adapter->num_tx_desc;
3071
3072         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3073             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3074         IXGBE_TX_UNLOCK(txr);
3075 }
3076
3077 /*********************************************************************
3078  *
3079  *  Initialize all transmit rings.
3080  *
3081  **********************************************************************/
3082 static int
3083 ixgbe_setup_transmit_structures(struct adapter *adapter)
3084 {
3085         struct tx_ring *txr = adapter->tx_rings;
3086
3087         for (int i = 0; i < adapter->num_queues; i++, txr++)
3088                 ixgbe_setup_transmit_ring(txr);
3089
3090         return (0);
3091 }
3092
3093 /*********************************************************************
3094  *
3095  *  Enable transmit unit.
3096  *
3097  **********************************************************************/
3098 static void
3099 ixgbe_initialize_transmit_units(struct adapter *adapter)
3100 {
3101         struct tx_ring  *txr = adapter->tx_rings;
3102         struct ixgbe_hw *hw = &adapter->hw;
3103
3104         /* Setup the Base and Length of the Tx Descriptor Ring */
3105
3106         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3107                 u64     tdba = txr->txdma.dma_paddr;
3108                 u32     txctrl;
3109
3110                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3111                        (tdba & 0x00000000ffffffffULL));
3112                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3113                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3114                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3115
3116                 /* Setup the HW Tx Head and Tail descriptor pointers */
3117                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3118                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3119
3120                 /* Setup Transmit Descriptor Cmd Settings */
3121                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3122                 txr->queue_status = IXGBE_QUEUE_IDLE;
3123
3124                 /* Disable Head Writeback */
3125                 switch (hw->mac.type) {
3126                 case ixgbe_mac_82598EB:
3127                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3128                         break;
3129                 case ixgbe_mac_82599EB:
3130                 case ixgbe_mac_X540:
3131                 default:
3132                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3133                         break;
3134                 }
3135                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3136                 switch (hw->mac.type) {
3137                 case ixgbe_mac_82598EB:
3138                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3139                         break;
3140                 case ixgbe_mac_82599EB:
3141                 case ixgbe_mac_X540:
3142                 default:
3143                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3144                         break;
3145                 }
3146
3147         }
3148
3149         if (hw->mac.type != ixgbe_mac_82598EB) {
3150                 u32 dmatxctl, rttdcs;
3151                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3152                 dmatxctl |= IXGBE_DMATXCTL_TE;
3153                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3154                 /* Disable arbiter to set MTQC */
3155                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3156                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3157                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3158                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3159                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3160                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3161         }
3162
3163         return;
3164 }
3165
3166 /*********************************************************************
3167  *
3168  *  Free all transmit rings.
3169  *
3170  **********************************************************************/
3171 static void
3172 ixgbe_free_transmit_structures(struct adapter *adapter)
3173 {
3174         struct tx_ring *txr = adapter->tx_rings;
3175
3176         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3177                 IXGBE_TX_LOCK(txr);
3178                 ixgbe_free_transmit_buffers(txr);
3179                 ixgbe_dma_free(adapter, &txr->txdma);
3180                 IXGBE_TX_UNLOCK(txr);
3181                 IXGBE_TX_LOCK_DESTROY(txr);
3182         }
3183         kfree(adapter->tx_rings, M_DEVBUF);
3184 }
3185
3186 /*********************************************************************
3187  *
3188  *  Free transmit ring related data structures.
3189  *
3190  **********************************************************************/
3191 static void
3192 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3193 {
3194         struct adapter *adapter = txr->adapter;
3195         struct ixgbe_tx_buf *tx_buffer;
3196         int             i;
3197
3198         INIT_DEBUGOUT("free_transmit_ring: begin");
3199
3200         if (txr->tx_buffers == NULL)
3201                 return;
3202
3203         tx_buffer = txr->tx_buffers;
3204         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3205                 if (tx_buffer->m_head != NULL) {
3206                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3207                             BUS_DMASYNC_POSTWRITE);
3208                         bus_dmamap_unload(txr->txtag,
3209                             tx_buffer->map);
3210                         m_freem(tx_buffer->m_head);
3211                         tx_buffer->m_head = NULL;
3212                         if (tx_buffer->map != NULL) {
3213                                 bus_dmamap_destroy(txr->txtag,
3214                                     tx_buffer->map);
3215                                 tx_buffer->map = NULL;
3216                         }
3217                 } else if (tx_buffer->map != NULL) {
3218                         bus_dmamap_unload(txr->txtag,
3219                             tx_buffer->map);
3220                         bus_dmamap_destroy(txr->txtag,
3221                             tx_buffer->map);
3222                         tx_buffer->map = NULL;
3223                 }
3224         }
3225 #if 0 /* __FreeBSD_version >= 800000 */
3226         if (txr->br != NULL)
3227                 buf_ring_free(txr->br, M_DEVBUF);
3228 #endif
3229         if (txr->tx_buffers != NULL) {
3230                 kfree(txr->tx_buffers, M_DEVBUF);
3231                 txr->tx_buffers = NULL;
3232         }
3233         if (txr->txtag != NULL) {
3234                 bus_dma_tag_destroy(txr->txtag);
3235                 txr->txtag = NULL;
3236         }
3237         return;
3238 }
3239
3240 /*********************************************************************
3241  *
3242  *  Advanced Context Descriptor setup for VLAN or CSUM
3243  *
3244  **********************************************************************/
3245
3246 static bool
3247 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3248 {
3249         struct adapter *adapter = txr->adapter;
3250         struct ixgbe_adv_tx_context_desc *TXD;
3251         struct ixgbe_tx_buf        *tx_buffer;
3252         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3253         struct ether_vlan_header *eh;
3254         struct ip *ip;
3255         struct ip6_hdr *ip6;
3256         int  ehdrlen, ip_hlen = 0;
3257         u16     etype;
3258         u8      ipproto = 0;
3259         bool    offload = TRUE;
3260         int ctxd = txr->next_avail_desc;
3261 #ifdef NET_VLAN
3262         u16 vtag = 0;
3263 #endif
3264
3265
3266         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3267                 offload = FALSE;
3268
3269         tx_buffer = &txr->tx_buffers[ctxd];
3270         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3271
3272         /*
3273         ** In advanced descriptors the vlan tag must
3274         ** be placed into the descriptor itself.
3275         */
3276 #ifdef NET_VLAN
3277         if (mp->m_flags & M_VLANTAG) {
3278                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3279                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3280         } else if (offload == FALSE)
3281                 return FALSE;
3282 #endif
3283
3284         /*
3285          * Determine where frame payload starts.
3286          * Jump over vlan headers if already present,
3287          * helpful for QinQ too.
3288          */
3289         eh = mtod(mp, struct ether_vlan_header *);
3290         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3291                 etype = ntohs(eh->evl_proto);
3292                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3293         } else {
3294                 etype = ntohs(eh->evl_encap_proto);
3295                 ehdrlen = ETHER_HDR_LEN;
3296         }
3297
3298         /* Set the ether header length */
3299         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3300
3301         switch (etype) {
3302                 case ETHERTYPE_IP:
3303                         ip = (struct ip *)(mp->m_data + ehdrlen);
3304                         ip_hlen = ip->ip_hl << 2;
3305                         ipproto = ip->ip_p;
3306                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3307                         break;
3308                 case ETHERTYPE_IPV6:
3309                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3310                         ip_hlen = sizeof(struct ip6_hdr);
3311                         /* XXX-BZ this will go badly in case of ext hdrs. */
3312                         ipproto = ip6->ip6_nxt;
3313                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3314                         break;
3315                 default:
3316                         offload = FALSE;
3317                         break;
3318         }
3319
3320         vlan_macip_lens |= ip_hlen;
3321         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3322
3323         switch (ipproto) {
3324                 case IPPROTO_TCP:
3325                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3326                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3327                         break;
3328
3329                 case IPPROTO_UDP:
3330                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3331                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3332                         break;
3333
3334 #if 0
3335                 case IPPROTO_SCTP:
3336                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3337                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3338                         break;
3339 #endif
3340                 default:
3341                         offload = FALSE;
3342                         break;
3343         }
3344
3345         /* Now copy bits into descriptor */
3346         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3347         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3348         TXD->seqnum_seed = htole32(0);
3349         TXD->mss_l4len_idx = htole32(0);
3350
3351         tx_buffer->m_head = NULL;
3352         tx_buffer->eop_index = -1;
3353
3354         /* We've consumed the first desc, adjust counters */
3355         if (++ctxd == adapter->num_tx_desc)
3356                 ctxd = 0;
3357         txr->next_avail_desc = ctxd;
3358         --txr->tx_avail;
3359
3360         return (offload);
3361 }
3362
3363 /**********************************************************************
3364  *
3365  *  Setup work for hardware segmentation offload (TSO) on
3366  *  adapters using advanced tx descriptors
3367  *
3368  **********************************************************************/
3369 #if 0   /* NET_TSO */
3370 static bool
3371 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3372     u32 *olinfo_status)
3373 {
3374         struct adapter *adapter = txr->adapter;
3375         struct ixgbe_adv_tx_context_desc *TXD;
3376         struct ixgbe_tx_buf        *tx_buffer;
3377 #ifdef NET_VLAN
3378         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3379         u16 vtag = 0, eh_type;
3380 #else
3381         u16 eh_type;
3382         u32 type_tucmd_mlhl = 0;
3383 #endif
3384         u32 mss_l4len_idx = 0, len;
3385         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3386         struct ether_vlan_header *eh;
3387 #ifdef INET6
3388         struct ip6_hdr *ip6;
3389 #endif
3390 #ifdef INET
3391         struct ip *ip;
3392 #endif
3393         struct tcphdr *th;
3394
3395
3396         /*
3397          * Determine where frame payload starts.
3398          * Jump over vlan headers if already present
3399          */
3400         eh = mtod(mp, struct ether_vlan_header *);
3401         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3402                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3403                 eh_type = eh->evl_proto;
3404         } else {
3405                 ehdrlen = ETHER_HDR_LEN;
3406                 eh_type = eh->evl_encap_proto;
3407         }
3408
3409         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3410         len = ehdrlen + sizeof(struct tcphdr);
3411         switch (ntohs(eh_type)) {
3412 #ifdef INET6
3413         case ETHERTYPE_IPV6:
3414                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3415                         return FALSE;
3416                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3417                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3418                 if (ip6->ip6_nxt != IPPROTO_TCP)
3419                         return FALSE;
3420                 ip_hlen = sizeof(struct ip6_hdr);
3421                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3422                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3423                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3424                 break;
3425 #endif
3426 #ifdef INET
3427         case ETHERTYPE_IP:
3428                 if (mp->m_len < len + sizeof(struct ip))
3429                         return FALSE;
3430                 ip = (struct ip *)(mp->m_data + ehdrlen);
3431                 if (ip->ip_p != IPPROTO_TCP)
3432                         return FALSE;
3433                 ip->ip_sum = 0;
3434                 ip_hlen = ip->ip_hl << 2;
3435                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3436                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3437                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3438                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3439                 /* Tell transmit desc to also do IPv4 checksum. */
3440                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3441                 break;
3442 #endif
3443         default:
3444                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3445                     __func__, ntohs(eh_type));
3446                 break;
3447         }
3448
3449         ctxd = txr->next_avail_desc;
3450         tx_buffer = &txr->tx_buffers[ctxd];
3451         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3452
3453         tcp_hlen = th->th_off << 2;
3454
3455         /* This is used in the transmit desc in encap */
3456         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3457
3458         /* VLAN MACLEN IPLEN */
3459 #ifdef NET_VLAN
3460         if (mp->m_flags & M_VLANTAG) {
3461                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3462                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3463         }
3464
3465         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3466         vlan_macip_lens |= ip_hlen;
3467         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3468 #endif
3469
3470         /* ADV DTYPE TUCMD */
3471         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3472         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3473         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3474
3475         /* MSS L4LEN IDX */
3476         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3477         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3478         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3479
3480         TXD->seqnum_seed = htole32(0);
3481         tx_buffer->m_head = NULL;
3482         tx_buffer->eop_index = -1;
3483
3484         if (++ctxd == adapter->num_tx_desc)
3485                 ctxd = 0;
3486
3487         txr->tx_avail--;
3488         txr->next_avail_desc = ctxd;
3489         return TRUE;
3490 }
3491 #endif
3492
3493 #ifdef IXGBE_FDIR
3494 /*
3495 ** This routine parses packet headers so that Flow
3496 ** Director can make a hashed filter table entry
3497 ** allowing traffic flows to be identified and kept
3498 ** on the same cpu.  This would be a performance
3499 ** hit, but we only do it at IXGBE_FDIR_RATE of
3500 ** packets.
3501 */
3502 static void
3503 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3504 {
3505         struct adapter                  *adapter = txr->adapter;
3506         struct ix_queue                 *que;
3507         struct ip                       *ip;
3508         struct tcphdr                   *th;
3509         struct udphdr                   *uh;
3510         struct ether_vlan_header        *eh;
3511         union ixgbe_atr_hash_dword      input = {.dword = 0};
3512         union ixgbe_atr_hash_dword      common = {.dword = 0};
3513         int                             ehdrlen, ip_hlen;
3514         u16                             etype;
3515
3516         eh = mtod(mp, struct ether_vlan_header *);
3517         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3518                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3519                 etype = eh->evl_proto;
3520         } else {
3521                 ehdrlen = ETHER_HDR_LEN;
3522                 etype = eh->evl_encap_proto;
3523         }
3524
3525         /* Only handling IPv4 */
3526         if (etype != htons(ETHERTYPE_IP))
3527                 return;
3528
3529         ip = (struct ip *)(mp->m_data + ehdrlen);
3530         ip_hlen = ip->ip_hl << 2;
3531
3532         /* check if we're UDP or TCP */
3533         switch (ip->ip_p) {
3534         case IPPROTO_TCP:
3535                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3536                 /* src and dst are inverted */
3537                 common.port.dst ^= th->th_sport;
3538                 common.port.src ^= th->th_dport;
3539                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3540                 break;
3541         case IPPROTO_UDP:
3542                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3543                 /* src and dst are inverted */
3544                 common.port.dst ^= uh->uh_sport;
3545                 common.port.src ^= uh->uh_dport;
3546                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3547                 break;
3548         default:
3549                 return;
3550         }
3551
3552         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3553         if (mp->m_pkthdr.ether_vtag)
3554                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3555         else
3556                 common.flex_bytes ^= etype;
3557         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3558
3559         que = &adapter->queues[txr->me];
3560         /*
3561         ** This assumes the Rx queue and Tx
3562         ** queue are bound to the same CPU
3563         */
3564         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3565             input, common, que->msix);
3566 }
3567 #endif /* IXGBE_FDIR */
3568
3569 /**********************************************************************
3570  *
3571  *  Examine each tx_buffer in the used queue. If the hardware is done
3572  *  processing the packet then free associated resources. The
3573  *  tx_buffer is put back on the free queue.
3574  *
3575  **********************************************************************/
3576 static bool
3577 ixgbe_txeof(struct tx_ring *txr)
3578 {
3579         struct adapter  *adapter = txr->adapter;
3580         struct ifnet    *ifp = adapter->ifp;
3581         u32     first, last, done, processed;
3582         struct ixgbe_tx_buf *tx_buffer;
3583         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3584
3585         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3586
3587 #ifdef DEV_NETMAP
3588         if (ifp->if_capenable & IFCAP_NETMAP) {
3589                 struct netmap_adapter *na = NA(ifp);
3590                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3591
3592                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3593
3594                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3595                     BUS_DMASYNC_POSTREAD);
3596                 /*
3597                  * In netmap mode, all the work is done in the context
3598                  * of the client thread. Interrupt handlers only wake up
3599                  * clients, which may be sleeping on individual rings
3600                  * or on a global resource for all rings.
3601                  * To implement tx interrupt mitigation, we wake up the client
3602                  * thread roughly every half ring, even if the NIC interrupts
3603                  * more frequently. This is implemented as follows:
3604                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3605                  *   the slot that should wake up the thread (nkr_num_slots
3606                  *   means the user thread should not be woken up);
3607                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3608                  *   or the slot has the DD bit set.
3609                  *
3610                  * When the driver has separate locks, we need to
3611                  * release and re-acquire txlock to avoid deadlocks.
3612                  * XXX see if we can find a better way.
3613                  */
3614                 if (!netmap_mitigate ||
3615                     (kring->nr_kflags < kring->nkr_num_slots &&
3616                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3617                         kring->nr_kflags = kring->nkr_num_slots;
3618                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3619                         IXGBE_TX_UNLOCK(txr);
3620                         IXGBE_CORE_LOCK(adapter);
3621                         selwakeuppri(&na->tx_si, PI_NET);
3622                         IXGBE_CORE_UNLOCK(adapter);
3623                         IXGBE_TX_LOCK(txr);
3624                 }
3625                 return FALSE;
3626         }
3627 #endif /* DEV_NETMAP */
3628
3629         if (txr->tx_avail == adapter->num_tx_desc) {
3630                 txr->queue_status = IXGBE_QUEUE_IDLE;
3631                 return FALSE;
3632         }
3633
3634         processed = 0;
3635         first = txr->next_to_clean;
3636         tx_buffer = &txr->tx_buffers[first];
3637         /* For cleanup we just use legacy struct */
3638         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3639         last = tx_buffer->eop_index;
3640         if (last == -1)
3641                 return FALSE;
3642         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3643
3644         /*
3645         ** Get the index of the first descriptor
3646         ** BEYOND the EOP and call that 'done'.
3647         ** I do this so the comparison in the
3648         ** inner while loop below can be simple
3649         */
3650         if (++last == adapter->num_tx_desc) last = 0;
3651         done = last;
3652
3653         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3654             BUS_DMASYNC_POSTREAD);
3655         /*
3656         ** Only the EOP descriptor of a packet now has the DD
3657         ** bit set, this is what we look for...
3658         */
3659         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3660                 /* We clean the range of the packet */
3661                 while (first != done) {
3662                         tx_desc->upper.data = 0;
3663                         tx_desc->lower.data = 0;
3664                         tx_desc->buffer_addr = 0;
3665                         ++txr->tx_avail;
3666                         ++processed;
3667
3668                         if (tx_buffer->m_head) {
3669                                 txr->bytes +=
3670                                     tx_buffer->m_head->m_pkthdr.len;
3671                                 bus_dmamap_sync(txr->txtag,
3672                                     tx_buffer->map,
3673                                     BUS_DMASYNC_POSTWRITE);
3674                                 bus_dmamap_unload(txr->txtag,
3675                                     tx_buffer->map);
3676                                 m_freem(tx_buffer->m_head);
3677                                 tx_buffer->m_head = NULL;
3678                                 tx_buffer->map = NULL;
3679                         }
3680                         tx_buffer->eop_index = -1;
3681                         txr->watchdog_time = ticks;
3682
3683                         if (++first == adapter->num_tx_desc)
3684                                 first = 0;
3685
3686                         tx_buffer = &txr->tx_buffers[first];
3687                         tx_desc =
3688                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3689                 }
3690                 ++txr->packets;
3691                 ++ifp->if_opackets;
3692                 /* See if there is more work now */
3693                 last = tx_buffer->eop_index;
3694                 if (last != -1) {
3695                         eop_desc =
3696                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3697                         /* Get next done point */
3698                         if (++last == adapter->num_tx_desc) last = 0;
3699                         done = last;
3700                 } else
3701                         break;
3702         }
3703         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3704             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3705
3706         txr->next_to_clean = first;
3707
3708         /*
3709         ** Watchdog calculation, we know there's
3710         ** work outstanding or the first return
3711         ** would have been taken, so none processed
3712         ** for too long indicates a hang.
3713         */
3714         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3715                 txr->queue_status = IXGBE_QUEUE_HUNG;
3716
3717         /* With a minimum free clear the depleted state bit.  */
3718         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3719                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3720
3721         if (txr->tx_avail == adapter->num_tx_desc) {
3722                 txr->queue_status = IXGBE_QUEUE_IDLE;
3723                 return (FALSE);
3724         }
3725
3726         return TRUE;
3727 }
3728
3729 /*********************************************************************
3730  *
3731  *  Refresh mbuf buffers for RX descriptor rings
3732  *   - now keeps its own state so discards due to resource
3733  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3734  *     it just returns, keeping its placeholder, thus it can simply
3735  *     be recalled to try again.
3736  *
3737  **********************************************************************/
3738 static void
3739 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3740 {
3741         struct adapter          *adapter = rxr->adapter;
3742         bus_dma_segment_t       hseg[1];
3743         bus_dma_segment_t       pseg[1];
3744         struct ixgbe_rx_buf     *rxbuf;
3745         struct mbuf             *mh, *mp;
3746         int                     i, j, nsegs, error;
3747         bool                    refreshed = FALSE;
3748
3749         i = j = rxr->next_to_refresh;
3750         /* Control the loop with one beyond */
3751         if (++j == adapter->num_rx_desc)
3752                 j = 0;
3753
3754         while (j != limit) {
3755                 rxbuf = &rxr->rx_buffers[i];
3756                 if (rxr->hdr_split == FALSE)
3757                         goto no_split;
3758
3759                 if (rxbuf->m_head == NULL) {
3760                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3761                         if (mh == NULL)
3762                                 goto update;
3763                 } else
3764                         mh = rxbuf->m_head;
3765
3766                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3767                 mh->m_len = MHLEN;
3768                 mh->m_flags |= M_PKTHDR;
3769                 /* Get the memory mapping */
3770                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3771                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3772                 if (error != 0) {
3773                         kprintf("Refresh mbufs: hdr dmamap load"
3774                             " failure - %d\n", error);
3775                         m_free(mh);
3776                         rxbuf->m_head = NULL;
3777                         goto update;
3778                 }
3779                 rxbuf->m_head = mh;
3780                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3781                     BUS_DMASYNC_PREREAD);
3782                 rxr->rx_base[i].read.hdr_addr =
3783                     htole64(hseg[0].ds_addr);
3784
3785 no_split:
3786                 if (rxbuf->m_pack == NULL) {
3787                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3788                             M_PKTHDR, adapter->rx_mbuf_sz);
3789                         if (mp == NULL)
3790                                 goto update;
3791                 } else
3792                         mp = rxbuf->m_pack;
3793
3794                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3795                 /* Get the memory mapping */
3796                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3797                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3798                 if (error != 0) {
3799                         kprintf("Refresh mbufs: payload dmamap load"
3800                             " failure - %d\n", error);
3801                         m_free(mp);
3802                         rxbuf->m_pack = NULL;
3803                         goto update;
3804                 }
3805                 rxbuf->m_pack = mp;
3806                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3807                     BUS_DMASYNC_PREREAD);
3808                 rxr->rx_base[i].read.pkt_addr =
3809                     htole64(pseg[0].ds_addr);
3810
3811                 refreshed = TRUE;
3812                 /* Next is precalculated */
3813                 i = j;
3814                 rxr->next_to_refresh = i;
3815                 if (++j == adapter->num_rx_desc)
3816                         j = 0;
3817         }
3818 update:
3819         if (refreshed) /* Update hardware tail index */
3820                 IXGBE_WRITE_REG(&adapter->hw,
3821                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3822         return;
3823 }
3824
3825 /*********************************************************************
3826  *
3827  *  Allocate memory for rx_buffer structures. Since we use one
3828  *  rx_buffer per received packet, the maximum number of rx_buffer's
3829  *  that we'll need is equal to the number of receive descriptors
3830  *  that we've allocated.
3831  *
3832  **********************************************************************/
3833 static int
3834 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3835 {
3836         struct  adapter         *adapter = rxr->adapter;
3837         device_t                dev = adapter->dev;
3838         struct ixgbe_rx_buf     *rxbuf;
3839         int                     i, bsize, error;
3840
3841         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3842         if (!(rxr->rx_buffers =
3843             (struct ixgbe_rx_buf *) kmalloc(bsize,
3844             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3845                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3846                 error = ENOMEM;
3847                 goto fail;
3848         }
3849
3850         if ((error = bus_dma_tag_create(NULL,   /* parent */
3851                                    1, 0,        /* alignment, bounds */
3852                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3853                                    BUS_SPACE_MAXADDR,   /* highaddr */
3854                                    NULL, NULL,          /* filter, filterarg */
3855                                    MSIZE,               /* maxsize */
3856                                    1,                   /* nsegments */
3857                                    MSIZE,               /* maxsegsize */
3858                                    0,                   /* flags */
3859                                    &rxr->htag))) {
3860                 device_printf(dev, "Unable to create RX DMA tag\n");
3861                 goto fail;
3862         }
3863
3864         if ((error = bus_dma_tag_create(NULL,   /* parent */
3865                                    1, 0,        /* alignment, bounds */
3866                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3867                                    BUS_SPACE_MAXADDR,   /* highaddr */
3868                                    NULL, NULL,          /* filter, filterarg */
3869                                    MJUM16BYTES,         /* maxsize */
3870                                    1,                   /* nsegments */
3871                                    MJUM16BYTES,         /* maxsegsize */
3872                                    0,                   /* flags */
3873                                    &rxr->ptag))) {
3874                 device_printf(dev, "Unable to create RX DMA tag\n");
3875                 goto fail;
3876         }
3877
3878         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3879                 rxbuf = &rxr->rx_buffers[i];
3880                 error = bus_dmamap_create(rxr->htag,
3881                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3882                 if (error) {
3883                         device_printf(dev, "Unable to create RX head map\n");
3884                         goto fail;
3885                 }
3886                 error = bus_dmamap_create(rxr->ptag,
3887                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3888                 if (error) {
3889                         device_printf(dev, "Unable to create RX pkt map\n");
3890                         goto fail;
3891                 }
3892         }
3893
3894         return (0);
3895
3896 fail:
3897         /* Frees all, but can handle partial completion */
3898         ixgbe_free_receive_structures(adapter);
3899         return (error);
3900 }
3901
3902 /*
3903 ** Used to detect a descriptor that has
3904 ** been merged by Hardware RSC.
3905 */
3906 static inline u32
3907 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3908 {
3909         return (le32toh(rx->wb.lower.lo_dword.data) &
3910             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3911 }
3912
3913 /*********************************************************************
3914  *
3915  *  Initialize Hardware RSC (LRO) feature on 82599
3916  *  for an RX ring, this is toggled by the LRO capability
3917  *  even though it is transparent to the stack.
3918  *
3919  **********************************************************************/
3920 #if 0   /* NET_LRO */
3921 static void
3922 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3923 {
3924         struct  adapter         *adapter = rxr->adapter;
3925         struct  ixgbe_hw        *hw = &adapter->hw;
3926         u32                     rscctrl, rdrxctl;
3927
3928         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3929         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3930 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3931         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3932 #endif /* DEV_NETMAP */
3933         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3934         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3935         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3936
3937         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3938         rscctrl |= IXGBE_RSCCTL_RSCEN;
3939         /*
3940         ** Limit the total number of descriptors that
3941         ** can be combined, so it does not exceed 64K
3942         */
3943         if (adapter->rx_mbuf_sz == MCLBYTES)
3944                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3945         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3946                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3947         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3948                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3949         else  /* Using 16K cluster */
3950                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3951
3952         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3953
3954         /* Enable TCP header recognition */
3955         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3956             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3957             IXGBE_PSRTYPE_TCPHDR));
3958
3959         /* Disable RSC for ACK packets */
3960         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3961             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3962
3963         rxr->hw_rsc = TRUE;
3964 }
3965 #endif
3966
3967 static void
3968 ixgbe_free_receive_ring(struct rx_ring *rxr)
3969 {
3970         struct  adapter         *adapter;
3971         struct ixgbe_rx_buf       *rxbuf;
3972         int i;
3973
3974         adapter = rxr->adapter;
3975         for (i = 0; i < adapter->num_rx_desc; i++) {
3976                 rxbuf = &rxr->rx_buffers[i];
3977                 if (rxbuf->m_head != NULL) {
3978                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3979                             BUS_DMASYNC_POSTREAD);
3980                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3981                         rxbuf->m_head->m_flags |= M_PKTHDR;
3982                         m_freem(rxbuf->m_head);
3983                 }
3984                 if (rxbuf->m_pack != NULL) {
3985                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3986                             BUS_DMASYNC_POSTREAD);
3987                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3988                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3989                         m_freem(rxbuf->m_pack);
3990                 }
3991                 rxbuf->m_head = NULL;
3992                 rxbuf->m_pack = NULL;
3993         }
3994 }
3995
3996
3997 /*********************************************************************
3998  *
3999  *  Initialize a receive ring and its buffers.
4000  *
4001  **********************************************************************/
4002 static int
4003 ixgbe_setup_receive_ring(struct rx_ring *rxr)
4004 {
4005         struct  adapter         *adapter;
4006         struct ifnet            *ifp;
4007         device_t                dev;
4008         struct ixgbe_rx_buf     *rxbuf;
4009         bus_dma_segment_t       pseg[1], hseg[1];
4010 #if 0   /* NET_LRO */
4011         struct lro_ctrl         *lro = &rxr->lro;
4012 #endif
4013         int                     rsize, nsegs, error = 0;
4014 #ifdef DEV_NETMAP
4015         struct netmap_adapter *na = NA(rxr->adapter->ifp);
4016         struct netmap_slot *slot;
4017 #endif /* DEV_NETMAP */
4018
4019         adapter = rxr->adapter;
4020         ifp = adapter->ifp;
4021         dev = adapter->dev;
4022
4023         /* Clear the ring contents */
4024         IXGBE_RX_LOCK(rxr);
4025 #ifdef DEV_NETMAP
4026         /* same as in ixgbe_setup_transmit_ring() */
4027         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4028 #endif /* DEV_NETMAP */
4029         rsize = roundup2(adapter->num_rx_desc *
4030             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4031         bzero((void *)rxr->rx_base, rsize);
4032
4033         /* Free current RX buffer structs and their mbufs */
4034         ixgbe_free_receive_ring(rxr);
4035
4036         /* Configure header split? */
4037         if (ixgbe_header_split)
4038                 rxr->hdr_split = TRUE;
4039
4040         /* Now replenish the mbufs */
4041         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4042                 struct mbuf     *mh, *mp;
4043
4044                 rxbuf = &rxr->rx_buffers[j];
4045 #ifdef DEV_NETMAP
4046                 /*
4047                  * In netmap mode, fill the map and set the buffer
4048                  * address in the NIC ring, considering the offset
4049                  * between the netmap and NIC rings (see comment in
4050                  * ixgbe_setup_transmit_ring() ). No need to allocate
4051                  * an mbuf, so end the block with a continue;
4052                  */
4053                 if (slot) {
4054                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4055                         uint64_t paddr;
4056                         void *addr;
4057
4058                         addr = PNMB(slot + sj, &paddr);
4059                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4060                         /* Update descriptor */
4061                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4062                         continue;
4063                 }
4064 #endif /* DEV_NETMAP */
4065                 /*
4066                 ** Don't allocate mbufs if not
4067                 ** doing header split, its wasteful
4068                 */
4069                 if (rxr->hdr_split == FALSE)
4070                         goto skip_head;
4071
4072                 /* First the header */
4073                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4074                 if (rxbuf->m_head == NULL) {
4075                         error = ENOBUFS;
4076                         goto fail;
4077                 }
4078                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4079                 mh = rxbuf->m_head;
4080                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4081                 mh->m_flags |= M_PKTHDR;
4082                 /* Get the memory mapping */
4083                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4084                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4085                     &nsegs, BUS_DMA_NOWAIT);
4086
4087                 if (error != 0) /* Nothing elegant to do here */
4088                         goto fail;
4089                 bus_dmamap_sync(rxr->htag,
4090                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4091                 /* Update descriptor */
4092                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4093
4094 skip_head:
4095                 /* Now the payload cluster */
4096                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4097                     M_PKTHDR, adapter->rx_mbuf_sz);
4098                 if (rxbuf->m_pack == NULL) {
4099                         error = ENOBUFS;
4100                         goto fail;
4101                 }
4102                 mp = rxbuf->m_pack;
4103                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4104                 /* Get the memory mapping */
4105                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4106                     rxbuf->pmap, mp, hseg, 1,
4107                     &nsegs, BUS_DMA_NOWAIT);
4108                 if (error != 0)
4109                         goto fail;
4110                 bus_dmamap_sync(rxr->ptag,
4111                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4112                 /* Update descriptor */
4113                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4114         }
4115
4116
4117         /* Setup our descriptor indices */
4118         rxr->next_to_check = 0;
4119         rxr->next_to_refresh = 0;
4120         rxr->lro_enabled = FALSE;
4121         rxr->rx_split_packets = 0;
4122         rxr->rx_bytes = 0;
4123         rxr->discard = FALSE;
4124         rxr->vtag_strip = FALSE;
4125
4126         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4127             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4128
4129         /*
4130         ** Now set up the LRO interface:
4131         ** 82598 uses software LRO, the
4132         ** 82599 and X540 use a hardware assist.
4133         */
4134 #if 0 /* NET_LRO */
4135         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4136             (ifp->if_capenable & IFCAP_RXCSUM) &&
4137             (ifp->if_capenable & IFCAP_LRO))
4138                 ixgbe_setup_hw_rsc(rxr);
4139         else if (ifp->if_capenable & IFCAP_LRO) {
4140                 int err = tcp_lro_init(lro);
4141                 if (err) {
4142                         device_printf(dev, "LRO Initialization failed!\n");
4143                         goto fail;
4144                 }
4145                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4146                 rxr->lro_enabled = TRUE;
4147                 lro->ifp = adapter->ifp;
4148         }
4149 #endif
4150
4151         IXGBE_RX_UNLOCK(rxr);
4152         return (0);
4153
4154 fail:
4155         ixgbe_free_receive_ring(rxr);
4156         IXGBE_RX_UNLOCK(rxr);
4157         return (error);
4158 }
4159
4160 /*********************************************************************
4161  *
4162  *  Initialize all receive rings.
4163  *
4164  **********************************************************************/
4165 static int
4166 ixgbe_setup_receive_structures(struct adapter *adapter)
4167 {
4168         struct rx_ring *rxr = adapter->rx_rings;
4169         int j;
4170
4171         for (j = 0; j < adapter->num_queues; j++, rxr++)
4172                 if (ixgbe_setup_receive_ring(rxr))
4173                         goto fail;
4174
4175         return (0);
4176 fail:
4177         /*
4178          * Free RX buffers allocated so far, we will only handle
4179          * the rings that completed, the failing case will have
4180          * cleaned up for itself. 'j' failed, so its the terminus.
4181          */
4182         for (int i = 0; i < j; ++i) {
4183                 rxr = &adapter->rx_rings[i];
4184                 ixgbe_free_receive_ring(rxr);
4185         }
4186
4187         return (ENOBUFS);
4188 }
4189
4190 /*********************************************************************
4191  *
4192  *  Setup receive registers and features.
4193  *
4194  **********************************************************************/
4195 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4196
4197 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4198
4199 static void
4200 ixgbe_initialize_receive_units(struct adapter *adapter)
4201 {
4202         struct  rx_ring *rxr = adapter->rx_rings;
4203         struct ixgbe_hw *hw = &adapter->hw;
4204         struct ifnet   *ifp = adapter->ifp;
4205         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4206         u32             reta, mrqc = 0, hlreg, random[10];
4207
4208
4209         /*
4210          * Make sure receives are disabled while
4211          * setting up the descriptor ring
4212          */
4213         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4214         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4215             rxctrl & ~IXGBE_RXCTRL_RXEN);
4216
4217         /* Enable broadcasts */
4218         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4219         fctrl |= IXGBE_FCTRL_BAM;
4220         fctrl |= IXGBE_FCTRL_DPF;
4221         fctrl |= IXGBE_FCTRL_PMCF;
4222         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4223
4224         /* Set for Jumbo Frames? */
4225         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4226         if (ifp->if_mtu > ETHERMTU)
4227                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4228         else
4229                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4230 #ifdef DEV_NETMAP
4231         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4232         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4233                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4234         else
4235                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4236 #endif /* DEV_NETMAP */
4237         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4238
4239         bufsz = (adapter->rx_mbuf_sz +
4240             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4241
4242         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4243                 u64 rdba = rxr->rxdma.dma_paddr;
4244
4245                 /* Setup the Base and Length of the Rx Descriptor Ring */
4246                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4247                                (rdba & 0x00000000ffffffffULL));
4248                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4249                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4250                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4251
4252                 /* Set up the SRRCTL register */
4253                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4254                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4255                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4256                 srrctl |= bufsz;
4257                 if (rxr->hdr_split) {
4258                         /* Use a standard mbuf for the header */
4259                         srrctl |= ((IXGBE_RX_HDR <<
4260                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4261                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4262                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4263                 } else
4264                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4265                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4266
4267                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4268                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4269                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4270         }
4271
4272         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4273                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4274                               IXGBE_PSRTYPE_UDPHDR |
4275                               IXGBE_PSRTYPE_IPV4HDR |
4276                               IXGBE_PSRTYPE_IPV6HDR;
4277                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4278         }
4279
4280         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4281
4282         /* Setup RSS */
4283         if (adapter->num_queues > 1) {
4284                 int i, j;
4285                 reta = 0;
4286
4287                 /* set up random bits */
4288                 karc4rand(&random, sizeof(random));
4289
4290                 /* Set up the redirection table */
4291                 for (i = 0, j = 0; i < 128; i++, j++) {
4292                         if (j == adapter->num_queues) j = 0;
4293                         reta = (reta << 8) | (j * 0x11);
4294                         if ((i & 3) == 3)
4295                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4296                 }
4297
4298                 /* Now fill our hash function seeds */
4299                 for (int i = 0; i < 10; i++)
4300                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4301
4302                 /* Perform hash on these packet types */
4303                 mrqc = IXGBE_MRQC_RSSEN
4304                      | IXGBE_MRQC_RSS_FIELD_IPV4
4305                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4306                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4307                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4308                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4309                      | IXGBE_MRQC_RSS_FIELD_IPV6
4310                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4311                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4312                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4313                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4314
4315                 /* RSS and RX IPP Checksum are mutually exclusive */
4316                 rxcsum |= IXGBE_RXCSUM_PCSD;
4317         }
4318
4319         if (ifp->if_capenable & IFCAP_RXCSUM)
4320                 rxcsum |= IXGBE_RXCSUM_PCSD;
4321
4322         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4323                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4324
4325         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4326
4327         return;
4328 }
4329
4330 /*********************************************************************
4331  *
4332  *  Free all receive rings.
4333  *
4334  **********************************************************************/
4335 static void
4336 ixgbe_free_receive_structures(struct adapter *adapter)
4337 {
4338         struct rx_ring *rxr = adapter->rx_rings;
4339
4340         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4341 #if 0   /* NET_LRO */
4342                 struct lro_ctrl         *lro = &rxr->lro;
4343 #endif
4344                 ixgbe_free_receive_buffers(rxr);
4345                 /* Free LRO memory */
4346 #if 0   /* NET_LRO */
4347                 tcp_lro_free(lro);
4348 #endif
4349                 /* Free the ring memory as well */
4350                 ixgbe_dma_free(adapter, &rxr->rxdma);
4351         }
4352
4353         kfree(adapter->rx_rings, M_DEVBUF);
4354 }
4355
4356
4357 /*********************************************************************
4358  *
4359  *  Free receive ring data structures
4360  *
4361  **********************************************************************/
4362 static void
4363 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4364 {
4365         struct adapter          *adapter = rxr->adapter;
4366         struct ixgbe_rx_buf     *rxbuf;
4367
4368         INIT_DEBUGOUT("free_receive_structures: begin");
4369
4370         /* Cleanup any existing buffers */
4371         if (rxr->rx_buffers != NULL) {
4372                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4373                         rxbuf = &rxr->rx_buffers[i];
4374                         if (rxbuf->m_head != NULL) {
4375                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4376                                     BUS_DMASYNC_POSTREAD);
4377                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4378                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4379                                 m_freem(rxbuf->m_head);
4380                         }
4381                         if (rxbuf->m_pack != NULL) {
4382                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4383                                     BUS_DMASYNC_POSTREAD);
4384                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4385                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4386                                 m_freem(rxbuf->m_pack);
4387                         }
4388                         rxbuf->m_head = NULL;
4389                         rxbuf->m_pack = NULL;
4390                         if (rxbuf->hmap != NULL) {
4391                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4392                                 rxbuf->hmap = NULL;
4393                         }
4394                         if (rxbuf->pmap != NULL) {
4395                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4396                                 rxbuf->pmap = NULL;
4397                         }
4398                 }
4399                 if (rxr->rx_buffers != NULL) {
4400                         kfree(rxr->rx_buffers, M_DEVBUF);
4401                         rxr->rx_buffers = NULL;
4402                 }
4403         }
4404
4405         if (rxr->htag != NULL) {
4406                 bus_dma_tag_destroy(rxr->htag);
4407                 rxr->htag = NULL;
4408         }
4409         if (rxr->ptag != NULL) {
4410                 bus_dma_tag_destroy(rxr->ptag);
4411                 rxr->ptag = NULL;
4412         }
4413
4414         return;
4415 }
4416
4417 static __inline void
4418 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4419 {
4420
4421         /*
4422          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4423          * should be computed by hardware. Also it should not have VLAN tag in
4424          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4425          */
4426 #if 0   /* NET_LRO */
4427         if (rxr->lro_enabled &&
4428             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4429             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4430             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4431             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4432             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4433             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4434             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4435             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4436                 /*
4437                  * Send to the stack if:
4438                  **  - LRO not enabled, or
4439                  **  - no LRO resources, or
4440                  **  - lro enqueue fails
4441                  */
4442                 if (rxr->lro.lro_cnt != 0)
4443                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4444                                 return;
4445         }
4446 #endif
4447         IXGBE_RX_UNLOCK(rxr);
4448         (*ifp->if_input)(ifp, m);
4449         IXGBE_RX_LOCK(rxr);
4450 }
4451
4452 static __inline void
4453 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4454 {
4455         struct ixgbe_rx_buf     *rbuf;
4456
4457         rbuf = &rxr->rx_buffers[i];
4458
4459         if (rbuf->fmp != NULL) {/* Partial chain ? */
4460                 rbuf->fmp->m_flags |= M_PKTHDR;
4461                 m_freem(rbuf->fmp);
4462                 rbuf->fmp = NULL;
4463         }
4464
4465         /*
4466         ** With advanced descriptors the writeback
4467         ** clobbers the buffer addrs, so its easier
4468         ** to just free the existing mbufs and take
4469         ** the normal refresh path to get new buffers
4470         ** and mapping.
4471         */
4472         if (rbuf->m_head) {
4473                 m_free(rbuf->m_head);
4474                 rbuf->m_head = NULL;
4475         }
4476
4477         if (rbuf->m_pack) {
4478                 m_free(rbuf->m_pack);
4479                 rbuf->m_pack = NULL;
4480         }
4481
4482         return;
4483 }
4484
4485
4486 /*********************************************************************
4487  *
4488  *  This routine executes in interrupt context. It replenishes
4489  *  the mbufs in the descriptor and sends data which has been
4490  *  dma'ed into host memory to upper layer.
4491  *
4492  *  We loop at most count times if count is > 0, or until done if
4493  *  count < 0.
4494  *
4495  *  Return TRUE for more work, FALSE for all clean.
4496  *********************************************************************/
4497 static bool
4498 ixgbe_rxeof(struct ix_queue *que, int count)
4499 {
4500         struct adapter          *adapter = que->adapter;
4501         struct rx_ring          *rxr = que->rxr;
4502         struct ifnet            *ifp = adapter->ifp;
4503 #if 0   /* NET_LRO */
4504         struct lro_ctrl         *lro = &rxr->lro;
4505         struct lro_entry        *queued;
4506 #endif
4507         int                     i, nextp, processed = 0;
4508         u32                     staterr = 0;
4509         union ixgbe_adv_rx_desc *cur;
4510         struct ixgbe_rx_buf     *rbuf, *nbuf;
4511
4512         IXGBE_RX_LOCK(rxr);
4513
4514 #ifdef DEV_NETMAP
4515         if (ifp->if_capenable & IFCAP_NETMAP) {
4516                 /*
4517                  * Same as the txeof routine: only wakeup clients on intr.
4518                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4519                  * mitigation (ixgbe_rxsync() will not look for new packets
4520                  * unless NKR_PENDINTR is set).
4521                  */
4522                 struct netmap_adapter *na = NA(ifp);
4523
4524                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4525                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4526                 IXGBE_RX_UNLOCK(rxr);
4527                 IXGBE_CORE_LOCK(adapter);
4528                 selwakeuppri(&na->rx_si, PI_NET);
4529                 IXGBE_CORE_UNLOCK(adapter);
4530                 return (FALSE);
4531         }
4532 #endif /* DEV_NETMAP */
4533         for (i = rxr->next_to_check; count != 0;) {
4534                 struct mbuf     *sendmp, *mh, *mp;
4535                 u32             rsc, ptype;
4536                 u16             hlen, plen, hdr;
4537 #ifdef NET_VLAN
4538                 u16             vtag = 0;
4539 #endif
4540                 bool            eop;
4541
4542                 /* Sync the ring. */
4543                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4544                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4545
4546                 cur = &rxr->rx_base[i];
4547                 staterr = le32toh(cur->wb.upper.status_error);
4548
4549                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4550                         break;
4551                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4552                         break;
4553
4554                 count--;
4555                 sendmp = NULL;
4556                 nbuf = NULL;
4557                 rsc = 0;
4558                 cur->wb.upper.status_error = 0;
4559                 rbuf = &rxr->rx_buffers[i];
4560                 mh = rbuf->m_head;
4561                 mp = rbuf->m_pack;
4562
4563                 plen = le16toh(cur->wb.upper.length);
4564                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4565                     IXGBE_RXDADV_PKTTYPE_MASK;
4566                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4567                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4568
4569                 /* Process vlan info */
4570 #ifdef NET_VLAN
4571                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4572                         vtag = le16toh(cur->wb.upper.vlan);
4573 #endif
4574
4575                 /* Make sure bad packets are discarded */
4576                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4577                     (rxr->discard)) {
4578                         ifp->if_ierrors++;
4579                         rxr->rx_discarded++;
4580                         if (eop)
4581                                 rxr->discard = FALSE;
4582                         else
4583                                 rxr->discard = TRUE;
4584                         ixgbe_rx_discard(rxr, i);
4585                         goto next_desc;
4586                 }
4587
4588                 /*
4589                 ** On 82599 which supports a hardware
4590                 ** LRO (called HW RSC), packets need
4591                 ** not be fragmented across sequential
4592                 ** descriptors, rather the next descriptor
4593                 ** is indicated in bits of the descriptor.
4594                 ** This also means that we might proceses
4595                 ** more than one packet at a time, something
4596                 ** that has never been true before, it
4597                 ** required eliminating global chain pointers
4598                 ** in favor of what we are doing here.  -jfv
4599                 */
4600                 if (!eop) {
4601                         /*
4602                         ** Figure out the next descriptor
4603                         ** of this frame.
4604                         */
4605                         if (rxr->hw_rsc == TRUE) {
4606                                 rsc = ixgbe_rsc_count(cur);
4607                                 rxr->rsc_num += (rsc - 1);
4608                         }
4609                         if (rsc) { /* Get hardware index */
4610                                 nextp = ((staterr &
4611                                     IXGBE_RXDADV_NEXTP_MASK) >>
4612                                     IXGBE_RXDADV_NEXTP_SHIFT);
4613                         } else { /* Just sequential */
4614                                 nextp = i + 1;
4615                                 if (nextp == adapter->num_rx_desc)
4616                                         nextp = 0;
4617                         }
4618                         nbuf = &rxr->rx_buffers[nextp];
4619                         prefetch(nbuf);
4620                 }
4621                 /*
4622                 ** The header mbuf is ONLY used when header
4623                 ** split is enabled, otherwise we get normal
4624                 ** behavior, ie, both header and payload
4625                 ** are DMA'd into the payload buffer.
4626                 **
4627                 ** Rather than using the fmp/lmp global pointers
4628                 ** we now keep the head of a packet chain in the
4629                 ** buffer struct and pass this along from one
4630                 ** descriptor to the next, until we get EOP.
4631                 */
4632                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4633                         /* This must be an initial descriptor */
4634                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4635                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4636                         if (hlen > IXGBE_RX_HDR)
4637                                 hlen = IXGBE_RX_HDR;
4638                         mh->m_len = hlen;
4639                         mh->m_flags |= M_PKTHDR;
4640                         mh->m_next = NULL;
4641                         mh->m_pkthdr.len = mh->m_len;
4642                         /* Null buf pointer so it is refreshed */
4643                         rbuf->m_head = NULL;
4644                         /*
4645                         ** Check the payload length, this
4646                         ** could be zero if its a small
4647                         ** packet.
4648                         */
4649                         if (plen > 0) {
4650                                 mp->m_len = plen;
4651                                 mp->m_next = NULL;
4652                                 mp->m_flags &= ~M_PKTHDR;
4653                                 mh->m_next = mp;
4654                                 mh->m_pkthdr.len += mp->m_len;
4655                                 /* Null buf pointer so it is refreshed */
4656                                 rbuf->m_pack = NULL;
4657                                 rxr->rx_split_packets++;
4658                         }
4659                         /*
4660                         ** Now create the forward
4661                         ** chain so when complete
4662                         ** we wont have to.
4663                         */
4664                         if (eop == 0) {
4665                                 /* stash the chain head */
4666                                 nbuf->fmp = mh;
4667                                 /* Make forward chain */
4668                                 if (plen)
4669                                         mp->m_next = nbuf->m_pack;
4670                                 else
4671                                         mh->m_next = nbuf->m_pack;
4672                         } else {
4673                                 /* Singlet, prepare to send */
4674                                 sendmp = mh;
4675                                 /* If hardware handled vtag */
4676 #ifdef NET_VLAN
4677                                 if (vtag) {
4678                                         sendmp->m_pkthdr.ether_vtag = vtag;
4679                                         sendmp->m_flags |= M_VLANTAG;
4680                                 }
4681 #endif
4682                         }
4683                 } else {
4684                         /*
4685                         ** Either no header split, or a
4686                         ** secondary piece of a fragmented
4687                         ** split packet.
4688                         */
4689                         mp->m_len = plen;
4690                         /*
4691                         ** See if there is a stored head
4692                         ** that determines what we are
4693                         */
4694                         sendmp = rbuf->fmp;
4695                         rbuf->m_pack = rbuf->fmp = NULL;
4696
4697                         if (sendmp != NULL) {  /* secondary frag */
4698                                 mp->m_flags &= ~M_PKTHDR;
4699                                 sendmp->m_pkthdr.len += mp->m_len;
4700                         } else {
4701                                 /* first desc of a non-ps chain */
4702                                 sendmp = mp;
4703                                 sendmp->m_flags |= M_PKTHDR;
4704                                 sendmp->m_pkthdr.len = mp->m_len;
4705 #ifdef NET_VLAN
4706                                 if (staterr & IXGBE_RXD_STAT_VP) {
4707                                         sendmp->m_pkthdr.ether_vtag = vtag;
4708                                         sendmp->m_flags |= M_VLANTAG;
4709                                 }
4710 #endif
4711                         }
4712                         /* Pass the head pointer on */
4713                         if (eop == 0) {
4714                                 nbuf->fmp = sendmp;
4715                                 sendmp = NULL;
4716                                 mp->m_next = nbuf->m_pack;
4717                         }
4718                 }
4719                 ++processed;
4720                 /* Sending this frame? */
4721                 if (eop) {
4722                         sendmp->m_pkthdr.rcvif = ifp;
4723                         ifp->if_ipackets++;
4724                         rxr->rx_packets++;
4725                         /* capture data for AIM */
4726                         rxr->bytes += sendmp->m_pkthdr.len;
4727                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4728                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4729                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4730 #if 0 /* __FreeBSD_version >= 800000 */
4731                         sendmp->m_pkthdr.flowid = que->msix;
4732                         sendmp->m_flags |= M_FLOWID;
4733 #endif
4734                 }
4735 next_desc:
4736                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4737                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4738
4739                 /* Advance our pointers to the next descriptor. */
4740                 if (++i == adapter->num_rx_desc)
4741                         i = 0;
4742
4743                 /* Now send to the stack or do LRO */
4744                 if (sendmp != NULL) {
4745                         rxr->next_to_check = i;
4746                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4747                         i = rxr->next_to_check;
4748                 }
4749
4750                /* Every 8 descriptors we go to refresh mbufs */
4751                 if (processed == 8) {
4752                         ixgbe_refresh_mbufs(rxr, i);
4753                         processed = 0;
4754                 }
4755         }
4756
4757         /* Refresh any remaining buf structs */
4758         if (ixgbe_rx_unrefreshed(rxr))
4759                 ixgbe_refresh_mbufs(rxr, i);
4760
4761         rxr->next_to_check = i;
4762
4763         /*
4764          * Flush any outstanding LRO work
4765          */
4766 #if 0   /* NET_LRO */
4767         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4768                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4769                 tcp_lro_flush(lro, queued);
4770         }
4771 #endif
4772
4773         IXGBE_RX_UNLOCK(rxr);
4774
4775         /*
4776         ** We still have cleaning to do?
4777         ** Schedule another interrupt if so.
4778         */
4779         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4780                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4781                 return (TRUE);
4782         }
4783
4784         return (FALSE);
4785 }
4786
4787
4788 /*********************************************************************
4789  *
4790  *  Verify that the hardware indicated that the checksum is valid.
4791  *  Inform the stack about the status of checksum so that stack
4792  *  doesn't spend time verifying the checksum.
4793  *
4794  *********************************************************************/
4795 static void
4796 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4797 {
4798         u16     status = (u16) staterr;
4799         u8      errors = (u8) (staterr >> 24);
4800         bool    sctp = FALSE;
4801
4802         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4803             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4804                 sctp = TRUE;
4805
4806         if (status & IXGBE_RXD_STAT_IPCS) {
4807                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4808                         /* IP Checksum Good */
4809                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4810                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4811
4812                 } else
4813                         mp->m_pkthdr.csum_flags = 0;
4814         }
4815         if (status & IXGBE_RXD_STAT_L4CS) {
4816                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4817 #if 0
4818                 if (sctp)
4819                         type = CSUM_SCTP_VALID;
4820 #endif
4821                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4822                         mp->m_pkthdr.csum_flags |= type;
4823                         if (!sctp)
4824                                 mp->m_pkthdr.csum_data = htons(0xffff);
4825                 }
4826         }
4827         return;
4828 }
4829
4830
4831 /*
4832 ** This routine is run via an vlan config EVENT,
4833 ** it enables us to use the HW Filter table since
4834 ** we can get the vlan id. This just creates the
4835 ** entry in the soft version of the VFTA, init will
4836 ** repopulate the real table.
4837 */
4838 static void
4839 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4840 {
4841         struct adapter  *adapter = ifp->if_softc;
4842         u16             index, bit;
4843
4844         if (ifp->if_softc !=  arg)   /* Not our event */
4845                 return;
4846
4847         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4848                 return;
4849
4850         IXGBE_CORE_LOCK(adapter);
4851         index = (vtag >> 5) & 0x7F;
4852         bit = vtag & 0x1F;
4853         adapter->shadow_vfta[index] |= (1 << bit);
4854         ++adapter->num_vlans;
4855         ixgbe_init_locked(adapter);
4856         IXGBE_CORE_UNLOCK(adapter);
4857 }
4858
4859 /*
4860 ** This routine is run via an vlan
4861 ** unconfig EVENT, remove our entry
4862 ** in the soft vfta.
4863 */
4864 static void
4865 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4866 {
4867         struct adapter  *adapter = ifp->if_softc;
4868         u16             index, bit;
4869
4870         if (ifp->if_softc !=  arg)
4871                 return;
4872
4873         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4874                 return;
4875
4876         IXGBE_CORE_LOCK(adapter);
4877         index = (vtag >> 5) & 0x7F;
4878         bit = vtag & 0x1F;
4879         adapter->shadow_vfta[index] &= ~(1 << bit);
4880         --adapter->num_vlans;
4881         /* Re-init to load the changes */
4882         ixgbe_init_locked(adapter);
4883         IXGBE_CORE_UNLOCK(adapter);
4884 }
4885
4886 static void
4887 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4888 {
4889 #ifdef NET_VLAN
4890         struct ifnet    *ifp = adapter->ifp;
4891         struct ixgbe_hw *hw = &adapter->hw;
4892         struct rx_ring  *rxr;
4893         u32             ctrl;
4894
4895         /*
4896         ** We get here thru init_locked, meaning
4897         ** a soft reset, this has already cleared
4898         ** the VFTA and other state, so if there
4899         ** have been no vlan's registered do nothing.
4900         */
4901         if (adapter->num_vlans == 0)
4902                 return;
4903
4904         /*
4905         ** A soft reset zero's out the VFTA, so
4906         ** we need to repopulate it now.
4907         */
4908         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4909                 if (adapter->shadow_vfta[i] != 0)
4910                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4911                             adapter->shadow_vfta[i]);
4912
4913         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4914         /* Enable the Filter Table if enabled */
4915         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4916                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4917                 ctrl |= IXGBE_VLNCTRL_VFE;
4918         }
4919         if (hw->mac.type == ixgbe_mac_82598EB)
4920                 ctrl |= IXGBE_VLNCTRL_VME;
4921         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4922
4923         /* Setup the queues for vlans */
4924         for (int i = 0; i < adapter->num_queues; i++) {
4925                 rxr = &adapter->rx_rings[i];
4926                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4927                 if (hw->mac.type != ixgbe_mac_82598EB) {
4928                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4929                         ctrl |= IXGBE_RXDCTL_VME;
4930                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4931                 }
4932                 rxr->vtag_strip = TRUE;
4933         }
4934 #endif
4935 }
4936
4937 static void
4938 ixgbe_enable_intr(struct adapter *adapter)
4939 {
4940         struct ixgbe_hw *hw = &adapter->hw;
4941         struct ix_queue *que = adapter->queues;
4942         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4943
4944
4945         /* Enable Fan Failure detection */
4946         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4947                     mask |= IXGBE_EIMS_GPI_SDP1;
4948         else {
4949                     mask |= IXGBE_EIMS_ECC;
4950                     mask |= IXGBE_EIMS_GPI_SDP0;
4951                     mask |= IXGBE_EIMS_GPI_SDP1;
4952                     mask |= IXGBE_EIMS_GPI_SDP2;
4953 #ifdef IXGBE_FDIR
4954                     mask |= IXGBE_EIMS_FLOW_DIR;
4955 #endif
4956         }
4957
4958         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4959
4960         /* With RSS we use auto clear */
4961         if (adapter->msix_mem) {
4962                 mask = IXGBE_EIMS_ENABLE_MASK;
4963                 /* Don't autoclear Link */
4964                 mask &= ~IXGBE_EIMS_OTHER;
4965                 mask &= ~IXGBE_EIMS_LSC;
4966                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4967         }
4968
4969         /*
4970         ** Now enable all queues, this is done separately to
4971         ** allow for handling the extended (beyond 32) MSIX
4972         ** vectors that can be used by 82599
4973         */
4974         for (int i = 0; i < adapter->num_queues; i++, que++)
4975                 ixgbe_enable_queue(adapter, que->msix);
4976
4977         IXGBE_WRITE_FLUSH(hw);
4978
4979         return;
4980 }
4981
4982 static void
4983 ixgbe_disable_intr(struct adapter *adapter)
4984 {
4985         if (adapter->msix_mem)
4986                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4987         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4988                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4989         } else {
4990                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4991                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4992                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4993         }
4994         IXGBE_WRITE_FLUSH(&adapter->hw);
4995         return;
4996 }
4997
4998 u16
4999 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5000 {
5001         u16 value;
5002
5003         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
5004             reg, 2);
5005
5006         return (value);
5007 }
5008
5009 void
5010 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5011 {
5012         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
5013             reg, value, 2);
5014
5015         return;
5016 }
5017
5018 /*
5019 ** Setup the correct IVAR register for a particular MSIX interrupt
5020 **   (yes this is all very magic and confusing :)
5021 **  - entry is the register array entry
5022 **  - vector is the MSIX vector for this queue
5023 **  - type is RX/TX/MISC
5024 */
5025 static void
5026 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5027 {
5028         struct ixgbe_hw *hw = &adapter->hw;
5029         u32 ivar, index;
5030
5031         vector |= IXGBE_IVAR_ALLOC_VAL;
5032
5033         switch (hw->mac.type) {
5034
5035         case ixgbe_mac_82598EB:
5036                 if (type == -1)
5037                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5038                 else
5039                         entry += (type * 64);
5040                 index = (entry >> 2) & 0x1F;
5041                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5042                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5043                 ivar |= (vector << (8 * (entry & 0x3)));
5044                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5045                 break;
5046
5047         case ixgbe_mac_82599EB:
5048         case ixgbe_mac_X540:
5049                 if (type == -1) { /* MISC IVAR */
5050                         index = (entry & 1) * 8;
5051                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5052                         ivar &= ~(0xFF << index);
5053                         ivar |= (vector << index);
5054                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5055                 } else {        /* RX/TX IVARS */
5056                         index = (16 * (entry & 1)) + (8 * type);
5057                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5058                         ivar &= ~(0xFF << index);
5059                         ivar |= (vector << index);
5060                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5061                 }
5062
5063         default:
5064                 break;
5065         }
5066 }
5067
5068 static void
5069 ixgbe_configure_ivars(struct adapter *adapter)
5070 {
5071         struct  ix_queue *que = adapter->queues;
5072         u32 newitr;
5073
5074         if (ixgbe_max_interrupt_rate > 0)
5075                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5076         else
5077                 newitr = 0;
5078
5079         for (int i = 0; i < adapter->num_queues; i++, que++) {
5080                 /* First the RX queue entry */
5081                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5082                 /* ... and the TX */
5083                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5084                 /* Set an Initial EITR value */
5085                 IXGBE_WRITE_REG(&adapter->hw,
5086                     IXGBE_EITR(que->msix), newitr);
5087         }
5088
5089         /* For the Link interrupt */
5090         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5091 }
5092
5093 /*
5094 ** ixgbe_sfp_probe - called in the local timer to
5095 ** determine if a port had optics inserted.
5096 */
5097 static bool ixgbe_sfp_probe(struct adapter *adapter)
5098 {
5099         struct ixgbe_hw *hw = &adapter->hw;
5100         device_t        dev = adapter->dev;
5101         bool            result = FALSE;
5102
5103         if ((hw->phy.type == ixgbe_phy_nl) &&
5104             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5105                 s32 ret = hw->phy.ops.identify_sfp(hw);
5106                 if (ret)
5107                         goto out;
5108                 ret = hw->phy.ops.reset(hw);
5109                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5110                         device_printf(dev,"Unsupported SFP+ module detected!");
5111                         kprintf(" Reload driver with supported module.\n");
5112                         adapter->sfp_probe = FALSE;
5113                         goto out;
5114                 } else
5115                         device_printf(dev,"SFP+ module detected!\n");
5116                 /* We now have supported optics */
5117                 adapter->sfp_probe = FALSE;
5118                 /* Set the optics type so system reports correctly */
5119                 ixgbe_setup_optics(adapter);
5120                 result = TRUE;
5121         }
5122 out:
5123         return (result);
5124 }
5125
5126 /*
5127 ** Tasklet handler for MSIX Link interrupts
5128 **  - do outside interrupt since it might sleep
5129 */
5130 static void
5131 ixgbe_handle_link(void *context, int pending)
5132 {
5133         struct adapter  *adapter = context;
5134
5135         ixgbe_check_link(&adapter->hw,
5136             &adapter->link_speed, &adapter->link_up, 0);
5137         ixgbe_update_link_status(adapter);
5138 }
5139
5140 /*
5141 ** Tasklet for handling SFP module interrupts
5142 */
5143 static void
5144 ixgbe_handle_mod(void *context, int pending)
5145 {
5146         struct adapter  *adapter = context;
5147         struct ixgbe_hw *hw = &adapter->hw;
5148         device_t        dev = adapter->dev;
5149         u32 err;
5150
5151         err = hw->phy.ops.identify_sfp(hw);
5152         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5153                 device_printf(dev,
5154                     "Unsupported SFP+ module type was detected.\n");
5155                 return;
5156         }
5157         err = hw->mac.ops.setup_sfp(hw);
5158         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5159                 device_printf(dev,
5160                     "Setup failure - unsupported SFP+ module type.\n");
5161                 return;
5162         }
5163         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5164         return;
5165 }
5166
5167
5168 /*
5169 ** Tasklet for handling MSF (multispeed fiber) interrupts
5170 */
5171 static void
5172 ixgbe_handle_msf(void *context, int pending)
5173 {
5174         struct adapter  *adapter = context;
5175         struct ixgbe_hw *hw = &adapter->hw;
5176         u32 autoneg;
5177         bool negotiate;
5178
5179         autoneg = hw->phy.autoneg_advertised;
5180         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5181                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5182         if (hw->mac.ops.setup_link)
5183                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5184         return;
5185 }
5186
5187 #ifdef IXGBE_FDIR
5188 /*
5189 ** Tasklet for reinitializing the Flow Director filter table
5190 */
5191 static void
5192 ixgbe_reinit_fdir(void *context, int pending)
5193 {
5194         struct adapter  *adapter = context;
5195         struct ifnet   *ifp = adapter->ifp;
5196
5197         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5198                 return;
5199         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5200         adapter->fdir_reinit = 0;
5201         /* re-enable flow director interrupts */
5202         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5203         /* Restart the interface */
5204         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5205         return;
5206 }
5207 #endif
5208
5209 /**********************************************************************
5210  *
5211  *  Update the board statistics counters.
5212  *
5213  **********************************************************************/
5214 static void
5215 ixgbe_update_stats_counters(struct adapter *adapter)
5216 {
5217         struct ifnet   *ifp = adapter->ifp;
5218         struct ixgbe_hw *hw = &adapter->hw;
5219         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5220         u64  total_missed_rx = 0;
5221
5222         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5223         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5224         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5225         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5226
5227         for (int i = 0; i < 8; i++) {
5228                 u32 mp;
5229                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5230                 /* missed_rx tallies misses for the gprc workaround */
5231                 missed_rx += mp;
5232                 /* global total per queue */
5233                 adapter->stats.mpc[i] += mp;
5234                 /* Running comprehensive total for stats display */
5235                 total_missed_rx += adapter->stats.mpc[i];
5236                 if (hw->mac.type == ixgbe_mac_82598EB)
5237                         adapter->stats.rnbc[i] +=
5238                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5239                 adapter->stats.pxontxc[i] +=
5240                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5241                 adapter->stats.pxonrxc[i] +=
5242                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5243                 adapter->stats.pxofftxc[i] +=
5244                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5245                 adapter->stats.pxoffrxc[i] +=
5246                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5247                 adapter->stats.pxon2offc[i] +=
5248                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5249         }
5250         for (int i = 0; i < 16; i++) {
5251                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5252                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5253                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5254                 adapter->stats.qbrc[i] +=
5255                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5256                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5257                 adapter->stats.qbtc[i] +=
5258                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5259                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5260         }
5261         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5262         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5263         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5264
5265         /* Hardware workaround, gprc counts missed packets */
5266         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5267         adapter->stats.gprc -= missed_rx;
5268
5269         if (hw->mac.type != ixgbe_mac_82598EB) {
5270                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5271                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5272                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5273                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5274                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5275                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5276                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5277                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5278         } else {
5279                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5280                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5281                 /* 82598 only has a counter in the high register */
5282                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5283                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5284                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5285         }
5286
5287         /*
5288          * Workaround: mprc hardware is incorrectly counting
5289          * broadcasts, so for now we subtract those.
5290          */
5291         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5292         adapter->stats.bprc += bprc;
5293         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5294         if (hw->mac.type == ixgbe_mac_82598EB)
5295                 adapter->stats.mprc -= bprc;
5296
5297         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5298         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5299         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5300         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5301         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5302         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5303
5304         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5305         adapter->stats.lxontxc += lxon;
5306         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5307         adapter->stats.lxofftxc += lxoff;
5308         total = lxon + lxoff;
5309
5310         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5311         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5312         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5313         adapter->stats.gptc -= total;
5314         adapter->stats.mptc -= total;
5315         adapter->stats.ptc64 -= total;
5316         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5317
5318         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5319         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5320         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5321         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5322         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5323         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5324         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5325         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5326         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5327         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5328         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5329         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5330         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5331         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5332         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5333         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5334         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5335         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5336         /* Only read FCOE on 82599 */
5337         if (hw->mac.type != ixgbe_mac_82598EB) {
5338                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5339                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5340                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5341                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5342                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5343         }
5344
5345         /* Fill out the OS statistics structure */
5346         ifp->if_ipackets = adapter->stats.gprc;
5347         ifp->if_opackets = adapter->stats.gptc;
5348         ifp->if_ibytes = adapter->stats.gorc;
5349         ifp->if_obytes = adapter->stats.gotc;
5350         ifp->if_imcasts = adapter->stats.mprc;
5351         ifp->if_collisions = 0;
5352
5353         /* Rx Errors */
5354         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5355                 adapter->stats.rlec;
5356 }
5357
5358 /** ixgbe_sysctl_tdh_handler - Handler function
5359  *  Retrieves the TDH value from the hardware
5360  */
5361 static int
5362 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5363 {
5364         int error;
5365
5366         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5367         if (!txr) return 0;
5368
5369         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5370         error = sysctl_handle_int(oidp, &val, 0, req);
5371         if (error || !req->newptr)
5372                 return error;
5373         return 0;
5374 }
5375
5376 /** ixgbe_sysctl_tdt_handler - Handler function
5377  *  Retrieves the TDT value from the hardware
5378  */
5379 static int
5380 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5381 {
5382         int error;
5383
5384         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5385         if (!txr) return 0;
5386
5387         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5388         error = sysctl_handle_int(oidp, &val, 0, req);
5389         if (error || !req->newptr)
5390                 return error;
5391         return 0;
5392 }
5393
5394 /** ixgbe_sysctl_rdh_handler - Handler function
5395  *  Retrieves the RDH value from the hardware
5396  */
5397 static int
5398 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5399 {
5400         int error;
5401
5402         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5403         if (!rxr) return 0;
5404
5405         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5406         error = sysctl_handle_int(oidp, &val, 0, req);
5407         if (error || !req->newptr)
5408                 return error;
5409         return 0;
5410 }
5411
5412 /** ixgbe_sysctl_rdt_handler - Handler function
5413  *  Retrieves the RDT value from the hardware
5414  */
5415 static int
5416 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5417 {
5418         int error;
5419
5420         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5421         if (!rxr) return 0;
5422
5423         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5424         error = sysctl_handle_int(oidp, &val, 0, req);
5425         if (error || !req->newptr)
5426                 return error;
5427         return 0;
5428 }
5429
5430 static int
5431 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5432 {
5433         int error;
5434         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5435         unsigned int reg, usec, rate;
5436
5437         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5438         usec = ((reg & 0x0FF8) >> 3);
5439         if (usec > 0)
5440                 rate = 500000 / usec;
5441         else
5442                 rate = 0;
5443         error = sysctl_handle_int(oidp, &rate, 0, req);
5444         if (error || !req->newptr)
5445                 return error;
5446         reg &= ~0xfff; /* default, no limitation */
5447         ixgbe_max_interrupt_rate = 0;
5448         if (rate > 0 && rate < 500000) {
5449                 if (rate < 1000)
5450                         rate = 1000;
5451                 ixgbe_max_interrupt_rate = rate;
5452                 reg |= ((4000000/rate) & 0xff8 );
5453         }
5454         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5455         return 0;
5456 }
5457
5458 /*
5459  * Add sysctl variables, one per statistic, to the system.
5460  */
5461 static void
5462 ixgbe_add_hw_stats(struct adapter *adapter)
5463 {
5464         struct tx_ring *txr = adapter->tx_rings;
5465         struct rx_ring *rxr = adapter->rx_rings;
5466
5467         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5468         struct sysctl_oid *tree = adapter->sysctl_tree;
5469         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5470         struct ixgbe_hw_stats *stats = &adapter->stats;
5471
5472         struct sysctl_oid *stat_node, *queue_node;
5473         struct sysctl_oid_list *stat_list, *queue_list;
5474
5475 #define QUEUE_NAME_LEN 32
5476         char namebuf[QUEUE_NAME_LEN];
5477
5478         /* Driver Statistics */
5479         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5480                         CTLFLAG_RD, &adapter->dropped_pkts,
5481                         "Driver dropped packets");
5482         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5483                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5484                         "m_defrag() failed");
5485         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5486                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5487                         "Driver tx dma failure in xmit");
5488         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5489                         CTLFLAG_RD, &adapter->watchdog_events,
5490                         "Watchdog timeouts");
5491 #if 0   /* NET_TSO */
5492         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5493                         CTLFLAG_RD, &adapter->tso_tx,
5494                         "TSO");
5495 #endif
5496         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5497                         CTLFLAG_RD, &adapter->link_irq,
5498                         "Link MSIX IRQ Handled");
5499
5500         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5501         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5502                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5503                                             CTLFLAG_RD, NULL, "Queue Name");
5504                 queue_list = SYSCTL_CHILDREN(queue_node);
5505
5506                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5507                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5508                                 sizeof(&adapter->queues[i]),
5509                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5510                                 "Interrupt Rate");
5511                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5512                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5513                                 "irqs on this queue");
5514                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5515                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5516                                 ixgbe_sysctl_tdh_handler, "IU",
5517                                 "Transmit Descriptor Head");
5518                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5519                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5520                                 ixgbe_sysctl_tdt_handler, "IU",
5521                                 "Transmit Descriptor Tail");
5522                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5523                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5524                                 "Queue No Descriptor Available");
5525                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5526                                 CTLFLAG_RD, &txr->total_packets, 0,
5527                                 "Queue Packets Transmitted");
5528         }
5529
5530         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5531         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5532                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5533                                             CTLFLAG_RD, NULL, "Queue Name");
5534                 queue_list = SYSCTL_CHILDREN(queue_node);
5535
5536 #if 0   /* NET_LRO */
5537                 struct lro_ctrl *lro = &rxr->lro;
5538 #endif
5539
5540         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5541                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5542                                             CTLFLAG_RD, NULL, "Queue Name");
5543                 queue_list = SYSCTL_CHILDREN(queue_node);
5544
5545                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5546                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5547                                 ixgbe_sysctl_rdh_handler, "IU",
5548                                 "Receive Descriptor Head");
5549                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5550                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5551                                 ixgbe_sysctl_rdt_handler, "IU",
5552                                 "Receive Descriptor Tail");
5553                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5554                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5555                                 "Queue Packets Received");
5556                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5557                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5558                                 "Queue Bytes Received");
5559 #if 0   /* NET_LRO */
5560                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5561                                 CTLFLAG_RD, &lro->lro_queued, 0,
5562                                 "LRO Queued");
5563                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5564                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5565                                 "LRO Flushed");
5566 #endif
5567         }
5568
5569         /* MAC stats get the own sub node */
5570
5571         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5572                                     CTLFLAG_RD, NULL, "MAC Statistics");
5573         stat_list = SYSCTL_CHILDREN(stat_node);
5574
5575         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5576                         CTLFLAG_RD, &stats->crcerrs, 0,
5577                         "CRC Errors");
5578         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5579                         CTLFLAG_RD, &stats->illerrc, 0,
5580                         "Illegal Byte Errors");
5581         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5582                         CTLFLAG_RD, &stats->errbc, 0,
5583                         "Byte Errors");
5584         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5585                         CTLFLAG_RD, &stats->mspdc, 0,
5586                         "MAC Short Packets Discarded");
5587         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5588                         CTLFLAG_RD, &stats->mlfc, 0,
5589                         "MAC Local Faults");
5590         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5591                         CTLFLAG_RD, &stats->mrfc, 0,
5592                         "MAC Remote Faults");
5593         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5594                         CTLFLAG_RD, &stats->rlec, 0,
5595                         "Receive Length Errors");
5596         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5597                         CTLFLAG_RD, &stats->lxontxc, 0,
5598                         "Link XON Transmitted");
5599         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5600                         CTLFLAG_RD, &stats->lxonrxc, 0,
5601                         "Link XON Received");
5602         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5603                         CTLFLAG_RD, &stats->lxofftxc, 0,
5604                         "Link XOFF Transmitted");
5605         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5606                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5607                         "Link XOFF Received");
5608
5609         /* Packet Reception Stats */
5610         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5611                         CTLFLAG_RD, &stats->tor, 0,
5612                         "Total Octets Received");
5613         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5614                         CTLFLAG_RD, &stats->gorc, 0,
5615                         "Good Octets Received");
5616         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5617                         CTLFLAG_RD, &stats->tpr, 0,
5618                         "Total Packets Received");
5619         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5620                         CTLFLAG_RD, &stats->gprc, 0,
5621                         "Good Packets Received");
5622         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5623                         CTLFLAG_RD, &stats->mprc, 0,
5624                         "Multicast Packets Received");
5625         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5626                         CTLFLAG_RD, &stats->bprc, 0,
5627                         "Broadcast Packets Received");
5628         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5629                         CTLFLAG_RD, &stats->prc64, 0,
5630                         "64 byte frames received ");
5631         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5632                         CTLFLAG_RD, &stats->prc127, 0,
5633                         "65-127 byte frames received");
5634         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5635                         CTLFLAG_RD, &stats->prc255, 0,
5636                         "128-255 byte frames received");
5637         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5638                         CTLFLAG_RD, &stats->prc511, 0,
5639                         "256-511 byte frames received");
5640         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5641                         CTLFLAG_RD, &stats->prc1023, 0,
5642                         "512-1023 byte frames received");
5643         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5644                         CTLFLAG_RD, &stats->prc1522, 0,
5645                         "1023-1522 byte frames received");
5646         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5647                         CTLFLAG_RD, &stats->ruc, 0,
5648                         "Receive Undersized");
5649         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5650                         CTLFLAG_RD, &stats->rfc, 0,
5651                         "Fragmented Packets Received ");
5652         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5653                         CTLFLAG_RD, &stats->roc, 0,
5654                         "Oversized Packets Received");
5655         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5656                         CTLFLAG_RD, &stats->rjc, 0,
5657                         "Received Jabber");
5658         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5659                         CTLFLAG_RD, &stats->mngprc, 0,
5660                         "Management Packets Received");
5661         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5662                         CTLFLAG_RD, &stats->mngptc, 0,
5663                         "Management Packets Dropped");
5664         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5665                         CTLFLAG_RD, &stats->xec, 0,
5666                         "Checksum Errors");
5667
5668         /* Packet Transmission Stats */
5669         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5670                         CTLFLAG_RD, &stats->gotc, 0,
5671                         "Good Octets Transmitted");
5672         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5673                         CTLFLAG_RD, &stats->tpt, 0,
5674                         "Total Packets Transmitted");
5675         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5676                         CTLFLAG_RD, &stats->gptc, 0,
5677                         "Good Packets Transmitted");
5678         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5679                         CTLFLAG_RD, &stats->bptc, 0,
5680                         "Broadcast Packets Transmitted");
5681         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5682                         CTLFLAG_RD, &stats->mptc, 0,
5683                         "Multicast Packets Transmitted");
5684         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5685                         CTLFLAG_RD, &stats->mngptc, 0,
5686                         "Management Packets Transmitted");
5687         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5688                         CTLFLAG_RD, &stats->ptc64, 0,
5689                         "64 byte frames transmitted ");
5690         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5691                         CTLFLAG_RD, &stats->ptc127, 0,
5692                         "65-127 byte frames transmitted");
5693         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5694                         CTLFLAG_RD, &stats->ptc255, 0,
5695                         "128-255 byte frames transmitted");
5696         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5697                         CTLFLAG_RD, &stats->ptc511, 0,
5698                         "256-511 byte frames transmitted");
5699         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5700                         CTLFLAG_RD, &stats->ptc1023, 0,
5701                         "512-1023 byte frames transmitted");
5702         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5703                         CTLFLAG_RD, &stats->ptc1522, 0,
5704                         "1024-1522 byte frames transmitted");
5705
5706         /* FC Stats */
5707         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5708                 CTLFLAG_RD, &stats->fccrc, 0,
5709                 "FC CRC Errors");
5710         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5711                 CTLFLAG_RD, &stats->fclast, 0,
5712                 "FC Last Error");
5713         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5714                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5715                 "FCoE Packets Dropped");
5716         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5717                 CTLFLAG_RD, &stats->fcoeprc, 0,
5718                 "FCoE Packets Received");
5719         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5720                 CTLFLAG_RD, &stats->fcoeptc, 0,
5721                 "FCoE Packets Transmitted");
5722         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5723                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5724                 "FCoE DWords Received");
5725         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5726                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5727                 "FCoE DWords Transmitted");
5728 }
5729
5730 /*
5731 ** Set flow control using sysctl:
5732 ** Flow control values:
5733 **      0 - off
5734 **      1 - rx pause
5735 **      2 - tx pause
5736 **      3 - full
5737 */
5738 static int
5739 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5740 {
5741         int error, last;
5742         struct adapter *adapter = (struct adapter *) arg1;
5743
5744         last = adapter->fc;
5745         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5746         if ((error) || (req->newptr == NULL))
5747                 return (error);
5748
5749         /* Don't bother if it's not changed */
5750         if (adapter->fc == last)
5751                 return (0);
5752
5753         switch (adapter->fc) {
5754                 case ixgbe_fc_rx_pause:
5755                 case ixgbe_fc_tx_pause:
5756                 case ixgbe_fc_full:
5757                         adapter->hw.fc.requested_mode = adapter->fc;
5758                         break;
5759                 case ixgbe_fc_none:
5760                 default:
5761                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5762         }
5763         /* Don't autoneg if forcing a value */
5764         adapter->hw.fc.disable_fc_autoneg = TRUE;
5765         ixgbe_fc_enable(&adapter->hw);
5766         return error;
5767 }
5768
5769 static void
5770 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5771         const char *description, int *limit, int value)
5772 {
5773         *limit = value;
5774         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5775             SYSCTL_CHILDREN(adapter->sysctl_tree),
5776             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5777 }
5778
5779 /*
5780 ** Control link advertise speed:
5781 **      1 - advertise only 1G
5782 **      2 - advertise 100Mb
5783 **      3 - advertise normal
5784 */
5785 static int
5786 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5787 {
5788         int                     error = 0;
5789         struct adapter          *adapter;
5790         device_t                dev;
5791         struct ixgbe_hw         *hw;
5792         ixgbe_link_speed        speed, last;
5793
5794         adapter = (struct adapter *) arg1;
5795         dev = adapter->dev;
5796         hw = &adapter->hw;
5797         last = adapter->advertise;
5798
5799         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5800         if ((error) || (adapter->advertise == -1))
5801                 return (error);
5802
5803         if (adapter->advertise == last) /* no change */
5804                 return (0);
5805
5806         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5807             (hw->phy.multispeed_fiber)))
5808                 return (error);
5809
5810         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5811                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5812                 return (error);
5813         }
5814
5815         if (adapter->advertise == 1)
5816                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5817         else if (adapter->advertise == 2)
5818                 speed = IXGBE_LINK_SPEED_100_FULL;
5819         else if (adapter->advertise == 3)
5820                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5821                         IXGBE_LINK_SPEED_10GB_FULL;
5822         else /* bogus value */
5823                 return (error);
5824
5825         hw->mac.autotry_restart = TRUE;
5826         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5827
5828         return (error);
5829 }
5830
5831 /*
5832 ** Thermal Shutdown Trigger
5833 **   - cause a Thermal Overtemp IRQ
5834 */
5835 static int
5836 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5837 {
5838         int             error, fire = 0;
5839         struct adapter  *adapter = (struct adapter *) arg1;
5840         struct ixgbe_hw *hw = &adapter->hw;
5841
5842
5843         if (hw->mac.type != ixgbe_mac_X540)
5844                 return (0);
5845
5846         error = sysctl_handle_int(oidp, &fire, 0, req);
5847         if ((error) || (req->newptr == NULL))
5848                 return (error);
5849
5850         if (fire) {
5851                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5852                 reg |= IXGBE_EICR_TS;
5853                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5854         }
5855
5856         return (0);
5857 }