sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 #if 0   /* HAVE_MSIX */
 125 static int      ixgbe_setup_msix(struct adapter *);
 126 #endif
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 #if 0   /* NET_LRO */
 146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 147 #endif
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *, int);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 167                     const char *, int *, int);
 168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 169 #if 0   /* NET_TSO */
 170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 171 #endif
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_que(void *, int);
 199 static void     ixgbe_handle_link(void *, int);
 200 static void     ixgbe_handle_msf(void *, int);
 201 static void     ixgbe_handle_mod(void *, int);
 202
 203 #ifdef IXGBE_FDIR
 204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 205 static void     ixgbe_reinit_fdir(void *, int);
 206 #endif
 207
 208 /*********************************************************************
 209  *  FreeBSD Device Interface Entry Points
 210  *********************************************************************/
 211
 212 static device_method_t ixgbe_methods[] = {
 213         /* Device interface */
 214         DEVMETHOD(device_probe, ixgbe_probe),
 215         DEVMETHOD(device_attach, ixgbe_attach),
 216         DEVMETHOD(device_detach, ixgbe_detach),
 217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 218         {0, 0}
 219 };
 220
 221 static driver_t ixgbe_driver = {
 222         "ix", ixgbe_methods, sizeof(struct adapter),
 223 };
 224
 225 devclass_t ixgbe_devclass;
 226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 227
 228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 230
 231 /*
 232 ** TUNEABLE PARAMETERS:
 233 */
 234
 235 /*
 236 ** AIM: Adaptive Interrupt Moderation
 237 ** which means that the interrupt rate
 238 ** is varied over time based on the
 239 ** traffic for that interrupt vector
 240 */
 241 static int ixgbe_enable_aim = TRUE;
 242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 243
 244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 246
 247 /* How many packets rxeof tries to clean at a time */
 248 static int ixgbe_rx_process_limit = 128;
 249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 static int ixgbe_msi_enable = 1;
 261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 262
 263 /*
 264  * MSIX should be the default for best performance,
 265  * but this allows it to be forced off for testing.
 266  */
 267 static int ixgbe_enable_msix = 1;
 268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 269
 270 /*
 271  * Header split: this causes the hardware to DMA
 272  * the header into a separate mbuf from the payload,
 273  * it can be a performance win in some workloads, but
 274  * in others it actually hurts, its off by default.
 275  */
 276 static int ixgbe_header_split = FALSE;
 277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 278
 279 /*
 280  * Number of Queues, can be set to 0,
 281  * it then autoconfigures based on the
 282  * number of cpus with a max of 8. This
 283  * can be overriden manually here.
 284  */
 285 static int ixgbe_num_queues = 0;
 286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 287
 288 /*
 289 ** Number of TX descriptors per ring,
 290 ** setting higher than RX as this seems
 291 ** the better performing choice.
 292 */
 293 static int ixgbe_txd = PERFORM_TXD;
 294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 295
 296 /* Number of RX descriptors per ring */
 297 static int ixgbe_rxd = PERFORM_RXD;
 298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** For Flow Director: this is the
 306 ** number of TX packets we sample
 307 ** for the filter pool, this means
 308 ** every 20th packet will be probed.
 309 **
 310 ** This feature can be disabled by
 311 ** setting this to 0.
 312 */
 313 static int atr_sample_rate = 20;
 314 /*
 315 ** Flow Director actually 'steals'
 316 ** part of the packet buffer as its
 317 ** filter pool, this variable controls
 318 ** how much it uses:
 319 **  0 = 64K, 1 = 128K, 2 = 256K
 320 */
 321 static int fdir_pballoc = 1;
 322 #endif
 323
 324 #ifdef DEV_NETMAP
 325 /*
 326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 327  * be a reference on how to implement netmap support in a driver.
 328  * Additional comments are in ixgbe_netmap.h .
 329  *
 330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 331  * that extend the standard driver.
 332  */
 333 #include <dev/netmap/ixgbe_netmap.h>
 334 #endif /* DEV_NETMAP */
 335
 336 /*********************************************************************
 337  *  Device identification routine
 338  *
 339  *  ixgbe_probe determines if the driver should be loaded on
 340  *  adapter based on PCI vendor/device id of the adapter.
 341  *
 342  *  return BUS_PROBE_DEFAULT on success, positive on failure
 343  *********************************************************************/
 344
 345 static int
 346 ixgbe_probe(device_t dev)
 347 {
 348         ixgbe_vendor_info_t *ent;
 349
 350         u16     pci_vendor_id = 0;
 351         u16     pci_device_id = 0;
 352         u16     pci_subvendor_id = 0;
 353         u16     pci_subdevice_id = 0;
 354         char    adapter_name[256];
 355
 356         INIT_DEBUGOUT("ixgbe_probe: begin");
 357
 358         pci_vendor_id = pci_get_vendor(dev);
 359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 360                 return (ENXIO);
 361
 362         pci_device_id = pci_get_device(dev);
 363         pci_subvendor_id = pci_get_subvendor(dev);
 364         pci_subdevice_id = pci_get_subdevice(dev);
 365
 366         ent = ixgbe_vendor_info_array;
 367         while (ent->vendor_id != 0) {
 368                 if ((pci_vendor_id == ent->vendor_id) &&
 369                     (pci_device_id == ent->device_id) &&
 370
 371                     ((pci_subvendor_id == ent->subvendor_id) ||
 372                      (ent->subvendor_id == 0)) &&
 373
 374                     ((pci_subdevice_id == ent->subdevice_id) ||
 375                      (ent->subdevice_id == 0))) {
 376                         ksprintf(adapter_name, "%s, Version - %s",
 377                                 ixgbe_strings[ent->index],
 378                                 ixgbe_driver_version);
 379                         device_set_desc_copy(dev, adapter_name);
 380                         ++ixgbe_total_ports;
 381                         return (BUS_PROBE_DEFAULT);
 382                 }
 383                 ent++;
 384         }
 385         return (ENXIO);
 386 }
 387
 388 /*********************************************************************
 389  *  Device initialization routine
 390  *
 391  *  The attach entry point is called when the driver is being loaded.
 392  *  This routine identifies the type of hardware, allocates all resources
 393  *  and initializes the hardware.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 ixgbe_attach(device_t dev)
 400 {
 401         struct adapter *adapter;
 402         struct ixgbe_hw *hw;
 403         int             error = 0;
 404         u16             csum;
 405         u32             ctrl_ext;
 406
 407         INIT_DEBUGOUT("ixgbe_attach: begin");
 408
 409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 410                 device_printf(dev, "Disabled by device hint\n");
 411                 return (ENXIO);
 412         }
 413
 414         /* Allocate, clear, and link in our adapter structure */
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         hw = &adapter->hw;
 418
 419         /* Core Lock Init*/
 420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 421
 422         /* SYSCTL APIs */
 423
 424         sysctl_ctx_init(&adapter->sysctl_ctx);
 425         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 426             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 427             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 428         if (adapter->sysctl_tree == NULL) {
 429                 device_printf(adapter->dev, "can't add sysctl node\n");
 430                 return (EINVAL);
 431         }
 432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 433                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 434                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 435                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 436
 437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 438                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 439                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 440                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 441
 442         /*
 443         ** Allow a kind of speed control by forcing the autoneg
 444         ** advertised speed list to only a certain value, this
 445         ** supports 1G on 82599 devices, and 100Mb on x540.
 446         */
 447         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 448                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 449                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 450                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 451
 452         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 453                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 454                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 455                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 456
 457         /* Set up the timer callout */
 458         callout_init_mp(&adapter->timer);
 459
 460         /* Determine hardware revision */
 461         ixgbe_identify_hardware(adapter);
 462
 463         /* Do base PCI setup - map BAR0 */
 464         if (ixgbe_allocate_pci_resources(adapter)) {
 465                 device_printf(dev, "Allocation of PCI resources failed\n");
 466                 error = ENXIO;
 467                 goto err_out;
 468         }
 469
 470         /* Do descriptor calc and sanity checks */
 471         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 472             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 473                 device_printf(dev, "TXD config issue, using default!\n");
 474                 adapter->num_tx_desc = DEFAULT_TXD;
 475         } else
 476                 adapter->num_tx_desc = ixgbe_txd;
 477
 478         /*
 479         ** With many RX rings it is easy to exceed the
 480         ** system mbuf allocation. Tuning nmbclusters
 481         ** can alleviate this.
 482         */
 483         if (nmbclusters > 0 ) {
 484                 int s;
 485                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 486                 if (s > nmbclusters) {
 487                         device_printf(dev, "RX Descriptors exceed "
 488                             "system mbuf max, using default instead!\n");
 489                         ixgbe_rxd = DEFAULT_RXD;
 490                 }
 491         }
 492
 493         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 494             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 495                 device_printf(dev, "RXD config issue, using default!\n");
 496                 adapter->num_rx_desc = DEFAULT_RXD;
 497         } else
 498                 adapter->num_rx_desc = ixgbe_rxd;
 499
 500         /* Allocate our TX/RX Queues */
 501         if (ixgbe_allocate_queues(adapter)) {
 502                 error = ENOMEM;
 503                 goto err_out;
 504         }
 505
 506         /* Allocate multicast array memory. */
 507         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 508             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 509         if (adapter->mta == NULL) {
 510                 device_printf(dev, "Can not allocate multicast setup array\n");
 511                 error = ENOMEM;
 512                 goto err_late;
 513         }
 514
 515         /* Initialize the shared code */
 516         error = ixgbe_init_shared_code(hw);
 517         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 518                 /*
 519                 ** No optics in this port, set up
 520                 ** so the timer routine will probe
 521                 ** for later insertion.
 522                 */
 523                 adapter->sfp_probe = TRUE;
 524                 error = 0;
 525         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 526                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 527                 error = EIO;
 528                 goto err_late;
 529         } else if (error) {
 530                 device_printf(dev,"Unable to initialize the shared code\n");
 531                 error = EIO;
 532                 goto err_late;
 533         }
 534
 535         /* Make sure we have a good EEPROM before we read from it */
 536         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 537                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 538                 error = EIO;
 539                 goto err_late;
 540         }
 541
 542         error = ixgbe_init_hw(hw);
 543         switch (error) {
 544         case IXGBE_ERR_EEPROM_VERSION:
 545                 device_printf(dev, "This device is a pre-production adapter/"
 546                     "LOM.  Please be aware there may be issues associated "
 547                     "with your hardware.\n If you are experiencing problems "
 548                     "please contact your Intel or hardware representative "
 549                     "who provided you with this hardware.\n");
 550                 break;
 551         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 552                 device_printf(dev,"Unsupported SFP+ Module\n");
 553                 error = EIO;
 554                 device_printf(dev,"Hardware Initialization Failure\n");
 555                 goto err_late;
 556         case IXGBE_ERR_SFP_NOT_PRESENT:
 557                 device_printf(dev,"No SFP+ Module found\n");
 558                 /* falls thru */
 559         default:
 560                 break;
 561         }
 562
 563         /* Detect and set physical type */
 564         ixgbe_setup_optics(adapter);
 565
 566         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 567                 error = ixgbe_allocate_msix(adapter);
 568         else
 569                 error = ixgbe_allocate_legacy(adapter);
 570         if (error)
 571                 goto err_late;
 572
 573         /* Setup OS specific network interface */
 574         if (ixgbe_setup_interface(dev, adapter) != 0)
 575                 goto err_late;
 576
 577         /* Sysctl for limiting the amount of work done in the taskqueue */
 578         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 579             "max number of rx packets to process", &adapter->rx_process_limit,
 580             ixgbe_rx_process_limit);
 581
 582         /* Initialize statistics */
 583         ixgbe_update_stats_counters(adapter);
 584
 585         /* Register for VLAN events */
 586         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 587             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 588         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 589             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 590
 591         /* Print PCIE bus type/speed/width info */
 592         ixgbe_get_bus_info(hw);
 593         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 594             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 595             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 596             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 597             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 598             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 599             ("Unknown"));
 600
 601         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 602             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 603                 device_printf(dev, "PCI-Express bandwidth available"
 604                     " for this card\n     is not sufficient for"
 605                     " optimal performance.\n");
 606                 device_printf(dev, "For optimal performance a x8 "
 607                     "PCIE, or x4 PCIE 2 slot is required.\n");
 608         }
 609
 610         /* let hardware know driver is loaded */
 611         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 612         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 613         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 614
 615         ixgbe_add_hw_stats(adapter);
 616
 617 #ifdef DEV_NETMAP
 618         ixgbe_netmap_attach(adapter);
 619 #endif /* DEV_NETMAP */
 620         INIT_DEBUGOUT("ixgbe_attach: end");
 621         return (0);
 622 err_late:
 623         ixgbe_free_transmit_structures(adapter);
 624         ixgbe_free_receive_structures(adapter);
 625 err_out:
 626         if (adapter->ifp != NULL)
 627                 if_free(adapter->ifp);
 628         ixgbe_free_pci_resources(adapter);
 629         kfree(adapter->mta, M_DEVBUF);
 630         return (error);
 631
 632 }
 633
 634 /*********************************************************************
 635  *  Device removal routine
 636  *
 637  *  The detach entry point is called when the driver is being removed.
 638  *  This routine stops the adapter and deallocates all the resources
 639  *  that were allocated for driver operation.
 640  *
 641  *  return 0 on success, positive on failure
 642  *********************************************************************/
 643
 644 static int
 645 ixgbe_detach(device_t dev)
 646 {
 647         struct adapter *adapter = device_get_softc(dev);
 648         struct ix_queue *que = adapter->queues;
 649         u32     ctrl_ext;
 650
 651         INIT_DEBUGOUT("ixgbe_detach: begin");
 652
 653 #ifdef NET_VLAN
 654         /* Make sure VLANS are not using driver */
 655         if (adapter->ifp->if_vlantrunk != NULL) {
 656                 device_printf(dev,"Vlan in use, detach first\n");
 657                 return (EBUSY);
 658         }
 659 #endif
 660
 661         IXGBE_CORE_LOCK(adapter);
 662         ixgbe_stop(adapter);
 663         IXGBE_CORE_UNLOCK(adapter);
 664
 665         for (int i = 0; i < adapter->num_queues; i++, que++) {
 666                 if (que->tq) {
 667                         taskqueue_drain(que->tq, &que->que_task);
 668                         taskqueue_free(que->tq);
 669                 }
 670         }
 671
 672         /* Drain the Link queue */
 673         if (adapter->tq) {
 674                 taskqueue_drain(adapter->tq, &adapter->link_task);
 675                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 676                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 677 #ifdef IXGBE_FDIR
 678                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 679 #endif
 680                 taskqueue_free(adapter->tq);
 681         }
 682
 683         /* let hardware know driver is unloading */
 684         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 685         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 686         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 687
 688         /* Unregister VLAN events */
 689 #ifdef NET_VLAN
 690         if (adapter->vlan_attach != NULL)
 691                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 692         if (adapter->vlan_detach != NULL)
 693                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 694 #endif
 695
 696         ether_ifdetach(adapter->ifp);
 697         callout_stop(&adapter->timer);
 698 #ifdef DEV_NETMAP
 699         netmap_detach(adapter->ifp);
 700 #endif /* DEV_NETMAP */
 701         ixgbe_free_pci_resources(adapter);
 702         bus_generic_detach(dev);
 703         if_free(adapter->ifp);
 704
 705         ixgbe_free_transmit_structures(adapter);
 706         ixgbe_free_receive_structures(adapter);
 707         kfree(adapter->mta, M_DEVBUF);
 708         sysctl_ctx_free(&adapter->sysctl_ctx);
 709
 710         IXGBE_CORE_LOCK_DESTROY(adapter);
 711         return (0);
 712 }
 713
 714 /*********************************************************************
 715  *
 716  *  Shutdown entry point
 717  *
 718  **********************************************************************/
 719
 720 static int
 721 ixgbe_shutdown(device_t dev)
 722 {
 723         struct adapter *adapter = device_get_softc(dev);
 724         IXGBE_CORE_LOCK(adapter);
 725         ixgbe_stop(adapter);
 726         IXGBE_CORE_UNLOCK(adapter);
 727         return (0);
 728 }
 729
 730
 731 /*********************************************************************
 732  *  Transmit entry point
 733  *
 734  *  ixgbe_start is called by the stack to initiate a transmit.
 735  *  The driver will remain in this routine as long as there are
 736  *  packets to transmit and transmit resources are available.
 737  *  In case resources are not available stack is notified and
 738  *  the packet is requeued.
 739  **********************************************************************/
 740
 741 static void
 742 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 743 {
 744         struct mbuf    *m_head;
 745         struct adapter *adapter = txr->adapter;
 746
 747         IXGBE_TX_LOCK_ASSERT(txr);
 748
 749         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 750                 return;
 751         if (!adapter->link_active)
 752                 return;
 753
 754         while (!ifq_is_empty(&ifp->if_snd)) {
 755                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 756                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 757                         break;
 758                 }
 759
 760                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 761                 if (m_head == NULL)
 762                         break;
 763
 764                 if (ixgbe_xmit(txr, &m_head)) {
 765 #if 0 /* XXX: prepend to an ALTQ queue ? */
 766                         if (m_head != NULL)
 767                                 IF_PREPEND(&ifp->if_snd, m_head);
 768 #endif
 769                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 770                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 771                         break;
 772                 }
 773                 /* Send a copy of the frame to the BPF listener */
 774                 ETHER_BPF_MTAP(ifp, m_head);
 775
 776                 /* Set watchdog on */
 777                 txr->watchdog_time = ticks;
 778                 txr->queue_status = IXGBE_QUEUE_WORKING;
 779
 780         }
 781         return;
 782 }
 783
 784 /*
 785  * Legacy TX start - called by the stack, this
 786  * always uses the first tx ring, and should
 787  * not be used with multiqueue tx enabled.
 788  */
 789 static void
 790 ixgbe_start(struct ifnet *ifp)
 791 {
 792         struct adapter *adapter = ifp->if_softc;
 793         struct tx_ring  *txr = adapter->tx_rings;
 794
 795         if (ifp->if_flags & IFF_RUNNING) {
 796                 IXGBE_TX_LOCK(txr);
 797                 ixgbe_start_locked(txr, ifp);
 798                 IXGBE_TX_UNLOCK(txr);
 799         }
 800         return;
 801 }
 802
 803 #if 0 /* __FreeBSD_version >= 800000 */
 804 /*
 805 ** Multiqueue Transmit driver
 806 **
 807 */
 808 static int
 809 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 810 {
 811         struct adapter  *adapter = ifp->if_softc;
 812         struct ix_queue *que;
 813         struct tx_ring  *txr;
 814         int             i = 0, err = 0;
 815
 816         /* Which queue to use */
 817         if ((m->m_flags & M_FLOWID) != 0)
 818                 i = m->m_pkthdr.flowid % adapter->num_queues;
 819         else
 820                 i = curcpu % adapter->num_queues;
 821
 822         txr = &adapter->tx_rings[i];
 823         que = &adapter->queues[i];
 824
 825         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 826             IXGBE_TX_TRYLOCK(txr)) {
 827                 err = ixgbe_mq_start_locked(ifp, txr, m);
 828                 IXGBE_TX_UNLOCK(txr);
 829         } else {
 830                 err = drbr_enqueue(ifp, txr->br, m);
 831                 taskqueue_enqueue(que->tq, &que->que_task);
 832         }
 833
 834         return (err);
 835 }
 836
 837 static int
 838 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 839 {
 840         struct adapter  *adapter = txr->adapter;
 841         struct mbuf     *next;
 842         int             enqueued, err = 0;
 843
 844         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 845             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 846             adapter->link_active == 0) {
 847                 if (m != NULL)
 848                         err = drbr_enqueue(ifp, txr->br, m);
 849                 return (err);
 850         }
 851
 852         enqueued = 0;
 853         if (m == NULL) {
 854                 next = drbr_dequeue(ifp, txr->br);
 855         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 856                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 857                         return (err);
 858                 next = drbr_dequeue(ifp, txr->br);
 859         } else
 860                 next = m;
 861
 862         /* Process the queue */
 863         while (next != NULL) {
 864                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 865                         if (next != NULL)
 866                                 err = drbr_enqueue(ifp, txr->br, next);
 867                         break;
 868                 }
 869                 enqueued++;
 870                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 871                 /* Send a copy of the frame to the BPF listener */
 872                 ETHER_BPF_MTAP(ifp, next);
 873                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 874                         break;
 875                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 876                         ixgbe_txeof(txr);
 877                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 878                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 879                         break;
 880                 }
 881                 next = drbr_dequeue(ifp, txr->br);
 882         }
 883
 884         if (enqueued > 0) {
 885                 /* Set watchdog on */
 886                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 887                 txr->watchdog_time = ticks;
 888         }
 889
 890         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 891                 ixgbe_txeof(txr);
 892
 893         return (err);
 894 }
 895
 896 /*
 897 ** Flush all ring buffers
 898 */
 899 static void
 900 ixgbe_qflush(struct ifnet *ifp)
 901 {
 902         struct adapter  *adapter = ifp->if_softc;
 903         struct tx_ring  *txr = adapter->tx_rings;
 904         struct mbuf     *m;
 905
 906         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 907                 IXGBE_TX_LOCK(txr);
 908                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 909                         m_freem(m);
 910                 IXGBE_TX_UNLOCK(txr);
 911         }
 912         if_qflush(ifp);
 913 }
 914 #endif /* __FreeBSD_version >= 800000 */
 915
 916 /*********************************************************************
 917  *  Ioctl entry point
 918  *
 919  *  ixgbe_ioctl is called when the user wants to configure the
 920  *  interface.
 921  *
 922  *  return 0 on success, positive on failure
 923  **********************************************************************/
 924
 925 static int
 926 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 927 {
 928         struct adapter  *adapter = ifp->if_softc;
 929         struct ifreq    *ifr = (struct ifreq *) data;
 930 #if defined(INET) || defined(INET6)
 931         struct ifaddr *ifa = (struct ifaddr *)data;
 932         bool            avoid_reset = FALSE;
 933 #endif
 934         int             error = 0;
 935
 936         switch (command) {
 937
 938         case SIOCSIFADDR:
 939 #ifdef INET
 940                 if (ifa->ifa_addr->sa_family == AF_INET)
 941                         avoid_reset = TRUE;
 942 #endif
 943 #ifdef INET6
 944                 if (ifa->ifa_addr->sa_family == AF_INET6)
 945                         avoid_reset = TRUE;
 946 #endif
 947 #if defined(INET) || defined(INET6)
 948                 /*
 949                 ** Calling init results in link renegotiation,
 950                 ** so we avoid doing it when possible.
 951                 */
 952                 if (avoid_reset) {
 953                         ifp->if_flags |= IFF_UP;
 954                         if (!(ifp->if_flags & IFF_RUNNING))
 955                                 ixgbe_init(adapter);
 956                         if (!(ifp->if_flags & IFF_NOARP))
 957                                 arp_ifinit(ifp, ifa);
 958                 } else
 959                         error = ether_ioctl(ifp, command, data);
 960 #endif
 961                 break;
 962         case SIOCSIFMTU:
 963                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 964                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 965                         error = EINVAL;
 966                 } else {
 967                         IXGBE_CORE_LOCK(adapter);
 968                         ifp->if_mtu = ifr->ifr_mtu;
 969                         adapter->max_frame_size =
 970                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 971                         ixgbe_init_locked(adapter);
 972                         IXGBE_CORE_UNLOCK(adapter);
 973                 }
 974                 break;
 975         case SIOCSIFFLAGS:
 976                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 977                 IXGBE_CORE_LOCK(adapter);
 978                 if (ifp->if_flags & IFF_UP) {
 979                         if ((ifp->if_flags & IFF_RUNNING)) {
 980                                 if ((ifp->if_flags ^ adapter->if_flags) &
 981                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 982                                         ixgbe_set_promisc(adapter);
 983                                 }
 984                         } else
 985                                 ixgbe_init_locked(adapter);
 986                 } else
 987                         if (ifp->if_flags & IFF_RUNNING)
 988                                 ixgbe_stop(adapter);
 989                 adapter->if_flags = ifp->if_flags;
 990                 IXGBE_CORE_UNLOCK(adapter);
 991                 break;
 992         case SIOCADDMULTI:
 993         case SIOCDELMULTI:
 994                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 995                 if (ifp->if_flags & IFF_RUNNING) {
 996                         IXGBE_CORE_LOCK(adapter);
 997                         ixgbe_disable_intr(adapter);
 998                         ixgbe_set_multi(adapter);
 999                         ixgbe_enable_intr(adapter);
1000                         IXGBE_CORE_UNLOCK(adapter);
1001                 }
1002                 break;
1003         case SIOCSIFMEDIA:
1004         case SIOCGIFMEDIA:
1005                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1006                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1007                 break;
1008         case SIOCSIFCAP:
1009         {
1010                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1011                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1012                 if (mask & IFCAP_HWCSUM)
1013                         ifp->if_capenable ^= IFCAP_HWCSUM;
1014 #if 0 /* NET_TSO */
1015                 if (mask & IFCAP_TSO4)
1016                         ifp->if_capenable ^= IFCAP_TSO4;
1017                 if (mask & IFCAP_TSO6)
1018                         ifp->if_capenable ^= IFCAP_TSO6;
1019 #endif
1020 #if 0 /* NET_LRO */
1021                 if (mask & IFCAP_LRO)
1022                         ifp->if_capenable ^= IFCAP_LRO;
1023 #endif
1024                 if (mask & IFCAP_VLAN_HWTAGGING)
1025                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1026                 if (mask & IFCAP_VLAN_HWFILTER)
1027                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1028 #if 0 /* NET_TSO */
1029                 if (mask & IFCAP_VLAN_HWTSO)
1030                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1031 #endif
1032                 if (ifp->if_flags & IFF_RUNNING) {
1033                         IXGBE_CORE_LOCK(adapter);
1034                         ixgbe_init_locked(adapter);
1035                         IXGBE_CORE_UNLOCK(adapter);
1036                 }
1037 #if 0
1038                 VLAN_CAPABILITIES(ifp);
1039 #endif
1040                 break;
1041         }
1042
1043         default:
1044                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1045                 error = ether_ioctl(ifp, command, data);
1046                 break;
1047         }
1048
1049         return (error);
1050 }
1051
1052 /*********************************************************************
1053  *  Init entry point
1054  *
1055  *  This routine is used in two ways. It is used by the stack as
1056  *  init entry point in network interface structure. It is also used
1057  *  by the driver as a hw/sw initialization routine to get to a
1058  *  consistent state.
1059  *
1060  *  return 0 on success, positive on failure
1061  **********************************************************************/
1062 #define IXGBE_MHADD_MFS_SHIFT 16
1063
1064 static void
1065 ixgbe_init_locked(struct adapter *adapter)
1066 {
1067         struct ifnet   *ifp = adapter->ifp;
1068         device_t        dev = adapter->dev;
1069         struct ixgbe_hw *hw = &adapter->hw;
1070         u32             k, txdctl, mhadd, gpie;
1071         u32             rxdctl, rxctrl;
1072
1073         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1074         INIT_DEBUGOUT("ixgbe_init: begin");
1075         hw->adapter_stopped = FALSE;
1076         ixgbe_stop_adapter(hw);
1077         callout_stop(&adapter->timer);
1078
1079         /* reprogram the RAR[0] in case user changed it. */
1080         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1081
1082         /* Get the latest mac address, User can use a LAA */
1083         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1084               IXGBE_ETH_LENGTH_OF_ADDRESS);
1085         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1086         hw->addr_ctrl.rar_used_count = 1;
1087
1088         /* Set the various hardware offload abilities */
1089         ifp->if_hwassist = 0;
1090 #if 0 /* NET_TSO */
1091         if (ifp->if_capenable & IFCAP_TSO)
1092                 ifp->if_hwassist |= CSUM_TSO;
1093 #endif
1094         if (ifp->if_capenable & IFCAP_TXCSUM) {
1095                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1096 #if 0
1097                 if (hw->mac.type != ixgbe_mac_82598EB)
1098                         ifp->if_hwassist |= CSUM_SCTP;
1099 #endif
1100         }
1101
1102         /* Prepare transmit descriptors and buffers */
1103         if (ixgbe_setup_transmit_structures(adapter)) {
1104                 device_printf(dev,"Could not setup transmit structures\n");
1105                 ixgbe_stop(adapter);
1106                 return;
1107         }
1108
1109         ixgbe_init_hw(hw);
1110         ixgbe_initialize_transmit_units(adapter);
1111
1112         /* Setup Multicast table */
1113         ixgbe_set_multi(adapter);
1114
1115         /*
1116         ** Determine the correct mbuf pool
1117         ** for doing jumbo/headersplit
1118         */
1119         if (adapter->max_frame_size <= 2048)
1120                 adapter->rx_mbuf_sz = MCLBYTES;
1121         else if (adapter->max_frame_size <= 4096)
1122                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1123         else if (adapter->max_frame_size <= 9216)
1124                 adapter->rx_mbuf_sz = MJUM9BYTES;
1125         else
1126                 adapter->rx_mbuf_sz = MJUM16BYTES;
1127
1128         /* Prepare receive descriptors and buffers */
1129         if (ixgbe_setup_receive_structures(adapter)) {
1130                 device_printf(dev,"Could not setup receive structures\n");
1131                 ixgbe_stop(adapter);
1132                 return;
1133         }
1134
1135         /* Configure RX settings */
1136         ixgbe_initialize_receive_units(adapter);
1137
1138         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1139
1140         /* Enable Fan Failure Interrupt */
1141         gpie |= IXGBE_SDP1_GPIEN;
1142
1143         /* Add for Module detection */
1144         if (hw->mac.type == ixgbe_mac_82599EB)
1145                 gpie |= IXGBE_SDP2_GPIEN;
1146
1147         /* Thermal Failure Detection */
1148         if (hw->mac.type == ixgbe_mac_X540)
1149                 gpie |= IXGBE_SDP0_GPIEN;
1150
1151         if (adapter->msix > 1) {
1152                 /* Enable Enhanced MSIX mode */
1153                 gpie |= IXGBE_GPIE_MSIX_MODE;
1154                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1155                     IXGBE_GPIE_OCD;
1156         }
1157         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1158
1159         /* Set MTU size */
1160         if (ifp->if_mtu > ETHERMTU) {
1161                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1162                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1163                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1164                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1165         }
1166
1167         /* Now enable all the queues */
1168
1169         for (int i = 0; i < adapter->num_queues; i++) {
1170                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1171                 txdctl |= IXGBE_TXDCTL_ENABLE;
1172                 /* Set WTHRESH to 8, burst writeback */
1173                 txdctl |= (8 << 16);
1174                 /*
1175                  * When the internal queue falls below PTHRESH (32),
1176                  * start prefetching as long as there are at least
1177                  * HTHRESH (1) buffers ready. The values are taken
1178                  * from the Intel linux driver 3.8.21.
1179                  * Prefetching enables tx line rate even with 1 queue.
1180                  */
1181                 txdctl |= (32 << 0) | (1 << 8);
1182                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1183         }
1184
1185         for (int i = 0; i < adapter->num_queues; i++) {
1186                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1187                 if (hw->mac.type == ixgbe_mac_82598EB) {
1188                         /*
1189                         ** PTHRESH = 21
1190                         ** HTHRESH = 4
1191                         ** WTHRESH = 8
1192                         */
1193                         rxdctl &= ~0x3FFFFF;
1194                         rxdctl |= 0x080420;
1195                 }
1196                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1197                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1198                 for (k = 0; k < 10; k++) {
1199                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1200                             IXGBE_RXDCTL_ENABLE)
1201                                 break;
1202                         else
1203                                 msec_delay(1);
1204                 }
1205                 wmb();
1206 #ifdef DEV_NETMAP
1207                 /*
1208                  * In netmap mode, we must preserve the buffers made
1209                  * available to userspace before the if_init()
1210                  * (this is true by default on the TX side, because
1211                  * init makes all buffers available to userspace).
1212                  *
1213                  * netmap_reset() and the device specific routines
1214                  * (e.g. ixgbe_setup_receive_rings()) map these
1215                  * buffers at the end of the NIC ring, so here we
1216                  * must set the RDT (tail) register to make sure
1217                  * they are not overwritten.
1218                  *
1219                  * In this driver the NIC ring starts at RDH = 0,
1220                  * RDT points to the last slot available for reception (?),
1221                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1222                  */
1223                 if (ifp->if_capenable & IFCAP_NETMAP) {
1224                         struct netmap_adapter *na = NA(adapter->ifp);
1225                         struct netmap_kring *kring = &na->rx_rings[i];
1226                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1227
1228                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1229                 } else
1230 #endif /* DEV_NETMAP */
1231                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1232         }
1233
1234         /* Set up VLAN support and filter */
1235         ixgbe_setup_vlan_hw_support(adapter);
1236
1237         /* Enable Receive engine */
1238         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1239         if (hw->mac.type == ixgbe_mac_82598EB)
1240                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1241         rxctrl |= IXGBE_RXCTRL_RXEN;
1242         ixgbe_enable_rx_dma(hw, rxctrl);
1243
1244         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1245
1246         /* Set up MSI/X routing */
1247         if (ixgbe_enable_msix)  {
1248                 ixgbe_configure_ivars(adapter);
1249                 /* Set up auto-mask */
1250                 if (hw->mac.type == ixgbe_mac_82598EB)
1251                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1252                 else {
1253                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1255                 }
1256         } else {  /* Simple settings for Legacy/MSI */
1257                 ixgbe_set_ivar(adapter, 0, 0, 0);
1258                 ixgbe_set_ivar(adapter, 0, 0, 1);
1259                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1260         }
1261
1262 #ifdef IXGBE_FDIR
1263         /* Init Flow director */
1264         if (hw->mac.type != ixgbe_mac_82598EB) {
1265                 u32 hdrm = 32 << fdir_pballoc;
1266
1267                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1268                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1269         }
1270 #endif
1271
1272         /*
1273         ** Check on any SFP devices that
1274         ** need to be kick-started
1275         */
1276         if (hw->phy.type == ixgbe_phy_none) {
1277                 int err = hw->phy.ops.identify(hw);
1278                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1279                         device_printf(dev,
1280                             "Unsupported SFP+ module type was detected.\n");
1281                         return;
1282                 }
1283         }
1284
1285         /* Set moderation on the Link interrupt */
1286         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1287
1288         /* Config/Enable Link */
1289         ixgbe_config_link(adapter);
1290
1291         /* Hardware Packet Buffer & Flow Control setup */
1292         {
1293                 u32 rxpb, frame, size, tmp;
1294
1295                 frame = adapter->max_frame_size;
1296
1297                 /* Calculate High Water */
1298                 if (hw->mac.type == ixgbe_mac_X540)
1299                         tmp = IXGBE_DV_X540(frame, frame);
1300                 else
1301                         tmp = IXGBE_DV(frame, frame);
1302                 size = IXGBE_BT2KB(tmp);
1303                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1304                 hw->fc.high_water[0] = rxpb - size;
1305
1306                 /* Now calculate Low Water */
1307                 if (hw->mac.type == ixgbe_mac_X540)
1308                         tmp = IXGBE_LOW_DV_X540(frame);
1309                 else
1310                         tmp = IXGBE_LOW_DV(frame);
1311                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1312
1313                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1314                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1315                 hw->fc.send_xon = TRUE;
1316         }
1317         /* Initialize the FC settings */
1318         ixgbe_start_hw(hw);
1319
1320         /* And now turn on interrupts */
1321         ixgbe_enable_intr(adapter);
1322
1323         /* Now inform the stack we're ready */
1324         ifp->if_flags |= IFF_RUNNING;
1325         ifp->if_flags &= ~IFF_OACTIVE;
1326
1327         return;
1328 }
1329
1330 static void
1331 ixgbe_init(void *arg)
1332 {
1333         struct adapter *adapter = arg;
1334
1335         IXGBE_CORE_LOCK(adapter);
1336         ixgbe_init_locked(adapter);
1337         IXGBE_CORE_UNLOCK(adapter);
1338         return;
1339 }
1340
1341
1342 /*
1343 **
1344 ** MSIX Interrupt Handlers and Tasklets
1345 **
1346 */
1347
1348 static inline void
1349 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1350 {
1351         struct ixgbe_hw *hw = &adapter->hw;
1352         u64     queue = (u64)(1 << vector);
1353         u32     mask;
1354
1355         if (hw->mac.type == ixgbe_mac_82598EB) {
1356                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1357                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1358         } else {
1359                 mask = (queue & 0xFFFFFFFF);
1360                 if (mask)
1361                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1362                 mask = (queue >> 32);
1363                 if (mask)
1364                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1365         }
1366 }
1367
1368 static inline void
1369 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1370 {
1371         struct ixgbe_hw *hw = &adapter->hw;
1372         u64     queue = (u64)(1 << vector);
1373         u32     mask;
1374
1375         if (hw->mac.type == ixgbe_mac_82598EB) {
1376                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1377                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1378         } else {
1379                 mask = (queue & 0xFFFFFFFF);
1380                 if (mask)
1381                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1382                 mask = (queue >> 32);
1383                 if (mask)
1384                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1385         }
1386 }
1387
1388 static inline void
1389 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1390 {
1391         u32 mask;
1392
1393         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1394                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1395                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1396         } else {
1397                 mask = (queues & 0xFFFFFFFF);
1398                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1399                 mask = (queues >> 32);
1400                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1401         }
1402 }
1403
1404
1405 static void
1406 ixgbe_handle_que(void *context, int pending)
1407 {
1408         struct ix_queue *que = context;
1409         struct adapter  *adapter = que->adapter;
1410         struct tx_ring  *txr = que->txr;
1411         struct ifnet    *ifp = adapter->ifp;
1412         bool            more;
1413
1414         if (ifp->if_flags & IFF_RUNNING) {
1415                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1416                 IXGBE_TX_LOCK(txr);
1417                 ixgbe_txeof(txr);
1418 #if 0 /*__FreeBSD_version >= 800000*/
1419                 if (!drbr_empty(ifp, txr->br))
1420                         ixgbe_mq_start_locked(ifp, txr, NULL);
1421 #else
1422                 if (!ifq_is_empty(&ifp->if_snd))
1423                         ixgbe_start_locked(txr, ifp);
1424 #endif
1425                 IXGBE_TX_UNLOCK(txr);
1426                 if (more) {
1427                         taskqueue_enqueue(que->tq, &que->que_task);
1428                         return;
1429                 }
1430         }
1431
1432         /* Reenable this interrupt */
1433         ixgbe_enable_queue(adapter, que->msix);
1434         return;
1435 }
1436
1437
1438 /*********************************************************************
1439  *
1440  *  Legacy Interrupt Service routine
1441  *
1442  **********************************************************************/
1443
1444 static void
1445 ixgbe_legacy_irq(void *arg)
1446 {
1447         struct ix_queue *que = arg;
1448         struct adapter  *adapter = que->adapter;
1449         struct ixgbe_hw *hw = &adapter->hw;
1450         struct          tx_ring *txr = adapter->tx_rings;
1451         bool            more_tx, more_rx;
1452         u32             reg_eicr, loop = MAX_LOOP;
1453
1454
1455         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1456
1457         ++que->irqs;
1458         if (reg_eicr == 0) {
1459                 ixgbe_enable_intr(adapter);
1460                 return;
1461         }
1462
1463         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1464
1465         IXGBE_TX_LOCK(txr);
1466         do {
1467                 more_tx = ixgbe_txeof(txr);
1468         } while (loop-- && more_tx);
1469         IXGBE_TX_UNLOCK(txr);
1470
1471         if (more_rx || more_tx)
1472                 taskqueue_enqueue(que->tq, &que->que_task);
1473
1474         /* Check for fan failure */
1475         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1476             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1477                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1478                     "REPLACE IMMEDIATELY!!\n");
1479                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1480         }
1481
1482         /* Link status change */
1483         if (reg_eicr & IXGBE_EICR_LSC)
1484                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1485
1486         ixgbe_enable_intr(adapter);
1487         return;
1488 }
1489
1490
1491 /*********************************************************************
1492  *
1493  *  MSIX Queue Interrupt Service routine
1494  *
1495  **********************************************************************/
1496 void
1497 ixgbe_msix_que(void *arg)
1498 {
1499         struct ix_queue *que = arg;
1500         struct adapter  *adapter = que->adapter;
1501         struct tx_ring  *txr = que->txr;
1502         struct rx_ring  *rxr = que->rxr;
1503         bool            more_tx, more_rx;
1504         u32             newitr = 0;
1505
1506         ixgbe_disable_queue(adapter, que->msix);
1507         ++que->irqs;
1508
1509         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1510
1511         IXGBE_TX_LOCK(txr);
1512         more_tx = ixgbe_txeof(txr);
1513         /*
1514         ** Make certain that if the stack
1515         ** has anything queued the task gets
1516         ** scheduled to handle it.
1517         */
1518 #if 0
1519 #if __FreeBSD_version < 800000
1520         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1521 #else
1522         if (!drbr_empty(adapter->ifp, txr->br))
1523 #endif
1524 #endif
1525         if (!ifq_is_empty(&adapter->ifp->if_snd))
1526                 more_tx = 1;
1527         IXGBE_TX_UNLOCK(txr);
1528
1529         /* Do AIM now? */
1530
1531         if (ixgbe_enable_aim == FALSE)
1532                 goto no_calc;
1533         /*
1534         ** Do Adaptive Interrupt Moderation:
1535         **  - Write out last calculated setting
1536         **  - Calculate based on average size over
1537         **    the last interval.
1538         */
1539         if (que->eitr_setting)
1540                 IXGBE_WRITE_REG(&adapter->hw,
1541                     IXGBE_EITR(que->msix), que->eitr_setting);
1542
1543         que->eitr_setting = 0;
1544
1545         /* Idle, do nothing */
1546         if ((txr->bytes == 0) && (rxr->bytes == 0))
1547                 goto no_calc;
1548
1549         if ((txr->bytes) && (txr->packets))
1550                 newitr = txr->bytes/txr->packets;
1551         if ((rxr->bytes) && (rxr->packets))
1552                 newitr = max(newitr,
1553                     (rxr->bytes / rxr->packets));
1554         newitr += 24; /* account for hardware frame, crc */
1555
1556         /* set an upper boundary */
1557         newitr = min(newitr, 3000);
1558
1559         /* Be nice to the mid range */
1560         if ((newitr > 300) && (newitr < 1200))
1561                 newitr = (newitr / 3);
1562         else
1563                 newitr = (newitr / 2);
1564
1565         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1566                 newitr |= newitr << 16;
1567         else
1568                 newitr |= IXGBE_EITR_CNT_WDIS;
1569
1570         /* save for next interrupt */
1571         que->eitr_setting = newitr;
1572
1573         /* Reset state */
1574         txr->bytes = 0;
1575         txr->packets = 0;
1576         rxr->bytes = 0;
1577         rxr->packets = 0;
1578
1579 no_calc:
1580         if (more_tx || more_rx)
1581                 taskqueue_enqueue(que->tq, &que->que_task);
1582         else /* Reenable this interrupt */
1583                 ixgbe_enable_queue(adapter, que->msix);
1584         return;
1585 }
1586
1587
1588 static void
1589 ixgbe_msix_link(void *arg)
1590 {
1591         struct adapter  *adapter = arg;
1592         struct ixgbe_hw *hw = &adapter->hw;
1593         u32             reg_eicr;
1594
1595         ++adapter->link_irq;
1596
1597         /* First get the cause */
1598         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1599         /* Clear interrupt with write */
1600         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1601
1602         /* Link status change */
1603         if (reg_eicr & IXGBE_EICR_LSC)
1604                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1605
1606         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1607 #ifdef IXGBE_FDIR
1608                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1609                         /* This is probably overkill :) */
1610                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1611                                 return;
1612                         /* Disable the interrupt */
1613                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1614                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1615                 } else
1616 #endif
1617                 if (reg_eicr & IXGBE_EICR_ECC) {
1618                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1619                             "Please Reboot!!\n");
1620                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1621                 } else
1622
1623                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1624                         /* Clear the interrupt */
1625                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1626                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1627                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1628                         /* Clear the interrupt */
1629                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1630                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1631                 }
1632         }
1633
1634         /* Check for fan failure */
1635         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1636             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1637                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1638                     "REPLACE IMMEDIATELY!!\n");
1639                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1640         }
1641
1642         /* Check for over temp condition */
1643         if ((hw->mac.type == ixgbe_mac_X540) &&
1644             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1645                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1646                     "PHY IS SHUT DOWN!!\n");
1647                 device_printf(adapter->dev, "System shutdown required\n");
1648                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1649         }
1650
1651         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1652         return;
1653 }
1654
1655 /*********************************************************************
1656  *
1657  *  Media Ioctl callback
1658  *
1659  *  This routine is called whenever the user queries the status of
1660  *  the interface using ifconfig.
1661  *
1662  **********************************************************************/
1663 static void
1664 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1665 {
1666         struct adapter *adapter = ifp->if_softc;
1667
1668         INIT_DEBUGOUT("ixgbe_media_status: begin");
1669         IXGBE_CORE_LOCK(adapter);
1670         ixgbe_update_link_status(adapter);
1671
1672         ifmr->ifm_status = IFM_AVALID;
1673         ifmr->ifm_active = IFM_ETHER;
1674
1675         if (!adapter->link_active) {
1676                 IXGBE_CORE_UNLOCK(adapter);
1677                 return;
1678         }
1679
1680         ifmr->ifm_status |= IFM_ACTIVE;
1681
1682         switch (adapter->link_speed) {
1683                 case IXGBE_LINK_SPEED_100_FULL:
1684                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1685                         break;
1686                 case IXGBE_LINK_SPEED_1GB_FULL:
1687                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1688                         break;
1689                 case IXGBE_LINK_SPEED_10GB_FULL:
1690                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1691                         break;
1692         }
1693
1694         IXGBE_CORE_UNLOCK(adapter);
1695
1696         return;
1697 }
1698
1699 /*********************************************************************
1700  *
1701  *  Media Ioctl callback
1702  *
1703  *  This routine is called when the user changes speed/duplex using
1704  *  media/mediopt option with ifconfig.
1705  *
1706  **********************************************************************/
1707 static int
1708 ixgbe_media_change(struct ifnet * ifp)
1709 {
1710         struct adapter *adapter = ifp->if_softc;
1711         struct ifmedia *ifm = &adapter->media;
1712
1713         INIT_DEBUGOUT("ixgbe_media_change: begin");
1714
1715         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1716                 return (EINVAL);
1717
1718         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1719         case IFM_AUTO:
1720                 adapter->hw.phy.autoneg_advertised =
1721                     IXGBE_LINK_SPEED_100_FULL |
1722                     IXGBE_LINK_SPEED_1GB_FULL |
1723                     IXGBE_LINK_SPEED_10GB_FULL;
1724                 break;
1725         default:
1726                 device_printf(adapter->dev, "Only auto media type\n");
1727                 return (EINVAL);
1728         }
1729
1730         return (0);
1731 }
1732
1733 /*********************************************************************
1734  *
1735  *  This routine maps the mbufs to tx descriptors, allowing the
1736  *  TX engine to transmit the packets.
1737  *      - return 0 on success, positive on failure
1738  *
1739  **********************************************************************/
1740
1741 static int
1742 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1743 {
1744         struct adapter  *adapter = txr->adapter;
1745         u32             olinfo_status = 0, cmd_type_len;
1746         u32             paylen = 0;
1747         int             i, j, error, nsegs, maxsegs;
1748         int             first, last = 0;
1749         struct mbuf     *m_head;
1750         bus_dma_segment_t segs[adapter->num_segs];
1751         bus_dmamap_t    map;
1752         struct ixgbe_tx_buf *txbuf;
1753         union ixgbe_adv_tx_desc *txd = NULL;
1754
1755         m_head = *m_headp;
1756
1757         /* Basic descriptor defines */
1758         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1759             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1760
1761         if (m_head->m_flags & M_VLANTAG)
1762                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1763
1764         /*
1765          * Important to capture the first descriptor
1766          * used because it will contain the index of
1767          * the one we tell the hardware to report back
1768          */
1769         first = txr->next_avail_desc;
1770         txbuf = &txr->tx_buffers[first];
1771         map = txbuf->map;
1772
1773         /*
1774          * Map the packet for DMA.
1775          */
1776         maxsegs = txr->tx_avail - IXGBE_TX_RESERVED;
1777         if (maxsegs > adapter->num_segs)
1778                 maxsegs = adapter->num_segs;
1779
1780         error = bus_dmamap_load_mbuf_defrag(txr->txtag, map, m_headp,
1781             segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1782         if (error) {
1783                 if (error == ENOBUFS)
1784                         adapter->mbuf_defrag_failed++;
1785                 else
1786                         adapter->no_tx_dma_setup++;
1787
1788                 m_freem(*m_headp);
1789                 *m_headp = NULL;
1790                 return (error);
1791         }
1792
1793         /* Make certain there are enough descriptors */
1794         if (nsegs > txr->tx_avail - 2) {
1795                 txr->no_desc_avail++;
1796                 error = ENOBUFS;
1797                 goto xmit_fail;
1798         }
1799         m_head = *m_headp;
1800
1801         /*
1802         ** Set up the appropriate offload context
1803         ** this becomes the first descriptor of
1804         ** a packet.
1805         */
1806 #if 0 /* NET_TSO */
1807         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1808                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1809                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1810                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1811                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1812                         ++adapter->tso_tx;
1813                 } else
1814                         return (ENXIO);
1815         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1816 #endif
1817         if (ixgbe_tx_ctx_setup(txr, m_head))
1818                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1819
1820 #ifdef IXGBE_IEEE1588
1821         /* This is changing soon to an mtag detection */
1822         if (we detect this mbuf has a TSTAMP mtag)
1823                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1824 #endif
1825
1826 #ifdef IXGBE_FDIR
1827         /* Do the flow director magic */
1828         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1829                 ++txr->atr_count;
1830                 if (txr->atr_count >= atr_sample_rate) {
1831                         ixgbe_atr(txr, m_head);
1832                         txr->atr_count = 0;
1833                 }
1834         }
1835 #endif
1836         /* Record payload length */
1837         if (paylen == 0)
1838                 olinfo_status |= m_head->m_pkthdr.len <<
1839                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1840
1841         i = txr->next_avail_desc;
1842         for (j = 0; j < nsegs; j++) {
1843                 bus_size_t seglen;
1844                 bus_addr_t segaddr;
1845
1846                 txbuf = &txr->tx_buffers[i];
1847                 txd = &txr->tx_base[i];
1848                 seglen = segs[j].ds_len;
1849                 segaddr = htole64(segs[j].ds_addr);
1850
1851                 txd->read.buffer_addr = segaddr;
1852                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1853                     cmd_type_len |seglen);
1854                 txd->read.olinfo_status = htole32(olinfo_status);
1855                 last = i; /* descriptor that will get completion IRQ */
1856
1857                 if (++i == adapter->num_tx_desc)
1858                         i = 0;
1859
1860                 txbuf->m_head = NULL;
1861                 txbuf->eop_index = -1;
1862         }
1863
1864         txd->read.cmd_type_len |=
1865             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1866         txr->tx_avail -= nsegs;
1867         txr->next_avail_desc = i;
1868
1869         txbuf->m_head = m_head;
1870         /* Swap the dma map between the first and last descriptor */
1871         txr->tx_buffers[first].map = txbuf->map;
1872         txbuf->map = map;
1873         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1874
1875         /* Set the index of the descriptor that will be marked done */
1876         txbuf = &txr->tx_buffers[first];
1877         txbuf->eop_index = last;
1878
1879         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1880             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1881         /*
1882          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1883          * hardware that this frame is available to transmit.
1884          */
1885         ++txr->total_packets;
1886         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1887
1888         return (0);
1889
1890 xmit_fail:
1891         bus_dmamap_unload(txr->txtag, txbuf->map);
1892         return (error);
1893
1894 }
1895
1896 static void
1897 ixgbe_set_promisc(struct adapter *adapter)
1898 {
1899         u_int32_t       reg_rctl;
1900         struct ifnet   *ifp = adapter->ifp;
1901
1902         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1903         reg_rctl &= (~IXGBE_FCTRL_UPE);
1904         reg_rctl &= (~IXGBE_FCTRL_MPE);
1905         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1906
1907         if (ifp->if_flags & IFF_PROMISC) {
1908                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1909                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1910         } else if (ifp->if_flags & IFF_ALLMULTI) {
1911                 reg_rctl |= IXGBE_FCTRL_MPE;
1912                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1913                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1914         }
1915         return;
1916 }
1917
1918
1919 /*********************************************************************
1920  *  Multicast Update
1921  *
1922  *  This routine is called whenever multicast address list is updated.
1923  *
1924  **********************************************************************/
1925 #define IXGBE_RAR_ENTRIES 16
1926
1927 static void
1928 ixgbe_set_multi(struct adapter *adapter)
1929 {
1930         u32     fctrl;
1931         u8      *mta;
1932         u8      *update_ptr;
1933         struct  ifmultiaddr *ifma;
1934         int     mcnt = 0;
1935         struct ifnet   *ifp = adapter->ifp;
1936
1937         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1938
1939         mta = adapter->mta;
1940         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1941             MAX_NUM_MULTICAST_ADDRESSES);
1942
1943         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1944         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1945         if (ifp->if_flags & IFF_PROMISC)
1946                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1947         else if (ifp->if_flags & IFF_ALLMULTI) {
1948                 fctrl |= IXGBE_FCTRL_MPE;
1949                 fctrl &= ~IXGBE_FCTRL_UPE;
1950         } else
1951                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1952
1953         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1954
1955         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1956                 if (ifma->ifma_addr->sa_family != AF_LINK)
1957                         continue;
1958                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1959                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1960                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1961                 mcnt++;
1962         }
1963
1964         update_ptr = mta;
1965         ixgbe_update_mc_addr_list(&adapter->hw,
1966             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1967
1968         return;
1969 }
1970
1971 /*
1972  * This is an iterator function now needed by the multicast
1973  * shared code. It simply feeds the shared code routine the
1974  * addresses in the array of ixgbe_set_multi() one by one.
1975  */
1976 static u8 *
1977 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
1978 {
1979         u8 *addr = *update_ptr;
1980         u8 *newptr;
1981         *vmdq = 0;
1982
1983         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1984         *update_ptr = newptr;
1985         return addr;
1986 }
1987
1988
1989 /*********************************************************************
1990  *  Timer routine
1991  *
1992  *  This routine checks for link status,updates statistics,
1993  *  and runs the watchdog check.
1994  *
1995  **********************************************************************/
1996
1997 static void
1998 ixgbe_local_timer(void *arg)
1999 {
2000         struct adapter  *adapter = arg;
2001         device_t        dev = adapter->dev;
2002         struct ifnet    *ifp = adapter->ifp;
2003         struct ix_queue *que = adapter->queues;
2004         struct tx_ring  *txr = adapter->tx_rings;
2005         int             hung, busy, paused;
2006
2007         IXGBE_CORE_LOCK(adapter);
2008         hung = busy = paused = 0;
2009
2010         /* Check for pluggable optics */
2011         if (adapter->sfp_probe)
2012                 if (!ixgbe_sfp_probe(adapter))
2013                         goto out; /* Nothing to do */
2014
2015         ixgbe_update_link_status(adapter);
2016         ixgbe_update_stats_counters(adapter);
2017
2018         /*
2019          * If the interface has been paused
2020          * then don't do the watchdog check
2021          */
2022         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2023                 paused = 1;
2024
2025         /*
2026         ** Check the TX queues status
2027         **      - central locked handling of OACTIVE
2028         **      - watchdog only if all queues show hung
2029         */
2030         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2031                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2032                     (paused == 0))
2033                         ++hung;
2034                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2035                         ++busy;
2036                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2037                         taskqueue_enqueue(que->tq, &que->que_task);
2038         }
2039         /* Only truely watchdog if all queues show hung */
2040         if (hung == adapter->num_queues)
2041                 goto watchdog;
2042         /* Only turn off the stack flow when ALL are depleted */
2043         if (busy == adapter->num_queues)
2044                 ifp->if_flags |= IFF_OACTIVE;
2045         else if ((ifp->if_flags & IFF_OACTIVE) &&
2046             (busy < adapter->num_queues))
2047                 ifp->if_flags &= ~IFF_OACTIVE;
2048
2049 out:
2050         ixgbe_rearm_queues(adapter, adapter->que_mask);
2051         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2052         IXGBE_CORE_UNLOCK(adapter);
2053         return;
2054
2055 watchdog:
2056         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2057         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2058             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2059             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2060         device_printf(dev,"TX(%d) desc avail = %d,"
2061             "Next TX to Clean = %d\n",
2062             txr->me, txr->tx_avail, txr->next_to_clean);
2063         adapter->ifp->if_flags &= ~IFF_RUNNING;
2064         adapter->watchdog_events++;
2065         ixgbe_init_locked(adapter);
2066
2067         IXGBE_CORE_UNLOCK(adapter);
2068 }
2069
2070 /*
2071 ** Note: this routine updates the OS on the link state
2072 **      the real check of the hardware only happens with
2073 **      a link interrupt.
2074 */
2075 static void
2076 ixgbe_update_link_status(struct adapter *adapter)
2077 {
2078         struct ifnet    *ifp = adapter->ifp;
2079         struct tx_ring *txr = adapter->tx_rings;
2080         device_t dev = adapter->dev;
2081
2082
2083         if (adapter->link_up){
2084                 if (adapter->link_active == FALSE) {
2085                         if (bootverbose)
2086                                 device_printf(dev,"Link is up %d Gbps %s \n",
2087                                     ((adapter->link_speed == 128)? 10:1),
2088                                     "Full Duplex");
2089                         adapter->link_active = TRUE;
2090                         /* Update any Flow Control changes */
2091                         ixgbe_fc_enable(&adapter->hw);
2092                         ifp->if_link_state = LINK_STATE_UP;
2093                         if_link_state_change(ifp);
2094                 }
2095         } else { /* Link down */
2096                 if (adapter->link_active == TRUE) {
2097                         if (bootverbose)
2098                                 device_printf(dev,"Link is Down\n");
2099                         ifp->if_link_state = LINK_STATE_DOWN;
2100                         if_link_state_change(ifp);
2101                         adapter->link_active = FALSE;
2102                         for (int i = 0; i < adapter->num_queues;
2103                             i++, txr++)
2104                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2105                 }
2106         }
2107
2108         return;
2109 }
2110
2111
2112 /*********************************************************************
2113  *
2114  *  This routine disables all traffic on the adapter by issuing a
2115  *  global reset on the MAC and deallocates TX/RX buffers.
2116  *
2117  **********************************************************************/
2118
2119 static void
2120 ixgbe_stop(void *arg)
2121 {
2122         struct ifnet   *ifp;
2123         struct adapter *adapter = arg;
2124         struct ixgbe_hw *hw = &adapter->hw;
2125         ifp = adapter->ifp;
2126
2127         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2128
2129         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2130         ixgbe_disable_intr(adapter);
2131         callout_stop(&adapter->timer);
2132
2133         /* Let the stack know...*/
2134         ifp->if_flags &= ~IFF_RUNNING;
2135         ifp->if_flags |= IFF_OACTIVE;
2136
2137         ixgbe_reset_hw(hw);
2138         hw->adapter_stopped = FALSE;
2139         ixgbe_stop_adapter(hw);
2140         /* Turn off the laser */
2141         if (hw->phy.multispeed_fiber)
2142                 ixgbe_disable_tx_laser(hw);
2143
2144         /* reprogram the RAR[0] in case user changed it. */
2145         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2146
2147         return;
2148 }
2149
2150
2151 /*********************************************************************
2152  *
2153  *  Determine hardware revision.
2154  *
2155  **********************************************************************/
2156 static void
2157 ixgbe_identify_hardware(struct adapter *adapter)
2158 {
2159         device_t        dev = adapter->dev;
2160         struct ixgbe_hw *hw = &adapter->hw;
2161
2162         /* Save off the information about this board */
2163         hw->vendor_id = pci_get_vendor(dev);
2164         hw->device_id = pci_get_device(dev);
2165         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2166         hw->subsystem_vendor_id =
2167             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2168         hw->subsystem_device_id =
2169             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2170
2171         /* We need this here to set the num_segs below */
2172         ixgbe_set_mac_type(hw);
2173
2174         /* Pick up the 82599 and VF settings */
2175         if (hw->mac.type != ixgbe_mac_82598EB) {
2176                 hw->phy.smart_speed = ixgbe_smart_speed;
2177                 adapter->num_segs = IXGBE_82599_SCATTER;
2178         } else
2179                 adapter->num_segs = IXGBE_82598_SCATTER;
2180
2181         return;
2182 }
2183
2184 /*********************************************************************
2185  *
2186  *  Determine optic type
2187  *
2188  **********************************************************************/
2189 static void
2190 ixgbe_setup_optics(struct adapter *adapter)
2191 {
2192         struct ixgbe_hw *hw = &adapter->hw;
2193         int             layer;
2194
2195         layer = ixgbe_get_supported_physical_layer(hw);
2196
2197         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2198                 adapter->optics = IFM_10G_T;
2199                 return;
2200         }
2201
2202         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2203                 adapter->optics = IFM_1000_T;
2204                 return;
2205         }
2206
2207         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2208             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2209                 adapter->optics = IFM_10G_LR;
2210                 return;
2211         }
2212
2213         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2214                 adapter->optics = IFM_10G_SR;
2215                 return;
2216         }
2217
2218         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2219                 adapter->optics = IFM_10G_TWINAX;
2220                 return;
2221         }
2222
2223         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2224             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2225                 adapter->optics = IFM_10G_CX4;
2226                 return;
2227         }
2228
2229         /* If we get here just set the default */
2230         adapter->optics = IFM_ETHER | IFM_AUTO;
2231         return;
2232 }
2233
2234 /*********************************************************************
2235  *
2236  *  Setup the Legacy or MSI Interrupt handler
2237  *
2238  **********************************************************************/
2239 static int
2240 ixgbe_allocate_legacy(struct adapter *adapter)
2241 {
2242         device_t dev = adapter->dev;
2243         struct          ix_queue *que = adapter->queues;
2244         int error, rid = 0;
2245         unsigned int intr_flags;
2246
2247         /* MSI RID at 1 */
2248         if (adapter->msix == 1)
2249                 rid = 1;
2250
2251         /* Try allocating a MSI interrupt first */
2252         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2253                 &rid, &intr_flags);
2254
2255         /* We allocate a single interrupt resource */
2256         adapter->res = bus_alloc_resource_any(dev,
2257             SYS_RES_IRQ, &rid, intr_flags);
2258         if (adapter->res == NULL) {
2259                 device_printf(dev, "Unable to allocate bus resource: "
2260                     "interrupt\n");
2261                 return (ENXIO);
2262         }
2263
2264         /*
2265          * Try allocating a fast interrupt and the associated deferred
2266          * processing contexts.
2267          */
2268         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2269         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2270             taskqueue_thread_enqueue, &que->tq);
2271         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2272             device_get_nameunit(adapter->dev));
2273
2274         /* Tasklets for Link, SFP and Multispeed Fiber */
2275         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2276         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2277         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2278 #ifdef IXGBE_FDIR
2279         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2280 #endif
2281         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2282             taskqueue_thread_enqueue, &adapter->tq);
2283         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2284             device_get_nameunit(adapter->dev));
2285
2286         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2287             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2288                 device_printf(dev, "Failed to register fast interrupt "
2289                     "handler: %d\n", error);
2290                 taskqueue_free(que->tq);
2291                 taskqueue_free(adapter->tq);
2292                 que->tq = NULL;
2293                 adapter->tq = NULL;
2294                 return (error);
2295         }
2296         /* For simplicity in the handlers */
2297         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2298
2299         return (0);
2300 }
2301
2302
2303 /*********************************************************************
2304  *
2305  *  Setup MSIX Interrupt resources and handlers
2306  *
2307  **********************************************************************/
2308 static int
2309 ixgbe_allocate_msix(struct adapter *adapter)
2310 {
2311         device_t        dev = adapter->dev;
2312         struct          ix_queue *que = adapter->queues;
2313         int             error, rid, vector = 0;
2314
2315         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2316                 rid = vector + 1;
2317                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2318                     RF_SHAREABLE | RF_ACTIVE);
2319                 if (que->res == NULL) {
2320                         device_printf(dev,"Unable to allocate"
2321                             " bus resource: que interrupt [%d]\n", vector);
2322                         return (ENXIO);
2323                 }
2324                 /* Set the handler function */
2325                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2326                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2327                 if (error) {
2328                         que->res = NULL;
2329                         device_printf(dev, "Failed to register QUE handler");
2330                         return (error);
2331                 }
2332 #if 0 /* __FreeBSD_version >= 800504 */
2333                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2334 #endif
2335                 que->msix = vector;
2336                 adapter->que_mask |= (u64)(1 << que->msix);
2337                 /*
2338                 ** Bind the msix vector, and thus the
2339                 ** ring to the corresponding cpu.
2340                 */
2341 #if 0 /* XXX */
2342                 if (adapter->num_queues > 1)
2343                         bus_bind_intr(dev, que->res, i);
2344 #endif
2345
2346                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2347                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2348                     taskqueue_thread_enqueue, &que->tq);
2349                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2350                     device_get_nameunit(adapter->dev));
2351         }
2352
2353         /* and Link */
2354         rid = vector + 1;
2355         adapter->res = bus_alloc_resource_any(dev,
2356             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2357         if (!adapter->res) {
2358                 device_printf(dev,"Unable to allocate"
2359             " bus resource: Link interrupt [%d]\n", rid);
2360                 return (ENXIO);
2361         }
2362         /* Set the link handler function */
2363         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2364             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2365         if (error) {
2366                 adapter->res = NULL;
2367                 device_printf(dev, "Failed to register LINK handler");
2368                 return (error);
2369         }
2370 #if 0 /* __FreeBSD_version >= 800504 */
2371         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2372 #endif
2373         adapter->linkvec = vector;
2374         /* Tasklets for Link, SFP and Multispeed Fiber */
2375         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2376         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2377         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2378 #ifdef IXGBE_FDIR
2379         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2380 #endif
2381         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2382             taskqueue_thread_enqueue, &adapter->tq);
2383         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2384             device_get_nameunit(adapter->dev));
2385
2386         return (0);
2387 }
2388
2389 #if 0   /* HAVE_MSIX */
2390 /*
2391  * Setup Either MSI/X or MSI
2392  */
2393 static int
2394 ixgbe_setup_msix(struct adapter *adapter)
2395 {
2396         device_t dev = adapter->dev;
2397         int rid, want, queues, msgs;
2398
2399         /* Override by tuneable */
2400         if (ixgbe_enable_msix == 0)
2401                 goto msi;
2402
2403         /* First try MSI/X */
2404         rid = PCIR_BAR(MSIX_82598_BAR);
2405         adapter->msix_mem = bus_alloc_resource_any(dev,
2406             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2407         if (!adapter->msix_mem) {
2408                 rid += 4;       /* 82599 maps in higher BAR */
2409                 adapter->msix_mem = bus_alloc_resource_any(dev,
2410                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2411         }
2412         if (!adapter->msix_mem) {
2413                 /* May not be enabled */
2414                 device_printf(adapter->dev,
2415                     "Unable to map MSIX table \n");
2416                 goto msi;
2417         }
2418
2419         msgs = pci_msix_count(dev);
2420         if (msgs == 0) { /* system has msix disabled */
2421                 bus_release_resource(dev, SYS_RES_MEMORY,
2422                     rid, adapter->msix_mem);
2423                 adapter->msix_mem = NULL;
2424                 goto msi;
2425         }
2426
2427         /* Figure out a reasonable auto config value */
2428         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2429
2430         if (ixgbe_num_queues != 0)
2431                 queues = ixgbe_num_queues;
2432         /* Set max queues to 8 when autoconfiguring */
2433         else if ((ixgbe_num_queues == 0) && (queues > 8))
2434                 queues = 8;
2435
2436         /*
2437         ** Want one vector (RX/TX pair) per queue
2438         ** plus an additional for Link.
2439         */
2440         want = queues + 1;
2441         if (msgs >= want)
2442                 msgs = want;
2443         else {
2444                 device_printf(adapter->dev,
2445                     "MSIX Configuration Problem, "
2446                     "%d vectors but %d queues wanted!\n",
2447                     msgs, want);
2448                 return (0); /* Will go to Legacy setup */
2449         }
2450         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2451                 device_printf(adapter->dev,
2452                     "Using MSIX interrupts with %d vectors\n", msgs);
2453                 adapter->num_queues = queues;
2454                 return (msgs);
2455         }
2456 msi:
2457         msgs = pci_msi_count(dev);
2458         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2459                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2460         else
2461                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2462         return (msgs);
2463 }
2464 #endif
2465
2466
2467 static int
2468 ixgbe_allocate_pci_resources(struct adapter *adapter)
2469 {
2470         int             rid;
2471         device_t        dev = adapter->dev;
2472
2473         rid = PCIR_BAR(0);
2474         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2475             &rid, RF_ACTIVE);
2476
2477         if (!(adapter->pci_mem)) {
2478                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2479                 return (ENXIO);
2480         }
2481
2482         adapter->osdep.mem_bus_space_tag =
2483                 rman_get_bustag(adapter->pci_mem);
2484         adapter->osdep.mem_bus_space_handle =
2485                 rman_get_bushandle(adapter->pci_mem);
2486         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2487
2488         /* Legacy defaults */
2489         adapter->num_queues = 1;
2490         adapter->hw.back = &adapter->osdep;
2491
2492         /*
2493         ** Now setup MSI or MSI/X, should
2494         ** return us the number of supported
2495         ** vectors. (Will be 1 for MSI)
2496         */
2497 #if 0   /* HAVE_MSIX */
2498         adapter->msix = ixgbe_setup_msix(adapter);
2499 #endif
2500         return (0);
2501 }
2502
2503 static void
2504 ixgbe_free_pci_resources(struct adapter * adapter)
2505 {
2506         struct          ix_queue *que = adapter->queues;
2507         device_t        dev = adapter->dev;
2508         int             rid, memrid;
2509
2510         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2511                 memrid = PCIR_BAR(MSIX_82598_BAR);
2512         else
2513                 memrid = PCIR_BAR(MSIX_82599_BAR);
2514
2515         /*
2516         ** There is a slight possibility of a failure mode
2517         ** in attach that will result in entering this function
2518         ** before interrupt resources have been initialized, and
2519         ** in that case we do not want to execute the loops below
2520         ** We can detect this reliably by the state of the adapter
2521         ** res pointer.
2522         */
2523         if (adapter->res == NULL)
2524                 goto mem;
2525
2526         /*
2527         **  Release all msix queue resources:
2528         */
2529         for (int i = 0; i < adapter->num_queues; i++, que++) {
2530                 rid = que->msix + 1;
2531                 if (que->tag != NULL) {
2532                         bus_teardown_intr(dev, que->res, que->tag);
2533                         que->tag = NULL;
2534                 }
2535                 if (que->res != NULL)
2536                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2537         }
2538
2539
2540         /* Clean the Legacy or Link interrupt last */
2541         if (adapter->linkvec) /* we are doing MSIX */
2542                 rid = adapter->linkvec + 1;
2543         else
2544                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2545
2546         if (adapter->tag != NULL) {
2547                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2548                 adapter->tag = NULL;
2549         }
2550         if (adapter->res != NULL)
2551                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2552         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2553                 pci_release_msi(adapter->dev);
2554
2555 mem:
2556         if (adapter->msix)
2557                 pci_release_msi(dev);
2558
2559         if (adapter->msix_mem != NULL)
2560                 bus_release_resource(dev, SYS_RES_MEMORY,
2561                     memrid, adapter->msix_mem);
2562
2563         if (adapter->pci_mem != NULL)
2564                 bus_release_resource(dev, SYS_RES_MEMORY,
2565                     PCIR_BAR(0), adapter->pci_mem);
2566
2567         return;
2568 }
2569
2570 /*********************************************************************
2571  *
2572  *  Setup networking device structure and register an interface.
2573  *
2574  **********************************************************************/
2575 static int
2576 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2577 {
2578         struct ixgbe_hw *hw = &adapter->hw;
2579         struct ifnet   *ifp;
2580
2581         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2582
2583         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2584         if (ifp == NULL) {
2585                 device_printf(dev, "can not allocate ifnet structure\n");
2586                 return (-1);
2587         }
2588         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2589         ifp->if_baudrate = 1000000000;
2590         ifp->if_init = ixgbe_init;
2591         ifp->if_softc = adapter;
2592         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2593         ifp->if_ioctl = ixgbe_ioctl;
2594         ifp->if_start = ixgbe_start;
2595 #if 0 /* __FreeBSD_version >= 800000 */
2596         ifp->if_transmit = ixgbe_mq_start;
2597         ifp->if_qflush = ixgbe_qflush;
2598 #endif
2599         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2600
2601         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2602
2603         adapter->max_frame_size =
2604             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2605
2606         /*
2607          * Tell the upper layer(s) we support long frames.
2608          */
2609         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2610
2611 #if 0 /* NET_TSO */
2612         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2613 #endif
2614         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2615         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2616         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2617 #if 0 /* NET_TSO */
2618                              |  IFCAP_VLAN_HWTSO
2619 #endif
2620                              |  IFCAP_VLAN_MTU;
2621         ifp->if_capenable = ifp->if_capabilities;
2622
2623         /* Don't enable LRO by default */
2624 #if 0 /* NET_LRO */
2625         ifp->if_capabilities |= IFCAP_LRO;
2626 #endif
2627
2628         /*
2629         ** Don't turn this on by default, if vlans are
2630         ** created on another pseudo device (eg. lagg)
2631         ** then vlan events are not passed thru, breaking
2632         ** operation, but with HW FILTER off it works. If
2633         ** using vlans directly on the ixgbe driver you can
2634         ** enable this and get full hardware tag filtering.
2635         */
2636         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2637
2638         /*
2639          * Specify the media types supported by this adapter and register
2640          * callbacks to update media and link information
2641          */
2642         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2643                      ixgbe_media_status);
2644         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2645         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2646         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2647                 ifmedia_add(&adapter->media,
2648                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2649                 ifmedia_add(&adapter->media,
2650                     IFM_ETHER | IFM_1000_T, 0, NULL);
2651         }
2652         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2653         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2654
2655         return (0);
2656 }
2657
2658 static void
2659 ixgbe_config_link(struct adapter *adapter)
2660 {
2661         struct ixgbe_hw *hw = &adapter->hw;
2662         u32     autoneg, err = 0;
2663         bool    sfp, negotiate;
2664
2665         sfp = ixgbe_is_sfp(hw);
2666
2667         if (sfp) {
2668                 if (hw->phy.multispeed_fiber) {
2669                         hw->mac.ops.setup_sfp(hw);
2670                         ixgbe_enable_tx_laser(hw);
2671                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2672                 } else
2673                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2674         } else {
2675                 if (hw->mac.ops.check_link)
2676                         err = ixgbe_check_link(hw, &autoneg,
2677                             &adapter->link_up, FALSE);
2678                 if (err)
2679                         goto out;
2680                 autoneg = hw->phy.autoneg_advertised;
2681                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2682                         err  = hw->mac.ops.get_link_capabilities(hw,
2683                             &autoneg, &negotiate);
2684                 if (err)
2685                         goto out;
2686                 if (hw->mac.ops.setup_link)
2687                         err = hw->mac.ops.setup_link(hw, autoneg,
2688                             negotiate, adapter->link_up);
2689         }
2690 out:
2691         return;
2692 }
2693
2694 /********************************************************************
2695  * Manage DMA'able memory.
2696  *******************************************************************/
2697 static void
2698 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2699 {
2700         if (error)
2701                 return;
2702         *(bus_addr_t *) arg = segs->ds_addr;
2703         return;
2704 }
2705
2706 static int
2707 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2708                 struct ixgbe_dma_alloc *dma, int mapflags)
2709 {
2710         device_t dev = adapter->dev;
2711         int             r;
2712
2713         r = bus_dma_tag_create(NULL,    /* parent */
2714                                DBA_ALIGN, 0,    /* alignment, bounds */
2715                                BUS_SPACE_MAXADDR,       /* lowaddr */
2716                                BUS_SPACE_MAXADDR,       /* highaddr */
2717                                NULL, NULL,      /* filter, filterarg */
2718                                size,    /* maxsize */
2719                                1,       /* nsegments */
2720                                size,    /* maxsegsize */
2721                                BUS_DMA_ALLOCNOW,        /* flags */
2722                                &dma->dma_tag);
2723         if (r != 0) {
2724                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2725                        "error %u\n", r);
2726                 goto fail_0;
2727         }
2728         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2729                              BUS_DMA_NOWAIT, &dma->dma_map);
2730         if (r != 0) {
2731                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2732                        "error %u\n", r);
2733                 goto fail_1;
2734         }
2735         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2736                             size,
2737                             ixgbe_dmamap_cb,
2738                             &dma->dma_paddr,
2739                             mapflags | BUS_DMA_NOWAIT);
2740         if (r != 0) {
2741                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2742                        "error %u\n", r);
2743                 goto fail_2;
2744         }
2745         dma->dma_size = size;
2746         return (0);
2747 fail_2:
2748         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2749 fail_1:
2750         bus_dma_tag_destroy(dma->dma_tag);
2751 fail_0:
2752         dma->dma_map = NULL;
2753         dma->dma_tag = NULL;
2754         return (r);
2755 }
2756
2757 static void
2758 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2759 {
2760         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2761             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2762         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2763         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2764         bus_dma_tag_destroy(dma->dma_tag);
2765 }
2766
2767
2768 /*********************************************************************
2769  *
2770  *  Allocate memory for the transmit and receive rings, and then
2771  *  the descriptors associated with each, called only once at attach.
2772  *
2773  **********************************************************************/
2774 static int
2775 ixgbe_allocate_queues(struct adapter *adapter)
2776 {
2777         device_t        dev = adapter->dev;
2778         struct ix_queue *que;
2779         struct tx_ring  *txr;
2780         struct rx_ring  *rxr;
2781         int rsize, tsize, error = IXGBE_SUCCESS;
2782         int txconf = 0, rxconf = 0;
2783
2784         /* First allocate the top level queue structs */
2785         if (!(adapter->queues =
2786             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2787             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2788                 device_printf(dev, "Unable to allocate queue memory\n");
2789                 error = ENOMEM;
2790                 goto fail;
2791         }
2792
2793         /* First allocate the TX ring struct memory */
2794         if (!(adapter->tx_rings =
2795             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2796             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2797                 device_printf(dev, "Unable to allocate TX ring memory\n");
2798                 error = ENOMEM;
2799                 goto tx_fail;
2800         }
2801
2802         /* Next allocate the RX */
2803         if (!(adapter->rx_rings =
2804             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2805             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2806                 device_printf(dev, "Unable to allocate RX ring memory\n");
2807                 error = ENOMEM;
2808                 goto rx_fail;
2809         }
2810
2811         /* For the ring itself */
2812         tsize = roundup2(adapter->num_tx_desc *
2813             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2814
2815         /*
2816          * Now set up the TX queues, txconf is needed to handle the
2817          * possibility that things fail midcourse and we need to
2818          * undo memory gracefully
2819          */
2820         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2821                 /* Set up some basics */
2822                 txr = &adapter->tx_rings[i];
2823                 txr->adapter = adapter;
2824                 txr->me = i;
2825
2826                 /* Initialize the TX side lock */
2827                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2828                     device_get_nameunit(dev), txr->me);
2829                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2830
2831                 if (ixgbe_dma_malloc(adapter, tsize,
2832                         &txr->txdma, BUS_DMA_NOWAIT)) {
2833                         device_printf(dev,
2834                             "Unable to allocate TX Descriptor memory\n");
2835                         error = ENOMEM;
2836                         goto err_tx_desc;
2837                 }
2838                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2839                 bzero((void *)txr->tx_base, tsize);
2840
2841                 /* Now allocate transmit buffers for the ring */
2842                 if (ixgbe_allocate_transmit_buffers(txr)) {
2843                         device_printf(dev,
2844                             "Critical Failure setting up transmit buffers\n");
2845                         error = ENOMEM;
2846                         goto err_tx_desc;
2847                 }
2848 #if 0 /* __FreeBSD_version >= 800000 */
2849                 /* Allocate a buf ring */
2850                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2851                     M_WAITOK, &txr->tx_mtx);
2852                 if (txr->br == NULL) {
2853                         device_printf(dev,
2854                             "Critical Failure setting up buf ring\n");
2855                         error = ENOMEM;
2856                         goto err_tx_desc;
2857                 }
2858 #endif
2859         }
2860
2861         /*
2862          * Next the RX queues...
2863          */
2864         rsize = roundup2(adapter->num_rx_desc *
2865             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2866         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2867                 rxr = &adapter->rx_rings[i];
2868                 /* Set up some basics */
2869                 rxr->adapter = adapter;
2870                 rxr->me = i;
2871
2872                 /* Initialize the RX side lock */
2873                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2874                     device_get_nameunit(dev), rxr->me);
2875                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2876
2877                 if (ixgbe_dma_malloc(adapter, rsize,
2878                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2879                         device_printf(dev,
2880                             "Unable to allocate RxDescriptor memory\n");
2881                         error = ENOMEM;
2882                         goto err_rx_desc;
2883                 }
2884                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2885                 bzero((void *)rxr->rx_base, rsize);
2886
2887                 /* Allocate receive buffers for the ring*/
2888                 if (ixgbe_allocate_receive_buffers(rxr)) {
2889                         device_printf(dev,
2890                             "Critical Failure setting up receive buffers\n");
2891                         error = ENOMEM;
2892                         goto err_rx_desc;
2893                 }
2894         }
2895
2896         /*
2897         ** Finally set up the queue holding structs
2898         */
2899         for (int i = 0; i < adapter->num_queues; i++) {
2900                 que = &adapter->queues[i];
2901                 que->adapter = adapter;
2902                 que->txr = &adapter->tx_rings[i];
2903                 que->rxr = &adapter->rx_rings[i];
2904         }
2905
2906         return (0);
2907
2908 err_rx_desc:
2909         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2910                 ixgbe_dma_free(adapter, &rxr->rxdma);
2911 err_tx_desc:
2912         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2913                 ixgbe_dma_free(adapter, &txr->txdma);
2914         kfree(adapter->rx_rings, M_DEVBUF);
2915 rx_fail:
2916         kfree(adapter->tx_rings, M_DEVBUF);
2917 tx_fail:
2918         kfree(adapter->queues, M_DEVBUF);
2919 fail:
2920         return (error);
2921 }
2922
2923 /*********************************************************************
2924  *
2925  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2926  *  the information needed to transmit a packet on the wire. This is
2927  *  called only once at attach, setup is done every reset.
2928  *
2929  **********************************************************************/
2930 static int
2931 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2932 {
2933         struct adapter *adapter = txr->adapter;
2934         device_t dev = adapter->dev;
2935         struct ixgbe_tx_buf *txbuf;
2936         int error, i;
2937
2938         /*
2939          * Setup DMA descriptor areas.
2940          */
2941         if ((error = bus_dma_tag_create(
2942                                NULL,    /* parent */
2943                                1, 0,            /* alignment, bounds */
2944                                BUS_SPACE_MAXADDR,       /* lowaddr */
2945                                BUS_SPACE_MAXADDR,       /* highaddr */
2946                                NULL, NULL,              /* filter, filterarg */
2947                                IXGBE_TSO_SIZE,          /* maxsize */
2948                                adapter->num_segs,       /* nsegments */
2949                                PAGE_SIZE,               /* maxsegsize */
2950                                0,                       /* flags */
2951                                &txr->txtag))) {
2952                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2953                 goto fail;
2954         }
2955
2956         if (!(txr->tx_buffers =
2957             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2958             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2959                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2960                 error = ENOMEM;
2961                 goto fail;
2962         }
2963
2964         /* Create the descriptor buffer dma maps */
2965         txbuf = txr->tx_buffers;
2966         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2967                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2968                 if (error != 0) {
2969                         device_printf(dev, "Unable to create TX DMA map\n");
2970                         goto fail;
2971                 }
2972         }
2973
2974         return 0;
2975 fail:
2976         /* We free all, it handles case where we are in the middle */
2977         ixgbe_free_transmit_structures(adapter);
2978         return (error);
2979 }
2980
2981 /*********************************************************************
2982  *
2983  *  Initialize a transmit ring.
2984  *
2985  **********************************************************************/
2986 static void
2987 ixgbe_setup_transmit_ring(struct tx_ring *txr)
2988 {
2989         struct adapter *adapter = txr->adapter;
2990         struct ixgbe_tx_buf *txbuf;
2991         int i;
2992 #ifdef DEV_NETMAP
2993         struct netmap_adapter *na = NA(adapter->ifp);
2994         struct netmap_slot *slot;
2995 #endif /* DEV_NETMAP */
2996
2997         /* Clear the old ring contents */
2998         IXGBE_TX_LOCK(txr);
2999 #ifdef DEV_NETMAP
3000         /*
3001          * (under lock): if in netmap mode, do some consistency
3002          * checks and set slot to entry 0 of the netmap ring.
3003          */
3004         slot = netmap_reset(na, NR_TX, txr->me, 0);
3005 #endif /* DEV_NETMAP */
3006         bzero((void *)txr->tx_base,
3007               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3008         /* Reset indices */
3009         txr->next_avail_desc = 0;
3010         txr->next_to_clean = 0;
3011
3012         /* Free any existing tx buffers. */
3013         txbuf = txr->tx_buffers;
3014         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3015                 if (txbuf->m_head != NULL) {
3016                         bus_dmamap_sync(txr->txtag, txbuf->map,
3017                             BUS_DMASYNC_POSTWRITE);
3018                         bus_dmamap_unload(txr->txtag, txbuf->map);
3019                         m_freem(txbuf->m_head);
3020                         txbuf->m_head = NULL;
3021                 }
3022 #ifdef DEV_NETMAP
3023                 /*
3024                  * In netmap mode, set the map for the packet buffer.
3025                  * NOTE: Some drivers (not this one) also need to set
3026                  * the physical buffer address in the NIC ring.
3027                  * Slots in the netmap ring (indexed by "si") are
3028                  * kring->nkr_hwofs positions "ahead" wrt the
3029                  * corresponding slot in the NIC ring. In some drivers
3030                  * (not here) nkr_hwofs can be negative. Function
3031                  * netmap_idx_n2k() handles wraparounds properly.
3032                  */
3033                 if (slot) {
3034                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3035                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3036                 }
3037 #endif /* DEV_NETMAP */
3038                 /* Clear the EOP index */
3039                 txbuf->eop_index = -1;
3040         }
3041
3042 #ifdef IXGBE_FDIR
3043         /* Set the rate at which we sample packets */
3044         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3045                 txr->atr_sample = atr_sample_rate;
3046 #endif
3047
3048         /* Set number of descriptors available */
3049         txr->tx_avail = adapter->num_tx_desc;
3050
3051         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3052             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3053         IXGBE_TX_UNLOCK(txr);
3054 }
3055
3056 /*********************************************************************
3057  *
3058  *  Initialize all transmit rings.
3059  *
3060  **********************************************************************/
3061 static int
3062 ixgbe_setup_transmit_structures(struct adapter *adapter)
3063 {
3064         struct tx_ring *txr = adapter->tx_rings;
3065
3066         for (int i = 0; i < adapter->num_queues; i++, txr++)
3067                 ixgbe_setup_transmit_ring(txr);
3068
3069         return (0);
3070 }
3071
3072 /*********************************************************************
3073  *
3074  *  Enable transmit unit.
3075  *
3076  **********************************************************************/
3077 static void
3078 ixgbe_initialize_transmit_units(struct adapter *adapter)
3079 {
3080         struct tx_ring  *txr = adapter->tx_rings;
3081         struct ixgbe_hw *hw = &adapter->hw;
3082
3083         /* Setup the Base and Length of the Tx Descriptor Ring */
3084
3085         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3086                 u64     tdba = txr->txdma.dma_paddr;
3087                 u32     txctrl;
3088
3089                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3090                        (tdba & 0x00000000ffffffffULL));
3091                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3092                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3093                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3094
3095                 /* Setup the HW Tx Head and Tail descriptor pointers */
3096                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3097                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3098
3099                 /* Setup Transmit Descriptor Cmd Settings */
3100                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3101                 txr->queue_status = IXGBE_QUEUE_IDLE;
3102
3103                 /* Disable Head Writeback */
3104                 switch (hw->mac.type) {
3105                 case ixgbe_mac_82598EB:
3106                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3107                         break;
3108                 case ixgbe_mac_82599EB:
3109                 case ixgbe_mac_X540:
3110                 default:
3111                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3112                         break;
3113                 }
3114                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3115                 switch (hw->mac.type) {
3116                 case ixgbe_mac_82598EB:
3117                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3118                         break;
3119                 case ixgbe_mac_82599EB:
3120                 case ixgbe_mac_X540:
3121                 default:
3122                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3123                         break;
3124                 }
3125
3126         }
3127
3128         if (hw->mac.type != ixgbe_mac_82598EB) {
3129                 u32 dmatxctl, rttdcs;
3130                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3131                 dmatxctl |= IXGBE_DMATXCTL_TE;
3132                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3133                 /* Disable arbiter to set MTQC */
3134                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3135                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3136                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3137                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3138                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3139                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3140         }
3141
3142         return;
3143 }
3144
3145 /*********************************************************************
3146  *
3147  *  Free all transmit rings.
3148  *
3149  **********************************************************************/
3150 static void
3151 ixgbe_free_transmit_structures(struct adapter *adapter)
3152 {
3153         struct tx_ring *txr = adapter->tx_rings;
3154
3155         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3156                 IXGBE_TX_LOCK(txr);
3157                 ixgbe_free_transmit_buffers(txr);
3158                 ixgbe_dma_free(adapter, &txr->txdma);
3159                 IXGBE_TX_UNLOCK(txr);
3160                 IXGBE_TX_LOCK_DESTROY(txr);
3161         }
3162         kfree(adapter->tx_rings, M_DEVBUF);
3163 }
3164
3165 /*********************************************************************
3166  *
3167  *  Free transmit ring related data structures.
3168  *
3169  **********************************************************************/
3170 static void
3171 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3172 {
3173         struct adapter *adapter = txr->adapter;
3174         struct ixgbe_tx_buf *tx_buffer;
3175         int             i;
3176
3177         INIT_DEBUGOUT("free_transmit_ring: begin");
3178
3179         if (txr->tx_buffers == NULL)
3180                 return;
3181
3182         tx_buffer = txr->tx_buffers;
3183         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3184                 if (tx_buffer->m_head != NULL) {
3185                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3186                             BUS_DMASYNC_POSTWRITE);
3187                         bus_dmamap_unload(txr->txtag,
3188                             tx_buffer->map);
3189                         m_freem(tx_buffer->m_head);
3190                         tx_buffer->m_head = NULL;
3191                         if (tx_buffer->map != NULL) {
3192                                 bus_dmamap_destroy(txr->txtag,
3193                                     tx_buffer->map);
3194                                 tx_buffer->map = NULL;
3195                         }
3196                 } else if (tx_buffer->map != NULL) {
3197                         bus_dmamap_unload(txr->txtag,
3198                             tx_buffer->map);
3199                         bus_dmamap_destroy(txr->txtag,
3200                             tx_buffer->map);
3201                         tx_buffer->map = NULL;
3202                 }
3203         }
3204 #if 0 /* __FreeBSD_version >= 800000 */
3205         if (txr->br != NULL)
3206                 buf_ring_free(txr->br, M_DEVBUF);
3207 #endif
3208         if (txr->tx_buffers != NULL) {
3209                 kfree(txr->tx_buffers, M_DEVBUF);
3210                 txr->tx_buffers = NULL;
3211         }
3212         if (txr->txtag != NULL) {
3213                 bus_dma_tag_destroy(txr->txtag);
3214                 txr->txtag = NULL;
3215         }
3216         return;
3217 }
3218
3219 /*********************************************************************
3220  *
3221  *  Advanced Context Descriptor setup for VLAN or CSUM
3222  *
3223  **********************************************************************/
3224
3225 static bool
3226 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3227 {
3228         struct adapter *adapter = txr->adapter;
3229         struct ixgbe_adv_tx_context_desc *TXD;
3230         struct ixgbe_tx_buf        *tx_buffer;
3231         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3232         struct ether_vlan_header *eh;
3233         struct ip *ip;
3234         struct ip6_hdr *ip6;
3235         int  ehdrlen, ip_hlen = 0;
3236         u16     etype;
3237         u8      ipproto = 0;
3238         bool    offload = TRUE;
3239         int ctxd = txr->next_avail_desc;
3240 #ifdef NET_VLAN
3241         u16 vtag = 0;
3242 #endif
3243
3244
3245         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3246                 offload = FALSE;
3247
3248         tx_buffer = &txr->tx_buffers[ctxd];
3249         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3250
3251         /*
3252         ** In advanced descriptors the vlan tag must
3253         ** be placed into the descriptor itself.
3254         */
3255 #ifdef NET_VLAN
3256         if (mp->m_flags & M_VLANTAG) {
3257                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3258                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3259         } else if (offload == FALSE)
3260                 return FALSE;
3261 #endif
3262
3263         /*
3264          * Determine where frame payload starts.
3265          * Jump over vlan headers if already present,
3266          * helpful for QinQ too.
3267          */
3268         eh = mtod(mp, struct ether_vlan_header *);
3269         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3270                 etype = ntohs(eh->evl_proto);
3271                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3272         } else {
3273                 etype = ntohs(eh->evl_encap_proto);
3274                 ehdrlen = ETHER_HDR_LEN;
3275         }
3276
3277         /* Set the ether header length */
3278         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3279
3280         switch (etype) {
3281                 case ETHERTYPE_IP:
3282                         ip = (struct ip *)(mp->m_data + ehdrlen);
3283                         ip_hlen = ip->ip_hl << 2;
3284                         ipproto = ip->ip_p;
3285                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3286                         break;
3287                 case ETHERTYPE_IPV6:
3288                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3289                         ip_hlen = sizeof(struct ip6_hdr);
3290                         /* XXX-BZ this will go badly in case of ext hdrs. */
3291                         ipproto = ip6->ip6_nxt;
3292                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3293                         break;
3294                 default:
3295                         offload = FALSE;
3296                         break;
3297         }
3298
3299         vlan_macip_lens |= ip_hlen;
3300         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3301
3302         switch (ipproto) {
3303                 case IPPROTO_TCP:
3304                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3305                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3306                         break;
3307
3308                 case IPPROTO_UDP:
3309                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3310                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3311                         break;
3312
3313 #if 0
3314                 case IPPROTO_SCTP:
3315                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3316                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3317                         break;
3318 #endif
3319                 default:
3320                         offload = FALSE;
3321                         break;
3322         }
3323
3324         /* Now copy bits into descriptor */
3325         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3326         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3327         TXD->seqnum_seed = htole32(0);
3328         TXD->mss_l4len_idx = htole32(0);
3329
3330         tx_buffer->m_head = NULL;
3331         tx_buffer->eop_index = -1;
3332
3333         /* We've consumed the first desc, adjust counters */
3334         if (++ctxd == adapter->num_tx_desc)
3335                 ctxd = 0;
3336         txr->next_avail_desc = ctxd;
3337         --txr->tx_avail;
3338
3339         return (offload);
3340 }
3341
3342 /**********************************************************************
3343  *
3344  *  Setup work for hardware segmentation offload (TSO) on
3345  *  adapters using advanced tx descriptors
3346  *
3347  **********************************************************************/
3348 #if 0   /* NET_TSO */
3349 static bool
3350 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3351     u32 *olinfo_status)
3352 {
3353         struct adapter *adapter = txr->adapter;
3354         struct ixgbe_adv_tx_context_desc *TXD;
3355         struct ixgbe_tx_buf        *tx_buffer;
3356 #ifdef NET_VLAN
3357         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3358         u16 vtag = 0, eh_type;
3359 #else
3360         u16 eh_type;
3361         u32 type_tucmd_mlhl = 0;
3362 #endif
3363         u32 mss_l4len_idx = 0, len;
3364         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3365         struct ether_vlan_header *eh;
3366 #ifdef INET6
3367         struct ip6_hdr *ip6;
3368 #endif
3369 #ifdef INET
3370         struct ip *ip;
3371 #endif
3372         struct tcphdr *th;
3373
3374
3375         /*
3376          * Determine where frame payload starts.
3377          * Jump over vlan headers if already present
3378          */
3379         eh = mtod(mp, struct ether_vlan_header *);
3380         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3381                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3382                 eh_type = eh->evl_proto;
3383         } else {
3384                 ehdrlen = ETHER_HDR_LEN;
3385                 eh_type = eh->evl_encap_proto;
3386         }
3387
3388         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3389         len = ehdrlen + sizeof(struct tcphdr);
3390         switch (ntohs(eh_type)) {
3391 #ifdef INET6
3392         case ETHERTYPE_IPV6:
3393                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3394                         return FALSE;
3395                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3396                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3397                 if (ip6->ip6_nxt != IPPROTO_TCP)
3398                         return FALSE;
3399                 ip_hlen = sizeof(struct ip6_hdr);
3400                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3401                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3402                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3403                 break;
3404 #endif
3405 #ifdef INET
3406         case ETHERTYPE_IP:
3407                 if (mp->m_len < len + sizeof(struct ip))
3408                         return FALSE;
3409                 ip = (struct ip *)(mp->m_data + ehdrlen);
3410                 if (ip->ip_p != IPPROTO_TCP)
3411                         return FALSE;
3412                 ip->ip_sum = 0;
3413                 ip_hlen = ip->ip_hl << 2;
3414                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3415                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3416                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3417                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3418                 /* Tell transmit desc to also do IPv4 checksum. */
3419                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3420                 break;
3421 #endif
3422         default:
3423                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3424                     __func__, ntohs(eh_type));
3425                 break;
3426         }
3427
3428         ctxd = txr->next_avail_desc;
3429         tx_buffer = &txr->tx_buffers[ctxd];
3430         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3431
3432         tcp_hlen = th->th_off << 2;
3433
3434         /* This is used in the transmit desc in encap */
3435         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3436
3437         /* VLAN MACLEN IPLEN */
3438 #ifdef NET_VLAN
3439         if (mp->m_flags & M_VLANTAG) {
3440                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3441                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3442         }
3443
3444         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3445         vlan_macip_lens |= ip_hlen;
3446         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3447 #endif
3448
3449         /* ADV DTYPE TUCMD */
3450         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3451         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3452         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3453
3454         /* MSS L4LEN IDX */
3455         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3456         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3457         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3458
3459         TXD->seqnum_seed = htole32(0);
3460         tx_buffer->m_head = NULL;
3461         tx_buffer->eop_index = -1;
3462
3463         if (++ctxd == adapter->num_tx_desc)
3464                 ctxd = 0;
3465
3466         txr->tx_avail--;
3467         txr->next_avail_desc = ctxd;
3468         return TRUE;
3469 }
3470 #endif
3471
3472 #ifdef IXGBE_FDIR
3473 /*
3474 ** This routine parses packet headers so that Flow
3475 ** Director can make a hashed filter table entry
3476 ** allowing traffic flows to be identified and kept
3477 ** on the same cpu.  This would be a performance
3478 ** hit, but we only do it at IXGBE_FDIR_RATE of
3479 ** packets.
3480 */
3481 static void
3482 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3483 {
3484         struct adapter                  *adapter = txr->adapter;
3485         struct ix_queue                 *que;
3486         struct ip                       *ip;
3487         struct tcphdr                   *th;
3488         struct udphdr                   *uh;
3489         struct ether_vlan_header        *eh;
3490         union ixgbe_atr_hash_dword      input = {.dword = 0};
3491         union ixgbe_atr_hash_dword      common = {.dword = 0};
3492         int                             ehdrlen, ip_hlen;
3493         u16                             etype;
3494
3495         eh = mtod(mp, struct ether_vlan_header *);
3496         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3497                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3498                 etype = eh->evl_proto;
3499         } else {
3500                 ehdrlen = ETHER_HDR_LEN;
3501                 etype = eh->evl_encap_proto;
3502         }
3503
3504         /* Only handling IPv4 */
3505         if (etype != htons(ETHERTYPE_IP))
3506                 return;
3507
3508         ip = (struct ip *)(mp->m_data + ehdrlen);
3509         ip_hlen = ip->ip_hl << 2;
3510
3511         /* check if we're UDP or TCP */
3512         switch (ip->ip_p) {
3513         case IPPROTO_TCP:
3514                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3515                 /* src and dst are inverted */
3516                 common.port.dst ^= th->th_sport;
3517                 common.port.src ^= th->th_dport;
3518                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3519                 break;
3520         case IPPROTO_UDP:
3521                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3522                 /* src and dst are inverted */
3523                 common.port.dst ^= uh->uh_sport;
3524                 common.port.src ^= uh->uh_dport;
3525                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3526                 break;
3527         default:
3528                 return;
3529         }
3530
3531         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3532         if (mp->m_pkthdr.ether_vtag)
3533                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3534         else
3535                 common.flex_bytes ^= etype;
3536         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3537
3538         que = &adapter->queues[txr->me];
3539         /*
3540         ** This assumes the Rx queue and Tx
3541         ** queue are bound to the same CPU
3542         */
3543         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3544             input, common, que->msix);
3545 }
3546 #endif /* IXGBE_FDIR */
3547
3548 /**********************************************************************
3549  *
3550  *  Examine each tx_buffer in the used queue. If the hardware is done
3551  *  processing the packet then free associated resources. The
3552  *  tx_buffer is put back on the free queue.
3553  *
3554  **********************************************************************/
3555 static bool
3556 ixgbe_txeof(struct tx_ring *txr)
3557 {
3558         struct adapter  *adapter = txr->adapter;
3559         struct ifnet    *ifp = adapter->ifp;
3560         u32     first, last, done, processed;
3561         struct ixgbe_tx_buf *tx_buffer;
3562         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3563
3564         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3565
3566 #ifdef DEV_NETMAP
3567         if (ifp->if_capenable & IFCAP_NETMAP) {
3568                 struct netmap_adapter *na = NA(ifp);
3569                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3570
3571                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3572
3573                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3574                     BUS_DMASYNC_POSTREAD);
3575                 /*
3576                  * In netmap mode, all the work is done in the context
3577                  * of the client thread. Interrupt handlers only wake up
3578                  * clients, which may be sleeping on individual rings
3579                  * or on a global resource for all rings.
3580                  * To implement tx interrupt mitigation, we wake up the client
3581                  * thread roughly every half ring, even if the NIC interrupts
3582                  * more frequently. This is implemented as follows:
3583                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3584                  *   the slot that should wake up the thread (nkr_num_slots
3585                  *   means the user thread should not be woken up);
3586                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3587                  *   or the slot has the DD bit set.
3588                  *
3589                  * When the driver has separate locks, we need to
3590                  * release and re-acquire txlock to avoid deadlocks.
3591                  * XXX see if we can find a better way.
3592                  */
3593                 if (!netmap_mitigate ||
3594                     (kring->nr_kflags < kring->nkr_num_slots &&
3595                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3596                         kring->nr_kflags = kring->nkr_num_slots;
3597                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3598                         IXGBE_TX_UNLOCK(txr);
3599                         IXGBE_CORE_LOCK(adapter);
3600                         selwakeuppri(&na->tx_si, PI_NET);
3601                         IXGBE_CORE_UNLOCK(adapter);
3602                         IXGBE_TX_LOCK(txr);
3603                 }
3604                 return FALSE;
3605         }
3606 #endif /* DEV_NETMAP */
3607
3608         if (txr->tx_avail == adapter->num_tx_desc) {
3609                 txr->queue_status = IXGBE_QUEUE_IDLE;
3610                 return FALSE;
3611         }
3612
3613         processed = 0;
3614         first = txr->next_to_clean;
3615         tx_buffer = &txr->tx_buffers[first];
3616         /* For cleanup we just use legacy struct */
3617         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3618         last = tx_buffer->eop_index;
3619         if (last == -1)
3620                 return FALSE;
3621         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3622
3623         /*
3624         ** Get the index of the first descriptor
3625         ** BEYOND the EOP and call that 'done'.
3626         ** I do this so the comparison in the
3627         ** inner while loop below can be simple
3628         */
3629         if (++last == adapter->num_tx_desc) last = 0;
3630         done = last;
3631
3632         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3633             BUS_DMASYNC_POSTREAD);
3634         /*
3635         ** Only the EOP descriptor of a packet now has the DD
3636         ** bit set, this is what we look for...
3637         */
3638         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3639                 /* We clean the range of the packet */
3640                 while (first != done) {
3641                         tx_desc->upper.data = 0;
3642                         tx_desc->lower.data = 0;
3643                         tx_desc->buffer_addr = 0;
3644                         ++txr->tx_avail;
3645                         ++processed;
3646
3647                         if (tx_buffer->m_head) {
3648                                 txr->bytes +=
3649                                     tx_buffer->m_head->m_pkthdr.len;
3650                                 bus_dmamap_sync(txr->txtag,
3651                                     tx_buffer->map,
3652                                     BUS_DMASYNC_POSTWRITE);
3653                                 bus_dmamap_unload(txr->txtag,
3654                                     tx_buffer->map);
3655                                 m_freem(tx_buffer->m_head);
3656                                 tx_buffer->m_head = NULL;
3657                                 tx_buffer->map = NULL;
3658                         }
3659                         tx_buffer->eop_index = -1;
3660                         txr->watchdog_time = ticks;
3661
3662                         if (++first == adapter->num_tx_desc)
3663                                 first = 0;
3664
3665                         tx_buffer = &txr->tx_buffers[first];
3666                         tx_desc =
3667                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3668                 }
3669                 ++txr->packets;
3670                 ++ifp->if_opackets;
3671                 /* See if there is more work now */
3672                 last = tx_buffer->eop_index;
3673                 if (last != -1) {
3674                         eop_desc =
3675                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3676                         /* Get next done point */
3677                         if (++last == adapter->num_tx_desc) last = 0;
3678                         done = last;
3679                 } else
3680                         break;
3681         }
3682         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3683             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3684
3685         txr->next_to_clean = first;
3686
3687         /*
3688         ** Watchdog calculation, we know there's
3689         ** work outstanding or the first return
3690         ** would have been taken, so none processed
3691         ** for too long indicates a hang.
3692         */
3693         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3694                 txr->queue_status = IXGBE_QUEUE_HUNG;
3695
3696         /* With a minimum free clear the depleted state bit.  */
3697         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3698                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3699
3700         if (txr->tx_avail == adapter->num_tx_desc) {
3701                 txr->queue_status = IXGBE_QUEUE_IDLE;
3702                 return (FALSE);
3703         }
3704
3705         return TRUE;
3706 }
3707
3708 /*********************************************************************
3709  *
3710  *  Refresh mbuf buffers for RX descriptor rings
3711  *   - now keeps its own state so discards due to resource
3712  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3713  *     it just returns, keeping its placeholder, thus it can simply
3714  *     be recalled to try again.
3715  *
3716  **********************************************************************/
3717 static void
3718 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3719 {
3720         struct adapter          *adapter = rxr->adapter;
3721         bus_dma_segment_t       hseg[1];
3722         bus_dma_segment_t       pseg[1];
3723         struct ixgbe_rx_buf     *rxbuf;
3724         struct mbuf             *mh, *mp;
3725         int                     i, j, nsegs, error;
3726         bool                    refreshed = FALSE;
3727
3728         i = j = rxr->next_to_refresh;
3729         /* Control the loop with one beyond */
3730         if (++j == adapter->num_rx_desc)
3731                 j = 0;
3732
3733         while (j != limit) {
3734                 rxbuf = &rxr->rx_buffers[i];
3735                 if (rxr->hdr_split == FALSE)
3736                         goto no_split;
3737
3738                 if (rxbuf->m_head == NULL) {
3739                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3740                         if (mh == NULL)
3741                                 goto update;
3742                 } else
3743                         mh = rxbuf->m_head;
3744
3745                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3746                 mh->m_len = MHLEN;
3747                 mh->m_flags |= M_PKTHDR;
3748                 /* Get the memory mapping */
3749                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3750                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3751                 if (error != 0) {
3752                         kprintf("Refresh mbufs: hdr dmamap load"
3753                             " failure - %d\n", error);
3754                         m_free(mh);
3755                         rxbuf->m_head = NULL;
3756                         goto update;
3757                 }
3758                 rxbuf->m_head = mh;
3759                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3760                     BUS_DMASYNC_PREREAD);
3761                 rxr->rx_base[i].read.hdr_addr =
3762                     htole64(hseg[0].ds_addr);
3763
3764 no_split:
3765                 if (rxbuf->m_pack == NULL) {
3766                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3767                             M_PKTHDR, adapter->rx_mbuf_sz);
3768                         if (mp == NULL)
3769                                 goto update;
3770                 } else
3771                         mp = rxbuf->m_pack;
3772
3773                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3774                 /* Get the memory mapping */
3775                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3776                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3777                 if (error != 0) {
3778                         kprintf("Refresh mbufs: payload dmamap load"
3779                             " failure - %d\n", error);
3780                         m_free(mp);
3781                         rxbuf->m_pack = NULL;
3782                         goto update;
3783                 }
3784                 rxbuf->m_pack = mp;
3785                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3786                     BUS_DMASYNC_PREREAD);
3787                 rxr->rx_base[i].read.pkt_addr =
3788                     htole64(pseg[0].ds_addr);
3789
3790                 refreshed = TRUE;
3791                 /* Next is precalculated */
3792                 i = j;
3793                 rxr->next_to_refresh = i;
3794                 if (++j == adapter->num_rx_desc)
3795                         j = 0;
3796         }
3797 update:
3798         if (refreshed) /* Update hardware tail index */
3799                 IXGBE_WRITE_REG(&adapter->hw,
3800                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3801         return;
3802 }
3803
3804 /*********************************************************************
3805  *
3806  *  Allocate memory for rx_buffer structures. Since we use one
3807  *  rx_buffer per received packet, the maximum number of rx_buffer's
3808  *  that we'll need is equal to the number of receive descriptors
3809  *  that we've allocated.
3810  *
3811  **********************************************************************/
3812 static int
3813 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3814 {
3815         struct  adapter         *adapter = rxr->adapter;
3816         device_t                dev = adapter->dev;
3817         struct ixgbe_rx_buf     *rxbuf;
3818         int                     i, bsize, error;
3819
3820         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3821         if (!(rxr->rx_buffers =
3822             (struct ixgbe_rx_buf *) kmalloc(bsize,
3823             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3824                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3825                 error = ENOMEM;
3826                 goto fail;
3827         }
3828
3829         if ((error = bus_dma_tag_create(NULL,   /* parent */
3830                                    1, 0,        /* alignment, bounds */
3831                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3832                                    BUS_SPACE_MAXADDR,   /* highaddr */
3833                                    NULL, NULL,          /* filter, filterarg */
3834                                    MSIZE,               /* maxsize */
3835                                    1,                   /* nsegments */
3836                                    MSIZE,               /* maxsegsize */
3837                                    0,                   /* flags */
3838                                    &rxr->htag))) {
3839                 device_printf(dev, "Unable to create RX DMA tag\n");
3840                 goto fail;
3841         }
3842
3843         if ((error = bus_dma_tag_create(NULL,   /* parent */
3844                                    1, 0,        /* alignment, bounds */
3845                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3846                                    BUS_SPACE_MAXADDR,   /* highaddr */
3847                                    NULL, NULL,          /* filter, filterarg */
3848                                    MJUM16BYTES,         /* maxsize */
3849                                    1,                   /* nsegments */
3850                                    MJUM16BYTES,         /* maxsegsize */
3851                                    0,                   /* flags */
3852                                    &rxr->ptag))) {
3853                 device_printf(dev, "Unable to create RX DMA tag\n");
3854                 goto fail;
3855         }
3856
3857         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3858                 rxbuf = &rxr->rx_buffers[i];
3859                 error = bus_dmamap_create(rxr->htag,
3860                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3861                 if (error) {
3862                         device_printf(dev, "Unable to create RX head map\n");
3863                         goto fail;
3864                 }
3865                 error = bus_dmamap_create(rxr->ptag,
3866                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3867                 if (error) {
3868                         device_printf(dev, "Unable to create RX pkt map\n");
3869                         goto fail;
3870                 }
3871         }
3872
3873         return (0);
3874
3875 fail:
3876         /* Frees all, but can handle partial completion */
3877         ixgbe_free_receive_structures(adapter);
3878         return (error);
3879 }
3880
3881 /*
3882 ** Used to detect a descriptor that has
3883 ** been merged by Hardware RSC.
3884 */
3885 static inline u32
3886 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3887 {
3888         return (le32toh(rx->wb.lower.lo_dword.data) &
3889             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3890 }
3891
3892 /*********************************************************************
3893  *
3894  *  Initialize Hardware RSC (LRO) feature on 82599
3895  *  for an RX ring, this is toggled by the LRO capability
3896  *  even though it is transparent to the stack.
3897  *
3898  **********************************************************************/
3899 #if 0   /* NET_LRO */
3900 static void
3901 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3902 {
3903         struct  adapter         *adapter = rxr->adapter;
3904         struct  ixgbe_hw        *hw = &adapter->hw;
3905         u32                     rscctrl, rdrxctl;
3906
3907         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3908         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3909 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3910         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3911 #endif /* DEV_NETMAP */
3912         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3913         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3914         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3915
3916         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3917         rscctrl |= IXGBE_RSCCTL_RSCEN;
3918         /*
3919         ** Limit the total number of descriptors that
3920         ** can be combined, so it does not exceed 64K
3921         */
3922         if (adapter->rx_mbuf_sz == MCLBYTES)
3923                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3924         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3925                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3926         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3927                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3928         else  /* Using 16K cluster */
3929                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3930
3931         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3932
3933         /* Enable TCP header recognition */
3934         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3935             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3936             IXGBE_PSRTYPE_TCPHDR));
3937
3938         /* Disable RSC for ACK packets */
3939         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3940             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3941
3942         rxr->hw_rsc = TRUE;
3943 }
3944 #endif
3945
3946 static void
3947 ixgbe_free_receive_ring(struct rx_ring *rxr)
3948 {
3949         struct  adapter         *adapter;
3950         struct ixgbe_rx_buf       *rxbuf;
3951         int i;
3952
3953         adapter = rxr->adapter;
3954         for (i = 0; i < adapter->num_rx_desc; i++) {
3955                 rxbuf = &rxr->rx_buffers[i];
3956                 if (rxbuf->m_head != NULL) {
3957                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3958                             BUS_DMASYNC_POSTREAD);
3959                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3960                         rxbuf->m_head->m_flags |= M_PKTHDR;
3961                         m_freem(rxbuf->m_head);
3962                 }
3963                 if (rxbuf->m_pack != NULL) {
3964                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3965                             BUS_DMASYNC_POSTREAD);
3966                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3967                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3968                         m_freem(rxbuf->m_pack);
3969                 }
3970                 rxbuf->m_head = NULL;
3971                 rxbuf->m_pack = NULL;
3972         }
3973 }
3974
3975
3976 /*********************************************************************
3977  *
3978  *  Initialize a receive ring and its buffers.
3979  *
3980  **********************************************************************/
3981 static int
3982 ixgbe_setup_receive_ring(struct rx_ring *rxr)
3983 {
3984         struct  adapter         *adapter;
3985         struct ifnet            *ifp;
3986         device_t                dev;
3987         struct ixgbe_rx_buf     *rxbuf;
3988         bus_dma_segment_t       pseg[1], hseg[1];
3989 #if 0   /* NET_LRO */
3990         struct lro_ctrl         *lro = &rxr->lro;
3991 #endif
3992         int                     rsize, nsegs, error = 0;
3993 #ifdef DEV_NETMAP
3994         struct netmap_adapter *na = NA(rxr->adapter->ifp);
3995         struct netmap_slot *slot;
3996 #endif /* DEV_NETMAP */
3997
3998         adapter = rxr->adapter;
3999         ifp = adapter->ifp;
4000         dev = adapter->dev;
4001
4002         /* Clear the ring contents */
4003         IXGBE_RX_LOCK(rxr);
4004 #ifdef DEV_NETMAP
4005         /* same as in ixgbe_setup_transmit_ring() */
4006         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4007 #endif /* DEV_NETMAP */
4008         rsize = roundup2(adapter->num_rx_desc *
4009             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4010         bzero((void *)rxr->rx_base, rsize);
4011
4012         /* Free current RX buffer structs and their mbufs */
4013         ixgbe_free_receive_ring(rxr);
4014
4015         /* Configure header split? */
4016         if (ixgbe_header_split)
4017                 rxr->hdr_split = TRUE;
4018
4019         /* Now replenish the mbufs */
4020         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4021                 struct mbuf     *mh, *mp;
4022
4023                 rxbuf = &rxr->rx_buffers[j];
4024 #ifdef DEV_NETMAP
4025                 /*
4026                  * In netmap mode, fill the map and set the buffer
4027                  * address in the NIC ring, considering the offset
4028                  * between the netmap and NIC rings (see comment in
4029                  * ixgbe_setup_transmit_ring() ). No need to allocate
4030                  * an mbuf, so end the block with a continue;
4031                  */
4032                 if (slot) {
4033                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4034                         uint64_t paddr;
4035                         void *addr;
4036
4037                         addr = PNMB(slot + sj, &paddr);
4038                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4039                         /* Update descriptor */
4040                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4041                         continue;
4042                 }
4043 #endif /* DEV_NETMAP */
4044                 /*
4045                 ** Don't allocate mbufs if not
4046                 ** doing header split, its wasteful
4047                 */
4048                 if (rxr->hdr_split == FALSE)
4049                         goto skip_head;
4050
4051                 /* First the header */
4052                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4053                 if (rxbuf->m_head == NULL) {
4054                         error = ENOBUFS;
4055                         goto fail;
4056                 }
4057                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4058                 mh = rxbuf->m_head;
4059                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4060                 mh->m_flags |= M_PKTHDR;
4061                 /* Get the memory mapping */
4062                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4063                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4064                     &nsegs, BUS_DMA_NOWAIT);
4065
4066                 if (error != 0) /* Nothing elegant to do here */
4067                         goto fail;
4068                 bus_dmamap_sync(rxr->htag,
4069                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4070                 /* Update descriptor */
4071                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4072
4073 skip_head:
4074                 /* Now the payload cluster */
4075                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4076                     M_PKTHDR, adapter->rx_mbuf_sz);
4077                 if (rxbuf->m_pack == NULL) {
4078                         error = ENOBUFS;
4079                         goto fail;
4080                 }
4081                 mp = rxbuf->m_pack;
4082                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4083                 /* Get the memory mapping */
4084                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4085                     rxbuf->pmap, mp, hseg, 1,
4086                     &nsegs, BUS_DMA_NOWAIT);
4087                 if (error != 0)
4088                         goto fail;
4089                 bus_dmamap_sync(rxr->ptag,
4090                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4091                 /* Update descriptor */
4092                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4093         }
4094
4095
4096         /* Setup our descriptor indices */
4097         rxr->next_to_check = 0;
4098         rxr->next_to_refresh = 0;
4099         rxr->lro_enabled = FALSE;
4100         rxr->rx_split_packets = 0;
4101         rxr->rx_bytes = 0;
4102         rxr->discard = FALSE;
4103         rxr->vtag_strip = FALSE;
4104
4105         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4106             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4107
4108         /*
4109         ** Now set up the LRO interface:
4110         ** 82598 uses software LRO, the
4111         ** 82599 and X540 use a hardware assist.
4112         */
4113 #if 0 /* NET_LRO */
4114         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4115             (ifp->if_capenable & IFCAP_RXCSUM) &&
4116             (ifp->if_capenable & IFCAP_LRO))
4117                 ixgbe_setup_hw_rsc(rxr);
4118         else if (ifp->if_capenable & IFCAP_LRO) {
4119                 int err = tcp_lro_init(lro);
4120                 if (err) {
4121                         device_printf(dev, "LRO Initialization failed!\n");
4122                         goto fail;
4123                 }
4124                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4125                 rxr->lro_enabled = TRUE;
4126                 lro->ifp = adapter->ifp;
4127         }
4128 #endif
4129
4130         IXGBE_RX_UNLOCK(rxr);
4131         return (0);
4132
4133 fail:
4134         ixgbe_free_receive_ring(rxr);
4135         IXGBE_RX_UNLOCK(rxr);
4136         return (error);
4137 }
4138
4139 /*********************************************************************
4140  *
4141  *  Initialize all receive rings.
4142  *
4143  **********************************************************************/
4144 static int
4145 ixgbe_setup_receive_structures(struct adapter *adapter)
4146 {
4147         struct rx_ring *rxr = adapter->rx_rings;
4148         int j;
4149
4150         for (j = 0; j < adapter->num_queues; j++, rxr++)
4151                 if (ixgbe_setup_receive_ring(rxr))
4152                         goto fail;
4153
4154         return (0);
4155 fail:
4156         /*
4157          * Free RX buffers allocated so far, we will only handle
4158          * the rings that completed, the failing case will have
4159          * cleaned up for itself. 'j' failed, so its the terminus.
4160          */
4161         for (int i = 0; i < j; ++i) {
4162                 rxr = &adapter->rx_rings[i];
4163                 ixgbe_free_receive_ring(rxr);
4164         }
4165
4166         return (ENOBUFS);
4167 }
4168
4169 /*********************************************************************
4170  *
4171  *  Setup receive registers and features.
4172  *
4173  **********************************************************************/
4174 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4175
4176 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4177
4178 static void
4179 ixgbe_initialize_receive_units(struct adapter *adapter)
4180 {
4181         struct  rx_ring *rxr = adapter->rx_rings;
4182         struct ixgbe_hw *hw = &adapter->hw;
4183         struct ifnet   *ifp = adapter->ifp;
4184         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4185         u32             reta, mrqc = 0, hlreg, random[10];
4186
4187
4188         /*
4189          * Make sure receives are disabled while
4190          * setting up the descriptor ring
4191          */
4192         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4193         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4194             rxctrl & ~IXGBE_RXCTRL_RXEN);
4195
4196         /* Enable broadcasts */
4197         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4198         fctrl |= IXGBE_FCTRL_BAM;
4199         fctrl |= IXGBE_FCTRL_DPF;
4200         fctrl |= IXGBE_FCTRL_PMCF;
4201         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4202
4203         /* Set for Jumbo Frames? */
4204         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4205         if (ifp->if_mtu > ETHERMTU)
4206                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4207         else
4208                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4209 #ifdef DEV_NETMAP
4210         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4211         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4212                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4213         else
4214                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4215 #endif /* DEV_NETMAP */
4216         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4217
4218         bufsz = (adapter->rx_mbuf_sz +
4219             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4220
4221         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4222                 u64 rdba = rxr->rxdma.dma_paddr;
4223
4224                 /* Setup the Base and Length of the Rx Descriptor Ring */
4225                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4226                                (rdba & 0x00000000ffffffffULL));
4227                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4228                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4229                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4230
4231                 /* Set up the SRRCTL register */
4232                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4233                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4234                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4235                 srrctl |= bufsz;
4236                 if (rxr->hdr_split) {
4237                         /* Use a standard mbuf for the header */
4238                         srrctl |= ((IXGBE_RX_HDR <<
4239                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4240                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4241                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4242                 } else
4243                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4244                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4245
4246                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4247                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4248                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4249         }
4250
4251         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4252                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4253                               IXGBE_PSRTYPE_UDPHDR |
4254                               IXGBE_PSRTYPE_IPV4HDR |
4255                               IXGBE_PSRTYPE_IPV6HDR;
4256                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4257         }
4258
4259         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4260
4261         /* Setup RSS */
4262         if (adapter->num_queues > 1) {
4263                 int i, j;
4264                 reta = 0;
4265
4266                 /* set up random bits */
4267                 karc4rand(&random, sizeof(random));
4268
4269                 /* Set up the redirection table */
4270                 for (i = 0, j = 0; i < 128; i++, j++) {
4271                         if (j == adapter->num_queues) j = 0;
4272                         reta = (reta << 8) | (j * 0x11);
4273                         if ((i & 3) == 3)
4274                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4275                 }
4276
4277                 /* Now fill our hash function seeds */
4278                 for (int i = 0; i < 10; i++)
4279                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4280
4281                 /* Perform hash on these packet types */
4282                 mrqc = IXGBE_MRQC_RSSEN
4283                      | IXGBE_MRQC_RSS_FIELD_IPV4
4284                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4285                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4286                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4287                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4288                      | IXGBE_MRQC_RSS_FIELD_IPV6
4289                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4290                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4291                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4292                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4293
4294                 /* RSS and RX IPP Checksum are mutually exclusive */
4295                 rxcsum |= IXGBE_RXCSUM_PCSD;
4296         }
4297
4298         if (ifp->if_capenable & IFCAP_RXCSUM)
4299                 rxcsum |= IXGBE_RXCSUM_PCSD;
4300
4301         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4302                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4303
4304         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4305
4306         return;
4307 }
4308
4309 /*********************************************************************
4310  *
4311  *  Free all receive rings.
4312  *
4313  **********************************************************************/
4314 static void
4315 ixgbe_free_receive_structures(struct adapter *adapter)
4316 {
4317         struct rx_ring *rxr = adapter->rx_rings;
4318
4319         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4320 #if 0   /* NET_LRO */
4321                 struct lro_ctrl         *lro = &rxr->lro;
4322 #endif
4323                 ixgbe_free_receive_buffers(rxr);
4324                 /* Free LRO memory */
4325 #if 0   /* NET_LRO */
4326                 tcp_lro_free(lro);
4327 #endif
4328                 /* Free the ring memory as well */
4329                 ixgbe_dma_free(adapter, &rxr->rxdma);
4330         }
4331
4332         kfree(adapter->rx_rings, M_DEVBUF);
4333 }
4334
4335
4336 /*********************************************************************
4337  *
4338  *  Free receive ring data structures
4339  *
4340  **********************************************************************/
4341 static void
4342 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4343 {
4344         struct adapter          *adapter = rxr->adapter;
4345         struct ixgbe_rx_buf     *rxbuf;
4346
4347         INIT_DEBUGOUT("free_receive_structures: begin");
4348
4349         /* Cleanup any existing buffers */
4350         if (rxr->rx_buffers != NULL) {
4351                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4352                         rxbuf = &rxr->rx_buffers[i];
4353                         if (rxbuf->m_head != NULL) {
4354                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4355                                     BUS_DMASYNC_POSTREAD);
4356                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4357                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4358                                 m_freem(rxbuf->m_head);
4359                         }
4360                         if (rxbuf->m_pack != NULL) {
4361                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4362                                     BUS_DMASYNC_POSTREAD);
4363                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4364                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4365                                 m_freem(rxbuf->m_pack);
4366                         }
4367                         rxbuf->m_head = NULL;
4368                         rxbuf->m_pack = NULL;
4369                         if (rxbuf->hmap != NULL) {
4370                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4371                                 rxbuf->hmap = NULL;
4372                         }
4373                         if (rxbuf->pmap != NULL) {
4374                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4375                                 rxbuf->pmap = NULL;
4376                         }
4377                 }
4378                 if (rxr->rx_buffers != NULL) {
4379                         kfree(rxr->rx_buffers, M_DEVBUF);
4380                         rxr->rx_buffers = NULL;
4381                 }
4382         }
4383
4384         if (rxr->htag != NULL) {
4385                 bus_dma_tag_destroy(rxr->htag);
4386                 rxr->htag = NULL;
4387         }
4388         if (rxr->ptag != NULL) {
4389                 bus_dma_tag_destroy(rxr->ptag);
4390                 rxr->ptag = NULL;
4391         }
4392
4393         return;
4394 }
4395
4396 static __inline void
4397 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4398 {
4399
4400         /*
4401          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4402          * should be computed by hardware. Also it should not have VLAN tag in
4403          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4404          */
4405 #if 0   /* NET_LRO */
4406         if (rxr->lro_enabled &&
4407             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4408             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4409             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4410             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4411             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4412             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4413             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4414             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4415                 /*
4416                  * Send to the stack if:
4417                  **  - LRO not enabled, or
4418                  **  - no LRO resources, or
4419                  **  - lro enqueue fails
4420                  */
4421                 if (rxr->lro.lro_cnt != 0)
4422                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4423                                 return;
4424         }
4425 #endif
4426         IXGBE_RX_UNLOCK(rxr);
4427         (*ifp->if_input)(ifp, m);
4428         IXGBE_RX_LOCK(rxr);
4429 }
4430
4431 static __inline void
4432 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4433 {
4434         struct ixgbe_rx_buf     *rbuf;
4435
4436         rbuf = &rxr->rx_buffers[i];
4437
4438         if (rbuf->fmp != NULL) {/* Partial chain ? */
4439                 rbuf->fmp->m_flags |= M_PKTHDR;
4440                 m_freem(rbuf->fmp);
4441                 rbuf->fmp = NULL;
4442         }
4443
4444         /*
4445         ** With advanced descriptors the writeback
4446         ** clobbers the buffer addrs, so its easier
4447         ** to just free the existing mbufs and take
4448         ** the normal refresh path to get new buffers
4449         ** and mapping.
4450         */
4451         if (rbuf->m_head) {
4452                 m_free(rbuf->m_head);
4453                 rbuf->m_head = NULL;
4454         }
4455
4456         if (rbuf->m_pack) {
4457                 m_free(rbuf->m_pack);
4458                 rbuf->m_pack = NULL;
4459         }
4460
4461         return;
4462 }
4463
4464
4465 /*********************************************************************
4466  *
4467  *  This routine executes in interrupt context. It replenishes
4468  *  the mbufs in the descriptor and sends data which has been
4469  *  dma'ed into host memory to upper layer.
4470  *
4471  *  We loop at most count times if count is > 0, or until done if
4472  *  count < 0.
4473  *
4474  *  Return TRUE for more work, FALSE for all clean.
4475  *********************************************************************/
4476 static bool
4477 ixgbe_rxeof(struct ix_queue *que, int count)
4478 {
4479         struct adapter          *adapter = que->adapter;
4480         struct rx_ring          *rxr = que->rxr;
4481         struct ifnet            *ifp = adapter->ifp;
4482 #if 0   /* NET_LRO */
4483         struct lro_ctrl         *lro = &rxr->lro;
4484         struct lro_entry        *queued;
4485 #endif
4486         int                     i, nextp, processed = 0;
4487         u32                     staterr = 0;
4488         union ixgbe_adv_rx_desc *cur;
4489         struct ixgbe_rx_buf     *rbuf, *nbuf;
4490
4491         IXGBE_RX_LOCK(rxr);
4492
4493 #ifdef DEV_NETMAP
4494         if (ifp->if_capenable & IFCAP_NETMAP) {
4495                 /*
4496                  * Same as the txeof routine: only wakeup clients on intr.
4497                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4498                  * mitigation (ixgbe_rxsync() will not look for new packets
4499                  * unless NKR_PENDINTR is set).
4500                  */
4501                 struct netmap_adapter *na = NA(ifp);
4502
4503                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4504                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4505                 IXGBE_RX_UNLOCK(rxr);
4506                 IXGBE_CORE_LOCK(adapter);
4507                 selwakeuppri(&na->rx_si, PI_NET);
4508                 IXGBE_CORE_UNLOCK(adapter);
4509                 return (FALSE);
4510         }
4511 #endif /* DEV_NETMAP */
4512         for (i = rxr->next_to_check; count != 0;) {
4513                 struct mbuf     *sendmp, *mh, *mp;
4514                 u32             rsc, ptype;
4515                 u16             hlen, plen, hdr;
4516 #ifdef NET_VLAN
4517                 u16             vtag = 0;
4518 #endif
4519                 bool            eop;
4520
4521                 /* Sync the ring. */
4522                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4523                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4524
4525                 cur = &rxr->rx_base[i];
4526                 staterr = le32toh(cur->wb.upper.status_error);
4527
4528                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4529                         break;
4530                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4531                         break;
4532
4533                 count--;
4534                 sendmp = NULL;
4535                 nbuf = NULL;
4536                 rsc = 0;
4537                 cur->wb.upper.status_error = 0;
4538                 rbuf = &rxr->rx_buffers[i];
4539                 mh = rbuf->m_head;
4540                 mp = rbuf->m_pack;
4541
4542                 plen = le16toh(cur->wb.upper.length);
4543                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4544                     IXGBE_RXDADV_PKTTYPE_MASK;
4545                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4546                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4547
4548                 /* Process vlan info */
4549 #ifdef NET_VLAN
4550                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4551                         vtag = le16toh(cur->wb.upper.vlan);
4552 #endif
4553
4554                 /* Make sure bad packets are discarded */
4555                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4556                     (rxr->discard)) {
4557                         ifp->if_ierrors++;
4558                         rxr->rx_discarded++;
4559                         if (eop)
4560                                 rxr->discard = FALSE;
4561                         else
4562                                 rxr->discard = TRUE;
4563                         ixgbe_rx_discard(rxr, i);
4564                         goto next_desc;
4565                 }
4566
4567                 /*
4568                 ** On 82599 which supports a hardware
4569                 ** LRO (called HW RSC), packets need
4570                 ** not be fragmented across sequential
4571                 ** descriptors, rather the next descriptor
4572                 ** is indicated in bits of the descriptor.
4573                 ** This also means that we might proceses
4574                 ** more than one packet at a time, something
4575                 ** that has never been true before, it
4576                 ** required eliminating global chain pointers
4577                 ** in favor of what we are doing here.  -jfv
4578                 */
4579                 if (!eop) {
4580                         /*
4581                         ** Figure out the next descriptor
4582                         ** of this frame.
4583                         */
4584                         if (rxr->hw_rsc == TRUE) {
4585                                 rsc = ixgbe_rsc_count(cur);
4586                                 rxr->rsc_num += (rsc - 1);
4587                         }
4588                         if (rsc) { /* Get hardware index */
4589                                 nextp = ((staterr &
4590                                     IXGBE_RXDADV_NEXTP_MASK) >>
4591                                     IXGBE_RXDADV_NEXTP_SHIFT);
4592                         } else { /* Just sequential */
4593                                 nextp = i + 1;
4594                                 if (nextp == adapter->num_rx_desc)
4595                                         nextp = 0;
4596                         }
4597                         nbuf = &rxr->rx_buffers[nextp];
4598                         prefetch(nbuf);
4599                 }
4600                 /*
4601                 ** The header mbuf is ONLY used when header
4602                 ** split is enabled, otherwise we get normal
4603                 ** behavior, ie, both header and payload
4604                 ** are DMA'd into the payload buffer.
4605                 **
4606                 ** Rather than using the fmp/lmp global pointers
4607                 ** we now keep the head of a packet chain in the
4608                 ** buffer struct and pass this along from one
4609                 ** descriptor to the next, until we get EOP.
4610                 */
4611                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4612                         /* This must be an initial descriptor */
4613                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4614                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4615                         if (hlen > IXGBE_RX_HDR)
4616                                 hlen = IXGBE_RX_HDR;
4617                         mh->m_len = hlen;
4618                         mh->m_flags |= M_PKTHDR;
4619                         mh->m_next = NULL;
4620                         mh->m_pkthdr.len = mh->m_len;
4621                         /* Null buf pointer so it is refreshed */
4622                         rbuf->m_head = NULL;
4623                         /*
4624                         ** Check the payload length, this
4625                         ** could be zero if its a small
4626                         ** packet.
4627                         */
4628                         if (plen > 0) {
4629                                 mp->m_len = plen;
4630                                 mp->m_next = NULL;
4631                                 mp->m_flags &= ~M_PKTHDR;
4632                                 mh->m_next = mp;
4633                                 mh->m_pkthdr.len += mp->m_len;
4634                                 /* Null buf pointer so it is refreshed */
4635                                 rbuf->m_pack = NULL;
4636                                 rxr->rx_split_packets++;
4637                         }
4638                         /*
4639                         ** Now create the forward
4640                         ** chain so when complete
4641                         ** we wont have to.
4642                         */
4643                         if (eop == 0) {
4644                                 /* stash the chain head */
4645                                 nbuf->fmp = mh;
4646                                 /* Make forward chain */
4647                                 if (plen)
4648                                         mp->m_next = nbuf->m_pack;
4649                                 else
4650                                         mh->m_next = nbuf->m_pack;
4651                         } else {
4652                                 /* Singlet, prepare to send */
4653                                 sendmp = mh;
4654                                 /* If hardware handled vtag */
4655 #ifdef NET_VLAN
4656                                 if (vtag) {
4657                                         sendmp->m_pkthdr.ether_vtag = vtag;
4658                                         sendmp->m_flags |= M_VLANTAG;
4659                                 }
4660 #endif
4661                         }
4662                 } else {
4663                         /*
4664                         ** Either no header split, or a
4665                         ** secondary piece of a fragmented
4666                         ** split packet.
4667                         */
4668                         mp->m_len = plen;
4669                         /*
4670                         ** See if there is a stored head
4671                         ** that determines what we are
4672                         */
4673                         sendmp = rbuf->fmp;
4674                         rbuf->m_pack = rbuf->fmp = NULL;
4675
4676                         if (sendmp != NULL) {  /* secondary frag */
4677                                 mp->m_flags &= ~M_PKTHDR;
4678                                 sendmp->m_pkthdr.len += mp->m_len;
4679                         } else {
4680                                 /* first desc of a non-ps chain */
4681                                 sendmp = mp;
4682                                 sendmp->m_flags |= M_PKTHDR;
4683                                 sendmp->m_pkthdr.len = mp->m_len;
4684 #ifdef NET_VLAN
4685                                 if (staterr & IXGBE_RXD_STAT_VP) {
4686                                         sendmp->m_pkthdr.ether_vtag = vtag;
4687                                         sendmp->m_flags |= M_VLANTAG;
4688                                 }
4689 #endif
4690                         }
4691                         /* Pass the head pointer on */
4692                         if (eop == 0) {
4693                                 nbuf->fmp = sendmp;
4694                                 sendmp = NULL;
4695                                 mp->m_next = nbuf->m_pack;
4696                         }
4697                 }
4698                 ++processed;
4699                 /* Sending this frame? */
4700                 if (eop) {
4701                         sendmp->m_pkthdr.rcvif = ifp;
4702                         ifp->if_ipackets++;
4703                         rxr->rx_packets++;
4704                         /* capture data for AIM */
4705                         rxr->bytes += sendmp->m_pkthdr.len;
4706                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4707                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4708                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4709 #if 0 /* __FreeBSD_version >= 800000 */
4710                         sendmp->m_pkthdr.flowid = que->msix;
4711                         sendmp->m_flags |= M_FLOWID;
4712 #endif
4713                 }
4714 next_desc:
4715                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4716                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4717
4718                 /* Advance our pointers to the next descriptor. */
4719                 if (++i == adapter->num_rx_desc)
4720                         i = 0;
4721
4722                 /* Now send to the stack or do LRO */
4723                 if (sendmp != NULL) {
4724                         rxr->next_to_check = i;
4725                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4726                         i = rxr->next_to_check;
4727                 }
4728
4729                /* Every 8 descriptors we go to refresh mbufs */
4730                 if (processed == 8) {
4731                         ixgbe_refresh_mbufs(rxr, i);
4732                         processed = 0;
4733                 }
4734         }
4735
4736         /* Refresh any remaining buf structs */
4737         if (ixgbe_rx_unrefreshed(rxr))
4738                 ixgbe_refresh_mbufs(rxr, i);
4739
4740         rxr->next_to_check = i;
4741
4742         /*
4743          * Flush any outstanding LRO work
4744          */
4745 #if 0   /* NET_LRO */
4746         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4747                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4748                 tcp_lro_flush(lro, queued);
4749         }
4750 #endif
4751
4752         IXGBE_RX_UNLOCK(rxr);
4753
4754         /*
4755         ** We still have cleaning to do?
4756         ** Schedule another interrupt if so.
4757         */
4758         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4759                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4760                 return (TRUE);
4761         }
4762
4763         return (FALSE);
4764 }
4765
4766
4767 /*********************************************************************
4768  *
4769  *  Verify that the hardware indicated that the checksum is valid.
4770  *  Inform the stack about the status of checksum so that stack
4771  *  doesn't spend time verifying the checksum.
4772  *
4773  *********************************************************************/
4774 static void
4775 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4776 {
4777         u16     status = (u16) staterr;
4778         u8      errors = (u8) (staterr >> 24);
4779         bool    sctp = FALSE;
4780
4781         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4782             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4783                 sctp = TRUE;
4784
4785         if (status & IXGBE_RXD_STAT_IPCS) {
4786                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4787                         /* IP Checksum Good */
4788                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4789                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4790
4791                 } else
4792                         mp->m_pkthdr.csum_flags = 0;
4793         }
4794         if (status & IXGBE_RXD_STAT_L4CS) {
4795                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4796 #if 0
4797                 if (sctp)
4798                         type = CSUM_SCTP_VALID;
4799 #endif
4800                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4801                         mp->m_pkthdr.csum_flags |= type;
4802                         if (!sctp)
4803                                 mp->m_pkthdr.csum_data = htons(0xffff);
4804                 }
4805         }
4806         return;
4807 }
4808
4809
4810 /*
4811 ** This routine is run via an vlan config EVENT,
4812 ** it enables us to use the HW Filter table since
4813 ** we can get the vlan id. This just creates the
4814 ** entry in the soft version of the VFTA, init will
4815 ** repopulate the real table.
4816 */
4817 static void
4818 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4819 {
4820         struct adapter  *adapter = ifp->if_softc;
4821         u16             index, bit;
4822
4823         if (ifp->if_softc !=  arg)   /* Not our event */
4824                 return;
4825
4826         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4827                 return;
4828
4829         IXGBE_CORE_LOCK(adapter);
4830         index = (vtag >> 5) & 0x7F;
4831         bit = vtag & 0x1F;
4832         adapter->shadow_vfta[index] |= (1 << bit);
4833         ++adapter->num_vlans;
4834         ixgbe_init_locked(adapter);
4835         IXGBE_CORE_UNLOCK(adapter);
4836 }
4837
4838 /*
4839 ** This routine is run via an vlan
4840 ** unconfig EVENT, remove our entry
4841 ** in the soft vfta.
4842 */
4843 static void
4844 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4845 {
4846         struct adapter  *adapter = ifp->if_softc;
4847         u16             index, bit;
4848
4849         if (ifp->if_softc !=  arg)
4850                 return;
4851
4852         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4853                 return;
4854
4855         IXGBE_CORE_LOCK(adapter);
4856         index = (vtag >> 5) & 0x7F;
4857         bit = vtag & 0x1F;
4858         adapter->shadow_vfta[index] &= ~(1 << bit);
4859         --adapter->num_vlans;
4860         /* Re-init to load the changes */
4861         ixgbe_init_locked(adapter);
4862         IXGBE_CORE_UNLOCK(adapter);
4863 }
4864
4865 static void
4866 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4867 {
4868 #ifdef NET_VLAN
4869         struct ifnet    *ifp = adapter->ifp;
4870         struct ixgbe_hw *hw = &adapter->hw;
4871         struct rx_ring  *rxr;
4872         u32             ctrl;
4873
4874         /*
4875         ** We get here thru init_locked, meaning
4876         ** a soft reset, this has already cleared
4877         ** the VFTA and other state, so if there
4878         ** have been no vlan's registered do nothing.
4879         */
4880         if (adapter->num_vlans == 0)
4881                 return;
4882
4883         /*
4884         ** A soft reset zero's out the VFTA, so
4885         ** we need to repopulate it now.
4886         */
4887         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4888                 if (adapter->shadow_vfta[i] != 0)
4889                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4890                             adapter->shadow_vfta[i]);
4891
4892         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4893         /* Enable the Filter Table if enabled */
4894         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4895                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4896                 ctrl |= IXGBE_VLNCTRL_VFE;
4897         }
4898         if (hw->mac.type == ixgbe_mac_82598EB)
4899                 ctrl |= IXGBE_VLNCTRL_VME;
4900         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4901
4902         /* Setup the queues for vlans */
4903         for (int i = 0; i < adapter->num_queues; i++) {
4904                 rxr = &adapter->rx_rings[i];
4905                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4906                 if (hw->mac.type != ixgbe_mac_82598EB) {
4907                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4908                         ctrl |= IXGBE_RXDCTL_VME;
4909                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4910                 }
4911                 rxr->vtag_strip = TRUE;
4912         }
4913 #endif
4914 }
4915
4916 static void
4917 ixgbe_enable_intr(struct adapter *adapter)
4918 {
4919         struct ixgbe_hw *hw = &adapter->hw;
4920         struct ix_queue *que = adapter->queues;
4921         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4922
4923
4924         /* Enable Fan Failure detection */
4925         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4926                     mask |= IXGBE_EIMS_GPI_SDP1;
4927         else {
4928                     mask |= IXGBE_EIMS_ECC;
4929                     mask |= IXGBE_EIMS_GPI_SDP0;
4930                     mask |= IXGBE_EIMS_GPI_SDP1;
4931                     mask |= IXGBE_EIMS_GPI_SDP2;
4932 #ifdef IXGBE_FDIR
4933                     mask |= IXGBE_EIMS_FLOW_DIR;
4934 #endif
4935         }
4936
4937         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4938
4939         /* With RSS we use auto clear */
4940         if (adapter->msix_mem) {
4941                 mask = IXGBE_EIMS_ENABLE_MASK;
4942                 /* Don't autoclear Link */
4943                 mask &= ~IXGBE_EIMS_OTHER;
4944                 mask &= ~IXGBE_EIMS_LSC;
4945                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4946         }
4947
4948         /*
4949         ** Now enable all queues, this is done separately to
4950         ** allow for handling the extended (beyond 32) MSIX
4951         ** vectors that can be used by 82599
4952         */
4953         for (int i = 0; i < adapter->num_queues; i++, que++)
4954                 ixgbe_enable_queue(adapter, que->msix);
4955
4956         IXGBE_WRITE_FLUSH(hw);
4957
4958         return;
4959 }
4960
4961 static void
4962 ixgbe_disable_intr(struct adapter *adapter)
4963 {
4964         if (adapter->msix_mem)
4965                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4966         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4967                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4968         } else {
4969                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4970                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4971                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4972         }
4973         IXGBE_WRITE_FLUSH(&adapter->hw);
4974         return;
4975 }
4976
4977 u16
4978 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
4979 {
4980         u16 value;
4981
4982         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
4983             reg, 2);
4984
4985         return (value);
4986 }
4987
4988 void
4989 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
4990 {
4991         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
4992             reg, value, 2);
4993
4994         return;
4995 }
4996
4997 /*
4998 ** Setup the correct IVAR register for a particular MSIX interrupt
4999 **   (yes this is all very magic and confusing :)
5000 **  - entry is the register array entry
5001 **  - vector is the MSIX vector for this queue
5002 **  - type is RX/TX/MISC
5003 */
5004 static void
5005 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5006 {
5007         struct ixgbe_hw *hw = &adapter->hw;
5008         u32 ivar, index;
5009
5010         vector |= IXGBE_IVAR_ALLOC_VAL;
5011
5012         switch (hw->mac.type) {
5013
5014         case ixgbe_mac_82598EB:
5015                 if (type == -1)
5016                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5017                 else
5018                         entry += (type * 64);
5019                 index = (entry >> 2) & 0x1F;
5020                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5021                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5022                 ivar |= (vector << (8 * (entry & 0x3)));
5023                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5024                 break;
5025
5026         case ixgbe_mac_82599EB:
5027         case ixgbe_mac_X540:
5028                 if (type == -1) { /* MISC IVAR */
5029                         index = (entry & 1) * 8;
5030                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5031                         ivar &= ~(0xFF << index);
5032                         ivar |= (vector << index);
5033                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5034                 } else {        /* RX/TX IVARS */
5035                         index = (16 * (entry & 1)) + (8 * type);
5036                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5037                         ivar &= ~(0xFF << index);
5038                         ivar |= (vector << index);
5039                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5040                 }
5041
5042         default:
5043                 break;
5044         }
5045 }
5046
5047 static void
5048 ixgbe_configure_ivars(struct adapter *adapter)
5049 {
5050         struct  ix_queue *que = adapter->queues;
5051         u32 newitr;
5052
5053         if (ixgbe_max_interrupt_rate > 0)
5054                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5055         else
5056                 newitr = 0;
5057
5058         for (int i = 0; i < adapter->num_queues; i++, que++) {
5059                 /* First the RX queue entry */
5060                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5061                 /* ... and the TX */
5062                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5063                 /* Set an Initial EITR value */
5064                 IXGBE_WRITE_REG(&adapter->hw,
5065                     IXGBE_EITR(que->msix), newitr);
5066         }
5067
5068         /* For the Link interrupt */
5069         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5070 }
5071
5072 /*
5073 ** ixgbe_sfp_probe - called in the local timer to
5074 ** determine if a port had optics inserted.
5075 */
5076 static bool ixgbe_sfp_probe(struct adapter *adapter)
5077 {
5078         struct ixgbe_hw *hw = &adapter->hw;
5079         device_t        dev = adapter->dev;
5080         bool            result = FALSE;
5081
5082         if ((hw->phy.type == ixgbe_phy_nl) &&
5083             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5084                 s32 ret = hw->phy.ops.identify_sfp(hw);
5085                 if (ret)
5086                         goto out;
5087                 ret = hw->phy.ops.reset(hw);
5088                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5089                         device_printf(dev,"Unsupported SFP+ module detected!");
5090                         kprintf(" Reload driver with supported module.\n");
5091                         adapter->sfp_probe = FALSE;
5092                         goto out;
5093                 } else
5094                         device_printf(dev,"SFP+ module detected!\n");
5095                 /* We now have supported optics */
5096                 adapter->sfp_probe = FALSE;
5097                 /* Set the optics type so system reports correctly */
5098                 ixgbe_setup_optics(adapter);
5099                 result = TRUE;
5100         }
5101 out:
5102         return (result);
5103 }
5104
5105 /*
5106 ** Tasklet handler for MSIX Link interrupts
5107 **  - do outside interrupt since it might sleep
5108 */
5109 static void
5110 ixgbe_handle_link(void *context, int pending)
5111 {
5112         struct adapter  *adapter = context;
5113
5114         ixgbe_check_link(&adapter->hw,
5115             &adapter->link_speed, &adapter->link_up, 0);
5116         ixgbe_update_link_status(adapter);
5117 }
5118
5119 /*
5120 ** Tasklet for handling SFP module interrupts
5121 */
5122 static void
5123 ixgbe_handle_mod(void *context, int pending)
5124 {
5125         struct adapter  *adapter = context;
5126         struct ixgbe_hw *hw = &adapter->hw;
5127         device_t        dev = adapter->dev;
5128         u32 err;
5129
5130         err = hw->phy.ops.identify_sfp(hw);
5131         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5132                 device_printf(dev,
5133                     "Unsupported SFP+ module type was detected.\n");
5134                 return;
5135         }
5136         err = hw->mac.ops.setup_sfp(hw);
5137         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5138                 device_printf(dev,
5139                     "Setup failure - unsupported SFP+ module type.\n");
5140                 return;
5141         }
5142         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5143         return;
5144 }
5145
5146
5147 /*
5148 ** Tasklet for handling MSF (multispeed fiber) interrupts
5149 */
5150 static void
5151 ixgbe_handle_msf(void *context, int pending)
5152 {
5153         struct adapter  *adapter = context;
5154         struct ixgbe_hw *hw = &adapter->hw;
5155         u32 autoneg;
5156         bool negotiate;
5157
5158         autoneg = hw->phy.autoneg_advertised;
5159         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5160                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5161         if (hw->mac.ops.setup_link)
5162                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5163         return;
5164 }
5165
5166 #ifdef IXGBE_FDIR
5167 /*
5168 ** Tasklet for reinitializing the Flow Director filter table
5169 */
5170 static void
5171 ixgbe_reinit_fdir(void *context, int pending)
5172 {
5173         struct adapter  *adapter = context;
5174         struct ifnet   *ifp = adapter->ifp;
5175
5176         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5177                 return;
5178         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5179         adapter->fdir_reinit = 0;
5180         /* re-enable flow director interrupts */
5181         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5182         /* Restart the interface */
5183         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5184         return;
5185 }
5186 #endif
5187
5188 /**********************************************************************
5189  *
5190  *  Update the board statistics counters.
5191  *
5192  **********************************************************************/
5193 static void
5194 ixgbe_update_stats_counters(struct adapter *adapter)
5195 {
5196         struct ifnet   *ifp = adapter->ifp;
5197         struct ixgbe_hw *hw = &adapter->hw;
5198         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5199         u64  total_missed_rx = 0;
5200
5201         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5202         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5203         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5204         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5205
5206         for (int i = 0; i < 8; i++) {
5207                 u32 mp;
5208                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5209                 /* missed_rx tallies misses for the gprc workaround */
5210                 missed_rx += mp;
5211                 /* global total per queue */
5212                 adapter->stats.mpc[i] += mp;
5213                 /* Running comprehensive total for stats display */
5214                 total_missed_rx += adapter->stats.mpc[i];
5215                 if (hw->mac.type == ixgbe_mac_82598EB)
5216                         adapter->stats.rnbc[i] +=
5217                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5218                 adapter->stats.pxontxc[i] +=
5219                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5220                 adapter->stats.pxonrxc[i] +=
5221                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5222                 adapter->stats.pxofftxc[i] +=
5223                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5224                 adapter->stats.pxoffrxc[i] +=
5225                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5226                 adapter->stats.pxon2offc[i] +=
5227                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5228         }
5229         for (int i = 0; i < 16; i++) {
5230                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5231                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5232                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5233                 adapter->stats.qbrc[i] +=
5234                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5235                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5236                 adapter->stats.qbtc[i] +=
5237                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5238                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5239         }
5240         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5241         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5242         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5243
5244         /* Hardware workaround, gprc counts missed packets */
5245         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5246         adapter->stats.gprc -= missed_rx;
5247
5248         if (hw->mac.type != ixgbe_mac_82598EB) {
5249                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5250                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5251                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5252                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5253                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5254                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5255                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5256                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5257         } else {
5258                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5259                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5260                 /* 82598 only has a counter in the high register */
5261                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5262                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5263                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5264         }
5265
5266         /*
5267          * Workaround: mprc hardware is incorrectly counting
5268          * broadcasts, so for now we subtract those.
5269          */
5270         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5271         adapter->stats.bprc += bprc;
5272         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5273         if (hw->mac.type == ixgbe_mac_82598EB)
5274                 adapter->stats.mprc -= bprc;
5275
5276         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5277         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5278         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5279         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5280         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5281         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5282
5283         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5284         adapter->stats.lxontxc += lxon;
5285         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5286         adapter->stats.lxofftxc += lxoff;
5287         total = lxon + lxoff;
5288
5289         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5290         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5291         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5292         adapter->stats.gptc -= total;
5293         adapter->stats.mptc -= total;
5294         adapter->stats.ptc64 -= total;
5295         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5296
5297         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5298         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5299         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5300         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5301         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5302         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5303         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5304         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5305         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5306         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5307         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5308         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5309         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5310         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5311         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5312         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5313         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5314         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5315         /* Only read FCOE on 82599 */
5316         if (hw->mac.type != ixgbe_mac_82598EB) {
5317                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5318                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5319                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5320                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5321                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5322         }
5323
5324         /* Fill out the OS statistics structure */
5325         ifp->if_ipackets = adapter->stats.gprc;
5326         ifp->if_opackets = adapter->stats.gptc;
5327         ifp->if_ibytes = adapter->stats.gorc;
5328         ifp->if_obytes = adapter->stats.gotc;
5329         ifp->if_imcasts = adapter->stats.mprc;
5330         ifp->if_collisions = 0;
5331
5332         /* Rx Errors */
5333         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5334                 adapter->stats.rlec;
5335 }
5336
5337 /** ixgbe_sysctl_tdh_handler - Handler function
5338  *  Retrieves the TDH value from the hardware
5339  */
5340 static int
5341 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5342 {
5343         int error;
5344
5345         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5346         if (!txr) return 0;
5347
5348         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5349         error = sysctl_handle_int(oidp, &val, 0, req);
5350         if (error || !req->newptr)
5351                 return error;
5352         return 0;
5353 }
5354
5355 /** ixgbe_sysctl_tdt_handler - Handler function
5356  *  Retrieves the TDT value from the hardware
5357  */
5358 static int
5359 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5360 {
5361         int error;
5362
5363         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5364         if (!txr) return 0;
5365
5366         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5367         error = sysctl_handle_int(oidp, &val, 0, req);
5368         if (error || !req->newptr)
5369                 return error;
5370         return 0;
5371 }
5372
5373 /** ixgbe_sysctl_rdh_handler - Handler function
5374  *  Retrieves the RDH value from the hardware
5375  */
5376 static int
5377 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5378 {
5379         int error;
5380
5381         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5382         if (!rxr) return 0;
5383
5384         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5385         error = sysctl_handle_int(oidp, &val, 0, req);
5386         if (error || !req->newptr)
5387                 return error;
5388         return 0;
5389 }
5390
5391 /** ixgbe_sysctl_rdt_handler - Handler function
5392  *  Retrieves the RDT value from the hardware
5393  */
5394 static int
5395 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5396 {
5397         int error;
5398
5399         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5400         if (!rxr) return 0;
5401
5402         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5403         error = sysctl_handle_int(oidp, &val, 0, req);
5404         if (error || !req->newptr)
5405                 return error;
5406         return 0;
5407 }
5408
5409 static int
5410 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5411 {
5412         int error;
5413         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5414         unsigned int reg, usec, rate;
5415
5416         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5417         usec = ((reg & 0x0FF8) >> 3);
5418         if (usec > 0)
5419                 rate = 500000 / usec;
5420         else
5421                 rate = 0;
5422         error = sysctl_handle_int(oidp, &rate, 0, req);
5423         if (error || !req->newptr)
5424                 return error;
5425         reg &= ~0xfff; /* default, no limitation */
5426         ixgbe_max_interrupt_rate = 0;
5427         if (rate > 0 && rate < 500000) {
5428                 if (rate < 1000)
5429                         rate = 1000;
5430                 ixgbe_max_interrupt_rate = rate;
5431                 reg |= ((4000000/rate) & 0xff8 );
5432         }
5433         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5434         return 0;
5435 }
5436
5437 /*
5438  * Add sysctl variables, one per statistic, to the system.
5439  */
5440 static void
5441 ixgbe_add_hw_stats(struct adapter *adapter)
5442 {
5443         struct tx_ring *txr = adapter->tx_rings;
5444         struct rx_ring *rxr = adapter->rx_rings;
5445
5446         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5447         struct sysctl_oid *tree = adapter->sysctl_tree;
5448         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5449         struct ixgbe_hw_stats *stats = &adapter->stats;
5450
5451         struct sysctl_oid *stat_node, *queue_node;
5452         struct sysctl_oid_list *stat_list, *queue_list;
5453
5454 #define QUEUE_NAME_LEN 32
5455         char namebuf[QUEUE_NAME_LEN];
5456
5457         /* Driver Statistics */
5458         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5459                         CTLFLAG_RD, &adapter->dropped_pkts,
5460                         "Driver dropped packets");
5461         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5462                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5463                         "m_defrag() failed");
5464         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5465                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5466                         "Driver tx dma failure in xmit");
5467         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5468                         CTLFLAG_RD, &adapter->watchdog_events,
5469                         "Watchdog timeouts");
5470 #if 0   /* NET_TSO */
5471         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5472                         CTLFLAG_RD, &adapter->tso_tx,
5473                         "TSO");
5474 #endif
5475         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5476                         CTLFLAG_RD, &adapter->link_irq,
5477                         "Link MSIX IRQ Handled");
5478
5479         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5480         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5481                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5482                                             CTLFLAG_RD, NULL, "Queue Name");
5483                 queue_list = SYSCTL_CHILDREN(queue_node);
5484
5485                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5486                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5487                                 sizeof(&adapter->queues[i]),
5488                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5489                                 "Interrupt Rate");
5490                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5491                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5492                                 "irqs on this queue");
5493                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5494                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5495                                 ixgbe_sysctl_tdh_handler, "IU",
5496                                 "Transmit Descriptor Head");
5497                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5498                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5499                                 ixgbe_sysctl_tdt_handler, "IU",
5500                                 "Transmit Descriptor Tail");
5501                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5502                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5503                                 "Queue No Descriptor Available");
5504                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5505                                 CTLFLAG_RD, &txr->total_packets, 0,
5506                                 "Queue Packets Transmitted");
5507         }
5508
5509         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5510         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5511                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5512                                             CTLFLAG_RD, NULL, "Queue Name");
5513                 queue_list = SYSCTL_CHILDREN(queue_node);
5514
5515 #if 0   /* NET_LRO */
5516                 struct lro_ctrl *lro = &rxr->lro;
5517 #endif
5518
5519         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5520                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5521                                             CTLFLAG_RD, NULL, "Queue Name");
5522                 queue_list = SYSCTL_CHILDREN(queue_node);
5523
5524                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5525                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5526                                 ixgbe_sysctl_rdh_handler, "IU",
5527                                 "Receive Descriptor Head");
5528                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5529                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5530                                 ixgbe_sysctl_rdt_handler, "IU",
5531                                 "Receive Descriptor Tail");
5532                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5533                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5534                                 "Queue Packets Received");
5535                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5536                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5537                                 "Queue Bytes Received");
5538 #if 0   /* NET_LRO */
5539                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5540                                 CTLFLAG_RD, &lro->lro_queued, 0,
5541                                 "LRO Queued");
5542                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5543                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5544                                 "LRO Flushed");
5545 #endif
5546         }
5547
5548         /* MAC stats get the own sub node */
5549
5550         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5551                                     CTLFLAG_RD, NULL, "MAC Statistics");
5552         stat_list = SYSCTL_CHILDREN(stat_node);
5553
5554         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5555                         CTLFLAG_RD, &stats->crcerrs, 0,
5556                         "CRC Errors");
5557         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5558                         CTLFLAG_RD, &stats->illerrc, 0,
5559                         "Illegal Byte Errors");
5560         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5561                         CTLFLAG_RD, &stats->errbc, 0,
5562                         "Byte Errors");
5563         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5564                         CTLFLAG_RD, &stats->mspdc, 0,
5565                         "MAC Short Packets Discarded");
5566         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5567                         CTLFLAG_RD, &stats->mlfc, 0,
5568                         "MAC Local Faults");
5569         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5570                         CTLFLAG_RD, &stats->mrfc, 0,
5571                         "MAC Remote Faults");
5572         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5573                         CTLFLAG_RD, &stats->rlec, 0,
5574                         "Receive Length Errors");
5575         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5576                         CTLFLAG_RD, &stats->lxontxc, 0,
5577                         "Link XON Transmitted");
5578         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5579                         CTLFLAG_RD, &stats->lxonrxc, 0,
5580                         "Link XON Received");
5581         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5582                         CTLFLAG_RD, &stats->lxofftxc, 0,
5583                         "Link XOFF Transmitted");
5584         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5585                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5586                         "Link XOFF Received");
5587
5588         /* Packet Reception Stats */
5589         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5590                         CTLFLAG_RD, &stats->tor, 0,
5591                         "Total Octets Received");
5592         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5593                         CTLFLAG_RD, &stats->gorc, 0,
5594                         "Good Octets Received");
5595         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5596                         CTLFLAG_RD, &stats->tpr, 0,
5597                         "Total Packets Received");
5598         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5599                         CTLFLAG_RD, &stats->gprc, 0,
5600                         "Good Packets Received");
5601         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5602                         CTLFLAG_RD, &stats->mprc, 0,
5603                         "Multicast Packets Received");
5604         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5605                         CTLFLAG_RD, &stats->bprc, 0,
5606                         "Broadcast Packets Received");
5607         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5608                         CTLFLAG_RD, &stats->prc64, 0,
5609                         "64 byte frames received ");
5610         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5611                         CTLFLAG_RD, &stats->prc127, 0,
5612                         "65-127 byte frames received");
5613         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5614                         CTLFLAG_RD, &stats->prc255, 0,
5615                         "128-255 byte frames received");
5616         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5617                         CTLFLAG_RD, &stats->prc511, 0,
5618                         "256-511 byte frames received");
5619         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5620                         CTLFLAG_RD, &stats->prc1023, 0,
5621                         "512-1023 byte frames received");
5622         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5623                         CTLFLAG_RD, &stats->prc1522, 0,
5624                         "1023-1522 byte frames received");
5625         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5626                         CTLFLAG_RD, &stats->ruc, 0,
5627                         "Receive Undersized");
5628         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5629                         CTLFLAG_RD, &stats->rfc, 0,
5630                         "Fragmented Packets Received ");
5631         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5632                         CTLFLAG_RD, &stats->roc, 0,
5633                         "Oversized Packets Received");
5634         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5635                         CTLFLAG_RD, &stats->rjc, 0,
5636                         "Received Jabber");
5637         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5638                         CTLFLAG_RD, &stats->mngprc, 0,
5639                         "Management Packets Received");
5640         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5641                         CTLFLAG_RD, &stats->mngptc, 0,
5642                         "Management Packets Dropped");
5643         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5644                         CTLFLAG_RD, &stats->xec, 0,
5645                         "Checksum Errors");
5646
5647         /* Packet Transmission Stats */
5648         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5649                         CTLFLAG_RD, &stats->gotc, 0,
5650                         "Good Octets Transmitted");
5651         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5652                         CTLFLAG_RD, &stats->tpt, 0,
5653                         "Total Packets Transmitted");
5654         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5655                         CTLFLAG_RD, &stats->gptc, 0,
5656                         "Good Packets Transmitted");
5657         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5658                         CTLFLAG_RD, &stats->bptc, 0,
5659                         "Broadcast Packets Transmitted");
5660         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5661                         CTLFLAG_RD, &stats->mptc, 0,
5662                         "Multicast Packets Transmitted");
5663         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5664                         CTLFLAG_RD, &stats->mngptc, 0,
5665                         "Management Packets Transmitted");
5666         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5667                         CTLFLAG_RD, &stats->ptc64, 0,
5668                         "64 byte frames transmitted ");
5669         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5670                         CTLFLAG_RD, &stats->ptc127, 0,
5671                         "65-127 byte frames transmitted");
5672         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5673                         CTLFLAG_RD, &stats->ptc255, 0,
5674                         "128-255 byte frames transmitted");
5675         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5676                         CTLFLAG_RD, &stats->ptc511, 0,
5677                         "256-511 byte frames transmitted");
5678         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5679                         CTLFLAG_RD, &stats->ptc1023, 0,
5680                         "512-1023 byte frames transmitted");
5681         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5682                         CTLFLAG_RD, &stats->ptc1522, 0,
5683                         "1024-1522 byte frames transmitted");
5684
5685         /* FC Stats */
5686         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5687                 CTLFLAG_RD, &stats->fccrc, 0,
5688                 "FC CRC Errors");
5689         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5690                 CTLFLAG_RD, &stats->fclast, 0,
5691                 "FC Last Error");
5692         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5693                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5694                 "FCoE Packets Dropped");
5695         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5696                 CTLFLAG_RD, &stats->fcoeprc, 0,
5697                 "FCoE Packets Received");
5698         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5699                 CTLFLAG_RD, &stats->fcoeptc, 0,
5700                 "FCoE Packets Transmitted");
5701         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5702                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5703                 "FCoE DWords Received");
5704         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5705                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5706                 "FCoE DWords Transmitted");
5707 }
5708
5709 /*
5710 ** Set flow control using sysctl:
5711 ** Flow control values:
5712 **      0 - off
5713 **      1 - rx pause
5714 **      2 - tx pause
5715 **      3 - full
5716 */
5717 static int
5718 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5719 {
5720         int error, last;
5721         struct adapter *adapter = (struct adapter *) arg1;
5722
5723         last = adapter->fc;
5724         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5725         if ((error) || (req->newptr == NULL))
5726                 return (error);
5727
5728         /* Don't bother if it's not changed */
5729         if (adapter->fc == last)
5730                 return (0);
5731
5732         switch (adapter->fc) {
5733                 case ixgbe_fc_rx_pause:
5734                 case ixgbe_fc_tx_pause:
5735                 case ixgbe_fc_full:
5736                         adapter->hw.fc.requested_mode = adapter->fc;
5737                         break;
5738                 case ixgbe_fc_none:
5739                 default:
5740                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5741         }
5742         /* Don't autoneg if forcing a value */
5743         adapter->hw.fc.disable_fc_autoneg = TRUE;
5744         ixgbe_fc_enable(&adapter->hw);
5745         return error;
5746 }
5747
5748 static void
5749 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5750         const char *description, int *limit, int value)
5751 {
5752         *limit = value;
5753         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5754             SYSCTL_CHILDREN(adapter->sysctl_tree),
5755             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5756 }
5757
5758 /*
5759 ** Control link advertise speed:
5760 **      1 - advertise only 1G
5761 **      2 - advertise 100Mb
5762 **      3 - advertise normal
5763 */
5764 static int
5765 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5766 {
5767         int                     error = 0;
5768         struct adapter          *adapter;
5769         device_t                dev;
5770         struct ixgbe_hw         *hw;
5771         ixgbe_link_speed        speed, last;
5772
5773         adapter = (struct adapter *) arg1;
5774         dev = adapter->dev;
5775         hw = &adapter->hw;
5776         last = adapter->advertise;
5777
5778         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5779         if ((error) || (adapter->advertise == -1))
5780                 return (error);
5781
5782         if (adapter->advertise == last) /* no change */
5783                 return (0);
5784
5785         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5786             (hw->phy.multispeed_fiber)))
5787                 return (error);
5788
5789         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5790                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5791                 return (error);
5792         }
5793
5794         if (adapter->advertise == 1)
5795                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5796         else if (adapter->advertise == 2)
5797                 speed = IXGBE_LINK_SPEED_100_FULL;
5798         else if (adapter->advertise == 3)
5799                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5800                         IXGBE_LINK_SPEED_10GB_FULL;
5801         else /* bogus value */
5802                 return (error);
5803
5804         hw->mac.autotry_restart = TRUE;
5805         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5806
5807         return (error);
5808 }
5809
5810 /*
5811 ** Thermal Shutdown Trigger
5812 **   - cause a Thermal Overtemp IRQ
5813 */
5814 static int
5815 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5816 {
5817         int             error, fire = 0;
5818         struct adapter  *adapter = (struct adapter *) arg1;
5819         struct ixgbe_hw *hw = &adapter->hw;
5820
5821
5822         if (hw->mac.type != ixgbe_mac_X540)
5823                 return (0);
5824
5825         error = sysctl_handle_int(oidp, &fire, 0, req);
5826         if ((error) || (req->newptr == NULL))
5827                 return (error);
5828
5829         if (fire) {
5830                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5831                 reg |= IXGBE_EICR_TS;
5832                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5833         }
5834
5835         return (0);
5836 }