sys/dev/netif/ixgbe/ixgbe.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2012, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD: src/sys/dev/ixgbe/ixgbe.c,v 1.70 2012/07/05 20:51:44 jfv Exp $*/
  34
  35 #include "opt_inet.h"
  36 #include "opt_inet6.h"
  37
  38 #include "ixgbe.h"
  39
  40 /*********************************************************************
  41  *  Set this to one to display debug statistics
  42  *********************************************************************/
  43 int             ixgbe_display_debug_stats = 0;
  44
  45 /*********************************************************************
  46  *  Driver version
  47  *********************************************************************/
  48 char ixgbe_driver_version[] = "2.4.8";
  49
  50 /*********************************************************************
  51  *  PCI Device ID Table
  52  *
  53  *  Used by probe to select devices to load on
  54  *  Last field stores an index into ixgbe_strings
  55  *  Last entry must be all 0s
  56  *
  57  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  58  *********************************************************************/
  59
  60 static ixgbe_vendor_info_t ixgbe_vendor_info_array[] =
  61 {
  62         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0},
  63         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0},
  64         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0},
  65         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0},
  66         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2, 0, 0, 0},
  67         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0},
  68         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0},
  69         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0},
  70         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0},
  71         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0},
  72         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0},
  73         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4, 0, 0, 0},
  74         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ, 0, 0, 0},
  75         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP, 0, 0, 0},
  76         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM, 0, 0, 0},
  77         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4, 0, 0, 0},
  78         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM, 0, 0, 0},
  79         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE, 0, 0, 0},
  80         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE, 0, 0, 0},
  81         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2, 0, 0, 0},
  82         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE, 0, 0, 0},
  83         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP, 0, 0, 0},
  84         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1, 0, 0, 0},
  85         {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T, 0, 0, 0},
  86         /* required last entry */
  87         {0, 0, 0, 0, 0}
  88 };
  89
  90 /*********************************************************************
  91  *  Table of branding strings
  92  *********************************************************************/
  93
  94 static char    *ixgbe_strings[] = {
  95         "Intel(R) PRO/10GbE PCI-Express Network Driver"
  96 };
  97
  98 /*********************************************************************
  99  *  Function prototypes
 100  *********************************************************************/
 101 static int      ixgbe_probe(device_t);
 102 static int      ixgbe_attach(device_t);
 103 static int      ixgbe_detach(device_t);
 104 static int      ixgbe_shutdown(device_t);
 105 static void     ixgbe_start(struct ifnet *);
 106 static void     ixgbe_start_locked(struct tx_ring *, struct ifnet *);
 107 #if 0 /* __FreeBSD_version >= 800000 */
 108 static int      ixgbe_mq_start(struct ifnet *, struct mbuf *);
 109 static int      ixgbe_mq_start_locked(struct ifnet *,
 110                     struct tx_ring *, struct mbuf *);
 111 static void     ixgbe_qflush(struct ifnet *);
 112 #endif
 113 static int      ixgbe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 114 static void     ixgbe_init(void *);
 115 static void     ixgbe_init_locked(struct adapter *);
 116 static void     ixgbe_stop(void *);
 117 static void     ixgbe_media_status(struct ifnet *, struct ifmediareq *);
 118 static int      ixgbe_media_change(struct ifnet *);
 119 static void     ixgbe_identify_hardware(struct adapter *);
 120 static int      ixgbe_allocate_pci_resources(struct adapter *);
 121 static int      ixgbe_allocate_msix(struct adapter *);
 122 static int      ixgbe_allocate_legacy(struct adapter *);
 123 static int      ixgbe_allocate_queues(struct adapter *);
 124 #if 0   /* HAVE_MSIX */
 125 static int      ixgbe_setup_msix(struct adapter *);
 126 #endif
 127 static void     ixgbe_free_pci_resources(struct adapter *);
 128 static void     ixgbe_local_timer(void *);
 129 static int      ixgbe_setup_interface(device_t, struct adapter *);
 130 static void     ixgbe_config_link(struct adapter *);
 131
 132 static int      ixgbe_allocate_transmit_buffers(struct tx_ring *);
 133 static int      ixgbe_setup_transmit_structures(struct adapter *);
 134 static void     ixgbe_setup_transmit_ring(struct tx_ring *);
 135 static void     ixgbe_initialize_transmit_units(struct adapter *);
 136 static void     ixgbe_free_transmit_structures(struct adapter *);
 137 static void     ixgbe_free_transmit_buffers(struct tx_ring *);
 138
 139 static int      ixgbe_allocate_receive_buffers(struct rx_ring *);
 140 static int      ixgbe_setup_receive_structures(struct adapter *);
 141 static int      ixgbe_setup_receive_ring(struct rx_ring *);
 142 static void     ixgbe_initialize_receive_units(struct adapter *);
 143 static void     ixgbe_free_receive_structures(struct adapter *);
 144 static void     ixgbe_free_receive_buffers(struct rx_ring *);
 145 #if 0   /* NET_LRO */
 146 static void     ixgbe_setup_hw_rsc(struct rx_ring *);
 147 #endif
 148
 149 static void     ixgbe_enable_intr(struct adapter *);
 150 static void     ixgbe_disable_intr(struct adapter *);
 151 static void     ixgbe_update_stats_counters(struct adapter *);
 152 static bool     ixgbe_txeof(struct tx_ring *);
 153 static bool     ixgbe_rxeof(struct ix_queue *, int);
 154 static void     ixgbe_rx_checksum(u32, struct mbuf *, u32);
 155 static void     ixgbe_set_promisc(struct adapter *);
 156 static void     ixgbe_set_multi(struct adapter *);
 157 static void     ixgbe_update_link_status(struct adapter *);
 158 static void     ixgbe_refresh_mbufs(struct rx_ring *, int);
 159 static int      ixgbe_xmit(struct tx_ring *, struct mbuf **);
 160 static int      ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS);
 161 static int      ixgbe_set_advertise(SYSCTL_HANDLER_ARGS);
 162 static int      ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS);
 163 static int      ixgbe_dma_malloc(struct adapter *, bus_size_t,
 164                     struct ixgbe_dma_alloc *, int);
 165 static void     ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *);
 166 static void     ixgbe_add_rx_process_limit(struct adapter *, const char *,
 167                     const char *, int *, int);
 168 static bool     ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 169 #if 0   /* NET_TSO */
 170 static bool     ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *);
 171 #endif
 172 static void     ixgbe_set_ivar(struct adapter *, u8, u8, s8);
 173 static void     ixgbe_configure_ivars(struct adapter *);
 174 static u8 *     ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *);
 175
 176 static void     ixgbe_setup_vlan_hw_support(struct adapter *);
 177 static void     ixgbe_register_vlan(void *, struct ifnet *, u16);
 178 static void     ixgbe_unregister_vlan(void *, struct ifnet *, u16);
 179
 180 static void     ixgbe_add_hw_stats(struct adapter *adapter);
 181
 182 static __inline void ixgbe_rx_discard(struct rx_ring *, int);
 183 static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *,
 184                     struct mbuf *, u32);
 185
 186 /* Support for pluggable optic modules */
 187 static bool     ixgbe_sfp_probe(struct adapter *);
 188 static void     ixgbe_setup_optics(struct adapter *);
 189
 190 /* Legacy (single vector interrupt handler */
 191 static void     ixgbe_legacy_irq(void *);
 192
 193 /* The MSI/X Interrupt handlers */
 194 static void     ixgbe_msix_que(void *);
 195 static void     ixgbe_msix_link(void *);
 196
 197 /* Deferred interrupt tasklets */
 198 static void     ixgbe_handle_que(void *, int);
 199 static void     ixgbe_handle_link(void *, int);
 200 static void     ixgbe_handle_msf(void *, int);
 201 static void     ixgbe_handle_mod(void *, int);
 202
 203 #ifdef IXGBE_FDIR
 204 static void     ixgbe_atr(struct tx_ring *, struct mbuf *);
 205 static void     ixgbe_reinit_fdir(void *, int);
 206 #endif
 207
 208 /*********************************************************************
 209  *  FreeBSD Device Interface Entry Points
 210  *********************************************************************/
 211
 212 static device_method_t ixgbe_methods[] = {
 213         /* Device interface */
 214         DEVMETHOD(device_probe, ixgbe_probe),
 215         DEVMETHOD(device_attach, ixgbe_attach),
 216         DEVMETHOD(device_detach, ixgbe_detach),
 217         DEVMETHOD(device_shutdown, ixgbe_shutdown),
 218         {0, 0}
 219 };
 220
 221 static driver_t ixgbe_driver = {
 222         "ix", ixgbe_methods, sizeof(struct adapter),
 223 };
 224
 225 devclass_t ixgbe_devclass;
 226 DRIVER_MODULE(ixgbe, pci, ixgbe_driver, ixgbe_devclass, 0, 0);
 227
 228 MODULE_DEPEND(ixgbe, pci, 1, 1, 1);
 229 MODULE_DEPEND(ixgbe, ether, 1, 1, 1);
 230
 231 /*
 232 ** TUNEABLE PARAMETERS:
 233 */
 234
 235 /*
 236 ** AIM: Adaptive Interrupt Moderation
 237 ** which means that the interrupt rate
 238 ** is varied over time based on the
 239 ** traffic for that interrupt vector
 240 */
 241 static int ixgbe_enable_aim = TRUE;
 242 TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim);
 243
 244 static int ixgbe_max_interrupt_rate = (4000000 / IXGBE_LOW_LATENCY);
 245 TUNABLE_INT("hw.ixgbe.max_interrupt_rate", &ixgbe_max_interrupt_rate);
 246
 247 /* How many packets rxeof tries to clean at a time */
 248 static int ixgbe_rx_process_limit = 128;
 249 TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit);
 250
 251 /*
 252 ** Smart speed setting, default to on
 253 ** this only works as a compile option
 254 ** right now as its during attach, set
 255 ** this to 'ixgbe_smart_speed_off' to
 256 ** disable.
 257 */
 258 static int ixgbe_smart_speed = ixgbe_smart_speed_on;
 259
 260 static int ixgbe_msi_enable = 1;
 261 TUNABLE_INT("hw.ixgbe.msi.enable", &ixgbe_msi_enable);
 262
 263 /*
 264  * MSIX should be the default for best performance,
 265  * but this allows it to be forced off for testing.
 266  */
 267 static int ixgbe_enable_msix = 1;
 268 TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix);
 269
 270 /*
 271  * Header split: this causes the hardware to DMA
 272  * the header into a separate mbuf from the payload,
 273  * it can be a performance win in some workloads, but
 274  * in others it actually hurts, its off by default.
 275  */
 276 static int ixgbe_header_split = FALSE;
 277 TUNABLE_INT("hw.ixgbe.hdr_split", &ixgbe_header_split);
 278
 279 /*
 280  * Number of Queues, can be set to 0,
 281  * it then autoconfigures based on the
 282  * number of cpus with a max of 8. This
 283  * can be overriden manually here.
 284  */
 285 static int ixgbe_num_queues = 0;
 286 TUNABLE_INT("hw.ixgbe.num_queues", &ixgbe_num_queues);
 287
 288 /*
 289 ** Number of TX descriptors per ring,
 290 ** setting higher than RX as this seems
 291 ** the better performing choice.
 292 */
 293 static int ixgbe_txd = PERFORM_TXD;
 294 TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd);
 295
 296 /* Number of RX descriptors per ring */
 297 static int ixgbe_rxd = PERFORM_RXD;
 298 TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd);
 299
 300 /* Keep running tab on them for sanity check */
 301 static int ixgbe_total_ports;
 302
 303 #ifdef IXGBE_FDIR
 304 /*
 305 ** For Flow Director: this is the
 306 ** number of TX packets we sample
 307 ** for the filter pool, this means
 308 ** every 20th packet will be probed.
 309 **
 310 ** This feature can be disabled by
 311 ** setting this to 0.
 312 */
 313 static int atr_sample_rate = 20;
 314 /*
 315 ** Flow Director actually 'steals'
 316 ** part of the packet buffer as its
 317 ** filter pool, this variable controls
 318 ** how much it uses:
 319 **  0 = 64K, 1 = 128K, 2 = 256K
 320 */
 321 static int fdir_pballoc = 1;
 322 #endif
 323
 324 #ifdef DEV_NETMAP
 325 /*
 326  * The #ifdef DEV_NETMAP / #endif blocks in this file are meant to
 327  * be a reference on how to implement netmap support in a driver.
 328  * Additional comments are in ixgbe_netmap.h .
 329  *
 330  * <dev/netmap/ixgbe_netmap.h> contains functions for netmap support
 331  * that extend the standard driver.
 332  */
 333 #include <dev/netmap/ixgbe_netmap.h>
 334 #endif /* DEV_NETMAP */
 335
 336 /*********************************************************************
 337  *  Device identification routine
 338  *
 339  *  ixgbe_probe determines if the driver should be loaded on
 340  *  adapter based on PCI vendor/device id of the adapter.
 341  *
 342  *  return BUS_PROBE_DEFAULT on success, positive on failure
 343  *********************************************************************/
 344
 345 static int
 346 ixgbe_probe(device_t dev)
 347 {
 348         ixgbe_vendor_info_t *ent;
 349
 350         u16     pci_vendor_id = 0;
 351         u16     pci_device_id = 0;
 352         u16     pci_subvendor_id = 0;
 353         u16     pci_subdevice_id = 0;
 354         char    adapter_name[256];
 355
 356         INIT_DEBUGOUT("ixgbe_probe: begin");
 357
 358         pci_vendor_id = pci_get_vendor(dev);
 359         if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID)
 360                 return (ENXIO);
 361
 362         pci_device_id = pci_get_device(dev);
 363         pci_subvendor_id = pci_get_subvendor(dev);
 364         pci_subdevice_id = pci_get_subdevice(dev);
 365
 366         ent = ixgbe_vendor_info_array;
 367         while (ent->vendor_id != 0) {
 368                 if ((pci_vendor_id == ent->vendor_id) &&
 369                     (pci_device_id == ent->device_id) &&
 370
 371                     ((pci_subvendor_id == ent->subvendor_id) ||
 372                      (ent->subvendor_id == 0)) &&
 373
 374                     ((pci_subdevice_id == ent->subdevice_id) ||
 375                      (ent->subdevice_id == 0))) {
 376                         ksprintf(adapter_name, "%s, Version - %s",
 377                                 ixgbe_strings[ent->index],
 378                                 ixgbe_driver_version);
 379                         device_set_desc_copy(dev, adapter_name);
 380                         ++ixgbe_total_ports;
 381                         return (BUS_PROBE_DEFAULT);
 382                 }
 383                 ent++;
 384         }
 385         return (ENXIO);
 386 }
 387
 388 /*********************************************************************
 389  *  Device initialization routine
 390  *
 391  *  The attach entry point is called when the driver is being loaded.
 392  *  This routine identifies the type of hardware, allocates all resources
 393  *  and initializes the hardware.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 ixgbe_attach(device_t dev)
 400 {
 401         struct adapter *adapter;
 402         struct ixgbe_hw *hw;
 403         int             error = 0;
 404         u16             csum;
 405         u32             ctrl_ext;
 406
 407         INIT_DEBUGOUT("ixgbe_attach: begin");
 408
 409         if (resource_disabled("ixgbe", device_get_unit(dev))) {
 410                 device_printf(dev, "Disabled by device hint\n");
 411                 return (ENXIO);
 412         }
 413
 414         /* Allocate, clear, and link in our adapter structure */
 415         adapter = device_get_softc(dev);
 416         adapter->dev = adapter->osdep.dev = dev;
 417         hw = &adapter->hw;
 418
 419         /* Core Lock Init*/
 420         IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 421         spin_init(&adapter->mcast_spin);
 422
 423         /* SYSCTL APIs */
 424
 425         sysctl_ctx_init(&adapter->sysctl_ctx);
 426         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 427             SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 428             device_get_nameunit(adapter->dev), CTLFLAG_RD, 0, "");
 429         if (adapter->sysctl_tree == NULL) {
 430                 device_printf(adapter->dev, "can't add sysctl node\n");
 431                 return (EINVAL);
 432         }
 433         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 434                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 435                         OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
 436                         adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control");
 437
 438         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 439                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 440                         OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 441                         &ixgbe_enable_aim, 1, "Interrupt Moderation");
 442
 443         /*
 444         ** Allow a kind of speed control by forcing the autoneg
 445         ** advertised speed list to only a certain value, this
 446         ** supports 1G on 82599 devices, and 100Mb on x540.
 447         */
 448         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 449                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 450                         OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
 451                         adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
 452
 453         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 454                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 455                         OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
 456                         0, ixgbe_set_thermal_test, "I", "Thermal Test");
 457
 458         /* Set up the timer callout */
 459         callout_init(&adapter->timer);
 460
 461         /* Determine hardware revision */
 462         ixgbe_identify_hardware(adapter);
 463
 464         /* Do base PCI setup - map BAR0 */
 465         if (ixgbe_allocate_pci_resources(adapter)) {
 466                 device_printf(dev, "Allocation of PCI resources failed\n");
 467                 error = ENXIO;
 468                 goto err_out;
 469         }
 470
 471         /* Do descriptor calc and sanity checks */
 472         if (((ixgbe_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 ||
 473             ixgbe_txd < MIN_TXD || ixgbe_txd > MAX_TXD) {
 474                 device_printf(dev, "TXD config issue, using default!\n");
 475                 adapter->num_tx_desc = DEFAULT_TXD;
 476         } else
 477                 adapter->num_tx_desc = ixgbe_txd;
 478
 479         /*
 480         ** With many RX rings it is easy to exceed the
 481         ** system mbuf allocation. Tuning nmbclusters
 482         ** can alleviate this.
 483         */
 484         if (nmbclusters > 0 ) {
 485                 int s;
 486                 s = (ixgbe_rxd * adapter->num_queues) * ixgbe_total_ports;
 487                 if (s > nmbclusters) {
 488                         device_printf(dev, "RX Descriptors exceed "
 489                             "system mbuf max, using default instead!\n");
 490                         ixgbe_rxd = DEFAULT_RXD;
 491                 }
 492         }
 493
 494         if (((ixgbe_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 ||
 495             ixgbe_rxd < MIN_TXD || ixgbe_rxd > MAX_TXD) {
 496                 device_printf(dev, "RXD config issue, using default!\n");
 497                 adapter->num_rx_desc = DEFAULT_RXD;
 498         } else
 499                 adapter->num_rx_desc = ixgbe_rxd;
 500
 501         /* Allocate our TX/RX Queues */
 502         if (ixgbe_allocate_queues(adapter)) {
 503                 error = ENOMEM;
 504                 goto err_out;
 505         }
 506
 507         /* Allocate multicast array memory. */
 508         adapter->mta = kmalloc(sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
 509             MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
 510         if (adapter->mta == NULL) {
 511                 device_printf(dev, "Can not allocate multicast setup array\n");
 512                 error = ENOMEM;
 513                 goto err_late;
 514         }
 515
 516         /* Initialize the shared code */
 517         error = ixgbe_init_shared_code(hw);
 518         if (error == IXGBE_ERR_SFP_NOT_PRESENT) {
 519                 /*
 520                 ** No optics in this port, set up
 521                 ** so the timer routine will probe
 522                 ** for later insertion.
 523                 */
 524                 adapter->sfp_probe = TRUE;
 525                 error = 0;
 526         } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) {
 527                 device_printf(dev,"Unsupported SFP+ module detected!\n");
 528                 error = EIO;
 529                 goto err_late;
 530         } else if (error) {
 531                 device_printf(dev,"Unable to initialize the shared code\n");
 532                 error = EIO;
 533                 goto err_late;
 534         }
 535
 536         /* Make sure we have a good EEPROM before we read from it */
 537         if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) {
 538                 device_printf(dev,"The EEPROM Checksum Is Not Valid\n");
 539                 error = EIO;
 540                 goto err_late;
 541         }
 542
 543         error = ixgbe_init_hw(hw);
 544         switch (error) {
 545         case IXGBE_ERR_EEPROM_VERSION:
 546                 device_printf(dev, "This device is a pre-production adapter/"
 547                     "LOM.  Please be aware there may be issues associated "
 548                     "with your hardware.\n If you are experiencing problems "
 549                     "please contact your Intel or hardware representative "
 550                     "who provided you with this hardware.\n");
 551                 break;
 552         case IXGBE_ERR_SFP_NOT_SUPPORTED:
 553                 device_printf(dev,"Unsupported SFP+ Module\n");
 554                 error = EIO;
 555                 device_printf(dev,"Hardware Initialization Failure\n");
 556                 goto err_late;
 557         case IXGBE_ERR_SFP_NOT_PRESENT:
 558                 device_printf(dev,"No SFP+ Module found\n");
 559                 /* falls thru */
 560         default:
 561                 break;
 562         }
 563
 564         /* Detect and set physical type */
 565         ixgbe_setup_optics(adapter);
 566
 567         if ((adapter->msix > 1) && (ixgbe_enable_msix))
 568                 error = ixgbe_allocate_msix(adapter);
 569         else
 570                 error = ixgbe_allocate_legacy(adapter);
 571         if (error)
 572                 goto err_late;
 573
 574         /* Setup OS specific network interface */
 575         if (ixgbe_setup_interface(dev, adapter) != 0)
 576                 goto err_late;
 577
 578         /* Sysctl for limiting the amount of work done in the taskqueue */
 579         ixgbe_add_rx_process_limit(adapter, "rx_processing_limit",
 580             "max number of rx packets to process", &adapter->rx_process_limit,
 581             ixgbe_rx_process_limit);
 582
 583         /* Initialize statistics */
 584         ixgbe_update_stats_counters(adapter);
 585
 586         /* Register for VLAN events */
 587         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 588             ixgbe_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 589         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 590             ixgbe_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 591
 592         /* Print PCIE bus type/speed/width info */
 593         ixgbe_get_bus_info(hw);
 594         device_printf(dev,"PCI Express Bus: Speed %s %s\n",
 595             ((hw->bus.speed == ixgbe_bus_speed_5000) ? "5.0Gb/s":
 596             (hw->bus.speed == ixgbe_bus_speed_2500) ? "2.5Gb/s":"Unknown"),
 597             (hw->bus.width == ixgbe_bus_width_pcie_x8) ? "Width x8" :
 598             (hw->bus.width == ixgbe_bus_width_pcie_x4) ? "Width x4" :
 599             (hw->bus.width == ixgbe_bus_width_pcie_x1) ? "Width x1" :
 600             ("Unknown"));
 601
 602         if ((hw->bus.width <= ixgbe_bus_width_pcie_x4) &&
 603             (hw->bus.speed == ixgbe_bus_speed_2500)) {
 604                 device_printf(dev, "PCI-Express bandwidth available"
 605                     " for this card\n     is not sufficient for"
 606                     " optimal performance.\n");
 607                 device_printf(dev, "For optimal performance a x8 "
 608                     "PCIE, or x4 PCIE 2 slot is required.\n");
 609         }
 610
 611         /* let hardware know driver is loaded */
 612         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
 613         ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
 614         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
 615
 616         ixgbe_add_hw_stats(adapter);
 617
 618 #ifdef DEV_NETMAP
 619         ixgbe_netmap_attach(adapter);
 620 #endif /* DEV_NETMAP */
 621         INIT_DEBUGOUT("ixgbe_attach: end");
 622         return (0);
 623 err_late:
 624         ixgbe_free_transmit_structures(adapter);
 625         ixgbe_free_receive_structures(adapter);
 626 err_out:
 627         if (adapter->ifp != NULL)
 628                 if_free(adapter->ifp);
 629         ixgbe_free_pci_resources(adapter);
 630         kfree(adapter->mta, M_DEVBUF);
 631         return (error);
 632
 633 }
 634
 635 /*********************************************************************
 636  *  Device removal routine
 637  *
 638  *  The detach entry point is called when the driver is being removed.
 639  *  This routine stops the adapter and deallocates all the resources
 640  *  that were allocated for driver operation.
 641  *
 642  *  return 0 on success, positive on failure
 643  *********************************************************************/
 644
 645 static int
 646 ixgbe_detach(device_t dev)
 647 {
 648         struct adapter *adapter = device_get_softc(dev);
 649         struct ix_queue *que = adapter->queues;
 650         u32     ctrl_ext;
 651
 652         INIT_DEBUGOUT("ixgbe_detach: begin");
 653
 654 #ifdef NET_VLAN
 655         /* Make sure VLANS are not using driver */
 656         if (adapter->ifp->if_vlantrunk != NULL) {
 657                 device_printf(dev,"Vlan in use, detach first\n");
 658                 return (EBUSY);
 659         }
 660 #endif
 661
 662         IXGBE_CORE_LOCK(adapter);
 663         ixgbe_stop(adapter);
 664         IXGBE_CORE_UNLOCK(adapter);
 665
 666         for (int i = 0; i < adapter->num_queues; i++, que++) {
 667                 if (que->tq) {
 668                         taskqueue_drain(que->tq, &que->que_task);
 669                         taskqueue_free(que->tq);
 670                 }
 671         }
 672
 673         /* Drain the Link queue */
 674         if (adapter->tq) {
 675                 taskqueue_drain(adapter->tq, &adapter->link_task);
 676                 taskqueue_drain(adapter->tq, &adapter->mod_task);
 677                 taskqueue_drain(adapter->tq, &adapter->msf_task);
 678 #ifdef IXGBE_FDIR
 679                 taskqueue_drain(adapter->tq, &adapter->fdir_task);
 680 #endif
 681                 taskqueue_free(adapter->tq);
 682         }
 683
 684         /* let hardware know driver is unloading */
 685         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 686         ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
 687         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext);
 688
 689         /* Unregister VLAN events */
 690 #ifdef NET_VLAN
 691         if (adapter->vlan_attach != NULL)
 692                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 693         if (adapter->vlan_detach != NULL)
 694                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 695 #endif
 696
 697         ether_ifdetach(adapter->ifp);
 698         callout_stop(&adapter->timer);
 699         lockuninit(&adapter->core_lock);
 700 #ifdef DEV_NETMAP
 701         netmap_detach(adapter->ifp);
 702 #endif /* DEV_NETMAP */
 703         ixgbe_free_pci_resources(adapter);
 704         bus_generic_detach(dev);
 705         if_free(adapter->ifp);
 706
 707         ixgbe_free_transmit_structures(adapter);
 708         ixgbe_free_receive_structures(adapter);
 709         kfree(adapter->mta, M_DEVBUF);
 710         sysctl_ctx_free(&adapter->sysctl_ctx);
 711
 712         spin_uninit(&adapter->mcast_spin);
 713         IXGBE_CORE_LOCK_DESTROY(adapter);
 714         return (0);
 715 }
 716
 717 /*********************************************************************
 718  *
 719  *  Shutdown entry point
 720  *
 721  **********************************************************************/
 722
 723 static int
 724 ixgbe_shutdown(device_t dev)
 725 {
 726         struct adapter *adapter = device_get_softc(dev);
 727         IXGBE_CORE_LOCK(adapter);
 728         ixgbe_stop(adapter);
 729         IXGBE_CORE_UNLOCK(adapter);
 730         return (0);
 731 }
 732
 733
 734 /*********************************************************************
 735  *  Transmit entry point
 736  *
 737  *  ixgbe_start is called by the stack to initiate a transmit.
 738  *  The driver will remain in this routine as long as there are
 739  *  packets to transmit and transmit resources are available.
 740  *  In case resources are not available stack is notified and
 741  *  the packet is requeued.
 742  **********************************************************************/
 743
 744 static void
 745 ixgbe_start_locked(struct tx_ring *txr, struct ifnet * ifp)
 746 {
 747         struct mbuf    *m_head;
 748         struct adapter *adapter = txr->adapter;
 749
 750         IXGBE_TX_LOCK_ASSERT(txr);
 751
 752         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
 753                 return;
 754         if (!adapter->link_active)
 755                 return;
 756
 757         while (!ifq_is_empty(&ifp->if_snd)) {
 758                 if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) {
 759                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 760                         break;
 761                 }
 762
 763                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 764                 if (m_head == NULL)
 765                         break;
 766
 767                 if (ixgbe_xmit(txr, &m_head)) {
 768 #if 0 /* XXX: prepend to an ALTQ queue ? */
 769                         if (m_head != NULL)
 770                                 IF_PREPEND(&ifp->if_snd, m_head);
 771 #endif
 772                         if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE)
 773                                 txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 774                         break;
 775                 }
 776                 /* Send a copy of the frame to the BPF listener */
 777                 ETHER_BPF_MTAP(ifp, m_head);
 778
 779                 /* Set watchdog on */
 780                 txr->watchdog_time = ticks;
 781                 txr->queue_status = IXGBE_QUEUE_WORKING;
 782
 783         }
 784         return;
 785 }
 786
 787 /*
 788  * Legacy TX start - called by the stack, this
 789  * always uses the first tx ring, and should
 790  * not be used with multiqueue tx enabled.
 791  */
 792 static void
 793 ixgbe_start(struct ifnet *ifp)
 794 {
 795         struct adapter *adapter = ifp->if_softc;
 796         struct tx_ring  *txr = adapter->tx_rings;
 797
 798         if (ifp->if_flags & IFF_RUNNING) {
 799                 IXGBE_TX_LOCK(txr);
 800                 ixgbe_start_locked(txr, ifp);
 801                 IXGBE_TX_UNLOCK(txr);
 802         }
 803         return;
 804 }
 805
 806 #if 0 /* __FreeBSD_version >= 800000 */
 807 /*
 808 ** Multiqueue Transmit driver
 809 **
 810 */
 811 static int
 812 ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
 813 {
 814         struct adapter  *adapter = ifp->if_softc;
 815         struct ix_queue *que;
 816         struct tx_ring  *txr;
 817         int             i = 0, err = 0;
 818
 819         /* Which queue to use */
 820         if ((m->m_flags & M_FLOWID) != 0)
 821                 i = m->m_pkthdr.flowid % adapter->num_queues;
 822         else
 823                 i = curcpu % adapter->num_queues;
 824
 825         txr = &adapter->tx_rings[i];
 826         que = &adapter->queues[i];
 827
 828         if (((txr->queue_status & IXGBE_QUEUE_DEPLETED) == 0) &&
 829             IXGBE_TX_TRYLOCK(txr)) {
 830                 err = ixgbe_mq_start_locked(ifp, txr, m);
 831                 IXGBE_TX_UNLOCK(txr);
 832         } else {
 833                 err = drbr_enqueue(ifp, txr->br, m);
 834                 taskqueue_enqueue(que->tq, &que->que_task);
 835         }
 836
 837         return (err);
 838 }
 839
 840 static int
 841 ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 842 {
 843         struct adapter  *adapter = txr->adapter;
 844         struct mbuf     *next;
 845         int             enqueued, err = 0;
 846
 847         if (((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) ||
 848             (txr->queue_status == IXGBE_QUEUE_DEPLETED) ||
 849             adapter->link_active == 0) {
 850                 if (m != NULL)
 851                         err = drbr_enqueue(ifp, txr->br, m);
 852                 return (err);
 853         }
 854
 855         enqueued = 0;
 856         if (m == NULL) {
 857                 next = drbr_dequeue(ifp, txr->br);
 858         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 859                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 860                         return (err);
 861                 next = drbr_dequeue(ifp, txr->br);
 862         } else
 863                 next = m;
 864
 865         /* Process the queue */
 866         while (next != NULL) {
 867                 if ((err = ixgbe_xmit(txr, &next)) != 0) {
 868                         if (next != NULL)
 869                                 err = drbr_enqueue(ifp, txr->br, next);
 870                         break;
 871                 }
 872                 enqueued++;
 873                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 874                 /* Send a copy of the frame to the BPF listener */
 875                 ETHER_BPF_MTAP(ifp, next);
 876                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 877                         break;
 878                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD)
 879                         ixgbe_txeof(txr);
 880                 if (txr->tx_avail < IXGBE_TX_OP_THRESHOLD) {
 881                         txr->queue_status |= IXGBE_QUEUE_DEPLETED;
 882                         break;
 883                 }
 884                 next = drbr_dequeue(ifp, txr->br);
 885         }
 886
 887         if (enqueued > 0) {
 888                 /* Set watchdog on */
 889                 txr->queue_status |= IXGBE_QUEUE_WORKING;
 890                 txr->watchdog_time = ticks;
 891         }
 892
 893         if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
 894                 ixgbe_txeof(txr);
 895
 896         return (err);
 897 }
 898
 899 /*
 900 ** Flush all ring buffers
 901 */
 902 static void
 903 ixgbe_qflush(struct ifnet *ifp)
 904 {
 905         struct adapter  *adapter = ifp->if_softc;
 906         struct tx_ring  *txr = adapter->tx_rings;
 907         struct mbuf     *m;
 908
 909         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 910                 IXGBE_TX_LOCK(txr);
 911                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 912                         m_freem(m);
 913                 IXGBE_TX_UNLOCK(txr);
 914         }
 915         if_qflush(ifp);
 916 }
 917 #endif /* __FreeBSD_version >= 800000 */
 918
 919 /*********************************************************************
 920  *  Ioctl entry point
 921  *
 922  *  ixgbe_ioctl is called when the user wants to configure the
 923  *  interface.
 924  *
 925  *  return 0 on success, positive on failure
 926  **********************************************************************/
 927
 928 static int
 929 ixgbe_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 930 {
 931         struct adapter  *adapter = ifp->if_softc;
 932         struct ifreq    *ifr = (struct ifreq *) data;
 933 #if defined(INET) || defined(INET6)
 934         struct ifaddr *ifa = (struct ifaddr *)data;
 935         bool            avoid_reset = FALSE;
 936 #endif
 937         int             error = 0;
 938
 939         switch (command) {
 940
 941         case SIOCSIFADDR:
 942 #ifdef INET
 943                 if (ifa->ifa_addr->sa_family == AF_INET)
 944                         avoid_reset = TRUE;
 945 #endif
 946 #ifdef INET6
 947                 if (ifa->ifa_addr->sa_family == AF_INET6)
 948                         avoid_reset = TRUE;
 949 #endif
 950 #if defined(INET) || defined(INET6)
 951                 /*
 952                 ** Calling init results in link renegotiation,
 953                 ** so we avoid doing it when possible.
 954                 */
 955                 if (avoid_reset) {
 956                         ifp->if_flags |= IFF_UP;
 957                         if (!(ifp->if_flags & IFF_RUNNING))
 958                                 ixgbe_init(adapter);
 959                         if (!(ifp->if_flags & IFF_NOARP))
 960                                 arp_ifinit(ifp, ifa);
 961                 } else
 962                         error = ether_ioctl(ifp, command, data);
 963 #endif
 964                 break;
 965         case SIOCSIFMTU:
 966                 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
 967                 if (ifr->ifr_mtu > IXGBE_MAX_FRAME_SIZE - ETHER_HDR_LEN) {
 968                         error = EINVAL;
 969                 } else {
 970                         IXGBE_CORE_LOCK(adapter);
 971                         ifp->if_mtu = ifr->ifr_mtu;
 972                         adapter->max_frame_size =
 973                                 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 974                         ixgbe_init_locked(adapter);
 975                         IXGBE_CORE_UNLOCK(adapter);
 976                 }
 977                 break;
 978         case SIOCSIFFLAGS:
 979                 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
 980                 IXGBE_CORE_LOCK(adapter);
 981                 if (ifp->if_flags & IFF_UP) {
 982                         if ((ifp->if_flags & IFF_RUNNING)) {
 983                                 if ((ifp->if_flags ^ adapter->if_flags) &
 984                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 985                                         ixgbe_set_promisc(adapter);
 986                                 }
 987                         } else
 988                                 ixgbe_init_locked(adapter);
 989                 } else
 990                         if (ifp->if_flags & IFF_RUNNING)
 991                                 ixgbe_stop(adapter);
 992                 adapter->if_flags = ifp->if_flags;
 993                 IXGBE_CORE_UNLOCK(adapter);
 994                 break;
 995         case SIOCADDMULTI:
 996         case SIOCDELMULTI:
 997                 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI");
 998                 if (ifp->if_flags & IFF_RUNNING) {
 999                         IXGBE_CORE_LOCK(adapter);
1000                         ixgbe_disable_intr(adapter);
1001                         ixgbe_set_multi(adapter);
1002                         ixgbe_enable_intr(adapter);
1003                         IXGBE_CORE_UNLOCK(adapter);
1004                 }
1005                 break;
1006         case SIOCSIFMEDIA:
1007         case SIOCGIFMEDIA:
1008                 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1009                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1010                 break;
1011         case SIOCSIFCAP:
1012         {
1013                 int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1014                 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1015                 if (mask & IFCAP_HWCSUM)
1016                         ifp->if_capenable ^= IFCAP_HWCSUM;
1017 #if 0 /* NET_TSO */
1018                 if (mask & IFCAP_TSO4)
1019                         ifp->if_capenable ^= IFCAP_TSO4;
1020                 if (mask & IFCAP_TSO6)
1021                         ifp->if_capenable ^= IFCAP_TSO6;
1022 #endif
1023 #if 0 /* NET_LRO */
1024                 if (mask & IFCAP_LRO)
1025                         ifp->if_capenable ^= IFCAP_LRO;
1026 #endif
1027                 if (mask & IFCAP_VLAN_HWTAGGING)
1028                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1029                 if (mask & IFCAP_VLAN_HWFILTER)
1030                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1031 #if 0 /* NET_TSO */
1032                 if (mask & IFCAP_VLAN_HWTSO)
1033                         ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1034 #endif
1035                 if (ifp->if_flags & IFF_RUNNING) {
1036                         IXGBE_CORE_LOCK(adapter);
1037                         ixgbe_init_locked(adapter);
1038                         IXGBE_CORE_UNLOCK(adapter);
1039                 }
1040 #if 0
1041                 VLAN_CAPABILITIES(ifp);
1042 #endif
1043                 break;
1044         }
1045
1046         default:
1047                 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command);
1048                 error = ether_ioctl(ifp, command, data);
1049                 break;
1050         }
1051
1052         return (error);
1053 }
1054
1055 /*********************************************************************
1056  *  Init entry point
1057  *
1058  *  This routine is used in two ways. It is used by the stack as
1059  *  init entry point in network interface structure. It is also used
1060  *  by the driver as a hw/sw initialization routine to get to a
1061  *  consistent state.
1062  *
1063  *  return 0 on success, positive on failure
1064  **********************************************************************/
1065 #define IXGBE_MHADD_MFS_SHIFT 16
1066
1067 static void
1068 ixgbe_init_locked(struct adapter *adapter)
1069 {
1070         struct ifnet   *ifp = adapter->ifp;
1071         device_t        dev = adapter->dev;
1072         struct ixgbe_hw *hw = &adapter->hw;
1073         u32             k, txdctl, mhadd, gpie;
1074         u32             rxdctl, rxctrl;
1075
1076         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
1077         INIT_DEBUGOUT("ixgbe_init: begin");
1078         hw->adapter_stopped = FALSE;
1079         ixgbe_stop_adapter(hw);
1080         callout_stop(&adapter->timer);
1081
1082         /* reprogram the RAR[0] in case user changed it. */
1083         ixgbe_set_rar(hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
1084
1085         /* Get the latest mac address, User can use a LAA */
1086         bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr,
1087               IXGBE_ETH_LENGTH_OF_ADDRESS);
1088         ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1);
1089         hw->addr_ctrl.rar_used_count = 1;
1090
1091         /* Set the various hardware offload abilities */
1092         ifp->if_hwassist = 0;
1093 #if 0 /* NET_TSO */
1094         if (ifp->if_capenable & IFCAP_TSO)
1095                 ifp->if_hwassist |= CSUM_TSO;
1096 #endif
1097         if (ifp->if_capenable & IFCAP_TXCSUM) {
1098                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1099 #if 0
1100                 if (hw->mac.type != ixgbe_mac_82598EB)
1101                         ifp->if_hwassist |= CSUM_SCTP;
1102 #endif
1103         }
1104
1105         /* Prepare transmit descriptors and buffers */
1106         if (ixgbe_setup_transmit_structures(adapter)) {
1107                 device_printf(dev,"Could not setup transmit structures\n");
1108                 ixgbe_stop(adapter);
1109                 return;
1110         }
1111
1112         ixgbe_init_hw(hw);
1113         ixgbe_initialize_transmit_units(adapter);
1114
1115         /* Setup Multicast table */
1116         ixgbe_set_multi(adapter);
1117
1118         /*
1119         ** Determine the correct mbuf pool
1120         ** for doing jumbo/headersplit
1121         */
1122         if (adapter->max_frame_size <= 2048)
1123                 adapter->rx_mbuf_sz = MCLBYTES;
1124         else if (adapter->max_frame_size <= 4096)
1125                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1126         else if (adapter->max_frame_size <= 9216)
1127                 adapter->rx_mbuf_sz = MJUM9BYTES;
1128         else
1129                 adapter->rx_mbuf_sz = MJUM16BYTES;
1130
1131         /* Prepare receive descriptors and buffers */
1132         if (ixgbe_setup_receive_structures(adapter)) {
1133                 device_printf(dev,"Could not setup receive structures\n");
1134                 ixgbe_stop(adapter);
1135                 return;
1136         }
1137
1138         /* Configure RX settings */
1139         ixgbe_initialize_receive_units(adapter);
1140
1141         gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE);
1142
1143         /* Enable Fan Failure Interrupt */
1144         gpie |= IXGBE_SDP1_GPIEN;
1145
1146         /* Add for Module detection */
1147         if (hw->mac.type == ixgbe_mac_82599EB)
1148                 gpie |= IXGBE_SDP2_GPIEN;
1149
1150         /* Thermal Failure Detection */
1151         if (hw->mac.type == ixgbe_mac_X540)
1152                 gpie |= IXGBE_SDP0_GPIEN;
1153
1154         if (adapter->msix > 1) {
1155                 /* Enable Enhanced MSIX mode */
1156                 gpie |= IXGBE_GPIE_MSIX_MODE;
1157                 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
1158                     IXGBE_GPIE_OCD;
1159         }
1160         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
1161
1162         /* Set MTU size */
1163         if (ifp->if_mtu > ETHERMTU) {
1164                 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
1165                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
1166                 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
1167                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
1168         }
1169
1170         /* Now enable all the queues */
1171
1172         for (int i = 0; i < adapter->num_queues; i++) {
1173                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
1174                 txdctl |= IXGBE_TXDCTL_ENABLE;
1175                 /* Set WTHRESH to 8, burst writeback */
1176                 txdctl |= (8 << 16);
1177                 /*
1178                  * When the internal queue falls below PTHRESH (32),
1179                  * start prefetching as long as there are at least
1180                  * HTHRESH (1) buffers ready. The values are taken
1181                  * from the Intel linux driver 3.8.21.
1182                  * Prefetching enables tx line rate even with 1 queue.
1183                  */
1184                 txdctl |= (32 << 0) | (1 << 8);
1185                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(i), txdctl);
1186         }
1187
1188         for (int i = 0; i < adapter->num_queues; i++) {
1189                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
1190                 if (hw->mac.type == ixgbe_mac_82598EB) {
1191                         /*
1192                         ** PTHRESH = 21
1193                         ** HTHRESH = 4
1194                         ** WTHRESH = 8
1195                         */
1196                         rxdctl &= ~0x3FFFFF;
1197                         rxdctl |= 0x080420;
1198                 }
1199                 rxdctl |= IXGBE_RXDCTL_ENABLE;
1200                 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), rxdctl);
1201                 for (k = 0; k < 10; k++) {
1202                         if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) &
1203                             IXGBE_RXDCTL_ENABLE)
1204                                 break;
1205                         else
1206                                 msec_delay(1);
1207                 }
1208                 wmb();
1209 #ifdef DEV_NETMAP
1210                 /*
1211                  * In netmap mode, we must preserve the buffers made
1212                  * available to userspace before the if_init()
1213                  * (this is true by default on the TX side, because
1214                  * init makes all buffers available to userspace).
1215                  *
1216                  * netmap_reset() and the device specific routines
1217                  * (e.g. ixgbe_setup_receive_rings()) map these
1218                  * buffers at the end of the NIC ring, so here we
1219                  * must set the RDT (tail) register to make sure
1220                  * they are not overwritten.
1221                  *
1222                  * In this driver the NIC ring starts at RDH = 0,
1223                  * RDT points to the last slot available for reception (?),
1224                  * so RDT = num_rx_desc - 1 means the whole ring is available.
1225                  */
1226                 if (ifp->if_capenable & IFCAP_NETMAP) {
1227                         struct netmap_adapter *na = NA(adapter->ifp);
1228                         struct netmap_kring *kring = &na->rx_rings[i];
1229                         int t = na->num_rx_desc - 1 - kring->nr_hwavail;
1230
1231                         IXGBE_WRITE_REG(hw, IXGBE_RDT(i), t);
1232                 } else
1233 #endif /* DEV_NETMAP */
1234                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1);
1235         }
1236
1237         /* Set up VLAN support and filter */
1238         ixgbe_setup_vlan_hw_support(adapter);
1239
1240         /* Enable Receive engine */
1241         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
1242         if (hw->mac.type == ixgbe_mac_82598EB)
1243                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
1244         rxctrl |= IXGBE_RXCTRL_RXEN;
1245         ixgbe_enable_rx_dma(hw, rxctrl);
1246
1247         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
1248
1249         /* Set up MSI/X routing */
1250         if (ixgbe_enable_msix)  {
1251                 ixgbe_configure_ivars(adapter);
1252                 /* Set up auto-mask */
1253                 if (hw->mac.type == ixgbe_mac_82598EB)
1254                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1255                 else {
1256                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
1257                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
1258                 }
1259         } else {  /* Simple settings for Legacy/MSI */
1260                 ixgbe_set_ivar(adapter, 0, 0, 0);
1261                 ixgbe_set_ivar(adapter, 0, 0, 1);
1262                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
1263         }
1264
1265 #ifdef IXGBE_FDIR
1266         /* Init Flow director */
1267         if (hw->mac.type != ixgbe_mac_82598EB) {
1268                 u32 hdrm = 32 << fdir_pballoc;
1269
1270                 hw->mac.ops.setup_rxpba(hw, 0, hdrm, PBA_STRATEGY_EQUAL);
1271                 ixgbe_init_fdir_signature_82599(&adapter->hw, fdir_pballoc);
1272         }
1273 #endif
1274
1275         /*
1276         ** Check on any SFP devices that
1277         ** need to be kick-started
1278         */
1279         if (hw->phy.type == ixgbe_phy_none) {
1280                 int err = hw->phy.ops.identify(hw);
1281                 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
1282                         device_printf(dev,
1283                             "Unsupported SFP+ module type was detected.\n");
1284                         return;
1285                 }
1286         }
1287
1288         /* Set moderation on the Link interrupt */
1289         IXGBE_WRITE_REG(hw, IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR);
1290
1291         /* Config/Enable Link */
1292         ixgbe_config_link(adapter);
1293
1294         /* Hardware Packet Buffer & Flow Control setup */
1295         {
1296                 u32 rxpb, frame, size, tmp;
1297
1298                 frame = adapter->max_frame_size;
1299
1300                 /* Calculate High Water */
1301                 if (hw->mac.type == ixgbe_mac_X540)
1302                         tmp = IXGBE_DV_X540(frame, frame);
1303                 else
1304                         tmp = IXGBE_DV(frame, frame);
1305                 size = IXGBE_BT2KB(tmp);
1306                 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
1307                 hw->fc.high_water[0] = rxpb - size;
1308
1309                 /* Now calculate Low Water */
1310                 if (hw->mac.type == ixgbe_mac_X540)
1311                         tmp = IXGBE_LOW_DV_X540(frame);
1312                 else
1313                         tmp = IXGBE_LOW_DV(frame);
1314                 hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1315
1316                 adapter->fc = hw->fc.requested_mode = ixgbe_fc_full;
1317                 hw->fc.pause_time = IXGBE_FC_PAUSE;
1318                 hw->fc.send_xon = TRUE;
1319         }
1320         /* Initialize the FC settings */
1321         ixgbe_start_hw(hw);
1322
1323         /* And now turn on interrupts */
1324         ixgbe_enable_intr(adapter);
1325
1326         /* Now inform the stack we're ready */
1327         ifp->if_flags |= IFF_RUNNING;
1328         ifp->if_flags &= ~IFF_OACTIVE;
1329
1330         return;
1331 }
1332
1333 static void
1334 ixgbe_init(void *arg)
1335 {
1336         struct adapter *adapter = arg;
1337
1338         IXGBE_CORE_LOCK(adapter);
1339         ixgbe_init_locked(adapter);
1340         IXGBE_CORE_UNLOCK(adapter);
1341         return;
1342 }
1343
1344
1345 /*
1346 **
1347 ** MSIX Interrupt Handlers and Tasklets
1348 **
1349 */
1350
1351 static inline void
1352 ixgbe_enable_queue(struct adapter *adapter, u32 vector)
1353 {
1354         struct ixgbe_hw *hw = &adapter->hw;
1355         u64     queue = (u64)(1 << vector);
1356         u32     mask;
1357
1358         if (hw->mac.type == ixgbe_mac_82598EB) {
1359                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1360                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1361         } else {
1362                 mask = (queue & 0xFFFFFFFF);
1363                 if (mask)
1364                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1365                 mask = (queue >> 32);
1366                 if (mask)
1367                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1368         }
1369 }
1370
1371 static inline void
1372 ixgbe_disable_queue(struct adapter *adapter, u32 vector)
1373 {
1374         struct ixgbe_hw *hw = &adapter->hw;
1375         u64     queue = (u64)(1 << vector);
1376         u32     mask;
1377
1378         if (hw->mac.type == ixgbe_mac_82598EB) {
1379                 mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1380                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1381         } else {
1382                 mask = (queue & 0xFFFFFFFF);
1383                 if (mask)
1384                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1385                 mask = (queue >> 32);
1386                 if (mask)
1387                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1388         }
1389 }
1390
1391 static inline void
1392 ixgbe_rearm_queues(struct adapter *adapter, u64 queues)
1393 {
1394         u32 mask;
1395
1396         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
1397                 mask = (IXGBE_EIMS_RTX_QUEUE & queues);
1398                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
1399         } else {
1400                 mask = (queues & 0xFFFFFFFF);
1401                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
1402                 mask = (queues >> 32);
1403                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
1404         }
1405 }
1406
1407
1408 static void
1409 ixgbe_handle_que(void *context, int pending)
1410 {
1411         struct ix_queue *que = context;
1412         struct adapter  *adapter = que->adapter;
1413         struct tx_ring  *txr = que->txr;
1414         struct ifnet    *ifp = adapter->ifp;
1415         bool            more;
1416
1417         if (ifp->if_flags & IFF_RUNNING) {
1418                 more = ixgbe_rxeof(que, adapter->rx_process_limit);
1419                 IXGBE_TX_LOCK(txr);
1420                 ixgbe_txeof(txr);
1421 #if 0 /*__FreeBSD_version >= 800000*/
1422                 if (!drbr_empty(ifp, txr->br))
1423                         ixgbe_mq_start_locked(ifp, txr, NULL);
1424 #else
1425                 if (!ifq_is_empty(&ifp->if_snd))
1426                         ixgbe_start_locked(txr, ifp);
1427 #endif
1428                 IXGBE_TX_UNLOCK(txr);
1429                 if (more) {
1430                         taskqueue_enqueue(que->tq, &que->que_task);
1431                         return;
1432                 }
1433         }
1434
1435         /* Reenable this interrupt */
1436         ixgbe_enable_queue(adapter, que->msix);
1437         return;
1438 }
1439
1440
1441 /*********************************************************************
1442  *
1443  *  Legacy Interrupt Service routine
1444  *
1445  **********************************************************************/
1446
1447 static void
1448 ixgbe_legacy_irq(void *arg)
1449 {
1450         struct ix_queue *que = arg;
1451         struct adapter  *adapter = que->adapter;
1452         struct ixgbe_hw *hw = &adapter->hw;
1453         struct          tx_ring *txr = adapter->tx_rings;
1454         bool            more_tx, more_rx;
1455         u32             reg_eicr, loop = MAX_LOOP;
1456
1457
1458         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1459
1460         ++que->irqs;
1461         if (reg_eicr == 0) {
1462                 ixgbe_enable_intr(adapter);
1463                 return;
1464         }
1465
1466         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1467
1468         IXGBE_TX_LOCK(txr);
1469         do {
1470                 more_tx = ixgbe_txeof(txr);
1471         } while (loop-- && more_tx);
1472         IXGBE_TX_UNLOCK(txr);
1473
1474         if (more_rx || more_tx)
1475                 taskqueue_enqueue(que->tq, &que->que_task);
1476
1477         /* Check for fan failure */
1478         if ((hw->phy.media_type == ixgbe_media_type_copper) &&
1479             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1480                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1481                     "REPLACE IMMEDIATELY!!\n");
1482                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1);
1483         }
1484
1485         /* Link status change */
1486         if (reg_eicr & IXGBE_EICR_LSC)
1487                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1488
1489         ixgbe_enable_intr(adapter);
1490         return;
1491 }
1492
1493
1494 /*********************************************************************
1495  *
1496  *  MSIX Queue Interrupt Service routine
1497  *
1498  **********************************************************************/
1499 void
1500 ixgbe_msix_que(void *arg)
1501 {
1502         struct ix_queue *que = arg;
1503         struct adapter  *adapter = que->adapter;
1504         struct tx_ring  *txr = que->txr;
1505         struct rx_ring  *rxr = que->rxr;
1506         bool            more_tx, more_rx;
1507         u32             newitr = 0;
1508
1509         ixgbe_disable_queue(adapter, que->msix);
1510         ++que->irqs;
1511
1512         more_rx = ixgbe_rxeof(que, adapter->rx_process_limit);
1513
1514         IXGBE_TX_LOCK(txr);
1515         more_tx = ixgbe_txeof(txr);
1516         /*
1517         ** Make certain that if the stack
1518         ** has anything queued the task gets
1519         ** scheduled to handle it.
1520         */
1521 #if 0
1522 #if __FreeBSD_version < 800000
1523         if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd))
1524 #else
1525         if (!drbr_empty(adapter->ifp, txr->br))
1526 #endif
1527 #endif
1528         if (!ifq_is_empty(&adapter->ifp->if_snd))
1529                 more_tx = 1;
1530         IXGBE_TX_UNLOCK(txr);
1531
1532         /* Do AIM now? */
1533
1534         if (ixgbe_enable_aim == FALSE)
1535                 goto no_calc;
1536         /*
1537         ** Do Adaptive Interrupt Moderation:
1538         **  - Write out last calculated setting
1539         **  - Calculate based on average size over
1540         **    the last interval.
1541         */
1542         if (que->eitr_setting)
1543                 IXGBE_WRITE_REG(&adapter->hw,
1544                     IXGBE_EITR(que->msix), que->eitr_setting);
1545
1546         que->eitr_setting = 0;
1547
1548         /* Idle, do nothing */
1549         if ((txr->bytes == 0) && (rxr->bytes == 0))
1550                 goto no_calc;
1551
1552         if ((txr->bytes) && (txr->packets))
1553                 newitr = txr->bytes/txr->packets;
1554         if ((rxr->bytes) && (rxr->packets))
1555                 newitr = max(newitr,
1556                     (rxr->bytes / rxr->packets));
1557         newitr += 24; /* account for hardware frame, crc */
1558
1559         /* set an upper boundary */
1560         newitr = min(newitr, 3000);
1561
1562         /* Be nice to the mid range */
1563         if ((newitr > 300) && (newitr < 1200))
1564                 newitr = (newitr / 3);
1565         else
1566                 newitr = (newitr / 2);
1567
1568         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
1569                 newitr |= newitr << 16;
1570         else
1571                 newitr |= IXGBE_EITR_CNT_WDIS;
1572
1573         /* save for next interrupt */
1574         que->eitr_setting = newitr;
1575
1576         /* Reset state */
1577         txr->bytes = 0;
1578         txr->packets = 0;
1579         rxr->bytes = 0;
1580         rxr->packets = 0;
1581
1582 no_calc:
1583         if (more_tx || more_rx)
1584                 taskqueue_enqueue(que->tq, &que->que_task);
1585         else /* Reenable this interrupt */
1586                 ixgbe_enable_queue(adapter, que->msix);
1587         return;
1588 }
1589
1590
1591 static void
1592 ixgbe_msix_link(void *arg)
1593 {
1594         struct adapter  *adapter = arg;
1595         struct ixgbe_hw *hw = &adapter->hw;
1596         u32             reg_eicr;
1597
1598         ++adapter->link_irq;
1599
1600         /* First get the cause */
1601         reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1602         /* Clear interrupt with write */
1603         IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1604
1605         /* Link status change */
1606         if (reg_eicr & IXGBE_EICR_LSC)
1607                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1608
1609         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
1610 #ifdef IXGBE_FDIR
1611                 if (reg_eicr & IXGBE_EICR_FLOW_DIR) {
1612                         /* This is probably overkill :) */
1613                         if (!atomic_cmpset_int(&adapter->fdir_reinit, 0, 1))
1614                                 return;
1615                         /* Disable the interrupt */
1616                         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EICR_FLOW_DIR);
1617                         taskqueue_enqueue(adapter->tq, &adapter->fdir_task);
1618                 } else
1619 #endif
1620                 if (reg_eicr & IXGBE_EICR_ECC) {
1621                         device_printf(adapter->dev, "\nCRITICAL: ECC ERROR!! "
1622                             "Please Reboot!!\n");
1623                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1624                 } else
1625
1626                 if (reg_eicr & IXGBE_EICR_GPI_SDP1) {
1627                         /* Clear the interrupt */
1628                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1629                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
1630                 } else if (reg_eicr & IXGBE_EICR_GPI_SDP2) {
1631                         /* Clear the interrupt */
1632                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1633                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
1634                 }
1635         }
1636
1637         /* Check for fan failure */
1638         if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1639             (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1640                 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! "
1641                     "REPLACE IMMEDIATELY!!\n");
1642                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1643         }
1644
1645         /* Check for over temp condition */
1646         if ((hw->mac.type == ixgbe_mac_X540) &&
1647             (reg_eicr & IXGBE_EICR_GPI_SDP0)) {
1648                 device_printf(adapter->dev, "\nCRITICAL: OVER TEMP!! "
1649                     "PHY IS SHUT DOWN!!\n");
1650                 device_printf(adapter->dev, "System shutdown required\n");
1651                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1652         }
1653
1654         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1655         return;
1656 }
1657
1658 /*********************************************************************
1659  *
1660  *  Media Ioctl callback
1661  *
1662  *  This routine is called whenever the user queries the status of
1663  *  the interface using ifconfig.
1664  *
1665  **********************************************************************/
1666 static void
1667 ixgbe_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1668 {
1669         struct adapter *adapter = ifp->if_softc;
1670
1671         INIT_DEBUGOUT("ixgbe_media_status: begin");
1672         IXGBE_CORE_LOCK(adapter);
1673         ixgbe_update_link_status(adapter);
1674
1675         ifmr->ifm_status = IFM_AVALID;
1676         ifmr->ifm_active = IFM_ETHER;
1677
1678         if (!adapter->link_active) {
1679                 IXGBE_CORE_UNLOCK(adapter);
1680                 return;
1681         }
1682
1683         ifmr->ifm_status |= IFM_ACTIVE;
1684
1685         switch (adapter->link_speed) {
1686                 case IXGBE_LINK_SPEED_100_FULL:
1687                         ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1688                         break;
1689                 case IXGBE_LINK_SPEED_1GB_FULL:
1690                         ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1691                         break;
1692                 case IXGBE_LINK_SPEED_10GB_FULL:
1693                         ifmr->ifm_active |= adapter->optics | IFM_FDX;
1694                         break;
1695         }
1696
1697         IXGBE_CORE_UNLOCK(adapter);
1698
1699         return;
1700 }
1701
1702 /*********************************************************************
1703  *
1704  *  Media Ioctl callback
1705  *
1706  *  This routine is called when the user changes speed/duplex using
1707  *  media/mediopt option with ifconfig.
1708  *
1709  **********************************************************************/
1710 static int
1711 ixgbe_media_change(struct ifnet * ifp)
1712 {
1713         struct adapter *adapter = ifp->if_softc;
1714         struct ifmedia *ifm = &adapter->media;
1715
1716         INIT_DEBUGOUT("ixgbe_media_change: begin");
1717
1718         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1719                 return (EINVAL);
1720
1721         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1722         case IFM_AUTO:
1723                 adapter->hw.phy.autoneg_advertised =
1724                     IXGBE_LINK_SPEED_100_FULL |
1725                     IXGBE_LINK_SPEED_1GB_FULL |
1726                     IXGBE_LINK_SPEED_10GB_FULL;
1727                 break;
1728         default:
1729                 device_printf(adapter->dev, "Only auto media type\n");
1730                 return (EINVAL);
1731         }
1732
1733         return (0);
1734 }
1735
1736 /*********************************************************************
1737  *
1738  *  This routine maps the mbufs to tx descriptors, allowing the
1739  *  TX engine to transmit the packets.
1740  *      - return 0 on success, positive on failure
1741  *
1742  **********************************************************************/
1743
1744 static int
1745 ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1746 {
1747         struct adapter  *adapter = txr->adapter;
1748         u32             olinfo_status = 0, cmd_type_len;
1749         u32             paylen = 0;
1750         int             i, j, error, nsegs;
1751         int             first, last = 0;
1752         struct mbuf     *m_head;
1753         bus_dma_segment_t segs[1];
1754         bus_dmamap_t    map;
1755         struct ixgbe_tx_buf *txbuf;
1756         union ixgbe_adv_tx_desc *txd = NULL;
1757
1758         m_head = *m_headp;
1759
1760         /* Basic descriptor defines */
1761         cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1762             IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1763
1764         if (m_head->m_flags & M_VLANTAG)
1765                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
1766
1767         /*
1768          * Important to capture the first descriptor
1769          * used because it will contain the index of
1770          * the one we tell the hardware to report back
1771          */
1772         first = txr->next_avail_desc;
1773         txbuf = &txr->tx_buffers[first];
1774         map = txbuf->map;
1775
1776         /*
1777          * Map the packet for DMA.
1778          */
1779         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1780             *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1781
1782         if (error == EFBIG) {
1783                 struct mbuf *m;
1784
1785                 m = m_defrag(*m_headp, MB_DONTWAIT);
1786                 if (m == NULL) {
1787                         adapter->mbuf_defrag_failed++;
1788                         m_freem(*m_headp);
1789                         *m_headp = NULL;
1790                         return (ENOBUFS);
1791                 }
1792                 *m_headp = m;
1793
1794                 /* Try it again */
1795                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1796                     *m_headp, segs, 1, &nsegs, BUS_DMA_NOWAIT);
1797
1798                 if (error == ENOMEM) {
1799                         adapter->no_tx_dma_setup++;
1800                         return (error);
1801                 } else if (error != 0) {
1802                         adapter->no_tx_dma_setup++;
1803                         m_freem(*m_headp);
1804                         *m_headp = NULL;
1805                         return (error);
1806                 }
1807         } else if (error == ENOMEM) {
1808                 adapter->no_tx_dma_setup++;
1809                 return (error);
1810         } else if (error != 0) {
1811                 adapter->no_tx_dma_setup++;
1812                 m_freem(*m_headp);
1813                 *m_headp = NULL;
1814                 return (error);
1815         }
1816
1817         /* Make certain there are enough descriptors */
1818         if (nsegs > txr->tx_avail - 2) {
1819                 txr->no_desc_avail++;
1820                 error = ENOBUFS;
1821                 goto xmit_fail;
1822         }
1823         m_head = *m_headp;
1824
1825         /*
1826         ** Set up the appropriate offload context
1827         ** this becomes the first descriptor of
1828         ** a packet.
1829         */
1830 #if 0 /* NET_TSO */
1831         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1832                 if (ixgbe_tso_setup(txr, m_head, &paylen, &olinfo_status)) {
1833                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
1834                         olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1835                         olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
1836                         ++adapter->tso_tx;
1837                 } else
1838                         return (ENXIO);
1839         } else if (ixgbe_tx_ctx_setup(txr, m_head))
1840 #endif
1841         if (ixgbe_tx_ctx_setup(txr, m_head))
1842                 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
1843
1844 #ifdef IXGBE_IEEE1588
1845         /* This is changing soon to an mtag detection */
1846         if (we detect this mbuf has a TSTAMP mtag)
1847                 cmd_type_len |= IXGBE_ADVTXD_MAC_TSTAMP;
1848 #endif
1849
1850 #ifdef IXGBE_FDIR
1851         /* Do the flow director magic */
1852         if ((txr->atr_sample) && (!adapter->fdir_reinit)) {
1853                 ++txr->atr_count;
1854                 if (txr->atr_count >= atr_sample_rate) {
1855                         ixgbe_atr(txr, m_head);
1856                         txr->atr_count = 0;
1857                 }
1858         }
1859 #endif
1860         /* Record payload length */
1861         if (paylen == 0)
1862                 olinfo_status |= m_head->m_pkthdr.len <<
1863                     IXGBE_ADVTXD_PAYLEN_SHIFT;
1864
1865         i = txr->next_avail_desc;
1866         for (j = 0; j < nsegs; j++) {
1867                 bus_size_t seglen;
1868                 bus_addr_t segaddr;
1869
1870                 txbuf = &txr->tx_buffers[i];
1871                 txd = &txr->tx_base[i];
1872                 seglen = segs[j].ds_len;
1873                 segaddr = htole64(segs[j].ds_addr);
1874
1875                 txd->read.buffer_addr = segaddr;
1876                 txd->read.cmd_type_len = htole32(txr->txd_cmd |
1877                     cmd_type_len |seglen);
1878                 txd->read.olinfo_status = htole32(olinfo_status);
1879                 last = i; /* descriptor that will get completion IRQ */
1880
1881                 if (++i == adapter->num_tx_desc)
1882                         i = 0;
1883
1884                 txbuf->m_head = NULL;
1885                 txbuf->eop_index = -1;
1886         }
1887
1888         txd->read.cmd_type_len |=
1889             htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1890         txr->tx_avail -= nsegs;
1891         txr->next_avail_desc = i;
1892
1893         txbuf->m_head = m_head;
1894         /* Swap the dma map between the first and last descriptor */
1895         txr->tx_buffers[first].map = txbuf->map;
1896         txbuf->map = map;
1897         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1898
1899         /* Set the index of the descriptor that will be marked done */
1900         txbuf = &txr->tx_buffers[first];
1901         txbuf->eop_index = last;
1902
1903         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1904             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1905         /*
1906          * Advance the Transmit Descriptor Tail (Tdt), this tells the
1907          * hardware that this frame is available to transmit.
1908          */
1909         ++txr->total_packets;
1910         IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i);
1911
1912         return (0);
1913
1914 xmit_fail:
1915         bus_dmamap_unload(txr->txtag, txbuf->map);
1916         return (error);
1917
1918 }
1919
1920 static void
1921 ixgbe_set_promisc(struct adapter *adapter)
1922 {
1923         u_int32_t       reg_rctl;
1924         struct ifnet   *ifp = adapter->ifp;
1925
1926         reg_rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1927         reg_rctl &= (~IXGBE_FCTRL_UPE);
1928         reg_rctl &= (~IXGBE_FCTRL_MPE);
1929         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1930
1931         if (ifp->if_flags & IFF_PROMISC) {
1932                 reg_rctl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1933                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1934         } else if (ifp->if_flags & IFF_ALLMULTI) {
1935                 reg_rctl |= IXGBE_FCTRL_MPE;
1936                 reg_rctl &= ~IXGBE_FCTRL_UPE;
1937                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, reg_rctl);
1938         }
1939         return;
1940 }
1941
1942
1943 /*********************************************************************
1944  *  Multicast Update
1945  *
1946  *  This routine is called whenever multicast address list is updated.
1947  *
1948  **********************************************************************/
1949 #define IXGBE_RAR_ENTRIES 16
1950
1951 static void
1952 ixgbe_set_multi(struct adapter *adapter)
1953 {
1954         u32     fctrl;
1955         u8      *mta;
1956         u8      *update_ptr;
1957         struct  ifmultiaddr *ifma;
1958         int     mcnt = 0;
1959         struct ifnet   *ifp = adapter->ifp;
1960
1961         IOCTL_DEBUGOUT("ixgbe_set_multi: begin");
1962
1963         mta = adapter->mta;
1964         bzero(mta, sizeof(u8) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1965             MAX_NUM_MULTICAST_ADDRESSES);
1966
1967         fctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_FCTRL);
1968         fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1969         if (ifp->if_flags & IFF_PROMISC)
1970                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1971         else if (ifp->if_flags & IFF_ALLMULTI) {
1972                 fctrl |= IXGBE_FCTRL_MPE;
1973                 fctrl &= ~IXGBE_FCTRL_UPE;
1974         } else
1975                 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
1976
1977         IXGBE_WRITE_REG(&adapter->hw, IXGBE_FCTRL, fctrl);
1978
1979         spin_lock(&adapter->mcast_spin);
1980         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1981                 if (ifma->ifma_addr->sa_family != AF_LINK)
1982                         continue;
1983                 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
1984                     &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1985                     IXGBE_ETH_LENGTH_OF_ADDRESS);
1986                 mcnt++;
1987         }
1988         spin_unlock(&adapter->mcast_spin);
1989
1990         update_ptr = mta;
1991         ixgbe_update_mc_addr_list(&adapter->hw,
1992             update_ptr, mcnt, ixgbe_mc_array_itr, TRUE);
1993
1994         return;
1995 }
1996
1997 /*
1998  * This is an iterator function now needed by the multicast
1999  * shared code. It simply feeds the shared code routine the
2000  * addresses in the array of ixgbe_set_multi() one by one.
2001  */
2002 static u8 *
2003 ixgbe_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq)
2004 {
2005         u8 *addr = *update_ptr;
2006         u8 *newptr;
2007         *vmdq = 0;
2008
2009         newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
2010         *update_ptr = newptr;
2011         return addr;
2012 }
2013
2014
2015 /*********************************************************************
2016  *  Timer routine
2017  *
2018  *  This routine checks for link status,updates statistics,
2019  *  and runs the watchdog check.
2020  *
2021  **********************************************************************/
2022
2023 static void
2024 ixgbe_local_timer(void *arg)
2025 {
2026         struct adapter  *adapter = arg;
2027         device_t        dev = adapter->dev;
2028         struct ifnet    *ifp = adapter->ifp;
2029         struct ix_queue *que = adapter->queues;
2030         struct tx_ring  *txr = adapter->tx_rings;
2031         int             hung, busy, paused;
2032
2033         IXGBE_CORE_LOCK(adapter);
2034         hung = busy = paused = 0;
2035
2036         /* Check for pluggable optics */
2037         if (adapter->sfp_probe)
2038                 if (!ixgbe_sfp_probe(adapter))
2039                         goto out; /* Nothing to do */
2040
2041         ixgbe_update_link_status(adapter);
2042         ixgbe_update_stats_counters(adapter);
2043
2044         /*
2045          * If the interface has been paused
2046          * then don't do the watchdog check
2047          */
2048         if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)
2049                 paused = 1;
2050
2051         /*
2052         ** Check the TX queues status
2053         **      - central locked handling of OACTIVE
2054         **      - watchdog only if all queues show hung
2055         */
2056         for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
2057                 if ((txr->queue_status & IXGBE_QUEUE_HUNG) &&
2058                     (paused == 0))
2059                         ++hung;
2060                 if (txr->queue_status & IXGBE_QUEUE_DEPLETED)
2061                         ++busy;
2062                 if ((txr->queue_status & IXGBE_QUEUE_IDLE) == 0)
2063                         taskqueue_enqueue(que->tq, &que->que_task);
2064         }
2065         /* Only truely watchdog if all queues show hung */
2066         if (hung == adapter->num_queues)
2067                 goto watchdog;
2068         /* Only turn off the stack flow when ALL are depleted */
2069         if (busy == adapter->num_queues)
2070                 ifp->if_flags |= IFF_OACTIVE;
2071         else if ((ifp->if_flags & IFF_OACTIVE) &&
2072             (busy < adapter->num_queues))
2073                 ifp->if_flags &= ~IFF_OACTIVE;
2074
2075 out:
2076         ixgbe_rearm_queues(adapter, adapter->que_mask);
2077         callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter);
2078         IXGBE_CORE_UNLOCK(adapter);
2079         return;
2080
2081 watchdog:
2082         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2083         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
2084             IXGBE_READ_REG(&adapter->hw, IXGBE_TDH(txr->me)),
2085             IXGBE_READ_REG(&adapter->hw, IXGBE_TDT(txr->me)));
2086         device_printf(dev,"TX(%d) desc avail = %d,"
2087             "Next TX to Clean = %d\n",
2088             txr->me, txr->tx_avail, txr->next_to_clean);
2089         adapter->ifp->if_flags &= ~IFF_RUNNING;
2090         adapter->watchdog_events++;
2091         ixgbe_init_locked(adapter);
2092
2093         IXGBE_CORE_UNLOCK(adapter);
2094 }
2095
2096 /*
2097 ** Note: this routine updates the OS on the link state
2098 **      the real check of the hardware only happens with
2099 **      a link interrupt.
2100 */
2101 static void
2102 ixgbe_update_link_status(struct adapter *adapter)
2103 {
2104         struct ifnet    *ifp = adapter->ifp;
2105         struct tx_ring *txr = adapter->tx_rings;
2106         device_t dev = adapter->dev;
2107
2108
2109         if (adapter->link_up){
2110                 if (adapter->link_active == FALSE) {
2111                         if (bootverbose)
2112                                 device_printf(dev,"Link is up %d Gbps %s \n",
2113                                     ((adapter->link_speed == 128)? 10:1),
2114                                     "Full Duplex");
2115                         adapter->link_active = TRUE;
2116                         /* Update any Flow Control changes */
2117                         ixgbe_fc_enable(&adapter->hw);
2118                         ifp->if_link_state = LINK_STATE_UP;
2119                         if_link_state_change(ifp);
2120                 }
2121         } else { /* Link down */
2122                 if (adapter->link_active == TRUE) {
2123                         if (bootverbose)
2124                                 device_printf(dev,"Link is Down\n");
2125                         ifp->if_link_state = LINK_STATE_DOWN;
2126                         if_link_state_change(ifp);
2127                         adapter->link_active = FALSE;
2128                         for (int i = 0; i < adapter->num_queues;
2129                             i++, txr++)
2130                                 txr->queue_status = IXGBE_QUEUE_IDLE;
2131                 }
2132         }
2133
2134         return;
2135 }
2136
2137
2138 /*********************************************************************
2139  *
2140  *  This routine disables all traffic on the adapter by issuing a
2141  *  global reset on the MAC and deallocates TX/RX buffers.
2142  *
2143  **********************************************************************/
2144
2145 static void
2146 ixgbe_stop(void *arg)
2147 {
2148         struct ifnet   *ifp;
2149         struct adapter *adapter = arg;
2150         struct ixgbe_hw *hw = &adapter->hw;
2151         ifp = adapter->ifp;
2152
2153         KKASSERT(lockstatus(&adapter->core_lock, curthread) != 0);
2154
2155         INIT_DEBUGOUT("ixgbe_stop: begin\n");
2156         ixgbe_disable_intr(adapter);
2157         callout_stop(&adapter->timer);
2158
2159         /* Let the stack know...*/
2160         ifp->if_flags &= ~IFF_RUNNING;
2161         ifp->if_flags |= IFF_OACTIVE;
2162
2163         ixgbe_reset_hw(hw);
2164         hw->adapter_stopped = FALSE;
2165         ixgbe_stop_adapter(hw);
2166         /* Turn off the laser */
2167         if (hw->phy.multispeed_fiber)
2168                 ixgbe_disable_tx_laser(hw);
2169
2170         /* reprogram the RAR[0] in case user changed it. */
2171         ixgbe_set_rar(&adapter->hw, 0, adapter->hw.mac.addr, 0, IXGBE_RAH_AV);
2172
2173         return;
2174 }
2175
2176
2177 /*********************************************************************
2178  *
2179  *  Determine hardware revision.
2180  *
2181  **********************************************************************/
2182 static void
2183 ixgbe_identify_hardware(struct adapter *adapter)
2184 {
2185         device_t        dev = adapter->dev;
2186         struct ixgbe_hw *hw = &adapter->hw;
2187
2188         /* Save off the information about this board */
2189         hw->vendor_id = pci_get_vendor(dev);
2190         hw->device_id = pci_get_device(dev);
2191         hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
2192         hw->subsystem_vendor_id =
2193             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2194         hw->subsystem_device_id =
2195             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2196
2197         /* We need this here to set the num_segs below */
2198         ixgbe_set_mac_type(hw);
2199
2200         /* Pick up the 82599 and VF settings */
2201         if (hw->mac.type != ixgbe_mac_82598EB) {
2202                 hw->phy.smart_speed = ixgbe_smart_speed;
2203                 adapter->num_segs = IXGBE_82599_SCATTER;
2204         } else
2205                 adapter->num_segs = IXGBE_82598_SCATTER;
2206
2207         return;
2208 }
2209
2210 /*********************************************************************
2211  *
2212  *  Determine optic type
2213  *
2214  **********************************************************************/
2215 static void
2216 ixgbe_setup_optics(struct adapter *adapter)
2217 {
2218         struct ixgbe_hw *hw = &adapter->hw;
2219         int             layer;
2220
2221         layer = ixgbe_get_supported_physical_layer(hw);
2222
2223         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) {
2224                 adapter->optics = IFM_10G_T;
2225                 return;
2226         }
2227
2228         if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) {
2229                 adapter->optics = IFM_1000_T;
2230                 return;
2231         }
2232
2233         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_LR |
2234             IXGBE_PHYSICAL_LAYER_10GBASE_LRM)) {
2235                 adapter->optics = IFM_10G_LR;
2236                 return;
2237         }
2238
2239         if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
2240                 adapter->optics = IFM_10G_SR;
2241                 return;
2242         }
2243
2244         if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU) {
2245                 adapter->optics = IFM_10G_TWINAX;
2246                 return;
2247         }
2248
2249         if (layer & (IXGBE_PHYSICAL_LAYER_10GBASE_KX4 |
2250             IXGBE_PHYSICAL_LAYER_10GBASE_CX4)) {
2251                 adapter->optics = IFM_10G_CX4;
2252                 return;
2253         }
2254
2255         /* If we get here just set the default */
2256         adapter->optics = IFM_ETHER | IFM_AUTO;
2257         return;
2258 }
2259
2260 /*********************************************************************
2261  *
2262  *  Setup the Legacy or MSI Interrupt handler
2263  *
2264  **********************************************************************/
2265 static int
2266 ixgbe_allocate_legacy(struct adapter *adapter)
2267 {
2268         device_t dev = adapter->dev;
2269         struct          ix_queue *que = adapter->queues;
2270         int error, rid = 0;
2271         unsigned int intr_flags;
2272
2273         /* MSI RID at 1 */
2274         if (adapter->msix == 1)
2275                 rid = 1;
2276
2277         /* Try allocating a MSI interrupt first */
2278         adapter->intr_type = pci_alloc_1intr(dev, ixgbe_msi_enable,
2279                 &rid, &intr_flags);
2280
2281         /* We allocate a single interrupt resource */
2282         adapter->res = bus_alloc_resource_any(dev,
2283             SYS_RES_IRQ, &rid, intr_flags);
2284         if (adapter->res == NULL) {
2285                 device_printf(dev, "Unable to allocate bus resource: "
2286                     "interrupt\n");
2287                 return (ENXIO);
2288         }
2289
2290         /*
2291          * Try allocating a fast interrupt and the associated deferred
2292          * processing contexts.
2293          */
2294         TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2295         que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2296             taskqueue_thread_enqueue, &que->tq);
2297         taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s ixq",
2298             device_get_nameunit(adapter->dev));
2299
2300         /* Tasklets for Link, SFP and Multispeed Fiber */
2301         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2302         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2303         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2304 #ifdef IXGBE_FDIR
2305         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2306 #endif
2307         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2308             taskqueue_thread_enqueue, &adapter->tq);
2309         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2310             device_get_nameunit(adapter->dev));
2311
2312         if ((error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2313             ixgbe_legacy_irq, que, &adapter->tag, &adapter->serializer)) != 0) {
2314                 device_printf(dev, "Failed to register fast interrupt "
2315                     "handler: %d\n", error);
2316                 taskqueue_free(que->tq);
2317                 taskqueue_free(adapter->tq);
2318                 que->tq = NULL;
2319                 adapter->tq = NULL;
2320                 return (error);
2321         }
2322         /* For simplicity in the handlers */
2323         adapter->que_mask = IXGBE_EIMS_ENABLE_MASK;
2324
2325         return (0);
2326 }
2327
2328
2329 /*********************************************************************
2330  *
2331  *  Setup MSIX Interrupt resources and handlers
2332  *
2333  **********************************************************************/
2334 static int
2335 ixgbe_allocate_msix(struct adapter *adapter)
2336 {
2337         device_t        dev = adapter->dev;
2338         struct          ix_queue *que = adapter->queues;
2339         int             error, rid, vector = 0;
2340
2341         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2342                 rid = vector + 1;
2343                 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2344                     RF_SHAREABLE | RF_ACTIVE);
2345                 if (que->res == NULL) {
2346                         device_printf(dev,"Unable to allocate"
2347                             " bus resource: que interrupt [%d]\n", vector);
2348                         return (ENXIO);
2349                 }
2350                 /* Set the handler function */
2351                 error = bus_setup_intr(dev, que->res, INTR_MPSAFE,
2352                     ixgbe_msix_que, que, &que->tag, &que->serializer);
2353                 if (error) {
2354                         que->res = NULL;
2355                         device_printf(dev, "Failed to register QUE handler");
2356                         return (error);
2357                 }
2358 #if 0 /* __FreeBSD_version >= 800504 */
2359                 bus_describe_intr(dev, que->res, que->tag, "que %d", i);
2360 #endif
2361                 que->msix = vector;
2362                 adapter->que_mask |= (u64)(1 << que->msix);
2363                 /*
2364                 ** Bind the msix vector, and thus the
2365                 ** ring to the corresponding cpu.
2366                 */
2367 #if 0 /* XXX */
2368                 if (adapter->num_queues > 1)
2369                         bus_bind_intr(dev, que->res, i);
2370 #endif
2371
2372                 TASK_INIT(&que->que_task, 0, ixgbe_handle_que, que);
2373                 que->tq = taskqueue_create("ixgbe_que", M_NOWAIT,
2374                     taskqueue_thread_enqueue, &que->tq);
2375                 taskqueue_start_threads(&que->tq, 1, PI_NET, -1, "%s que",
2376                     device_get_nameunit(adapter->dev));
2377         }
2378
2379         /* and Link */
2380         rid = vector + 1;
2381         adapter->res = bus_alloc_resource_any(dev,
2382             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2383         if (!adapter->res) {
2384                 device_printf(dev,"Unable to allocate"
2385             " bus resource: Link interrupt [%d]\n", rid);
2386                 return (ENXIO);
2387         }
2388         /* Set the link handler function */
2389         error = bus_setup_intr(dev, adapter->res, INTR_MPSAFE,
2390             ixgbe_msix_link, adapter, &adapter->tag, &adapter->serializer);
2391         if (error) {
2392                 adapter->res = NULL;
2393                 device_printf(dev, "Failed to register LINK handler");
2394                 return (error);
2395         }
2396 #if 0 /* __FreeBSD_version >= 800504 */
2397         bus_describe_intr(dev, adapter->res, adapter->tag, "link");
2398 #endif
2399         adapter->linkvec = vector;
2400         /* Tasklets for Link, SFP and Multispeed Fiber */
2401         TASK_INIT(&adapter->link_task, 0, ixgbe_handle_link, adapter);
2402         TASK_INIT(&adapter->mod_task, 0, ixgbe_handle_mod, adapter);
2403         TASK_INIT(&adapter->msf_task, 0, ixgbe_handle_msf, adapter);
2404 #ifdef IXGBE_FDIR
2405         TASK_INIT(&adapter->fdir_task, 0, ixgbe_reinit_fdir, adapter);
2406 #endif
2407         adapter->tq = taskqueue_create("ixgbe_link", M_NOWAIT,
2408             taskqueue_thread_enqueue, &adapter->tq);
2409         taskqueue_start_threads(&adapter->tq, 1, PI_NET, -1, "%s linkq",
2410             device_get_nameunit(adapter->dev));
2411
2412         return (0);
2413 }
2414
2415 #if 0   /* HAVE_MSIX */
2416 /*
2417  * Setup Either MSI/X or MSI
2418  */
2419 static int
2420 ixgbe_setup_msix(struct adapter *adapter)
2421 {
2422         device_t dev = adapter->dev;
2423         int rid, want, queues, msgs;
2424
2425         /* Override by tuneable */
2426         if (ixgbe_enable_msix == 0)
2427                 goto msi;
2428
2429         /* First try MSI/X */
2430         rid = PCIR_BAR(MSIX_82598_BAR);
2431         adapter->msix_mem = bus_alloc_resource_any(dev,
2432             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2433         if (!adapter->msix_mem) {
2434                 rid += 4;       /* 82599 maps in higher BAR */
2435                 adapter->msix_mem = bus_alloc_resource_any(dev,
2436                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2437         }
2438         if (!adapter->msix_mem) {
2439                 /* May not be enabled */
2440                 device_printf(adapter->dev,
2441                     "Unable to map MSIX table \n");
2442                 goto msi;
2443         }
2444
2445         msgs = pci_msix_count(dev);
2446         if (msgs == 0) { /* system has msix disabled */
2447                 bus_release_resource(dev, SYS_RES_MEMORY,
2448                     rid, adapter->msix_mem);
2449                 adapter->msix_mem = NULL;
2450                 goto msi;
2451         }
2452
2453         /* Figure out a reasonable auto config value */
2454         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2455
2456         if (ixgbe_num_queues != 0)
2457                 queues = ixgbe_num_queues;
2458         /* Set max queues to 8 when autoconfiguring */
2459         else if ((ixgbe_num_queues == 0) && (queues > 8))
2460                 queues = 8;
2461
2462         /*
2463         ** Want one vector (RX/TX pair) per queue
2464         ** plus an additional for Link.
2465         */
2466         want = queues + 1;
2467         if (msgs >= want)
2468                 msgs = want;
2469         else {
2470                 device_printf(adapter->dev,
2471                     "MSIX Configuration Problem, "
2472                     "%d vectors but %d queues wanted!\n",
2473                     msgs, want);
2474                 return (0); /* Will go to Legacy setup */
2475         }
2476         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2477                 device_printf(adapter->dev,
2478                     "Using MSIX interrupts with %d vectors\n", msgs);
2479                 adapter->num_queues = queues;
2480                 return (msgs);
2481         }
2482 msi:
2483         msgs = pci_msi_count(dev);
2484         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2485                 device_printf(adapter->dev,"Using an MSI interrupt\n");
2486         else
2487                 device_printf(adapter->dev,"Using a Legacy interrupt\n");
2488         return (msgs);
2489 }
2490 #endif
2491
2492
2493 static int
2494 ixgbe_allocate_pci_resources(struct adapter *adapter)
2495 {
2496         int             rid;
2497         device_t        dev = adapter->dev;
2498
2499         rid = PCIR_BAR(0);
2500         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2501             &rid, RF_ACTIVE);
2502
2503         if (!(adapter->pci_mem)) {
2504                 device_printf(dev,"Unable to allocate bus resource: memory\n");
2505                 return (ENXIO);
2506         }
2507
2508         adapter->osdep.mem_bus_space_tag =
2509                 rman_get_bustag(adapter->pci_mem);
2510         adapter->osdep.mem_bus_space_handle =
2511                 rman_get_bushandle(adapter->pci_mem);
2512         adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle;
2513
2514         /* Legacy defaults */
2515         adapter->num_queues = 1;
2516         adapter->hw.back = &adapter->osdep;
2517
2518         /*
2519         ** Now setup MSI or MSI/X, should
2520         ** return us the number of supported
2521         ** vectors. (Will be 1 for MSI)
2522         */
2523 #if 0   /* HAVE_MSIX */
2524         adapter->msix = ixgbe_setup_msix(adapter);
2525 #endif
2526         return (0);
2527 }
2528
2529 static void
2530 ixgbe_free_pci_resources(struct adapter * adapter)
2531 {
2532         struct          ix_queue *que = adapter->queues;
2533         device_t        dev = adapter->dev;
2534         int             rid, memrid;
2535
2536         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2537                 memrid = PCIR_BAR(MSIX_82598_BAR);
2538         else
2539                 memrid = PCIR_BAR(MSIX_82599_BAR);
2540
2541         /*
2542         ** There is a slight possibility of a failure mode
2543         ** in attach that will result in entering this function
2544         ** before interrupt resources have been initialized, and
2545         ** in that case we do not want to execute the loops below
2546         ** We can detect this reliably by the state of the adapter
2547         ** res pointer.
2548         */
2549         if (adapter->res == NULL)
2550                 goto mem;
2551
2552         /*
2553         **  Release all msix queue resources:
2554         */
2555         for (int i = 0; i < adapter->num_queues; i++, que++) {
2556                 rid = que->msix + 1;
2557                 if (que->tag != NULL) {
2558                         bus_teardown_intr(dev, que->res, que->tag);
2559                         que->tag = NULL;
2560                 }
2561                 if (que->res != NULL)
2562                         bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2563         }
2564
2565
2566         /* Clean the Legacy or Link interrupt last */
2567         if (adapter->linkvec) /* we are doing MSIX */
2568                 rid = adapter->linkvec + 1;
2569         else
2570                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2571
2572         if (adapter->tag != NULL) {
2573                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2574                 adapter->tag = NULL;
2575         }
2576         if (adapter->res != NULL)
2577                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2578         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2579                 pci_release_msi(adapter->dev);
2580
2581 mem:
2582         if (adapter->msix)
2583                 pci_release_msi(dev);
2584
2585         if (adapter->msix_mem != NULL)
2586                 bus_release_resource(dev, SYS_RES_MEMORY,
2587                     memrid, adapter->msix_mem);
2588
2589         if (adapter->pci_mem != NULL)
2590                 bus_release_resource(dev, SYS_RES_MEMORY,
2591                     PCIR_BAR(0), adapter->pci_mem);
2592
2593         return;
2594 }
2595
2596 /*********************************************************************
2597  *
2598  *  Setup networking device structure and register an interface.
2599  *
2600  **********************************************************************/
2601 static int
2602 ixgbe_setup_interface(device_t dev, struct adapter *adapter)
2603 {
2604         struct ixgbe_hw *hw = &adapter->hw;
2605         struct ifnet   *ifp;
2606
2607         INIT_DEBUGOUT("ixgbe_setup_interface: begin");
2608
2609         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2610         if (ifp == NULL) {
2611                 device_printf(dev, "can not allocate ifnet structure\n");
2612                 return (-1);
2613         }
2614         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2615         ifp->if_baudrate = 1000000000;
2616         ifp->if_init = ixgbe_init;
2617         ifp->if_softc = adapter;
2618         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2619         ifp->if_ioctl = ixgbe_ioctl;
2620         ifp->if_start = ixgbe_start;
2621 #if 0 /* __FreeBSD_version >= 800000 */
2622         ifp->if_transmit = ixgbe_mq_start;
2623         ifp->if_qflush = ixgbe_qflush;
2624 #endif
2625         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
2626
2627         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2628
2629         adapter->max_frame_size =
2630             ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
2631
2632         /*
2633          * Tell the upper layer(s) we support long frames.
2634          */
2635         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2636
2637 #if 0 /* NET_TSO */
2638         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO | IFCAP_VLAN_HWCSUM;
2639 #endif
2640         ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2641         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2642         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2643 #if 0 /* NET_TSO */
2644                              |  IFCAP_VLAN_HWTSO
2645 #endif
2646                              |  IFCAP_VLAN_MTU;
2647         ifp->if_capenable = ifp->if_capabilities;
2648
2649         /* Don't enable LRO by default */
2650 #if 0 /* NET_LRO */
2651         ifp->if_capabilities |= IFCAP_LRO;
2652 #endif
2653
2654         /*
2655         ** Don't turn this on by default, if vlans are
2656         ** created on another pseudo device (eg. lagg)
2657         ** then vlan events are not passed thru, breaking
2658         ** operation, but with HW FILTER off it works. If
2659         ** using vlans directly on the ixgbe driver you can
2660         ** enable this and get full hardware tag filtering.
2661         */
2662         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2663
2664         /*
2665          * Specify the media types supported by this adapter and register
2666          * callbacks to update media and link information
2667          */
2668         ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change,
2669                      ixgbe_media_status);
2670         ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics, 0, NULL);
2671         ifmedia_set(&adapter->media, IFM_ETHER | adapter->optics);
2672         if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2673                 ifmedia_add(&adapter->media,
2674                     IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2675                 ifmedia_add(&adapter->media,
2676                     IFM_ETHER | IFM_1000_T, 0, NULL);
2677         }
2678         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2679         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2680
2681         return (0);
2682 }
2683
2684 static void
2685 ixgbe_config_link(struct adapter *adapter)
2686 {
2687         struct ixgbe_hw *hw = &adapter->hw;
2688         u32     autoneg, err = 0;
2689         bool    sfp, negotiate;
2690
2691         sfp = ixgbe_is_sfp(hw);
2692
2693         if (sfp) {
2694                 if (hw->phy.multispeed_fiber) {
2695                         hw->mac.ops.setup_sfp(hw);
2696                         ixgbe_enable_tx_laser(hw);
2697                         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
2698                 } else
2699                         taskqueue_enqueue(adapter->tq, &adapter->mod_task);
2700         } else {
2701                 if (hw->mac.ops.check_link)
2702                         err = ixgbe_check_link(hw, &autoneg,
2703                             &adapter->link_up, FALSE);
2704                 if (err)
2705                         goto out;
2706                 autoneg = hw->phy.autoneg_advertised;
2707                 if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
2708                         err  = hw->mac.ops.get_link_capabilities(hw,
2709                             &autoneg, &negotiate);
2710                 if (err)
2711                         goto out;
2712                 if (hw->mac.ops.setup_link)
2713                         err = hw->mac.ops.setup_link(hw, autoneg,
2714                             negotiate, adapter->link_up);
2715         }
2716 out:
2717         return;
2718 }
2719
2720 /********************************************************************
2721  * Manage DMA'able memory.
2722  *******************************************************************/
2723 static void
2724 ixgbe_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error)
2725 {
2726         if (error)
2727                 return;
2728         *(bus_addr_t *) arg = segs->ds_addr;
2729         return;
2730 }
2731
2732 static int
2733 ixgbe_dma_malloc(struct adapter *adapter, bus_size_t size,
2734                 struct ixgbe_dma_alloc *dma, int mapflags)
2735 {
2736         device_t dev = adapter->dev;
2737         int             r;
2738
2739         r = bus_dma_tag_create(NULL,    /* parent */
2740                                DBA_ALIGN, 0,    /* alignment, bounds */
2741                                BUS_SPACE_MAXADDR,       /* lowaddr */
2742                                BUS_SPACE_MAXADDR,       /* highaddr */
2743                                NULL, NULL,      /* filter, filterarg */
2744                                size,    /* maxsize */
2745                                1,       /* nsegments */
2746                                size,    /* maxsegsize */
2747                                BUS_DMA_ALLOCNOW,        /* flags */
2748                                &dma->dma_tag);
2749         if (r != 0) {
2750                 device_printf(dev,"ixgbe_dma_malloc: bus_dma_tag_create failed; "
2751                        "error %u\n", r);
2752                 goto fail_0;
2753         }
2754         r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
2755                              BUS_DMA_NOWAIT, &dma->dma_map);
2756         if (r != 0) {
2757                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2758                        "error %u\n", r);
2759                 goto fail_1;
2760         }
2761         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2762                             size,
2763                             ixgbe_dmamap_cb,
2764                             &dma->dma_paddr,
2765                             mapflags | BUS_DMA_NOWAIT);
2766         if (r != 0) {
2767                 device_printf(dev,"ixgbe_dma_malloc: bus_dmamap_load failed; "
2768                        "error %u\n", r);
2769                 goto fail_2;
2770         }
2771         dma->dma_size = size;
2772         return (0);
2773 fail_2:
2774         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2775 fail_1:
2776         bus_dma_tag_destroy(dma->dma_tag);
2777 fail_0:
2778         dma->dma_map = NULL;
2779         dma->dma_tag = NULL;
2780         return (r);
2781 }
2782
2783 static void
2784 ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma)
2785 {
2786         bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2787             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2788         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2789         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2790         bus_dma_tag_destroy(dma->dma_tag);
2791 }
2792
2793
2794 /*********************************************************************
2795  *
2796  *  Allocate memory for the transmit and receive rings, and then
2797  *  the descriptors associated with each, called only once at attach.
2798  *
2799  **********************************************************************/
2800 static int
2801 ixgbe_allocate_queues(struct adapter *adapter)
2802 {
2803         device_t        dev = adapter->dev;
2804         struct ix_queue *que;
2805         struct tx_ring  *txr;
2806         struct rx_ring  *rxr;
2807         int rsize, tsize, error = IXGBE_SUCCESS;
2808         int txconf = 0, rxconf = 0;
2809
2810         /* First allocate the top level queue structs */
2811         if (!(adapter->queues =
2812             (struct ix_queue *) kmalloc(sizeof(struct ix_queue) *
2813             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2814                 device_printf(dev, "Unable to allocate queue memory\n");
2815                 error = ENOMEM;
2816                 goto fail;
2817         }
2818
2819         /* First allocate the TX ring struct memory */
2820         if (!(adapter->tx_rings =
2821             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2822             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2823                 device_printf(dev, "Unable to allocate TX ring memory\n");
2824                 error = ENOMEM;
2825                 goto tx_fail;
2826         }
2827
2828         /* Next allocate the RX */
2829         if (!(adapter->rx_rings =
2830             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2831             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2832                 device_printf(dev, "Unable to allocate RX ring memory\n");
2833                 error = ENOMEM;
2834                 goto rx_fail;
2835         }
2836
2837         /* For the ring itself */
2838         tsize = roundup2(adapter->num_tx_desc *
2839             sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2840
2841         /*
2842          * Now set up the TX queues, txconf is needed to handle the
2843          * possibility that things fail midcourse and we need to
2844          * undo memory gracefully
2845          */
2846         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2847                 /* Set up some basics */
2848                 txr = &adapter->tx_rings[i];
2849                 txr->adapter = adapter;
2850                 txr->me = i;
2851
2852                 /* Initialize the TX side lock */
2853                 ksnprintf(txr->lock_name, sizeof(txr->lock_name), "%s:tx(%d)",
2854                     device_get_nameunit(dev), txr->me);
2855                 lockinit(&txr->tx_lock, txr->lock_name, 0, LK_CANRECURSE);
2856
2857                 if (ixgbe_dma_malloc(adapter, tsize,
2858                         &txr->txdma, BUS_DMA_NOWAIT)) {
2859                         device_printf(dev,
2860                             "Unable to allocate TX Descriptor memory\n");
2861                         error = ENOMEM;
2862                         goto err_tx_desc;
2863                 }
2864                 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2865                 bzero((void *)txr->tx_base, tsize);
2866
2867                 /* Now allocate transmit buffers for the ring */
2868                 if (ixgbe_allocate_transmit_buffers(txr)) {
2869                         device_printf(dev,
2870                             "Critical Failure setting up transmit buffers\n");
2871                         error = ENOMEM;
2872                         goto err_tx_desc;
2873                 }
2874 #if 0 /* __FreeBSD_version >= 800000 */
2875                 /* Allocate a buf ring */
2876                 txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
2877                     M_WAITOK, &txr->tx_mtx);
2878                 if (txr->br == NULL) {
2879                         device_printf(dev,
2880                             "Critical Failure setting up buf ring\n");
2881                         error = ENOMEM;
2882                         goto err_tx_desc;
2883                 }
2884 #endif
2885         }
2886
2887         /*
2888          * Next the RX queues...
2889          */
2890         rsize = roundup2(adapter->num_rx_desc *
2891             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
2892         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2893                 rxr = &adapter->rx_rings[i];
2894                 /* Set up some basics */
2895                 rxr->adapter = adapter;
2896                 rxr->me = i;
2897
2898                 /* Initialize the RX side lock */
2899                 ksnprintf(rxr->lock_name, sizeof(rxr->lock_name), "%s:rx(%d)",
2900                     device_get_nameunit(dev), rxr->me);
2901                 lockinit(&rxr->rx_lock, rxr->lock_name, 0, LK_CANRECURSE);
2902
2903                 if (ixgbe_dma_malloc(adapter, rsize,
2904                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2905                         device_printf(dev,
2906                             "Unable to allocate RxDescriptor memory\n");
2907                         error = ENOMEM;
2908                         goto err_rx_desc;
2909                 }
2910                 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2911                 bzero((void *)rxr->rx_base, rsize);
2912
2913                 /* Allocate receive buffers for the ring*/
2914                 if (ixgbe_allocate_receive_buffers(rxr)) {
2915                         device_printf(dev,
2916                             "Critical Failure setting up receive buffers\n");
2917                         error = ENOMEM;
2918                         goto err_rx_desc;
2919                 }
2920         }
2921
2922         /*
2923         ** Finally set up the queue holding structs
2924         */
2925         for (int i = 0; i < adapter->num_queues; i++) {
2926                 que = &adapter->queues[i];
2927                 que->adapter = adapter;
2928                 que->txr = &adapter->tx_rings[i];
2929                 que->rxr = &adapter->rx_rings[i];
2930         }
2931
2932         return (0);
2933
2934 err_rx_desc:
2935         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2936                 ixgbe_dma_free(adapter, &rxr->rxdma);
2937 err_tx_desc:
2938         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2939                 ixgbe_dma_free(adapter, &txr->txdma);
2940         kfree(adapter->rx_rings, M_DEVBUF);
2941 rx_fail:
2942         kfree(adapter->tx_rings, M_DEVBUF);
2943 tx_fail:
2944         kfree(adapter->queues, M_DEVBUF);
2945 fail:
2946         return (error);
2947 }
2948
2949 /*********************************************************************
2950  *
2951  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2952  *  the information needed to transmit a packet on the wire. This is
2953  *  called only once at attach, setup is done every reset.
2954  *
2955  **********************************************************************/
2956 static int
2957 ixgbe_allocate_transmit_buffers(struct tx_ring *txr)
2958 {
2959         struct adapter *adapter = txr->adapter;
2960         device_t dev = adapter->dev;
2961         struct ixgbe_tx_buf *txbuf;
2962         int error, i;
2963
2964         /*
2965          * Setup DMA descriptor areas.
2966          */
2967         if ((error = bus_dma_tag_create(
2968                                NULL,    /* parent */
2969                                1, 0,            /* alignment, bounds */
2970                                BUS_SPACE_MAXADDR,       /* lowaddr */
2971                                BUS_SPACE_MAXADDR,       /* highaddr */
2972                                NULL, NULL,              /* filter, filterarg */
2973                                IXGBE_TSO_SIZE,          /* maxsize */
2974                                adapter->num_segs,       /* nsegments */
2975                                PAGE_SIZE,               /* maxsegsize */
2976                                0,                       /* flags */
2977                                &txr->txtag))) {
2978                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2979                 goto fail;
2980         }
2981
2982         if (!(txr->tx_buffers =
2983             (struct ixgbe_tx_buf *) kmalloc(sizeof(struct ixgbe_tx_buf) *
2984             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2985                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2986                 error = ENOMEM;
2987                 goto fail;
2988         }
2989
2990         /* Create the descriptor buffer dma maps */
2991         txbuf = txr->tx_buffers;
2992         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2993                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2994                 if (error != 0) {
2995                         device_printf(dev, "Unable to create TX DMA map\n");
2996                         goto fail;
2997                 }
2998         }
2999
3000         return 0;
3001 fail:
3002         /* We free all, it handles case where we are in the middle */
3003         ixgbe_free_transmit_structures(adapter);
3004         return (error);
3005 }
3006
3007 /*********************************************************************
3008  *
3009  *  Initialize a transmit ring.
3010  *
3011  **********************************************************************/
3012 static void
3013 ixgbe_setup_transmit_ring(struct tx_ring *txr)
3014 {
3015         struct adapter *adapter = txr->adapter;
3016         struct ixgbe_tx_buf *txbuf;
3017         int i;
3018 #ifdef DEV_NETMAP
3019         struct netmap_adapter *na = NA(adapter->ifp);
3020         struct netmap_slot *slot;
3021 #endif /* DEV_NETMAP */
3022
3023         /* Clear the old ring contents */
3024         IXGBE_TX_LOCK(txr);
3025 #ifdef DEV_NETMAP
3026         /*
3027          * (under lock): if in netmap mode, do some consistency
3028          * checks and set slot to entry 0 of the netmap ring.
3029          */
3030         slot = netmap_reset(na, NR_TX, txr->me, 0);
3031 #endif /* DEV_NETMAP */
3032         bzero((void *)txr->tx_base,
3033               (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc);
3034         /* Reset indices */
3035         txr->next_avail_desc = 0;
3036         txr->next_to_clean = 0;
3037
3038         /* Free any existing tx buffers. */
3039         txbuf = txr->tx_buffers;
3040         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3041                 if (txbuf->m_head != NULL) {
3042                         bus_dmamap_sync(txr->txtag, txbuf->map,
3043                             BUS_DMASYNC_POSTWRITE);
3044                         bus_dmamap_unload(txr->txtag, txbuf->map);
3045                         m_freem(txbuf->m_head);
3046                         txbuf->m_head = NULL;
3047                 }
3048 #ifdef DEV_NETMAP
3049                 /*
3050                  * In netmap mode, set the map for the packet buffer.
3051                  * NOTE: Some drivers (not this one) also need to set
3052                  * the physical buffer address in the NIC ring.
3053                  * Slots in the netmap ring (indexed by "si") are
3054                  * kring->nkr_hwofs positions "ahead" wrt the
3055                  * corresponding slot in the NIC ring. In some drivers
3056                  * (not here) nkr_hwofs can be negative. Function
3057                  * netmap_idx_n2k() handles wraparounds properly.
3058                  */
3059                 if (slot) {
3060                         int si = netmap_idx_n2k(&na->tx_rings[txr->me], i);
3061                         netmap_load_map(txr->txtag, txbuf->map, NMB(slot + si));
3062                 }
3063 #endif /* DEV_NETMAP */
3064                 /* Clear the EOP index */
3065                 txbuf->eop_index = -1;
3066         }
3067
3068 #ifdef IXGBE_FDIR
3069         /* Set the rate at which we sample packets */
3070         if (adapter->hw.mac.type != ixgbe_mac_82598EB)
3071                 txr->atr_sample = atr_sample_rate;
3072 #endif
3073
3074         /* Set number of descriptors available */
3075         txr->tx_avail = adapter->num_tx_desc;
3076
3077         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3078             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3079         IXGBE_TX_UNLOCK(txr);
3080 }
3081
3082 /*********************************************************************
3083  *
3084  *  Initialize all transmit rings.
3085  *
3086  **********************************************************************/
3087 static int
3088 ixgbe_setup_transmit_structures(struct adapter *adapter)
3089 {
3090         struct tx_ring *txr = adapter->tx_rings;
3091
3092         for (int i = 0; i < adapter->num_queues; i++, txr++)
3093                 ixgbe_setup_transmit_ring(txr);
3094
3095         return (0);
3096 }
3097
3098 /*********************************************************************
3099  *
3100  *  Enable transmit unit.
3101  *
3102  **********************************************************************/
3103 static void
3104 ixgbe_initialize_transmit_units(struct adapter *adapter)
3105 {
3106         struct tx_ring  *txr = adapter->tx_rings;
3107         struct ixgbe_hw *hw = &adapter->hw;
3108
3109         /* Setup the Base and Length of the Tx Descriptor Ring */
3110
3111         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3112                 u64     tdba = txr->txdma.dma_paddr;
3113                 u32     txctrl;
3114
3115                 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
3116                        (tdba & 0x00000000ffffffffULL));
3117                 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
3118                 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
3119                     adapter->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
3120
3121                 /* Setup the HW Tx Head and Tail descriptor pointers */
3122                 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
3123                 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
3124
3125                 /* Setup Transmit Descriptor Cmd Settings */
3126                 txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
3127                 txr->queue_status = IXGBE_QUEUE_IDLE;
3128
3129                 /* Disable Head Writeback */
3130                 switch (hw->mac.type) {
3131                 case ixgbe_mac_82598EB:
3132                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
3133                         break;
3134                 case ixgbe_mac_82599EB:
3135                 case ixgbe_mac_X540:
3136                 default:
3137                         txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
3138                         break;
3139                 }
3140                 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
3141                 switch (hw->mac.type) {
3142                 case ixgbe_mac_82598EB:
3143                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
3144                         break;
3145                 case ixgbe_mac_82599EB:
3146                 case ixgbe_mac_X540:
3147                 default:
3148                         IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
3149                         break;
3150                 }
3151
3152         }
3153
3154         if (hw->mac.type != ixgbe_mac_82598EB) {
3155                 u32 dmatxctl, rttdcs;
3156                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
3157                 dmatxctl |= IXGBE_DMATXCTL_TE;
3158                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
3159                 /* Disable arbiter to set MTQC */
3160                 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
3161                 rttdcs |= IXGBE_RTTDCS_ARBDIS;
3162                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3163                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
3164                 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
3165                 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
3166         }
3167
3168         return;
3169 }
3170
3171 /*********************************************************************
3172  *
3173  *  Free all transmit rings.
3174  *
3175  **********************************************************************/
3176 static void
3177 ixgbe_free_transmit_structures(struct adapter *adapter)
3178 {
3179         struct tx_ring *txr = adapter->tx_rings;
3180
3181         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3182                 IXGBE_TX_LOCK(txr);
3183                 ixgbe_free_transmit_buffers(txr);
3184                 ixgbe_dma_free(adapter, &txr->txdma);
3185                 IXGBE_TX_UNLOCK(txr);
3186                 IXGBE_TX_LOCK_DESTROY(txr);
3187         }
3188         kfree(adapter->tx_rings, M_DEVBUF);
3189 }
3190
3191 /*********************************************************************
3192  *
3193  *  Free transmit ring related data structures.
3194  *
3195  **********************************************************************/
3196 static void
3197 ixgbe_free_transmit_buffers(struct tx_ring *txr)
3198 {
3199         struct adapter *adapter = txr->adapter;
3200         struct ixgbe_tx_buf *tx_buffer;
3201         int             i;
3202
3203         INIT_DEBUGOUT("free_transmit_ring: begin");
3204
3205         if (txr->tx_buffers == NULL)
3206                 return;
3207
3208         tx_buffer = txr->tx_buffers;
3209         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3210                 if (tx_buffer->m_head != NULL) {
3211                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3212                             BUS_DMASYNC_POSTWRITE);
3213                         bus_dmamap_unload(txr->txtag,
3214                             tx_buffer->map);
3215                         m_freem(tx_buffer->m_head);
3216                         tx_buffer->m_head = NULL;
3217                         if (tx_buffer->map != NULL) {
3218                                 bus_dmamap_destroy(txr->txtag,
3219                                     tx_buffer->map);
3220                                 tx_buffer->map = NULL;
3221                         }
3222                 } else if (tx_buffer->map != NULL) {
3223                         bus_dmamap_unload(txr->txtag,
3224                             tx_buffer->map);
3225                         bus_dmamap_destroy(txr->txtag,
3226                             tx_buffer->map);
3227                         tx_buffer->map = NULL;
3228                 }
3229         }
3230 #if 0 /* __FreeBSD_version >= 800000 */
3231         if (txr->br != NULL)
3232                 buf_ring_free(txr->br, M_DEVBUF);
3233 #endif
3234         if (txr->tx_buffers != NULL) {
3235                 kfree(txr->tx_buffers, M_DEVBUF);
3236                 txr->tx_buffers = NULL;
3237         }
3238         if (txr->txtag != NULL) {
3239                 bus_dma_tag_destroy(txr->txtag);
3240                 txr->txtag = NULL;
3241         }
3242         return;
3243 }
3244
3245 /*********************************************************************
3246  *
3247  *  Advanced Context Descriptor setup for VLAN or CSUM
3248  *
3249  **********************************************************************/
3250
3251 static bool
3252 ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3253 {
3254         struct adapter *adapter = txr->adapter;
3255         struct ixgbe_adv_tx_context_desc *TXD;
3256         struct ixgbe_tx_buf        *tx_buffer;
3257         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3258         struct ether_vlan_header *eh;
3259         struct ip *ip;
3260         struct ip6_hdr *ip6;
3261         int  ehdrlen, ip_hlen = 0;
3262         u16     etype;
3263         u8      ipproto = 0;
3264         bool    offload = TRUE;
3265         int ctxd = txr->next_avail_desc;
3266 #ifdef NET_VLAN
3267         u16 vtag = 0;
3268 #endif
3269
3270
3271         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3272                 offload = FALSE;
3273
3274         tx_buffer = &txr->tx_buffers[ctxd];
3275         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3276
3277         /*
3278         ** In advanced descriptors the vlan tag must
3279         ** be placed into the descriptor itself.
3280         */
3281 #ifdef NET_VLAN
3282         if (mp->m_flags & M_VLANTAG) {
3283                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3284                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3285         } else if (offload == FALSE)
3286                 return FALSE;
3287 #endif
3288
3289         /*
3290          * Determine where frame payload starts.
3291          * Jump over vlan headers if already present,
3292          * helpful for QinQ too.
3293          */
3294         eh = mtod(mp, struct ether_vlan_header *);
3295         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3296                 etype = ntohs(eh->evl_proto);
3297                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3298         } else {
3299                 etype = ntohs(eh->evl_encap_proto);
3300                 ehdrlen = ETHER_HDR_LEN;
3301         }
3302
3303         /* Set the ether header length */
3304         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3305
3306         switch (etype) {
3307                 case ETHERTYPE_IP:
3308                         ip = (struct ip *)(mp->m_data + ehdrlen);
3309                         ip_hlen = ip->ip_hl << 2;
3310                         ipproto = ip->ip_p;
3311                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3312                         break;
3313                 case ETHERTYPE_IPV6:
3314                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3315                         ip_hlen = sizeof(struct ip6_hdr);
3316                         /* XXX-BZ this will go badly in case of ext hdrs. */
3317                         ipproto = ip6->ip6_nxt;
3318                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3319                         break;
3320                 default:
3321                         offload = FALSE;
3322                         break;
3323         }
3324
3325         vlan_macip_lens |= ip_hlen;
3326         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3327
3328         switch (ipproto) {
3329                 case IPPROTO_TCP:
3330                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3331                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3332                         break;
3333
3334                 case IPPROTO_UDP:
3335                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3336                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
3337                         break;
3338
3339 #if 0
3340                 case IPPROTO_SCTP:
3341                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3342                                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3343                         break;
3344 #endif
3345                 default:
3346                         offload = FALSE;
3347                         break;
3348         }
3349
3350         /* Now copy bits into descriptor */
3351         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3352         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3353         TXD->seqnum_seed = htole32(0);
3354         TXD->mss_l4len_idx = htole32(0);
3355
3356         tx_buffer->m_head = NULL;
3357         tx_buffer->eop_index = -1;
3358
3359         /* We've consumed the first desc, adjust counters */
3360         if (++ctxd == adapter->num_tx_desc)
3361                 ctxd = 0;
3362         txr->next_avail_desc = ctxd;
3363         --txr->tx_avail;
3364
3365         return (offload);
3366 }
3367
3368 /**********************************************************************
3369  *
3370  *  Setup work for hardware segmentation offload (TSO) on
3371  *  adapters using advanced tx descriptors
3372  *
3373  **********************************************************************/
3374 #if 0   /* NET_TSO */
3375 static bool
3376 ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen,
3377     u32 *olinfo_status)
3378 {
3379         struct adapter *adapter = txr->adapter;
3380         struct ixgbe_adv_tx_context_desc *TXD;
3381         struct ixgbe_tx_buf        *tx_buffer;
3382 #ifdef NET_VLAN
3383         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3384         u16 vtag = 0, eh_type;
3385 #else
3386         u16 eh_type;
3387         u32 type_tucmd_mlhl = 0;
3388 #endif
3389         u32 mss_l4len_idx = 0, len;
3390         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3391         struct ether_vlan_header *eh;
3392 #ifdef INET6
3393         struct ip6_hdr *ip6;
3394 #endif
3395 #ifdef INET
3396         struct ip *ip;
3397 #endif
3398         struct tcphdr *th;
3399
3400
3401         /*
3402          * Determine where frame payload starts.
3403          * Jump over vlan headers if already present
3404          */
3405         eh = mtod(mp, struct ether_vlan_header *);
3406         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3407                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3408                 eh_type = eh->evl_proto;
3409         } else {
3410                 ehdrlen = ETHER_HDR_LEN;
3411                 eh_type = eh->evl_encap_proto;
3412         }
3413
3414         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3415         len = ehdrlen + sizeof(struct tcphdr);
3416         switch (ntohs(eh_type)) {
3417 #ifdef INET6
3418         case ETHERTYPE_IPV6:
3419                 if (mp->m_len < len + sizeof(struct ip6_hdr))
3420                         return FALSE;
3421                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3422                 /* XXX-BZ For now we do not pretend to support ext. hdrs. */
3423                 if (ip6->ip6_nxt != IPPROTO_TCP)
3424                         return FALSE;
3425                 ip_hlen = sizeof(struct ip6_hdr);
3426                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3427                 th->th_sum = in6_cksum_pseudo(ip6, 0, IPPROTO_TCP, 0);
3428                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
3429                 break;
3430 #endif
3431 #ifdef INET
3432         case ETHERTYPE_IP:
3433                 if (mp->m_len < len + sizeof(struct ip))
3434                         return FALSE;
3435                 ip = (struct ip *)(mp->m_data + ehdrlen);
3436                 if (ip->ip_p != IPPROTO_TCP)
3437                         return FALSE;
3438                 ip->ip_sum = 0;
3439                 ip_hlen = ip->ip_hl << 2;
3440                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3441                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3442                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3443                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
3444                 /* Tell transmit desc to also do IPv4 checksum. */
3445                 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
3446                 break;
3447 #endif
3448         default:
3449                 panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
3450                     __func__, ntohs(eh_type));
3451                 break;
3452         }
3453
3454         ctxd = txr->next_avail_desc;
3455         tx_buffer = &txr->tx_buffers[ctxd];
3456         TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd];
3457
3458         tcp_hlen = th->th_off << 2;
3459
3460         /* This is used in the transmit desc in encap */
3461         *paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen;
3462
3463         /* VLAN MACLEN IPLEN */
3464 #ifdef NET_VLAN
3465         if (mp->m_flags & M_VLANTAG) {
3466                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3467                 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
3468         }
3469
3470         vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
3471         vlan_macip_lens |= ip_hlen;
3472         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3473 #endif
3474
3475         /* ADV DTYPE TUCMD */
3476         type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3477         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3478         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3479
3480         /* MSS L4LEN IDX */
3481         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
3482         mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
3483         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3484
3485         TXD->seqnum_seed = htole32(0);
3486         tx_buffer->m_head = NULL;
3487         tx_buffer->eop_index = -1;
3488
3489         if (++ctxd == adapter->num_tx_desc)
3490                 ctxd = 0;
3491
3492         txr->tx_avail--;
3493         txr->next_avail_desc = ctxd;
3494         return TRUE;
3495 }
3496 #endif
3497
3498 #ifdef IXGBE_FDIR
3499 /*
3500 ** This routine parses packet headers so that Flow
3501 ** Director can make a hashed filter table entry
3502 ** allowing traffic flows to be identified and kept
3503 ** on the same cpu.  This would be a performance
3504 ** hit, but we only do it at IXGBE_FDIR_RATE of
3505 ** packets.
3506 */
3507 static void
3508 ixgbe_atr(struct tx_ring *txr, struct mbuf *mp)
3509 {
3510         struct adapter                  *adapter = txr->adapter;
3511         struct ix_queue                 *que;
3512         struct ip                       *ip;
3513         struct tcphdr                   *th;
3514         struct udphdr                   *uh;
3515         struct ether_vlan_header        *eh;
3516         union ixgbe_atr_hash_dword      input = {.dword = 0};
3517         union ixgbe_atr_hash_dword      common = {.dword = 0};
3518         int                             ehdrlen, ip_hlen;
3519         u16                             etype;
3520
3521         eh = mtod(mp, struct ether_vlan_header *);
3522         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3523                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3524                 etype = eh->evl_proto;
3525         } else {
3526                 ehdrlen = ETHER_HDR_LEN;
3527                 etype = eh->evl_encap_proto;
3528         }
3529
3530         /* Only handling IPv4 */
3531         if (etype != htons(ETHERTYPE_IP))
3532                 return;
3533
3534         ip = (struct ip *)(mp->m_data + ehdrlen);
3535         ip_hlen = ip->ip_hl << 2;
3536
3537         /* check if we're UDP or TCP */
3538         switch (ip->ip_p) {
3539         case IPPROTO_TCP:
3540                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3541                 /* src and dst are inverted */
3542                 common.port.dst ^= th->th_sport;
3543                 common.port.src ^= th->th_dport;
3544                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_TCPV4;
3545                 break;
3546         case IPPROTO_UDP:
3547                 uh = (struct udphdr *)((caddr_t)ip + ip_hlen);
3548                 /* src and dst are inverted */
3549                 common.port.dst ^= uh->uh_sport;
3550                 common.port.src ^= uh->uh_dport;
3551                 input.formatted.flow_type ^= IXGBE_ATR_FLOW_TYPE_UDPV4;
3552                 break;
3553         default:
3554                 return;
3555         }
3556
3557         input.formatted.vlan_id = htobe16(mp->m_pkthdr.ether_vtag);
3558         if (mp->m_pkthdr.ether_vtag)
3559                 common.flex_bytes ^= htons(ETHERTYPE_VLAN);
3560         else
3561                 common.flex_bytes ^= etype;
3562         common.ip ^= ip->ip_src.s_addr ^ ip->ip_dst.s_addr;
3563
3564         que = &adapter->queues[txr->me];
3565         /*
3566         ** This assumes the Rx queue and Tx
3567         ** queue are bound to the same CPU
3568         */
3569         ixgbe_fdir_add_signature_filter_82599(&adapter->hw,
3570             input, common, que->msix);
3571 }
3572 #endif /* IXGBE_FDIR */
3573
3574 /**********************************************************************
3575  *
3576  *  Examine each tx_buffer in the used queue. If the hardware is done
3577  *  processing the packet then free associated resources. The
3578  *  tx_buffer is put back on the free queue.
3579  *
3580  **********************************************************************/
3581 static bool
3582 ixgbe_txeof(struct tx_ring *txr)
3583 {
3584         struct adapter  *adapter = txr->adapter;
3585         struct ifnet    *ifp = adapter->ifp;
3586         u32     first, last, done, processed;
3587         struct ixgbe_tx_buf *tx_buffer;
3588         struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc;
3589
3590         KKASSERT(lockstatus(&txr->tx_lock, curthread) != 0);
3591
3592 #ifdef DEV_NETMAP
3593         if (ifp->if_capenable & IFCAP_NETMAP) {
3594                 struct netmap_adapter *na = NA(ifp);
3595                 struct netmap_kring *kring = &na->tx_rings[txr->me];
3596
3597                 tx_desc = (struct ixgbe_legacy_tx_desc *)txr->tx_base;
3598
3599                 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3600                     BUS_DMASYNC_POSTREAD);
3601                 /*
3602                  * In netmap mode, all the work is done in the context
3603                  * of the client thread. Interrupt handlers only wake up
3604                  * clients, which may be sleeping on individual rings
3605                  * or on a global resource for all rings.
3606                  * To implement tx interrupt mitigation, we wake up the client
3607                  * thread roughly every half ring, even if the NIC interrupts
3608                  * more frequently. This is implemented as follows:
3609                  * - ixgbe_txsync() sets kring->nr_kflags with the index of
3610                  *   the slot that should wake up the thread (nkr_num_slots
3611                  *   means the user thread should not be woken up);
3612                  * - the driver ignores tx interrupts unless netmap_mitigate=0
3613                  *   or the slot has the DD bit set.
3614                  *
3615                  * When the driver has separate locks, we need to
3616                  * release and re-acquire txlock to avoid deadlocks.
3617                  * XXX see if we can find a better way.
3618                  */
3619                 if (!netmap_mitigate ||
3620                     (kring->nr_kflags < kring->nkr_num_slots &&
3621                      tx_desc[kring->nr_kflags].upper.fields.status & IXGBE_TXD_STAT_DD)) {
3622                         kring->nr_kflags = kring->nkr_num_slots;
3623                         selwakeuppri(&na->tx_rings[txr->me].si, PI_NET);
3624                         IXGBE_TX_UNLOCK(txr);
3625                         IXGBE_CORE_LOCK(adapter);
3626                         selwakeuppri(&na->tx_si, PI_NET);
3627                         IXGBE_CORE_UNLOCK(adapter);
3628                         IXGBE_TX_LOCK(txr);
3629                 }
3630                 return FALSE;
3631         }
3632 #endif /* DEV_NETMAP */
3633
3634         if (txr->tx_avail == adapter->num_tx_desc) {
3635                 txr->queue_status = IXGBE_QUEUE_IDLE;
3636                 return FALSE;
3637         }
3638
3639         processed = 0;
3640         first = txr->next_to_clean;
3641         tx_buffer = &txr->tx_buffers[first];
3642         /* For cleanup we just use legacy struct */
3643         tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3644         last = tx_buffer->eop_index;
3645         if (last == -1)
3646                 return FALSE;
3647         eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3648
3649         /*
3650         ** Get the index of the first descriptor
3651         ** BEYOND the EOP and call that 'done'.
3652         ** I do this so the comparison in the
3653         ** inner while loop below can be simple
3654         */
3655         if (++last == adapter->num_tx_desc) last = 0;
3656         done = last;
3657
3658         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3659             BUS_DMASYNC_POSTREAD);
3660         /*
3661         ** Only the EOP descriptor of a packet now has the DD
3662         ** bit set, this is what we look for...
3663         */
3664         while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) {
3665                 /* We clean the range of the packet */
3666                 while (first != done) {
3667                         tx_desc->upper.data = 0;
3668                         tx_desc->lower.data = 0;
3669                         tx_desc->buffer_addr = 0;
3670                         ++txr->tx_avail;
3671                         ++processed;
3672
3673                         if (tx_buffer->m_head) {
3674                                 txr->bytes +=
3675                                     tx_buffer->m_head->m_pkthdr.len;
3676                                 bus_dmamap_sync(txr->txtag,
3677                                     tx_buffer->map,
3678                                     BUS_DMASYNC_POSTWRITE);
3679                                 bus_dmamap_unload(txr->txtag,
3680                                     tx_buffer->map);
3681                                 m_freem(tx_buffer->m_head);
3682                                 tx_buffer->m_head = NULL;
3683                                 tx_buffer->map = NULL;
3684                         }
3685                         tx_buffer->eop_index = -1;
3686                         txr->watchdog_time = ticks;
3687
3688                         if (++first == adapter->num_tx_desc)
3689                                 first = 0;
3690
3691                         tx_buffer = &txr->tx_buffers[first];
3692                         tx_desc =
3693                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first];
3694                 }
3695                 ++txr->packets;
3696                 ++ifp->if_opackets;
3697                 /* See if there is more work now */
3698                 last = tx_buffer->eop_index;
3699                 if (last != -1) {
3700                         eop_desc =
3701                             (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
3702                         /* Get next done point */
3703                         if (++last == adapter->num_tx_desc) last = 0;
3704                         done = last;
3705                 } else
3706                         break;
3707         }
3708         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3709             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3710
3711         txr->next_to_clean = first;
3712
3713         /*
3714         ** Watchdog calculation, we know there's
3715         ** work outstanding or the first return
3716         ** would have been taken, so none processed
3717         ** for too long indicates a hang.
3718         */
3719         if ((!processed) && ((ticks - txr->watchdog_time) > IXGBE_WATCHDOG))
3720                 txr->queue_status = IXGBE_QUEUE_HUNG;
3721
3722         /* With a minimum free clear the depleted state bit.  */
3723         if (txr->tx_avail > IXGBE_TX_CLEANUP_THRESHOLD)
3724                 txr->queue_status &= ~IXGBE_QUEUE_DEPLETED;
3725
3726         if (txr->tx_avail == adapter->num_tx_desc) {
3727                 txr->queue_status = IXGBE_QUEUE_IDLE;
3728                 return (FALSE);
3729         }
3730
3731         return TRUE;
3732 }
3733
3734 /*********************************************************************
3735  *
3736  *  Refresh mbuf buffers for RX descriptor rings
3737  *   - now keeps its own state so discards due to resource
3738  *     exhaustion are unnecessary, if an mbuf cannot be obtained
3739  *     it just returns, keeping its placeholder, thus it can simply
3740  *     be recalled to try again.
3741  *
3742  **********************************************************************/
3743 static void
3744 ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit)
3745 {
3746         struct adapter          *adapter = rxr->adapter;
3747         bus_dma_segment_t       hseg[1];
3748         bus_dma_segment_t       pseg[1];
3749         struct ixgbe_rx_buf     *rxbuf;
3750         struct mbuf             *mh, *mp;
3751         int                     i, j, nsegs, error;
3752         bool                    refreshed = FALSE;
3753
3754         i = j = rxr->next_to_refresh;
3755         /* Control the loop with one beyond */
3756         if (++j == adapter->num_rx_desc)
3757                 j = 0;
3758
3759         while (j != limit) {
3760                 rxbuf = &rxr->rx_buffers[i];
3761                 if (rxr->hdr_split == FALSE)
3762                         goto no_split;
3763
3764                 if (rxbuf->m_head == NULL) {
3765                         mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3766                         if (mh == NULL)
3767                                 goto update;
3768                 } else
3769                         mh = rxbuf->m_head;
3770
3771                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3772                 mh->m_len = MHLEN;
3773                 mh->m_flags |= M_PKTHDR;
3774                 /* Get the memory mapping */
3775                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
3776                     rxbuf->hmap, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3777                 if (error != 0) {
3778                         kprintf("Refresh mbufs: hdr dmamap load"
3779                             " failure - %d\n", error);
3780                         m_free(mh);
3781                         rxbuf->m_head = NULL;
3782                         goto update;
3783                 }
3784                 rxbuf->m_head = mh;
3785                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3786                     BUS_DMASYNC_PREREAD);
3787                 rxr->rx_base[i].read.hdr_addr =
3788                     htole64(hseg[0].ds_addr);
3789
3790 no_split:
3791                 if (rxbuf->m_pack == NULL) {
3792                         mp = m_getjcl(MB_DONTWAIT, MT_DATA,
3793                             M_PKTHDR, adapter->rx_mbuf_sz);
3794                         if (mp == NULL)
3795                                 goto update;
3796                 } else
3797                         mp = rxbuf->m_pack;
3798
3799                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3800                 /* Get the memory mapping */
3801                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
3802                     rxbuf->pmap, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3803                 if (error != 0) {
3804                         kprintf("Refresh mbufs: payload dmamap load"
3805                             " failure - %d\n", error);
3806                         m_free(mp);
3807                         rxbuf->m_pack = NULL;
3808                         goto update;
3809                 }
3810                 rxbuf->m_pack = mp;
3811                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3812                     BUS_DMASYNC_PREREAD);
3813                 rxr->rx_base[i].read.pkt_addr =
3814                     htole64(pseg[0].ds_addr);
3815
3816                 refreshed = TRUE;
3817                 /* Next is precalculated */
3818                 i = j;
3819                 rxr->next_to_refresh = i;
3820                 if (++j == adapter->num_rx_desc)
3821                         j = 0;
3822         }
3823 update:
3824         if (refreshed) /* Update hardware tail index */
3825                 IXGBE_WRITE_REG(&adapter->hw,
3826                     IXGBE_RDT(rxr->me), rxr->next_to_refresh);
3827         return;
3828 }
3829
3830 /*********************************************************************
3831  *
3832  *  Allocate memory for rx_buffer structures. Since we use one
3833  *  rx_buffer per received packet, the maximum number of rx_buffer's
3834  *  that we'll need is equal to the number of receive descriptors
3835  *  that we've allocated.
3836  *
3837  **********************************************************************/
3838 static int
3839 ixgbe_allocate_receive_buffers(struct rx_ring *rxr)
3840 {
3841         struct  adapter         *adapter = rxr->adapter;
3842         device_t                dev = adapter->dev;
3843         struct ixgbe_rx_buf     *rxbuf;
3844         int                     i, bsize, error;
3845
3846         bsize = sizeof(struct ixgbe_rx_buf) * adapter->num_rx_desc;
3847         if (!(rxr->rx_buffers =
3848             (struct ixgbe_rx_buf *) kmalloc(bsize,
3849             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3850                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3851                 error = ENOMEM;
3852                 goto fail;
3853         }
3854
3855         if ((error = bus_dma_tag_create(NULL,   /* parent */
3856                                    1, 0,        /* alignment, bounds */
3857                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3858                                    BUS_SPACE_MAXADDR,   /* highaddr */
3859                                    NULL, NULL,          /* filter, filterarg */
3860                                    MSIZE,               /* maxsize */
3861                                    1,                   /* nsegments */
3862                                    MSIZE,               /* maxsegsize */
3863                                    0,                   /* flags */
3864                                    &rxr->htag))) {
3865                 device_printf(dev, "Unable to create RX DMA tag\n");
3866                 goto fail;
3867         }
3868
3869         if ((error = bus_dma_tag_create(NULL,   /* parent */
3870                                    1, 0,        /* alignment, bounds */
3871                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3872                                    BUS_SPACE_MAXADDR,   /* highaddr */
3873                                    NULL, NULL,          /* filter, filterarg */
3874                                    MJUM16BYTES,         /* maxsize */
3875                                    1,                   /* nsegments */
3876                                    MJUM16BYTES,         /* maxsegsize */
3877                                    0,                   /* flags */
3878                                    &rxr->ptag))) {
3879                 device_printf(dev, "Unable to create RX DMA tag\n");
3880                 goto fail;
3881         }
3882
3883         for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) {
3884                 rxbuf = &rxr->rx_buffers[i];
3885                 error = bus_dmamap_create(rxr->htag,
3886                     BUS_DMA_NOWAIT, &rxbuf->hmap);
3887                 if (error) {
3888                         device_printf(dev, "Unable to create RX head map\n");
3889                         goto fail;
3890                 }
3891                 error = bus_dmamap_create(rxr->ptag,
3892                     BUS_DMA_NOWAIT, &rxbuf->pmap);
3893                 if (error) {
3894                         device_printf(dev, "Unable to create RX pkt map\n");
3895                         goto fail;
3896                 }
3897         }
3898
3899         return (0);
3900
3901 fail:
3902         /* Frees all, but can handle partial completion */
3903         ixgbe_free_receive_structures(adapter);
3904         return (error);
3905 }
3906
3907 /*
3908 ** Used to detect a descriptor that has
3909 ** been merged by Hardware RSC.
3910 */
3911 static inline u32
3912 ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx)
3913 {
3914         return (le32toh(rx->wb.lower.lo_dword.data) &
3915             IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT;
3916 }
3917
3918 /*********************************************************************
3919  *
3920  *  Initialize Hardware RSC (LRO) feature on 82599
3921  *  for an RX ring, this is toggled by the LRO capability
3922  *  even though it is transparent to the stack.
3923  *
3924  **********************************************************************/
3925 #if 0   /* NET_LRO */
3926 static void
3927 ixgbe_setup_hw_rsc(struct rx_ring *rxr)
3928 {
3929         struct  adapter         *adapter = rxr->adapter;
3930         struct  ixgbe_hw        *hw = &adapter->hw;
3931         u32                     rscctrl, rdrxctl;
3932
3933         rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3934         rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3935 #ifdef DEV_NETMAP /* crcstrip is optional in netmap */
3936         if (adapter->ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
3937 #endif /* DEV_NETMAP */
3938         rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3939         rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
3940         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3941
3942         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me));
3943         rscctrl |= IXGBE_RSCCTL_RSCEN;
3944         /*
3945         ** Limit the total number of descriptors that
3946         ** can be combined, so it does not exceed 64K
3947         */
3948         if (adapter->rx_mbuf_sz == MCLBYTES)
3949                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
3950         else if (adapter->rx_mbuf_sz == MJUMPAGESIZE)
3951                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
3952         else if (adapter->rx_mbuf_sz == MJUM9BYTES)
3953                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
3954         else  /* Using 16K cluster */
3955                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
3956
3957         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl);
3958
3959         /* Enable TCP header recognition */
3960         IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0),
3961             (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) |
3962             IXGBE_PSRTYPE_TCPHDR));
3963
3964         /* Disable RSC for ACK packets */
3965         IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3966             (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3967
3968         rxr->hw_rsc = TRUE;
3969 }
3970 #endif
3971
3972 static void
3973 ixgbe_free_receive_ring(struct rx_ring *rxr)
3974 {
3975         struct  adapter         *adapter;
3976         struct ixgbe_rx_buf       *rxbuf;
3977         int i;
3978
3979         adapter = rxr->adapter;
3980         for (i = 0; i < adapter->num_rx_desc; i++) {
3981                 rxbuf = &rxr->rx_buffers[i];
3982                 if (rxbuf->m_head != NULL) {
3983                         bus_dmamap_sync(rxr->htag, rxbuf->hmap,
3984                             BUS_DMASYNC_POSTREAD);
3985                         bus_dmamap_unload(rxr->htag, rxbuf->hmap);
3986                         rxbuf->m_head->m_flags |= M_PKTHDR;
3987                         m_freem(rxbuf->m_head);
3988                 }
3989                 if (rxbuf->m_pack != NULL) {
3990                         bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
3991                             BUS_DMASYNC_POSTREAD);
3992                         bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
3993                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3994                         m_freem(rxbuf->m_pack);
3995                 }
3996                 rxbuf->m_head = NULL;
3997                 rxbuf->m_pack = NULL;
3998         }
3999 }
4000
4001
4002 /*********************************************************************
4003  *
4004  *  Initialize a receive ring and its buffers.
4005  *
4006  **********************************************************************/
4007 static int
4008 ixgbe_setup_receive_ring(struct rx_ring *rxr)
4009 {
4010         struct  adapter         *adapter;
4011         struct ifnet            *ifp;
4012         device_t                dev;
4013         struct ixgbe_rx_buf     *rxbuf;
4014         bus_dma_segment_t       pseg[1], hseg[1];
4015 #if 0   /* NET_LRO */
4016         struct lro_ctrl         *lro = &rxr->lro;
4017 #endif
4018         int                     rsize, nsegs, error = 0;
4019 #ifdef DEV_NETMAP
4020         struct netmap_adapter *na = NA(rxr->adapter->ifp);
4021         struct netmap_slot *slot;
4022 #endif /* DEV_NETMAP */
4023
4024         adapter = rxr->adapter;
4025         ifp = adapter->ifp;
4026         dev = adapter->dev;
4027
4028         /* Clear the ring contents */
4029         IXGBE_RX_LOCK(rxr);
4030 #ifdef DEV_NETMAP
4031         /* same as in ixgbe_setup_transmit_ring() */
4032         slot = netmap_reset(na, NR_RX, rxr->me, 0);
4033 #endif /* DEV_NETMAP */
4034         rsize = roundup2(adapter->num_rx_desc *
4035             sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN);
4036         bzero((void *)rxr->rx_base, rsize);
4037
4038         /* Free current RX buffer structs and their mbufs */
4039         ixgbe_free_receive_ring(rxr);
4040
4041         /* Configure header split? */
4042         if (ixgbe_header_split)
4043                 rxr->hdr_split = TRUE;
4044
4045         /* Now replenish the mbufs */
4046         for (int j = 0; j != adapter->num_rx_desc; ++j) {
4047                 struct mbuf     *mh, *mp;
4048
4049                 rxbuf = &rxr->rx_buffers[j];
4050 #ifdef DEV_NETMAP
4051                 /*
4052                  * In netmap mode, fill the map and set the buffer
4053                  * address in the NIC ring, considering the offset
4054                  * between the netmap and NIC rings (see comment in
4055                  * ixgbe_setup_transmit_ring() ). No need to allocate
4056                  * an mbuf, so end the block with a continue;
4057                  */
4058                 if (slot) {
4059                         int sj = netmap_idx_n2k(&na->rx_rings[rxr->me], j);
4060                         uint64_t paddr;
4061                         void *addr;
4062
4063                         addr = PNMB(slot + sj, &paddr);
4064                         netmap_load_map(rxr->ptag, rxbuf->pmap, addr);
4065                         /* Update descriptor */
4066                         rxr->rx_base[j].read.pkt_addr = htole64(paddr);
4067                         continue;
4068                 }
4069 #endif /* DEV_NETMAP */
4070                 /*
4071                 ** Don't allocate mbufs if not
4072                 ** doing header split, its wasteful
4073                 */
4074                 if (rxr->hdr_split == FALSE)
4075                         goto skip_head;
4076
4077                 /* First the header */
4078                 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA);
4079                 if (rxbuf->m_head == NULL) {
4080                         error = ENOBUFS;
4081                         goto fail;
4082                 }
4083                 m_adj(rxbuf->m_head, ETHER_ALIGN);
4084                 mh = rxbuf->m_head;
4085                 mh->m_len = mh->m_pkthdr.len = MHLEN;
4086                 mh->m_flags |= M_PKTHDR;
4087                 /* Get the memory mapping */
4088                 error = bus_dmamap_load_mbuf_segment(rxr->htag,
4089                     rxbuf->hmap, rxbuf->m_head, hseg, 1,
4090                     &nsegs, BUS_DMA_NOWAIT);
4091
4092                 if (error != 0) /* Nothing elegant to do here */
4093                         goto fail;
4094                 bus_dmamap_sync(rxr->htag,
4095                     rxbuf->hmap, BUS_DMASYNC_PREREAD);
4096                 /* Update descriptor */
4097                 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr);
4098
4099 skip_head:
4100                 /* Now the payload cluster */
4101                 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA,
4102                     M_PKTHDR, adapter->rx_mbuf_sz);
4103                 if (rxbuf->m_pack == NULL) {
4104                         error = ENOBUFS;
4105                         goto fail;
4106                 }
4107                 mp = rxbuf->m_pack;
4108                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
4109                 /* Get the memory mapping */
4110                 error = bus_dmamap_load_mbuf_segment(rxr->ptag,
4111                     rxbuf->pmap, mp, hseg, 1,
4112                     &nsegs, BUS_DMA_NOWAIT);
4113                 if (error != 0)
4114                         goto fail;
4115                 bus_dmamap_sync(rxr->ptag,
4116                     rxbuf->pmap, BUS_DMASYNC_PREREAD);
4117                 /* Update descriptor */
4118                 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr);
4119         }
4120
4121
4122         /* Setup our descriptor indices */
4123         rxr->next_to_check = 0;
4124         rxr->next_to_refresh = 0;
4125         rxr->lro_enabled = FALSE;
4126         rxr->rx_split_packets = 0;
4127         rxr->rx_bytes = 0;
4128         rxr->discard = FALSE;
4129         rxr->vtag_strip = FALSE;
4130
4131         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4132             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4133
4134         /*
4135         ** Now set up the LRO interface:
4136         ** 82598 uses software LRO, the
4137         ** 82599 and X540 use a hardware assist.
4138         */
4139 #if 0 /* NET_LRO */
4140         if ((adapter->hw.mac.type != ixgbe_mac_82598EB) &&
4141             (ifp->if_capenable & IFCAP_RXCSUM) &&
4142             (ifp->if_capenable & IFCAP_LRO))
4143                 ixgbe_setup_hw_rsc(rxr);
4144         else if (ifp->if_capenable & IFCAP_LRO) {
4145                 int err = tcp_lro_init(lro);
4146                 if (err) {
4147                         device_printf(dev, "LRO Initialization failed!\n");
4148                         goto fail;
4149                 }
4150                 INIT_DEBUGOUT("RX Soft LRO Initialized\n");
4151                 rxr->lro_enabled = TRUE;
4152                 lro->ifp = adapter->ifp;
4153         }
4154 #endif
4155
4156         IXGBE_RX_UNLOCK(rxr);
4157         return (0);
4158
4159 fail:
4160         ixgbe_free_receive_ring(rxr);
4161         IXGBE_RX_UNLOCK(rxr);
4162         return (error);
4163 }
4164
4165 /*********************************************************************
4166  *
4167  *  Initialize all receive rings.
4168  *
4169  **********************************************************************/
4170 static int
4171 ixgbe_setup_receive_structures(struct adapter *adapter)
4172 {
4173         struct rx_ring *rxr = adapter->rx_rings;
4174         int j;
4175
4176         for (j = 0; j < adapter->num_queues; j++, rxr++)
4177                 if (ixgbe_setup_receive_ring(rxr))
4178                         goto fail;
4179
4180         return (0);
4181 fail:
4182         /*
4183          * Free RX buffers allocated so far, we will only handle
4184          * the rings that completed, the failing case will have
4185          * cleaned up for itself. 'j' failed, so its the terminus.
4186          */
4187         for (int i = 0; i < j; ++i) {
4188                 rxr = &adapter->rx_rings[i];
4189                 ixgbe_free_receive_ring(rxr);
4190         }
4191
4192         return (ENOBUFS);
4193 }
4194
4195 /*********************************************************************
4196  *
4197  *  Setup receive registers and features.
4198  *
4199  **********************************************************************/
4200 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
4201
4202 #define BSIZEPKT_ROUNDUP ((1<<IXGBE_SRRCTL_BSIZEPKT_SHIFT)-1)
4203
4204 static void
4205 ixgbe_initialize_receive_units(struct adapter *adapter)
4206 {
4207         struct  rx_ring *rxr = adapter->rx_rings;
4208         struct ixgbe_hw *hw = &adapter->hw;
4209         struct ifnet   *ifp = adapter->ifp;
4210         u32             bufsz, rxctrl, fctrl, srrctl, rxcsum;
4211         u32             reta, mrqc = 0, hlreg, random[10];
4212
4213
4214         /*
4215          * Make sure receives are disabled while
4216          * setting up the descriptor ring
4217          */
4218         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4219         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL,
4220             rxctrl & ~IXGBE_RXCTRL_RXEN);
4221
4222         /* Enable broadcasts */
4223         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
4224         fctrl |= IXGBE_FCTRL_BAM;
4225         fctrl |= IXGBE_FCTRL_DPF;
4226         fctrl |= IXGBE_FCTRL_PMCF;
4227         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
4228
4229         /* Set for Jumbo Frames? */
4230         hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4231         if (ifp->if_mtu > ETHERMTU)
4232                 hlreg |= IXGBE_HLREG0_JUMBOEN;
4233         else
4234                 hlreg &= ~IXGBE_HLREG0_JUMBOEN;
4235 #ifdef DEV_NETMAP
4236         /* crcstrip is conditional in netmap (in RDRXCTL too ?) */
4237         if (ifp->if_capenable & IFCAP_NETMAP && !ix_crcstrip)
4238                 hlreg &= ~IXGBE_HLREG0_RXCRCSTRP;
4239         else
4240                 hlreg |= IXGBE_HLREG0_RXCRCSTRP;
4241 #endif /* DEV_NETMAP */
4242         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
4243
4244         bufsz = (adapter->rx_mbuf_sz +
4245             BSIZEPKT_ROUNDUP) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
4246
4247         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4248                 u64 rdba = rxr->rxdma.dma_paddr;
4249
4250                 /* Setup the Base and Length of the Rx Descriptor Ring */
4251                 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
4252                                (rdba & 0x00000000ffffffffULL));
4253                 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
4254                 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
4255                     adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
4256
4257                 /* Set up the SRRCTL register */
4258                 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
4259                 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
4260                 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
4261                 srrctl |= bufsz;
4262                 if (rxr->hdr_split) {
4263                         /* Use a standard mbuf for the header */
4264                         srrctl |= ((IXGBE_RX_HDR <<
4265                             IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT)
4266                             & IXGBE_SRRCTL_BSIZEHDR_MASK);
4267                         srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
4268                 } else
4269                         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
4270                 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
4271
4272                 /* Setup the HW Rx Head and Tail Descriptor Pointers */
4273                 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
4274                 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
4275         }
4276
4277         if (adapter->hw.mac.type != ixgbe_mac_82598EB) {
4278                 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
4279                               IXGBE_PSRTYPE_UDPHDR |
4280                               IXGBE_PSRTYPE_IPV4HDR |
4281                               IXGBE_PSRTYPE_IPV6HDR;
4282                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
4283         }
4284
4285         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
4286
4287         /* Setup RSS */
4288         if (adapter->num_queues > 1) {
4289                 int i, j;
4290                 reta = 0;
4291
4292                 /* set up random bits */
4293                 karc4rand(&random, sizeof(random));
4294
4295                 /* Set up the redirection table */
4296                 for (i = 0, j = 0; i < 128; i++, j++) {
4297                         if (j == adapter->num_queues) j = 0;
4298                         reta = (reta << 8) | (j * 0x11);
4299                         if ((i & 3) == 3)
4300                                 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
4301                 }
4302
4303                 /* Now fill our hash function seeds */
4304                 for (int i = 0; i < 10; i++)
4305                         IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]);
4306
4307                 /* Perform hash on these packet types */
4308                 mrqc = IXGBE_MRQC_RSSEN
4309                      | IXGBE_MRQC_RSS_FIELD_IPV4
4310                      | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
4311                      | IXGBE_MRQC_RSS_FIELD_IPV4_UDP
4312                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
4313                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX
4314                      | IXGBE_MRQC_RSS_FIELD_IPV6
4315                      | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
4316                      | IXGBE_MRQC_RSS_FIELD_IPV6_UDP
4317                      | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP;
4318                 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
4319
4320                 /* RSS and RX IPP Checksum are mutually exclusive */
4321                 rxcsum |= IXGBE_RXCSUM_PCSD;
4322         }
4323
4324         if (ifp->if_capenable & IFCAP_RXCSUM)
4325                 rxcsum |= IXGBE_RXCSUM_PCSD;
4326
4327         if (!(rxcsum & IXGBE_RXCSUM_PCSD))
4328                 rxcsum |= IXGBE_RXCSUM_IPPCSE;
4329
4330         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
4331
4332         return;
4333 }
4334
4335 /*********************************************************************
4336  *
4337  *  Free all receive rings.
4338  *
4339  **********************************************************************/
4340 static void
4341 ixgbe_free_receive_structures(struct adapter *adapter)
4342 {
4343         struct rx_ring *rxr = adapter->rx_rings;
4344
4345         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4346 #if 0   /* NET_LRO */
4347                 struct lro_ctrl         *lro = &rxr->lro;
4348 #endif
4349                 ixgbe_free_receive_buffers(rxr);
4350                 /* Free LRO memory */
4351 #if 0   /* NET_LRO */
4352                 tcp_lro_free(lro);
4353 #endif
4354                 /* Free the ring memory as well */
4355                 ixgbe_dma_free(adapter, &rxr->rxdma);
4356         }
4357
4358         kfree(adapter->rx_rings, M_DEVBUF);
4359 }
4360
4361
4362 /*********************************************************************
4363  *
4364  *  Free receive ring data structures
4365  *
4366  **********************************************************************/
4367 static void
4368 ixgbe_free_receive_buffers(struct rx_ring *rxr)
4369 {
4370         struct adapter          *adapter = rxr->adapter;
4371         struct ixgbe_rx_buf     *rxbuf;
4372
4373         INIT_DEBUGOUT("free_receive_structures: begin");
4374
4375         /* Cleanup any existing buffers */
4376         if (rxr->rx_buffers != NULL) {
4377                 for (int i = 0; i < adapter->num_rx_desc; i++) {
4378                         rxbuf = &rxr->rx_buffers[i];
4379                         if (rxbuf->m_head != NULL) {
4380                                 bus_dmamap_sync(rxr->htag, rxbuf->hmap,
4381                                     BUS_DMASYNC_POSTREAD);
4382                                 bus_dmamap_unload(rxr->htag, rxbuf->hmap);
4383                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4384                                 m_freem(rxbuf->m_head);
4385                         }
4386                         if (rxbuf->m_pack != NULL) {
4387                                 bus_dmamap_sync(rxr->ptag, rxbuf->pmap,
4388                                     BUS_DMASYNC_POSTREAD);
4389                                 bus_dmamap_unload(rxr->ptag, rxbuf->pmap);
4390                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4391                                 m_freem(rxbuf->m_pack);
4392                         }
4393                         rxbuf->m_head = NULL;
4394                         rxbuf->m_pack = NULL;
4395                         if (rxbuf->hmap != NULL) {
4396                                 bus_dmamap_destroy(rxr->htag, rxbuf->hmap);
4397                                 rxbuf->hmap = NULL;
4398                         }
4399                         if (rxbuf->pmap != NULL) {
4400                                 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap);
4401                                 rxbuf->pmap = NULL;
4402                         }
4403                 }
4404                 if (rxr->rx_buffers != NULL) {
4405                         kfree(rxr->rx_buffers, M_DEVBUF);
4406                         rxr->rx_buffers = NULL;
4407                 }
4408         }
4409
4410         if (rxr->htag != NULL) {
4411                 bus_dma_tag_destroy(rxr->htag);
4412                 rxr->htag = NULL;
4413         }
4414         if (rxr->ptag != NULL) {
4415                 bus_dma_tag_destroy(rxr->ptag);
4416                 rxr->ptag = NULL;
4417         }
4418
4419         return;
4420 }
4421
4422 static __inline void
4423 ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4424 {
4425
4426         /*
4427          * ATM LRO is only for IP/TCP packets and TCP checksum of the packet
4428          * should be computed by hardware. Also it should not have VLAN tag in
4429          * ethernet header.  In case of IPv6 we do not yet support ext. hdrs.
4430          */
4431 #if 0   /* NET_LRO */
4432         if (rxr->lro_enabled &&
4433             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4434             (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4435             ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4436             (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) ||
4437             (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) ==
4438             (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) &&
4439             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4440             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4441                 /*
4442                  * Send to the stack if:
4443                  **  - LRO not enabled, or
4444                  **  - no LRO resources, or
4445                  **  - lro enqueue fails
4446                  */
4447                 if (rxr->lro.lro_cnt != 0)
4448                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4449                                 return;
4450         }
4451 #endif
4452         IXGBE_RX_UNLOCK(rxr);
4453         (*ifp->if_input)(ifp, m);
4454         IXGBE_RX_LOCK(rxr);
4455 }
4456
4457 static __inline void
4458 ixgbe_rx_discard(struct rx_ring *rxr, int i)
4459 {
4460         struct ixgbe_rx_buf     *rbuf;
4461
4462         rbuf = &rxr->rx_buffers[i];
4463
4464         if (rbuf->fmp != NULL) {/* Partial chain ? */
4465                 rbuf->fmp->m_flags |= M_PKTHDR;
4466                 m_freem(rbuf->fmp);
4467                 rbuf->fmp = NULL;
4468         }
4469
4470         /*
4471         ** With advanced descriptors the writeback
4472         ** clobbers the buffer addrs, so its easier
4473         ** to just free the existing mbufs and take
4474         ** the normal refresh path to get new buffers
4475         ** and mapping.
4476         */
4477         if (rbuf->m_head) {
4478                 m_free(rbuf->m_head);
4479                 rbuf->m_head = NULL;
4480         }
4481
4482         if (rbuf->m_pack) {
4483                 m_free(rbuf->m_pack);
4484                 rbuf->m_pack = NULL;
4485         }
4486
4487         return;
4488 }
4489
4490
4491 /*********************************************************************
4492  *
4493  *  This routine executes in interrupt context. It replenishes
4494  *  the mbufs in the descriptor and sends data which has been
4495  *  dma'ed into host memory to upper layer.
4496  *
4497  *  We loop at most count times if count is > 0, or until done if
4498  *  count < 0.
4499  *
4500  *  Return TRUE for more work, FALSE for all clean.
4501  *********************************************************************/
4502 static bool
4503 ixgbe_rxeof(struct ix_queue *que, int count)
4504 {
4505         struct adapter          *adapter = que->adapter;
4506         struct rx_ring          *rxr = que->rxr;
4507         struct ifnet            *ifp = adapter->ifp;
4508 #if 0   /* NET_LRO */
4509         struct lro_ctrl         *lro = &rxr->lro;
4510         struct lro_entry        *queued;
4511 #endif
4512         int                     i, nextp, processed = 0;
4513         u32                     staterr = 0;
4514         union ixgbe_adv_rx_desc *cur;
4515         struct ixgbe_rx_buf     *rbuf, *nbuf;
4516
4517         IXGBE_RX_LOCK(rxr);
4518
4519 #ifdef DEV_NETMAP
4520         if (ifp->if_capenable & IFCAP_NETMAP) {
4521                 /*
4522                  * Same as the txeof routine: only wakeup clients on intr.
4523                  * NKR_PENDINTR in nr_kflags is used to implement interrupt
4524                  * mitigation (ixgbe_rxsync() will not look for new packets
4525                  * unless NKR_PENDINTR is set).
4526                  */
4527                 struct netmap_adapter *na = NA(ifp);
4528
4529                 na->rx_rings[rxr->me].nr_kflags |= NKR_PENDINTR;
4530                 selwakeuppri(&na->rx_rings[rxr->me].si, PI_NET);
4531                 IXGBE_RX_UNLOCK(rxr);
4532                 IXGBE_CORE_LOCK(adapter);
4533                 selwakeuppri(&na->rx_si, PI_NET);
4534                 IXGBE_CORE_UNLOCK(adapter);
4535                 return (FALSE);
4536         }
4537 #endif /* DEV_NETMAP */
4538         for (i = rxr->next_to_check; count != 0;) {
4539                 struct mbuf     *sendmp, *mh, *mp;
4540                 u32             rsc, ptype;
4541                 u16             hlen, plen, hdr;
4542 #ifdef NET_VLAN
4543                 u16             vtag = 0;
4544 #endif
4545                 bool            eop;
4546
4547                 /* Sync the ring. */
4548                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4549                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4550
4551                 cur = &rxr->rx_base[i];
4552                 staterr = le32toh(cur->wb.upper.status_error);
4553
4554                 if ((staterr & IXGBE_RXD_STAT_DD) == 0)
4555                         break;
4556                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4557                         break;
4558
4559                 count--;
4560                 sendmp = NULL;
4561                 nbuf = NULL;
4562                 rsc = 0;
4563                 cur->wb.upper.status_error = 0;
4564                 rbuf = &rxr->rx_buffers[i];
4565                 mh = rbuf->m_head;
4566                 mp = rbuf->m_pack;
4567
4568                 plen = le16toh(cur->wb.upper.length);
4569                 ptype = le32toh(cur->wb.lower.lo_dword.data) &
4570                     IXGBE_RXDADV_PKTTYPE_MASK;
4571                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4572                 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
4573
4574                 /* Process vlan info */
4575 #ifdef NET_VLAN
4576                 if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP))
4577                         vtag = le16toh(cur->wb.upper.vlan);
4578 #endif
4579
4580                 /* Make sure bad packets are discarded */
4581                 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) ||
4582                     (rxr->discard)) {
4583                         ifp->if_ierrors++;
4584                         rxr->rx_discarded++;
4585                         if (eop)
4586                                 rxr->discard = FALSE;
4587                         else
4588                                 rxr->discard = TRUE;
4589                         ixgbe_rx_discard(rxr, i);
4590                         goto next_desc;
4591                 }
4592
4593                 /*
4594                 ** On 82599 which supports a hardware
4595                 ** LRO (called HW RSC), packets need
4596                 ** not be fragmented across sequential
4597                 ** descriptors, rather the next descriptor
4598                 ** is indicated in bits of the descriptor.
4599                 ** This also means that we might proceses
4600                 ** more than one packet at a time, something
4601                 ** that has never been true before, it
4602                 ** required eliminating global chain pointers
4603                 ** in favor of what we are doing here.  -jfv
4604                 */
4605                 if (!eop) {
4606                         /*
4607                         ** Figure out the next descriptor
4608                         ** of this frame.
4609                         */
4610                         if (rxr->hw_rsc == TRUE) {
4611                                 rsc = ixgbe_rsc_count(cur);
4612                                 rxr->rsc_num += (rsc - 1);
4613                         }
4614                         if (rsc) { /* Get hardware index */
4615                                 nextp = ((staterr &
4616                                     IXGBE_RXDADV_NEXTP_MASK) >>
4617                                     IXGBE_RXDADV_NEXTP_SHIFT);
4618                         } else { /* Just sequential */
4619                                 nextp = i + 1;
4620                                 if (nextp == adapter->num_rx_desc)
4621                                         nextp = 0;
4622                         }
4623                         nbuf = &rxr->rx_buffers[nextp];
4624                         prefetch(nbuf);
4625                 }
4626                 /*
4627                 ** The header mbuf is ONLY used when header
4628                 ** split is enabled, otherwise we get normal
4629                 ** behavior, ie, both header and payload
4630                 ** are DMA'd into the payload buffer.
4631                 **
4632                 ** Rather than using the fmp/lmp global pointers
4633                 ** we now keep the head of a packet chain in the
4634                 ** buffer struct and pass this along from one
4635                 ** descriptor to the next, until we get EOP.
4636                 */
4637                 if (rxr->hdr_split && (rbuf->fmp == NULL)) {
4638                         /* This must be an initial descriptor */
4639                         hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
4640                             IXGBE_RXDADV_HDRBUFLEN_SHIFT;
4641                         if (hlen > IXGBE_RX_HDR)
4642                                 hlen = IXGBE_RX_HDR;
4643                         mh->m_len = hlen;
4644                         mh->m_flags |= M_PKTHDR;
4645                         mh->m_next = NULL;
4646                         mh->m_pkthdr.len = mh->m_len;
4647                         /* Null buf pointer so it is refreshed */
4648                         rbuf->m_head = NULL;
4649                         /*
4650                         ** Check the payload length, this
4651                         ** could be zero if its a small
4652                         ** packet.
4653                         */
4654                         if (plen > 0) {
4655                                 mp->m_len = plen;
4656                                 mp->m_next = NULL;
4657                                 mp->m_flags &= ~M_PKTHDR;
4658                                 mh->m_next = mp;
4659                                 mh->m_pkthdr.len += mp->m_len;
4660                                 /* Null buf pointer so it is refreshed */
4661                                 rbuf->m_pack = NULL;
4662                                 rxr->rx_split_packets++;
4663                         }
4664                         /*
4665                         ** Now create the forward
4666                         ** chain so when complete
4667                         ** we wont have to.
4668                         */
4669                         if (eop == 0) {
4670                                 /* stash the chain head */
4671                                 nbuf->fmp = mh;
4672                                 /* Make forward chain */
4673                                 if (plen)
4674                                         mp->m_next = nbuf->m_pack;
4675                                 else
4676                                         mh->m_next = nbuf->m_pack;
4677                         } else {
4678                                 /* Singlet, prepare to send */
4679                                 sendmp = mh;
4680                                 /* If hardware handled vtag */
4681 #ifdef NET_VLAN
4682                                 if (vtag) {
4683                                         sendmp->m_pkthdr.ether_vtag = vtag;
4684                                         sendmp->m_flags |= M_VLANTAG;
4685                                 }
4686 #endif
4687                         }
4688                 } else {
4689                         /*
4690                         ** Either no header split, or a
4691                         ** secondary piece of a fragmented
4692                         ** split packet.
4693                         */
4694                         mp->m_len = plen;
4695                         /*
4696                         ** See if there is a stored head
4697                         ** that determines what we are
4698                         */
4699                         sendmp = rbuf->fmp;
4700                         rbuf->m_pack = rbuf->fmp = NULL;
4701
4702                         if (sendmp != NULL) {  /* secondary frag */
4703                                 mp->m_flags &= ~M_PKTHDR;
4704                                 sendmp->m_pkthdr.len += mp->m_len;
4705                         } else {
4706                                 /* first desc of a non-ps chain */
4707                                 sendmp = mp;
4708                                 sendmp->m_flags |= M_PKTHDR;
4709                                 sendmp->m_pkthdr.len = mp->m_len;
4710 #ifdef NET_VLAN
4711                                 if (staterr & IXGBE_RXD_STAT_VP) {
4712                                         sendmp->m_pkthdr.ether_vtag = vtag;
4713                                         sendmp->m_flags |= M_VLANTAG;
4714                                 }
4715 #endif
4716                         }
4717                         /* Pass the head pointer on */
4718                         if (eop == 0) {
4719                                 nbuf->fmp = sendmp;
4720                                 sendmp = NULL;
4721                                 mp->m_next = nbuf->m_pack;
4722                         }
4723                 }
4724                 ++processed;
4725                 /* Sending this frame? */
4726                 if (eop) {
4727                         sendmp->m_pkthdr.rcvif = ifp;
4728                         ifp->if_ipackets++;
4729                         rxr->rx_packets++;
4730                         /* capture data for AIM */
4731                         rxr->bytes += sendmp->m_pkthdr.len;
4732                         rxr->rx_bytes += sendmp->m_pkthdr.len;
4733                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4734                                 ixgbe_rx_checksum(staterr, sendmp, ptype);
4735 #if 0 /* __FreeBSD_version >= 800000 */
4736                         sendmp->m_pkthdr.flowid = que->msix;
4737                         sendmp->m_flags |= M_FLOWID;
4738 #endif
4739                 }
4740 next_desc:
4741                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4742                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4743
4744                 /* Advance our pointers to the next descriptor. */
4745                 if (++i == adapter->num_rx_desc)
4746                         i = 0;
4747
4748                 /* Now send to the stack or do LRO */
4749                 if (sendmp != NULL) {
4750                         rxr->next_to_check = i;
4751                         ixgbe_rx_input(rxr, ifp, sendmp, ptype);
4752                         i = rxr->next_to_check;
4753                 }
4754
4755                /* Every 8 descriptors we go to refresh mbufs */
4756                 if (processed == 8) {
4757                         ixgbe_refresh_mbufs(rxr, i);
4758                         processed = 0;
4759                 }
4760         }
4761
4762         /* Refresh any remaining buf structs */
4763         if (ixgbe_rx_unrefreshed(rxr))
4764                 ixgbe_refresh_mbufs(rxr, i);
4765
4766         rxr->next_to_check = i;
4767
4768         /*
4769          * Flush any outstanding LRO work
4770          */
4771 #if 0   /* NET_LRO */
4772         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4773                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4774                 tcp_lro_flush(lro, queued);
4775         }
4776 #endif
4777
4778         IXGBE_RX_UNLOCK(rxr);
4779
4780         /*
4781         ** We still have cleaning to do?
4782         ** Schedule another interrupt if so.
4783         */
4784         if ((staterr & IXGBE_RXD_STAT_DD) != 0) {
4785                 ixgbe_rearm_queues(adapter, (u64)(1 << que->msix));
4786                 return (TRUE);
4787         }
4788
4789         return (FALSE);
4790 }
4791
4792
4793 /*********************************************************************
4794  *
4795  *  Verify that the hardware indicated that the checksum is valid.
4796  *  Inform the stack about the status of checksum so that stack
4797  *  doesn't spend time verifying the checksum.
4798  *
4799  *********************************************************************/
4800 static void
4801 ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype)
4802 {
4803         u16     status = (u16) staterr;
4804         u8      errors = (u8) (staterr >> 24);
4805         bool    sctp = FALSE;
4806
4807         if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
4808             (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)
4809                 sctp = TRUE;
4810
4811         if (status & IXGBE_RXD_STAT_IPCS) {
4812                 if (!(errors & IXGBE_RXD_ERR_IPE)) {
4813                         /* IP Checksum Good */
4814                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4815                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4816
4817                 } else
4818                         mp->m_pkthdr.csum_flags = 0;
4819         }
4820         if (status & IXGBE_RXD_STAT_L4CS) {
4821                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4822 #if 0
4823                 if (sctp)
4824                         type = CSUM_SCTP_VALID;
4825 #endif
4826                 if (!(errors & IXGBE_RXD_ERR_TCPE)) {
4827                         mp->m_pkthdr.csum_flags |= type;
4828                         if (!sctp)
4829                                 mp->m_pkthdr.csum_data = htons(0xffff);
4830                 }
4831         }
4832         return;
4833 }
4834
4835
4836 /*
4837 ** This routine is run via an vlan config EVENT,
4838 ** it enables us to use the HW Filter table since
4839 ** we can get the vlan id. This just creates the
4840 ** entry in the soft version of the VFTA, init will
4841 ** repopulate the real table.
4842 */
4843 static void
4844 ixgbe_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4845 {
4846         struct adapter  *adapter = ifp->if_softc;
4847         u16             index, bit;
4848
4849         if (ifp->if_softc !=  arg)   /* Not our event */
4850                 return;
4851
4852         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4853                 return;
4854
4855         IXGBE_CORE_LOCK(adapter);
4856         index = (vtag >> 5) & 0x7F;
4857         bit = vtag & 0x1F;
4858         adapter->shadow_vfta[index] |= (1 << bit);
4859         ++adapter->num_vlans;
4860         ixgbe_init_locked(adapter);
4861         IXGBE_CORE_UNLOCK(adapter);
4862 }
4863
4864 /*
4865 ** This routine is run via an vlan
4866 ** unconfig EVENT, remove our entry
4867 ** in the soft vfta.
4868 */
4869 static void
4870 ixgbe_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4871 {
4872         struct adapter  *adapter = ifp->if_softc;
4873         u16             index, bit;
4874
4875         if (ifp->if_softc !=  arg)
4876                 return;
4877
4878         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4879                 return;
4880
4881         IXGBE_CORE_LOCK(adapter);
4882         index = (vtag >> 5) & 0x7F;
4883         bit = vtag & 0x1F;
4884         adapter->shadow_vfta[index] &= ~(1 << bit);
4885         --adapter->num_vlans;
4886         /* Re-init to load the changes */
4887         ixgbe_init_locked(adapter);
4888         IXGBE_CORE_UNLOCK(adapter);
4889 }
4890
4891 static void
4892 ixgbe_setup_vlan_hw_support(struct adapter *adapter)
4893 {
4894 #ifdef NET_VLAN
4895         struct ifnet    *ifp = adapter->ifp;
4896         struct ixgbe_hw *hw = &adapter->hw;
4897         struct rx_ring  *rxr;
4898         u32             ctrl;
4899
4900         /*
4901         ** We get here thru init_locked, meaning
4902         ** a soft reset, this has already cleared
4903         ** the VFTA and other state, so if there
4904         ** have been no vlan's registered do nothing.
4905         */
4906         if (adapter->num_vlans == 0)
4907                 return;
4908
4909         /*
4910         ** A soft reset zero's out the VFTA, so
4911         ** we need to repopulate it now.
4912         */
4913         for (int i = 0; i < IXGBE_VFTA_SIZE; i++)
4914                 if (adapter->shadow_vfta[i] != 0)
4915                         IXGBE_WRITE_REG(hw, IXGBE_VFTA(i),
4916                             adapter->shadow_vfta[i]);
4917
4918         ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
4919         /* Enable the Filter Table if enabled */
4920         if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
4921                 ctrl &= ~IXGBE_VLNCTRL_CFIEN;
4922                 ctrl |= IXGBE_VLNCTRL_VFE;
4923         }
4924         if (hw->mac.type == ixgbe_mac_82598EB)
4925                 ctrl |= IXGBE_VLNCTRL_VME;
4926         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
4927
4928         /* Setup the queues for vlans */
4929         for (int i = 0; i < adapter->num_queues; i++) {
4930                 rxr = &adapter->rx_rings[i];
4931                 /* On 82599 the VLAN enable is per/queue in RXDCTL */
4932                 if (hw->mac.type != ixgbe_mac_82598EB) {
4933                         ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
4934                         ctrl |= IXGBE_RXDCTL_VME;
4935                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(i), ctrl);
4936                 }
4937                 rxr->vtag_strip = TRUE;
4938         }
4939 #endif
4940 }
4941
4942 static void
4943 ixgbe_enable_intr(struct adapter *adapter)
4944 {
4945         struct ixgbe_hw *hw = &adapter->hw;
4946         struct ix_queue *que = adapter->queues;
4947         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
4948
4949
4950         /* Enable Fan Failure detection */
4951         if (hw->device_id == IXGBE_DEV_ID_82598AT)
4952                     mask |= IXGBE_EIMS_GPI_SDP1;
4953         else {
4954                     mask |= IXGBE_EIMS_ECC;
4955                     mask |= IXGBE_EIMS_GPI_SDP0;
4956                     mask |= IXGBE_EIMS_GPI_SDP1;
4957                     mask |= IXGBE_EIMS_GPI_SDP2;
4958 #ifdef IXGBE_FDIR
4959                     mask |= IXGBE_EIMS_FLOW_DIR;
4960 #endif
4961         }
4962
4963         IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
4964
4965         /* With RSS we use auto clear */
4966         if (adapter->msix_mem) {
4967                 mask = IXGBE_EIMS_ENABLE_MASK;
4968                 /* Don't autoclear Link */
4969                 mask &= ~IXGBE_EIMS_OTHER;
4970                 mask &= ~IXGBE_EIMS_LSC;
4971                 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
4972         }
4973
4974         /*
4975         ** Now enable all queues, this is done separately to
4976         ** allow for handling the extended (beyond 32) MSIX
4977         ** vectors that can be used by 82599
4978         */
4979         for (int i = 0; i < adapter->num_queues; i++, que++)
4980                 ixgbe_enable_queue(adapter, que->msix);
4981
4982         IXGBE_WRITE_FLUSH(hw);
4983
4984         return;
4985 }
4986
4987 static void
4988 ixgbe_disable_intr(struct adapter *adapter)
4989 {
4990         if (adapter->msix_mem)
4991                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, 0);
4992         if (adapter->hw.mac.type == ixgbe_mac_82598EB) {
4993                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
4994         } else {
4995                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
4996                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
4997                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
4998         }
4999         IXGBE_WRITE_FLUSH(&adapter->hw);
5000         return;
5001 }
5002
5003 u16
5004 ixgbe_read_pci_cfg(struct ixgbe_hw *hw, u32 reg)
5005 {
5006         u16 value;
5007
5008         value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev,
5009             reg, 2);
5010
5011         return (value);
5012 }
5013
5014 void
5015 ixgbe_write_pci_cfg(struct ixgbe_hw *hw, u32 reg, u16 value)
5016 {
5017         pci_write_config(((struct ixgbe_osdep *)hw->back)->dev,
5018             reg, value, 2);
5019
5020         return;
5021 }
5022
5023 /*
5024 ** Setup the correct IVAR register for a particular MSIX interrupt
5025 **   (yes this is all very magic and confusing :)
5026 **  - entry is the register array entry
5027 **  - vector is the MSIX vector for this queue
5028 **  - type is RX/TX/MISC
5029 */
5030 static void
5031 ixgbe_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type)
5032 {
5033         struct ixgbe_hw *hw = &adapter->hw;
5034         u32 ivar, index;
5035
5036         vector |= IXGBE_IVAR_ALLOC_VAL;
5037
5038         switch (hw->mac.type) {
5039
5040         case ixgbe_mac_82598EB:
5041                 if (type == -1)
5042                         entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
5043                 else
5044                         entry += (type * 64);
5045                 index = (entry >> 2) & 0x1F;
5046                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
5047                 ivar &= ~(0xFF << (8 * (entry & 0x3)));
5048                 ivar |= (vector << (8 * (entry & 0x3)));
5049                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar);
5050                 break;
5051
5052         case ixgbe_mac_82599EB:
5053         case ixgbe_mac_X540:
5054                 if (type == -1) { /* MISC IVAR */
5055                         index = (entry & 1) * 8;
5056                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5057                         ivar &= ~(0xFF << index);
5058                         ivar |= (vector << index);
5059                         IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
5060                 } else {        /* RX/TX IVARS */
5061                         index = (16 * (entry & 1)) + (8 * type);
5062                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
5063                         ivar &= ~(0xFF << index);
5064                         ivar |= (vector << index);
5065                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
5066                 }
5067
5068         default:
5069                 break;
5070         }
5071 }
5072
5073 static void
5074 ixgbe_configure_ivars(struct adapter *adapter)
5075 {
5076         struct  ix_queue *que = adapter->queues;
5077         u32 newitr;
5078
5079         if (ixgbe_max_interrupt_rate > 0)
5080                 newitr = (4000000 / ixgbe_max_interrupt_rate) & 0x0FF8;
5081         else
5082                 newitr = 0;
5083
5084         for (int i = 0; i < adapter->num_queues; i++, que++) {
5085                 /* First the RX queue entry */
5086                 ixgbe_set_ivar(adapter, i, que->msix, 0);
5087                 /* ... and the TX */
5088                 ixgbe_set_ivar(adapter, i, que->msix, 1);
5089                 /* Set an Initial EITR value */
5090                 IXGBE_WRITE_REG(&adapter->hw,
5091                     IXGBE_EITR(que->msix), newitr);
5092         }
5093
5094         /* For the Link interrupt */
5095         ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1);
5096 }
5097
5098 /*
5099 ** ixgbe_sfp_probe - called in the local timer to
5100 ** determine if a port had optics inserted.
5101 */
5102 static bool ixgbe_sfp_probe(struct adapter *adapter)
5103 {
5104         struct ixgbe_hw *hw = &adapter->hw;
5105         device_t        dev = adapter->dev;
5106         bool            result = FALSE;
5107
5108         if ((hw->phy.type == ixgbe_phy_nl) &&
5109             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5110                 s32 ret = hw->phy.ops.identify_sfp(hw);
5111                 if (ret)
5112                         goto out;
5113                 ret = hw->phy.ops.reset(hw);
5114                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5115                         device_printf(dev,"Unsupported SFP+ module detected!");
5116                         kprintf(" Reload driver with supported module.\n");
5117                         adapter->sfp_probe = FALSE;
5118                         goto out;
5119                 } else
5120                         device_printf(dev,"SFP+ module detected!\n");
5121                 /* We now have supported optics */
5122                 adapter->sfp_probe = FALSE;
5123                 /* Set the optics type so system reports correctly */
5124                 ixgbe_setup_optics(adapter);
5125                 result = TRUE;
5126         }
5127 out:
5128         return (result);
5129 }
5130
5131 /*
5132 ** Tasklet handler for MSIX Link interrupts
5133 **  - do outside interrupt since it might sleep
5134 */
5135 static void
5136 ixgbe_handle_link(void *context, int pending)
5137 {
5138         struct adapter  *adapter = context;
5139
5140         ixgbe_check_link(&adapter->hw,
5141             &adapter->link_speed, &adapter->link_up, 0);
5142         ixgbe_update_link_status(adapter);
5143 }
5144
5145 /*
5146 ** Tasklet for handling SFP module interrupts
5147 */
5148 static void
5149 ixgbe_handle_mod(void *context, int pending)
5150 {
5151         struct adapter  *adapter = context;
5152         struct ixgbe_hw *hw = &adapter->hw;
5153         device_t        dev = adapter->dev;
5154         u32 err;
5155
5156         err = hw->phy.ops.identify_sfp(hw);
5157         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5158                 device_printf(dev,
5159                     "Unsupported SFP+ module type was detected.\n");
5160                 return;
5161         }
5162         err = hw->mac.ops.setup_sfp(hw);
5163         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5164                 device_printf(dev,
5165                     "Setup failure - unsupported SFP+ module type.\n");
5166                 return;
5167         }
5168         taskqueue_enqueue(adapter->tq, &adapter->msf_task);
5169         return;
5170 }
5171
5172
5173 /*
5174 ** Tasklet for handling MSF (multispeed fiber) interrupts
5175 */
5176 static void
5177 ixgbe_handle_msf(void *context, int pending)
5178 {
5179         struct adapter  *adapter = context;
5180         struct ixgbe_hw *hw = &adapter->hw;
5181         u32 autoneg;
5182         bool negotiate;
5183
5184         autoneg = hw->phy.autoneg_advertised;
5185         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
5186                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate);
5187         if (hw->mac.ops.setup_link)
5188                 hw->mac.ops.setup_link(hw, autoneg, negotiate, TRUE);
5189         return;
5190 }
5191
5192 #ifdef IXGBE_FDIR
5193 /*
5194 ** Tasklet for reinitializing the Flow Director filter table
5195 */
5196 static void
5197 ixgbe_reinit_fdir(void *context, int pending)
5198 {
5199         struct adapter  *adapter = context;
5200         struct ifnet   *ifp = adapter->ifp;
5201
5202         if (adapter->fdir_reinit != 1) /* Shouldn't happen */
5203                 return;
5204         ixgbe_reinit_fdir_tables_82599(&adapter->hw);
5205         adapter->fdir_reinit = 0;
5206         /* re-enable flow director interrupts */
5207         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5208         /* Restart the interface */
5209         ifp->if_drv_flags |= IFF_DRV_RUNNING;
5210         return;
5211 }
5212 #endif
5213
5214 /**********************************************************************
5215  *
5216  *  Update the board statistics counters.
5217  *
5218  **********************************************************************/
5219 static void
5220 ixgbe_update_stats_counters(struct adapter *adapter)
5221 {
5222         struct ifnet   *ifp = adapter->ifp;
5223         struct ixgbe_hw *hw = &adapter->hw;
5224         u32  missed_rx = 0, bprc, lxon, lxoff, total;
5225         u64  total_missed_rx = 0;
5226
5227         adapter->stats.crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5228         adapter->stats.illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
5229         adapter->stats.errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
5230         adapter->stats.mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
5231
5232         for (int i = 0; i < 8; i++) {
5233                 u32 mp;
5234                 mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5235                 /* missed_rx tallies misses for the gprc workaround */
5236                 missed_rx += mp;
5237                 /* global total per queue */
5238                 adapter->stats.mpc[i] += mp;
5239                 /* Running comprehensive total for stats display */
5240                 total_missed_rx += adapter->stats.mpc[i];
5241                 if (hw->mac.type == ixgbe_mac_82598EB)
5242                         adapter->stats.rnbc[i] +=
5243                             IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5244                 adapter->stats.pxontxc[i] +=
5245                     IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5246                 adapter->stats.pxonrxc[i] +=
5247                     IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5248                 adapter->stats.pxofftxc[i] +=
5249                     IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5250                 adapter->stats.pxoffrxc[i] +=
5251                     IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
5252                 adapter->stats.pxon2offc[i] +=
5253                     IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
5254         }
5255         for (int i = 0; i < 16; i++) {
5256                 adapter->stats.qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5257                 adapter->stats.qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5258                 adapter->stats.qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5259                 adapter->stats.qbrc[i] +=
5260                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBRC(i)) << 32);
5261                 adapter->stats.qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5262                 adapter->stats.qbtc[i] +=
5263                     ((u64)IXGBE_READ_REG(hw, IXGBE_QBTC(i)) << 32);
5264                 adapter->stats.qprdc[i] += IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5265         }
5266         adapter->stats.mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
5267         adapter->stats.mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
5268         adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5269
5270         /* Hardware workaround, gprc counts missed packets */
5271         adapter->stats.gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5272         adapter->stats.gprc -= missed_rx;
5273
5274         if (hw->mac.type != ixgbe_mac_82598EB) {
5275                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCL) +
5276                     ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
5277                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL) +
5278                     ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
5279                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORL) +
5280                     ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
5281                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5282                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
5283         } else {
5284                 adapter->stats.lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5285                 adapter->stats.lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
5286                 /* 82598 only has a counter in the high register */
5287                 adapter->stats.gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5288                 adapter->stats.gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5289                 adapter->stats.tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5290         }
5291
5292         /*
5293          * Workaround: mprc hardware is incorrectly counting
5294          * broadcasts, so for now we subtract those.
5295          */
5296         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5297         adapter->stats.bprc += bprc;
5298         adapter->stats.mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5299         if (hw->mac.type == ixgbe_mac_82598EB)
5300                 adapter->stats.mprc -= bprc;
5301
5302         adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5303         adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5304         adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5305         adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5306         adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5307         adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5308
5309         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5310         adapter->stats.lxontxc += lxon;
5311         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5312         adapter->stats.lxofftxc += lxoff;
5313         total = lxon + lxoff;
5314
5315         adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5316         adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5317         adapter->stats.ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5318         adapter->stats.gptc -= total;
5319         adapter->stats.mptc -= total;
5320         adapter->stats.ptc64 -= total;
5321         adapter->stats.gotc -= total * ETHER_MIN_LEN;
5322
5323         adapter->stats.ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5324         adapter->stats.rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5325         adapter->stats.roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5326         adapter->stats.rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5327         adapter->stats.mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
5328         adapter->stats.mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
5329         adapter->stats.mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
5330         adapter->stats.tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5331         adapter->stats.tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
5332         adapter->stats.ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5333         adapter->stats.ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5334         adapter->stats.ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5335         adapter->stats.ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5336         adapter->stats.ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5337         adapter->stats.bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5338         adapter->stats.xec += IXGBE_READ_REG(hw, IXGBE_XEC);
5339         adapter->stats.fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5340         adapter->stats.fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
5341         /* Only read FCOE on 82599 */
5342         if (hw->mac.type != ixgbe_mac_82598EB) {
5343                 adapter->stats.fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5344                 adapter->stats.fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5345                 adapter->stats.fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5346                 adapter->stats.fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5347                 adapter->stats.fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5348         }
5349
5350         /* Fill out the OS statistics structure */
5351         ifp->if_ipackets = adapter->stats.gprc;
5352         ifp->if_opackets = adapter->stats.gptc;
5353         ifp->if_ibytes = adapter->stats.gorc;
5354         ifp->if_obytes = adapter->stats.gotc;
5355         ifp->if_imcasts = adapter->stats.mprc;
5356         ifp->if_collisions = 0;
5357
5358         /* Rx Errors */
5359         ifp->if_ierrors = total_missed_rx + adapter->stats.crcerrs +
5360                 adapter->stats.rlec;
5361 }
5362
5363 /** ixgbe_sysctl_tdh_handler - Handler function
5364  *  Retrieves the TDH value from the hardware
5365  */
5366 static int
5367 ixgbe_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS)
5368 {
5369         int error;
5370
5371         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5372         if (!txr) return 0;
5373
5374         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDH(txr->me));
5375         error = sysctl_handle_int(oidp, &val, 0, req);
5376         if (error || !req->newptr)
5377                 return error;
5378         return 0;
5379 }
5380
5381 /** ixgbe_sysctl_tdt_handler - Handler function
5382  *  Retrieves the TDT value from the hardware
5383  */
5384 static int
5385 ixgbe_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS)
5386 {
5387         int error;
5388
5389         struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1);
5390         if (!txr) return 0;
5391
5392         unsigned val = IXGBE_READ_REG(&txr->adapter->hw, IXGBE_TDT(txr->me));
5393         error = sysctl_handle_int(oidp, &val, 0, req);
5394         if (error || !req->newptr)
5395                 return error;
5396         return 0;
5397 }
5398
5399 /** ixgbe_sysctl_rdh_handler - Handler function
5400  *  Retrieves the RDH value from the hardware
5401  */
5402 static int
5403 ixgbe_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS)
5404 {
5405         int error;
5406
5407         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5408         if (!rxr) return 0;
5409
5410         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDH(rxr->me));
5411         error = sysctl_handle_int(oidp, &val, 0, req);
5412         if (error || !req->newptr)
5413                 return error;
5414         return 0;
5415 }
5416
5417 /** ixgbe_sysctl_rdt_handler - Handler function
5418  *  Retrieves the RDT value from the hardware
5419  */
5420 static int
5421 ixgbe_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS)
5422 {
5423         int error;
5424
5425         struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1);
5426         if (!rxr) return 0;
5427
5428         unsigned val = IXGBE_READ_REG(&rxr->adapter->hw, IXGBE_RDT(rxr->me));
5429         error = sysctl_handle_int(oidp, &val, 0, req);
5430         if (error || !req->newptr)
5431                 return error;
5432         return 0;
5433 }
5434
5435 static int
5436 ixgbe_sysctl_interrupt_rate_handler(SYSCTL_HANDLER_ARGS)
5437 {
5438         int error;
5439         struct ix_queue *que = ((struct ix_queue *)oidp->oid_arg1);
5440         unsigned int reg, usec, rate;
5441
5442         reg = IXGBE_READ_REG(&que->adapter->hw, IXGBE_EITR(que->msix));
5443         usec = ((reg & 0x0FF8) >> 3);
5444         if (usec > 0)
5445                 rate = 500000 / usec;
5446         else
5447                 rate = 0;
5448         error = sysctl_handle_int(oidp, &rate, 0, req);
5449         if (error || !req->newptr)
5450                 return error;
5451         reg &= ~0xfff; /* default, no limitation */
5452         ixgbe_max_interrupt_rate = 0;
5453         if (rate > 0 && rate < 500000) {
5454                 if (rate < 1000)
5455                         rate = 1000;
5456                 ixgbe_max_interrupt_rate = rate;
5457                 reg |= ((4000000/rate) & 0xff8 );
5458         }
5459         IXGBE_WRITE_REG(&que->adapter->hw, IXGBE_EITR(que->msix), reg);
5460         return 0;
5461 }
5462
5463 /*
5464  * Add sysctl variables, one per statistic, to the system.
5465  */
5466 static void
5467 ixgbe_add_hw_stats(struct adapter *adapter)
5468 {
5469         struct tx_ring *txr = adapter->tx_rings;
5470         struct rx_ring *rxr = adapter->rx_rings;
5471
5472         struct sysctl_ctx_list *ctx = &adapter->sysctl_ctx;
5473         struct sysctl_oid *tree = adapter->sysctl_tree;
5474         struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
5475         struct ixgbe_hw_stats *stats = &adapter->stats;
5476
5477         struct sysctl_oid *stat_node, *queue_node;
5478         struct sysctl_oid_list *stat_list, *queue_list;
5479
5480 #define QUEUE_NAME_LEN 32
5481         char namebuf[QUEUE_NAME_LEN];
5482
5483         /* Driver Statistics */
5484         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped",
5485                         CTLFLAG_RD, &adapter->dropped_pkts,
5486                         "Driver dropped packets");
5487         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_failed",
5488                         CTLFLAG_RD, &adapter->mbuf_defrag_failed,
5489                         "m_defrag() failed");
5490         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "no_tx_dma_setup",
5491                         CTLFLAG_RD, &adapter->no_tx_dma_setup,
5492                         "Driver tx dma failure in xmit");
5493         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
5494                         CTLFLAG_RD, &adapter->watchdog_events,
5495                         "Watchdog timeouts");
5496 #if 0   /* NET_TSO */
5497         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tso_tx",
5498                         CTLFLAG_RD, &adapter->tso_tx,
5499                         "TSO");
5500 #endif
5501         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq",
5502                         CTLFLAG_RD, &adapter->link_irq,
5503                         "Link MSIX IRQ Handled");
5504
5505         for (int i = 0; i < adapter->num_queues; i++, txr++) {
5506         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5507                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5508                                             CTLFLAG_RD, NULL, "Queue Name");
5509                 queue_list = SYSCTL_CHILDREN(queue_node);
5510
5511                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "interrupt_rate",
5512                                 CTLTYPE_UINT | CTLFLAG_RW, &adapter->queues[i],
5513                                 sizeof(&adapter->queues[i]),
5514                                 ixgbe_sysctl_interrupt_rate_handler, "IU",
5515                                 "Interrupt Rate");
5516                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
5517                                 CTLFLAG_RD, &(adapter->queues[i].irqs), 0,
5518                                 "irqs on this queue");
5519                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head",
5520                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5521                                 ixgbe_sysctl_tdh_handler, "IU",
5522                                 "Transmit Descriptor Head");
5523                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail",
5524                                 CTLTYPE_UINT | CTLFLAG_RD, txr, sizeof(txr),
5525                                 ixgbe_sysctl_tdt_handler, "IU",
5526                                 "Transmit Descriptor Tail");
5527                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
5528                                 CTLFLAG_RD, &txr->no_desc_avail, 0,
5529                                 "Queue No Descriptor Available");
5530                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
5531                                 CTLFLAG_RD, &txr->total_packets, 0,
5532                                 "Queue Packets Transmitted");
5533         }
5534
5535         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
5536         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5537                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5538                                             CTLFLAG_RD, NULL, "Queue Name");
5539                 queue_list = SYSCTL_CHILDREN(queue_node);
5540
5541 #if 0   /* NET_LRO */
5542                 struct lro_ctrl *lro = &rxr->lro;
5543 #endif
5544
5545         ksnprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i);
5546                 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf,
5547                                             CTLFLAG_RD, NULL, "Queue Name");
5548                 queue_list = SYSCTL_CHILDREN(queue_node);
5549
5550                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head",
5551                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5552                                 ixgbe_sysctl_rdh_handler, "IU",
5553                                 "Receive Descriptor Head");
5554                 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail",
5555                                 CTLTYPE_UINT | CTLFLAG_RD, rxr, sizeof(rxr),
5556                                 ixgbe_sysctl_rdt_handler, "IU",
5557                                 "Receive Descriptor Tail");
5558                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
5559                                 CTLFLAG_RD, &rxr->rx_packets, 0,
5560                                 "Queue Packets Received");
5561                 SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
5562                                 CTLFLAG_RD, &rxr->rx_bytes, 0,
5563                                 "Queue Bytes Received");
5564 #if 0   /* NET_LRO */
5565                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_queued",
5566                                 CTLFLAG_RD, &lro->lro_queued, 0,
5567                                 "LRO Queued");
5568                 SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "lro_flushed",
5569                                 CTLFLAG_RD, &lro->lro_flushed, 0,
5570                                 "LRO Flushed");
5571 #endif
5572         }
5573
5574         /* MAC stats get the own sub node */
5575
5576         stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats",
5577                                     CTLFLAG_RD, NULL, "MAC Statistics");
5578         stat_list = SYSCTL_CHILDREN(stat_node);
5579
5580         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "crc_errs",
5581                         CTLFLAG_RD, &stats->crcerrs, 0,
5582                         "CRC Errors");
5583         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "ill_errs",
5584                         CTLFLAG_RD, &stats->illerrc, 0,
5585                         "Illegal Byte Errors");
5586         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "byte_errs",
5587                         CTLFLAG_RD, &stats->errbc, 0,
5588                         "Byte Errors");
5589         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "short_discards",
5590                         CTLFLAG_RD, &stats->mspdc, 0,
5591                         "MAC Short Packets Discarded");
5592         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "local_faults",
5593                         CTLFLAG_RD, &stats->mlfc, 0,
5594                         "MAC Local Faults");
5595         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "remote_faults",
5596                         CTLFLAG_RD, &stats->mrfc, 0,
5597                         "MAC Remote Faults");
5598         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rec_len_errs",
5599                         CTLFLAG_RD, &stats->rlec, 0,
5600                         "Receive Length Errors");
5601         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_txd",
5602                         CTLFLAG_RD, &stats->lxontxc, 0,
5603                         "Link XON Transmitted");
5604         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xon_rcvd",
5605                         CTLFLAG_RD, &stats->lxonrxc, 0,
5606                         "Link XON Received");
5607         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_txd",
5608                         CTLFLAG_RD, &stats->lxofftxc, 0,
5609                         "Link XOFF Transmitted");
5610         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "link_xoff_rcvd",
5611                         CTLFLAG_RD, &stats->lxoffrxc, 0,
5612                         "Link XOFF Received");
5613
5614         /* Packet Reception Stats */
5615         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_octets_rcvd",
5616                         CTLFLAG_RD, &stats->tor, 0,
5617                         "Total Octets Received");
5618         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_rcvd",
5619                         CTLFLAG_RD, &stats->gorc, 0,
5620                         "Good Octets Received");
5621         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_rcvd",
5622                         CTLFLAG_RD, &stats->tpr, 0,
5623                         "Total Packets Received");
5624         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_rcvd",
5625                         CTLFLAG_RD, &stats->gprc, 0,
5626                         "Good Packets Received");
5627         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_rcvd",
5628                         CTLFLAG_RD, &stats->mprc, 0,
5629                         "Multicast Packets Received");
5630         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_rcvd",
5631                         CTLFLAG_RD, &stats->bprc, 0,
5632                         "Broadcast Packets Received");
5633         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_64",
5634                         CTLFLAG_RD, &stats->prc64, 0,
5635                         "64 byte frames received ");
5636         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127",
5637                         CTLFLAG_RD, &stats->prc127, 0,
5638                         "65-127 byte frames received");
5639         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255",
5640                         CTLFLAG_RD, &stats->prc255, 0,
5641                         "128-255 byte frames received");
5642         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511",
5643                         CTLFLAG_RD, &stats->prc511, 0,
5644                         "256-511 byte frames received");
5645         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023",
5646                         CTLFLAG_RD, &stats->prc1023, 0,
5647                         "512-1023 byte frames received");
5648         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522",
5649                         CTLFLAG_RD, &stats->prc1522, 0,
5650                         "1023-1522 byte frames received");
5651         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_undersized",
5652                         CTLFLAG_RD, &stats->ruc, 0,
5653                         "Receive Undersized");
5654         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_fragmented",
5655                         CTLFLAG_RD, &stats->rfc, 0,
5656                         "Fragmented Packets Received ");
5657         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_oversized",
5658                         CTLFLAG_RD, &stats->roc, 0,
5659                         "Oversized Packets Received");
5660         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "recv_jabberd",
5661                         CTLFLAG_RD, &stats->rjc, 0,
5662                         "Received Jabber");
5663         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_rcvd",
5664                         CTLFLAG_RD, &stats->mngprc, 0,
5665                         "Management Packets Received");
5666         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_drpd",
5667                         CTLFLAG_RD, &stats->mngptc, 0,
5668                         "Management Packets Dropped");
5669         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "checksum_errs",
5670                         CTLFLAG_RD, &stats->xec, 0,
5671                         "Checksum Errors");
5672
5673         /* Packet Transmission Stats */
5674         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_octets_txd",
5675                         CTLFLAG_RD, &stats->gotc, 0,
5676                         "Good Octets Transmitted");
5677         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd",
5678                         CTLFLAG_RD, &stats->tpt, 0,
5679                         "Total Packets Transmitted");
5680         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd",
5681                         CTLFLAG_RD, &stats->gptc, 0,
5682                         "Good Packets Transmitted");
5683         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd",
5684                         CTLFLAG_RD, &stats->bptc, 0,
5685                         "Broadcast Packets Transmitted");
5686         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd",
5687                         CTLFLAG_RD, &stats->mptc, 0,
5688                         "Multicast Packets Transmitted");
5689         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "management_pkts_txd",
5690                         CTLFLAG_RD, &stats->mngptc, 0,
5691                         "Management Packets Transmitted");
5692         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_64",
5693                         CTLFLAG_RD, &stats->ptc64, 0,
5694                         "64 byte frames transmitted ");
5695         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127",
5696                         CTLFLAG_RD, &stats->ptc127, 0,
5697                         "65-127 byte frames transmitted");
5698         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255",
5699                         CTLFLAG_RD, &stats->ptc255, 0,
5700                         "128-255 byte frames transmitted");
5701         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511",
5702                         CTLFLAG_RD, &stats->ptc511, 0,
5703                         "256-511 byte frames transmitted");
5704         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023",
5705                         CTLFLAG_RD, &stats->ptc1023, 0,
5706                         "512-1023 byte frames transmitted");
5707         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522",
5708                         CTLFLAG_RD, &stats->ptc1522, 0,
5709                         "1024-1522 byte frames transmitted");
5710
5711         /* FC Stats */
5712         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_crc",
5713                 CTLFLAG_RD, &stats->fccrc, 0,
5714                 "FC CRC Errors");
5715         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_last",
5716                 CTLFLAG_RD, &stats->fclast, 0,
5717                 "FC Last Error");
5718         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_drpd",
5719                 CTLFLAG_RD, &stats->fcoerpdc, 0,
5720                 "FCoE Packets Dropped");
5721         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_rcvd",
5722                 CTLFLAG_RD, &stats->fcoeprc, 0,
5723                 "FCoE Packets Received");
5724         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_pkts_txd",
5725                 CTLFLAG_RD, &stats->fcoeptc, 0,
5726                 "FCoE Packets Transmitted");
5727         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_rcvd",
5728                 CTLFLAG_RD, &stats->fcoedwrc, 0,
5729                 "FCoE DWords Received");
5730         SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, "fc_dword_txd",
5731                 CTLFLAG_RD, &stats->fcoedwtc, 0,
5732                 "FCoE DWords Transmitted");
5733 }
5734
5735 /*
5736 ** Set flow control using sysctl:
5737 ** Flow control values:
5738 **      0 - off
5739 **      1 - rx pause
5740 **      2 - tx pause
5741 **      3 - full
5742 */
5743 static int
5744 ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
5745 {
5746         int error, last;
5747         struct adapter *adapter = (struct adapter *) arg1;
5748
5749         last = adapter->fc;
5750         error = sysctl_handle_int(oidp, &adapter->fc, 0, req);
5751         if ((error) || (req->newptr == NULL))
5752                 return (error);
5753
5754         /* Don't bother if it's not changed */
5755         if (adapter->fc == last)
5756                 return (0);
5757
5758         switch (adapter->fc) {
5759                 case ixgbe_fc_rx_pause:
5760                 case ixgbe_fc_tx_pause:
5761                 case ixgbe_fc_full:
5762                         adapter->hw.fc.requested_mode = adapter->fc;
5763                         break;
5764                 case ixgbe_fc_none:
5765                 default:
5766                         adapter->hw.fc.requested_mode = ixgbe_fc_none;
5767         }
5768         /* Don't autoneg if forcing a value */
5769         adapter->hw.fc.disable_fc_autoneg = TRUE;
5770         ixgbe_fc_enable(&adapter->hw);
5771         return error;
5772 }
5773
5774 static void
5775 ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name,
5776         const char *description, int *limit, int value)
5777 {
5778         *limit = value;
5779         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5780             SYSCTL_CHILDREN(adapter->sysctl_tree),
5781             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5782 }
5783
5784 /*
5785 ** Control link advertise speed:
5786 **      1 - advertise only 1G
5787 **      2 - advertise 100Mb
5788 **      3 - advertise normal
5789 */
5790 static int
5791 ixgbe_set_advertise(SYSCTL_HANDLER_ARGS)
5792 {
5793         int                     error = 0;
5794         struct adapter          *adapter;
5795         device_t                dev;
5796         struct ixgbe_hw         *hw;
5797         ixgbe_link_speed        speed, last;
5798
5799         adapter = (struct adapter *) arg1;
5800         dev = adapter->dev;
5801         hw = &adapter->hw;
5802         last = adapter->advertise;
5803
5804         error = sysctl_handle_int(oidp, &adapter->advertise, 0, req);
5805         if ((error) || (adapter->advertise == -1))
5806                 return (error);
5807
5808         if (adapter->advertise == last) /* no change */
5809                 return (0);
5810
5811         if (!((hw->phy.media_type == ixgbe_media_type_copper) ||
5812             (hw->phy.multispeed_fiber)))
5813                 return (error);
5814
5815         if ((adapter->advertise == 2) && (hw->mac.type != ixgbe_mac_X540)) {
5816                 device_printf(dev, "Set Advertise: 100Mb on X540 only\n");
5817                 return (error);
5818         }
5819
5820         if (adapter->advertise == 1)
5821                 speed = IXGBE_LINK_SPEED_1GB_FULL;
5822         else if (adapter->advertise == 2)
5823                 speed = IXGBE_LINK_SPEED_100_FULL;
5824         else if (adapter->advertise == 3)
5825                 speed = IXGBE_LINK_SPEED_1GB_FULL |
5826                         IXGBE_LINK_SPEED_10GB_FULL;
5827         else /* bogus value */
5828                 return (error);
5829
5830         hw->mac.autotry_restart = TRUE;
5831         hw->mac.ops.setup_link(hw, speed, TRUE, TRUE);
5832
5833         return (error);
5834 }
5835
5836 /*
5837 ** Thermal Shutdown Trigger
5838 **   - cause a Thermal Overtemp IRQ
5839 */
5840 static int
5841 ixgbe_set_thermal_test(SYSCTL_HANDLER_ARGS)
5842 {
5843         int             error, fire = 0;
5844         struct adapter  *adapter = (struct adapter *) arg1;
5845         struct ixgbe_hw *hw = &adapter->hw;
5846
5847
5848         if (hw->mac.type != ixgbe_mac_X540)
5849                 return (0);
5850
5851         error = sysctl_handle_int(oidp, &fire, 0, req);
5852         if ((error) || (req->newptr == NULL))
5853                 return (error);
5854
5855         if (fire) {
5856                 u32 reg = IXGBE_READ_REG(hw, IXGBE_EICS);
5857                 reg |= IXGBE_EICR_TS;
5858                 IXGBE_WRITE_REG(hw, IXGBE_EICS, reg);
5859         }
5860
5861         return (0);
5862 }