sys/dev/netif/e1000/if_igb.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #include "opt_altq.h"
  39 #endif
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #if __FreeBSD_version >= 800000
  44 #include <sys/buf_ring.h>
  45 #endif
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <sys/kernel.h>
  49 #include <sys/kthread.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/module.h>
  53 #include <sys/rman.h>
  54 #include <sys/socket.h>
  55 #include <sys/sockio.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/taskqueue.h>
  58 #include <sys/eventhandler.h>
  59
  60 #ifdef IGB_IEEE1588
  61 #include <sys/ieee1588.h>
  62 #endif
  63
  64 #include <net/bpf.h>
  65 #include <net/ethernet.h>
  66 #include <net/if.h>
  67 #include <net/if_arp.h>
  68 #include <net/if_dl.h>
  69 #include <net/if_media.h>
  70 #include <net/ifq_var.h>
  71
  72 #include <net/if_types.h>
  73 #include <net/vlan/if_vlan_var.h>
  74 #include <net/vlan/if_vlan_ether.h>
  75
  76 #include <netinet/in_systm.h>
  77 #include <netinet/in.h>
  78 #include <netinet/if_ether.h>
  79 #include <netinet/ip.h>
  80 #include <netinet/ip6.h>
  81 #include <netinet/tcp.h>
  82 #ifdef NET_LRO
  83 #include <netinet/tcp_lro.h>
  84 #endif
  85 #include <netinet/udp.h>
  86
  87 #include <sys/in_cksum.h>
  88 #include <bus/pci/pcivar.h>
  89 #include <bus/pci/pcireg.h>
  90
  91 #include "e1000_api.h"
  92 #include "e1000_82575.h"
  93 #include "if_igb.h"
  94
  95 /*********************************************************************
  96  *  Set this to one to display debug statistics
  97  *********************************************************************/
  98 int     igb_display_debug_stats = 0;
  99
 100 /*********************************************************************
 101  *  Driver version:
 102  *********************************************************************/
 103 char igb_driver_version[] = "version - 1.9.1";
 104
 105
 106 /*********************************************************************
 107  *  PCI Device ID Table
 108  *
 109  *  Used by probe to select devices to load on
 110  *  Last field stores an index into e1000_strings
 111  *  Last entry must be all 0s
 112  *
 113  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 114  *********************************************************************/
 115
 116 static igb_vendor_info_t igb_vendor_info_array[] =
 117 {
 118         { 0x8086, E1000_DEV_ID_82575EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 119         { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
 120                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
 122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82576,           PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82576_NS,        PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
 126         { 0x8086, E1000_DEV_ID_82576_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82576_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 128         { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
 129                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
 131                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 132         { 0x8086, E1000_DEV_ID_82580_COPPER,    PCI_ANY_ID, PCI_ANY_ID, 0},
 133         { 0x8086, E1000_DEV_ID_82580_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82580_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82580_SGMII,     PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
 137                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 138         /* required last entry */
 139         { 0, 0, 0, 0, 0}
 140 };
 141
 142 /*********************************************************************
 143  *  Table of branding strings for all supported NICs.
 144  *********************************************************************/
 145
 146 static char *igb_strings[] = {
 147         "Intel(R) PRO/1000 Network Connection"
 148 };
 149
 150 /*********************************************************************
 151  *  Function prototypes
 152  *********************************************************************/
 153 static int      igb_probe(device_t);
 154 static int      igb_attach(device_t);
 155 static int      igb_detach(device_t);
 156 static int      igb_shutdown(device_t);
 157 static int      igb_suspend(device_t);
 158 static int      igb_resume(device_t);
 159 static void     igb_start(struct ifnet *);
 160 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
 161 #if __FreeBSD_version >= 800000
 162 static int      igb_mq_start(struct ifnet *, struct mbuf *);
 163 static int      igb_mq_start_locked(struct ifnet *,
 164                     struct tx_ring *, struct mbuf *);
 165 static void     igb_qflush(struct ifnet *);
 166 #endif
 167 static int      igb_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 168 static void     igb_init(void *);
 169 static void     igb_init_locked(struct adapter *);
 170 static void     igb_stop(void *);
 171 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
 172 static int      igb_media_change(struct ifnet *);
 173 static void     igb_identify_hardware(struct adapter *);
 174 static int      igb_allocate_pci_resources(struct adapter *);
 175 static int      igb_allocate_msix(struct adapter *);
 176 static int      igb_allocate_legacy(struct adapter *);
 177 static int      igb_setup_msix(struct adapter *);
 178 static void     igb_free_pci_resources(struct adapter *);
 179 static void     igb_local_timer(void *);
 180 static void     igb_reset(struct adapter *);
 181 static void     igb_setup_interface(device_t, struct adapter *);
 182 static int      igb_allocate_queues(struct adapter *);
 183 static void     igb_configure_queues(struct adapter *);
 184
 185 static int      igb_allocate_transmit_buffers(struct tx_ring *);
 186 static void     igb_setup_transmit_structures(struct adapter *);
 187 static void     igb_setup_transmit_ring(struct tx_ring *);
 188 static void     igb_initialize_transmit_units(struct adapter *);
 189 static void     igb_free_transmit_structures(struct adapter *);
 190 static void     igb_free_transmit_buffers(struct tx_ring *);
 191
 192 static int      igb_allocate_receive_buffers(struct rx_ring *);
 193 static int      igb_setup_receive_structures(struct adapter *);
 194 static int      igb_setup_receive_ring(struct rx_ring *);
 195 static void     igb_initialize_receive_units(struct adapter *);
 196 static void     igb_free_receive_structures(struct adapter *);
 197 static void     igb_free_receive_buffers(struct rx_ring *);
 198 static void     igb_free_receive_ring(struct rx_ring *);
 199
 200 static void     igb_enable_intr(struct adapter *);
 201 static void     igb_disable_intr(struct adapter *);
 202 static void     igb_update_stats_counters(struct adapter *);
 203 static bool     igb_txeof(struct tx_ring *);
 204
 205 static __inline void igb_rx_discard(struct rx_ring *,
 206                     union e1000_adv_rx_desc *, int);
 207 static __inline void igb_rx_input(struct rx_ring *,
 208                     struct ifnet *, struct mbuf *, u32);
 209
 210 static bool     igb_rxeof(struct rx_ring *, int);
 211 static void     igb_rx_checksum(u32, struct mbuf *, u32);
 212 static int      igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 213 #if NET_TSO
 214 static bool     igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 215 #endif
 216 static void     igb_set_promisc(struct adapter *);
 217 static void     igb_disable_promisc(struct adapter *);
 218 static void     igb_set_multi(struct adapter *);
 219 static void     igb_print_hw_stats(struct adapter *);
 220 static void     igb_update_link_status(struct adapter *);
 221 static int      igb_get_buf(struct rx_ring *, int, u8);
 222
 223 static void     igb_register_vlan(void *, struct ifnet *, u16);
 224 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
 225 static void     igb_setup_vlan_hw_support(struct adapter *);
 226
 227 static int      igb_xmit(struct tx_ring *, struct mbuf **);
 228 static int      igb_dma_malloc(struct adapter *, bus_size_t,
 229                     struct igb_dma_alloc *, int);
 230 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
 231 static void     igb_print_debug_info(struct adapter *);
 232 static void     igb_print_nvm_info(struct adapter *);
 233 static int      igb_is_valid_ether_addr(u8 *);
 234 static int      igb_sysctl_stats(SYSCTL_HANDLER_ARGS);
 235 static int      igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 236 /* Management and WOL Support */
 237 static void     igb_init_manageability(struct adapter *);
 238 static void     igb_release_manageability(struct adapter *);
 239 static void     igb_get_hw_control(struct adapter *);
 240 static void     igb_release_hw_control(struct adapter *);
 241 static void     igb_enable_wakeup(device_t);
 242
 243 static void     igb_irq_fast(void *);
 244 static void     igb_add_rx_process_limit(struct adapter *, const char *,
 245                     const char *, int *, int);
 246 static void     igb_handle_rxtx(void *context, int pending);
 247 static void     igb_handle_que(void *context, int pending);
 248 static void     igb_handle_link(void *context, int pending);
 249
 250 /* These are MSIX only irq handlers */
 251 static void     igb_msix_que(void *);
 252 static void     igb_msix_link(void *);
 253
 254 #ifdef DEVICE_POLLING
 255 static poll_handler_t igb_poll;
 256 #endif /* POLLING */
 257
 258 /*********************************************************************
 259  *  FreeBSD Device Interface Entry Points
 260  *********************************************************************/
 261
 262 static device_method_t igb_methods[] = {
 263         /* Device interface */
 264         DEVMETHOD(device_probe, igb_probe),
 265         DEVMETHOD(device_attach, igb_attach),
 266         DEVMETHOD(device_detach, igb_detach),
 267         DEVMETHOD(device_shutdown, igb_shutdown),
 268         DEVMETHOD(device_suspend, igb_suspend),
 269         DEVMETHOD(device_resume, igb_resume),
 270         {0, 0}
 271 };
 272
 273 static driver_t igb_driver = {
 274         "igb", igb_methods, sizeof(struct adapter),
 275 };
 276
 277 static devclass_t igb_devclass;
 278 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
 279 MODULE_DEPEND(igb, pci, 1, 1, 1);
 280 MODULE_DEPEND(igb, ether, 1, 1, 1);
 281
 282 /*********************************************************************
 283  *  Tunable default values.
 284  *********************************************************************/
 285
 286 /* Descriptor defaults */
 287 static int igb_rxd = IGB_DEFAULT_RXD;
 288 static int igb_txd = IGB_DEFAULT_TXD;
 289 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
 290 TUNABLE_INT("hw.igb.txd", &igb_txd);
 291
 292 /*
 293 ** AIM: Adaptive Interrupt Moderation
 294 ** which means that the interrupt rate
 295 ** is varied over time based on the
 296 ** traffic for that interrupt vector
 297 */
 298 static int igb_enable_aim = TRUE;
 299 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
 300
 301 /*
 302  * MSIX should be the default for best performance,
 303  * but this allows it to be forced off for testing.
 304  */
 305 static int igb_enable_msix = 1;
 306 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
 307
 308 /*
 309  * Header split has seemed to be beneficial in
 310  * many circumstances tested, however there have
 311  * been some stability issues, so the default is
 312  * off.
 313  */
 314 static bool igb_header_split = FALSE;
 315 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
 316
 317 /*
 318 ** This will autoconfigure based on
 319 ** the number of CPUs if left at 0.
 320 */
 321 static int igb_num_queues = 0;
 322 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
 323
 324 /* How many packets rxeof tries to clean at a time */
 325 static int igb_rx_process_limit = 100;
 326 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
 327
 328 /* Flow control setting - default to FULL */
 329 static int igb_fc_setting = e1000_fc_full;
 330 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
 331
 332 /*
 333 ** Shadow VFTA table, this is needed because
 334 ** the real filter table gets cleared during
 335 ** a soft reset and the driver needs to be able
 336 ** to repopulate it.
 337 */
 338 static u32 igb_shadow_vfta[IGB_VFTA_SIZE];
 339
 340
 341 /*********************************************************************
 342  *  Device identification routine
 343  *
 344  *  igb_probe determines if the driver should be loaded on
 345  *  adapter based on PCI vendor/device id of the adapter.
 346  *
 347  *  return BUS_PROBE_DEFAULT on success, positive on failure
 348  *********************************************************************/
 349
 350 static int
 351 igb_probe(device_t dev)
 352 {
 353         char            adapter_name[60];
 354         uint16_t        pci_vendor_id = 0;
 355         uint16_t        pci_device_id = 0;
 356         uint16_t        pci_subvendor_id = 0;
 357         uint16_t        pci_subdevice_id = 0;
 358         igb_vendor_info_t *ent;
 359
 360         INIT_DEBUGOUT("igb_probe: begin");
 361
 362         pci_vendor_id = pci_get_vendor(dev);
 363         if (pci_vendor_id != IGB_VENDOR_ID)
 364                 return (ENXIO);
 365
 366         pci_device_id = pci_get_device(dev);
 367         pci_subvendor_id = pci_get_subvendor(dev);
 368         pci_subdevice_id = pci_get_subdevice(dev);
 369
 370         ent = igb_vendor_info_array;
 371         while (ent->vendor_id != 0) {
 372                 if ((pci_vendor_id == ent->vendor_id) &&
 373                     (pci_device_id == ent->device_id) &&
 374
 375                     ((pci_subvendor_id == ent->subvendor_id) ||
 376                     (ent->subvendor_id == PCI_ANY_ID)) &&
 377
 378                     ((pci_subdevice_id == ent->subdevice_id) ||
 379                     (ent->subdevice_id == PCI_ANY_ID))) {
 380                         ksprintf(adapter_name, "%s %s",
 381                                 igb_strings[ent->index],
 382                                 igb_driver_version);
 383                         device_set_desc_copy(dev, adapter_name);
 384                         return (BUS_PROBE_DEFAULT);
 385                 }
 386                 ent++;
 387         }
 388
 389         return (ENXIO);
 390 }
 391
 392 /*********************************************************************
 393  *  Device initialization routine
 394  *
 395  *  The attach entry point is called when the driver is being loaded.
 396  *  This routine identifies the type of hardware, allocates all resources
 397  *  and initializes the hardware.
 398  *
 399  *  return 0 on success, positive on failure
 400  *********************************************************************/
 401
 402 static int
 403 igb_attach(device_t dev)
 404 {
 405         struct adapter  *adapter;
 406         int             error = 0;
 407         u16             eeprom_data;
 408
 409         INIT_DEBUGOUT("igb_attach: begin");
 410
 411         adapter = device_get_softc(dev);
 412         adapter->dev = adapter->osdep.dev = dev;
 413         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 414
 415         /* SYSCTL stuff */
 416         sysctl_ctx_init(&adapter->sysctl_ctx);
 417         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 418                                         SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
 419                                         device_get_nameunit(adapter->dev),
 420                                         CTLFLAG_RD, 0, "");
 421         if (adapter->sysctl_tree == NULL) {
 422                 device_printf(adapter->dev, "can't add sysctl node\n");
 423                 error = ENOMEM;
 424                 goto err_sysctl;
 425         }
 426
 427         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 428             SYSCTL_CHILDREN(adapter->sysctl_tree),
 429             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 430             igb_sysctl_debug_info, "I", "Debug Information");
 431
 432         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 433             SYSCTL_CHILDREN(adapter->sysctl_tree),
 434             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 435             igb_sysctl_stats, "I", "Statistics");
 436
 437         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 438             SYSCTL_CHILDREN(adapter->sysctl_tree),
 439             OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
 440             &igb_fc_setting, 0, "Flow Control");
 441
 442         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
 443             SYSCTL_CHILDREN(adapter->sysctl_tree),
 444             OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 445             &igb_enable_aim, 1, "Interrupt Moderation");
 446
 447         callout_init(&adapter->timer);
 448
 449         /* Determine hardware and mac info */
 450         igb_identify_hardware(adapter);
 451
 452         /* Setup PCI resources */
 453         if (igb_allocate_pci_resources(adapter)) {
 454                 device_printf(dev, "Allocation of PCI resources failed\n");
 455                 error = ENXIO;
 456                 goto err_pci;
 457         }
 458
 459         /* Do Shared Code initialization */
 460         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 461                 device_printf(dev, "Setup of Shared code failed\n");
 462                 error = ENXIO;
 463                 goto err_pci;
 464         }
 465
 466         e1000_get_bus_info(&adapter->hw);
 467
 468         /* Sysctls for limiting the amount of work done in the taskqueue */
 469         igb_add_rx_process_limit(adapter, "rx_processing_limit",
 470             "max number of rx packets to process", &adapter->rx_process_limit,
 471             igb_rx_process_limit);
 472
 473         /*
 474          * Validate number of transmit and receive descriptors. It
 475          * must not exceed hardware maximum, and must be multiple
 476          * of E1000_DBA_ALIGN.
 477          */
 478         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
 479             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
 480                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 481                     IGB_DEFAULT_TXD, igb_txd);
 482                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
 483         } else
 484                 adapter->num_tx_desc = igb_txd;
 485         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
 486             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
 487                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 488                     IGB_DEFAULT_RXD, igb_rxd);
 489                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
 490         } else
 491                 adapter->num_rx_desc = igb_rxd;
 492
 493         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 494         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 495         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 496
 497         /* Copper options */
 498         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 499                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 500                 adapter->hw.phy.disable_polarity_correction = FALSE;
 501                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
 502         }
 503
 504         /*
 505          * Set the frame limits assuming
 506          * standard ethernet sized frames.
 507          */
 508         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 509         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 510
 511         /*
 512         ** Allocate and Setup Queues
 513         */
 514         if (igb_allocate_queues(adapter)) {
 515                 error = ENOMEM;
 516                 goto err_pci;
 517         }
 518
 519         /*
 520         ** Start from a known state, this is
 521         ** important in reading the nvm and
 522         ** mac from that.
 523         */
 524         e1000_reset_hw(&adapter->hw);
 525
 526         /* Make sure we have a good EEPROM before we read from it */
 527         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 528                 /*
 529                 ** Some PCI-E parts fail the first check due to
 530                 ** the link being in sleep state, call it again,
 531                 ** if it fails a second time its a real issue.
 532                 */
 533                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 534                         device_printf(dev,
 535                             "The EEPROM Checksum Is Not Valid\n");
 536                         error = EIO;
 537                         goto err_late;
 538                 }
 539         }
 540
 541         /*
 542         ** Copy the permanent MAC address out of the EEPROM
 543         */
 544         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 545                 device_printf(dev, "EEPROM read error while reading MAC"
 546                     " address\n");
 547                 error = EIO;
 548                 goto err_late;
 549         }
 550         /* Check its sanity */
 551         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
 552                 device_printf(dev, "Invalid MAC address\n");
 553                 error = EIO;
 554                 goto err_late;
 555         }
 556
 557         /*
 558         ** Configure Interrupts
 559         */
 560         if ((adapter->msix > 1) && (igb_enable_msix))
 561                 error = igb_allocate_msix(adapter);
 562         else /* MSI or Legacy */
 563                 error = igb_allocate_legacy(adapter);
 564         if (error)
 565                 goto err_late;
 566
 567         /* Setup OS specific network interface */
 568         igb_setup_interface(dev, adapter);
 569
 570         /* Now get a good starting state */
 571         igb_reset(adapter);
 572
 573         /* Initialize statistics */
 574         igb_update_stats_counters(adapter);
 575
 576         adapter->hw.mac.get_link_status = 1;
 577         igb_update_link_status(adapter);
 578
 579         /* Indicate SOL/IDER usage */
 580         if (e1000_check_reset_block(&adapter->hw))
 581                 device_printf(dev,
 582                     "PHY reset is blocked due to SOL/IDER session.\n");
 583
 584         /* Determine if we have to control management hardware */
 585         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 586
 587         /*
 588          * Setup Wake-on-Lan
 589          */
 590         /* APME bit in EEPROM is mapped to WUC.APME */
 591         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
 592         if (eeprom_data)
 593                 adapter->wol = E1000_WUFC_MAG;
 594
 595         /* Register for VLAN events */
 596         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 597              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 598         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 599              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 600
 601         /* Tell the stack that the interface is not active */
 602         adapter->ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
 603
 604         INIT_DEBUGOUT("igb_attach: end");
 605
 606         return (0);
 607
 608 err_late:
 609         igb_free_transmit_structures(adapter);
 610         igb_free_receive_structures(adapter);
 611         igb_release_hw_control(adapter);
 612 err_pci:
 613         igb_free_pci_resources(adapter);
 614 err_sysctl:
 615         sysctl_ctx_free(&adapter->sysctl_ctx);
 616         IGB_CORE_LOCK_DESTROY(adapter);
 617
 618         return (error);
 619 }
 620
 621 /*********************************************************************
 622  *  Device removal routine
 623  *
 624  *  The detach entry point is called when the driver is being removed.
 625  *  This routine stops the adapter and deallocates all the resources
 626  *  that were allocated for driver operation.
 627  *
 628  *  return 0 on success, positive on failure
 629  *********************************************************************/
 630
 631 static int
 632 igb_detach(device_t dev)
 633 {
 634         struct adapter  *adapter = device_get_softc(dev);
 635
 636         INIT_DEBUGOUT("igb_detach: begin");
 637
 638         /* Make sure VLANS are not using driver */
 639         if (adapter->ifp->if_vlantrunks != NULL) {
 640                 device_printf(dev,"Vlan in use, detach first\n");
 641                 return (EBUSY);
 642         }
 643
 644 #ifdef DEVICE_POLLING
 645         if (adapter->ifp->if_capenable & IFCAP_POLLING)
 646                 ether_poll_deregister(adapter->ifp);
 647 #endif
 648
 649         IGB_CORE_LOCK(adapter);
 650         adapter->in_detach = 1;
 651         igb_stop(adapter);
 652         IGB_CORE_UNLOCK(adapter);
 653
 654         e1000_phy_hw_reset(&adapter->hw);
 655
 656         /* Give control back to firmware */
 657         igb_release_manageability(adapter);
 658         igb_release_hw_control(adapter);
 659
 660         if (adapter->wol) {
 661                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 662                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 663                 igb_enable_wakeup(dev);
 664         }
 665
 666         /* Unregister VLAN events */
 667         if (adapter->vlan_attach != NULL)
 668                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 669         if (adapter->vlan_detach != NULL)
 670                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 671
 672         ether_ifdetach(adapter->ifp);
 673
 674         //callout_drain(&adapter->timer);
 675         callout_stop(&adapter->timer);
 676
 677         igb_free_pci_resources(adapter);
 678         bus_generic_detach(dev);
 679
 680         igb_free_transmit_structures(adapter);
 681         igb_free_receive_structures(adapter);
 682
 683         sysctl_ctx_free(&adapter->sysctl_ctx);
 684         IGB_CORE_LOCK_DESTROY(adapter);
 685
 686         return (0);
 687 }
 688
 689 /*********************************************************************
 690  *
 691  *  Shutdown entry point
 692  *
 693  **********************************************************************/
 694
 695 static int
 696 igb_shutdown(device_t dev)
 697 {
 698         return igb_suspend(dev);
 699 }
 700
 701 /*
 702  * Suspend/resume device methods.
 703  */
 704 static int
 705 igb_suspend(device_t dev)
 706 {
 707         struct adapter *adapter = device_get_softc(dev);
 708
 709         IGB_CORE_LOCK(adapter);
 710
 711         igb_stop(adapter);
 712
 713         igb_release_manageability(adapter);
 714         igb_release_hw_control(adapter);
 715
 716         if (adapter->wol) {
 717                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 718                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 719                 igb_enable_wakeup(dev);
 720         }
 721
 722         IGB_CORE_UNLOCK(adapter);
 723
 724         return bus_generic_suspend(dev);
 725 }
 726
 727 static int
 728 igb_resume(device_t dev)
 729 {
 730         struct adapter *adapter = device_get_softc(dev);
 731         struct ifnet *ifp = adapter->ifp;
 732
 733         IGB_CORE_LOCK(adapter);
 734         igb_init_locked(adapter);
 735         igb_init_manageability(adapter);
 736
 737         if ((ifp->if_flags & IFF_UP) &&
 738             (ifp->if_flags & IFF_RUNNING))
 739                 igb_start(ifp);
 740
 741         IGB_CORE_UNLOCK(adapter);
 742
 743         return bus_generic_resume(dev);
 744 }
 745
 746
 747 /*********************************************************************
 748  *  Transmit entry point
 749  *
 750  *  igb_start is called by the stack to initiate a transmit.
 751  *  The driver will remain in this routine as long as there are
 752  *  packets to transmit and transmit resources are available.
 753  *  In case resources are not available stack is notified and
 754  *  the packet is requeued.
 755  **********************************************************************/
 756
 757 static void
 758 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
 759 {
 760         struct adapter  *adapter = ifp->if_softc;
 761         struct mbuf     *m_head;
 762
 763         IGB_TX_LOCK_ASSERT(txr);
 764
 765         if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) !=
 766             IFF_RUNNING)
 767                 return;
 768         if (!adapter->link_active)
 769                 return;
 770
 771         while (!ifq_is_empty(&ifp->if_snd)) {
 772
 773                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
 774                 if (m_head == NULL)
 775                         break;
 776                 /*
 777                  *  Encapsulation can modify our pointer, and or make it
 778                  *  NULL on failure.  In that event, we can't requeue.
 779                  */
 780                 if (igb_xmit(txr, &m_head)) {
 781                         if (m_head == NULL)
 782                                 break;
 783                         ifp->if_flags |= IFF_OACTIVE;
 784                         ifq_prepend(&ifp->if_snd, m_head);
 785                         break;
 786                 }
 787
 788                 /* Send a copy of the frame to the BPF listener */
 789                 ETHER_BPF_MTAP(ifp, m_head);
 790
 791                 /* Set watchdog on */
 792                 txr->watchdog_check = TRUE;
 793         }
 794 }
 795
 796 /*
 797  * Legacy TX driver routine, called from the
 798  * stack, always uses tx[0], and spins for it.
 799  * Should not be used with multiqueue tx
 800  */
 801 static void
 802 igb_start(struct ifnet *ifp)
 803 {
 804         struct adapter  *adapter = ifp->if_softc;
 805         struct tx_ring  *txr = adapter->tx_rings;
 806
 807         if (ifp->if_flags & IFF_RUNNING) {
 808                 IGB_TX_LOCK(txr);
 809                 igb_start_locked(txr, ifp);
 810                 IGB_TX_UNLOCK(txr);
 811         }
 812         return;
 813 }
 814
 815 #if __FreeBSD_version >= 800000
 816 /*
 817 ** Multiqueue Transmit driver
 818 **
 819 */
 820 static int
 821 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
 822 {
 823         struct adapter  *adapter = ifp->if_softc;
 824         struct tx_ring  *txr;
 825         int             i = 0, err = 0;
 826
 827         /* Which queue to use */
 828         if ((m->m_flags & M_FLOWID) != 0)
 829                 i = m->m_pkthdr.flowid % adapter->num_queues;
 830         txr = &adapter->tx_rings[i];
 831
 832         if (IGB_TX_TRYLOCK(txr)) {
 833                 err = igb_mq_start_locked(ifp, txr, m);
 834                 IGB_TX_UNLOCK(txr);
 835         } else
 836                 err = drbr_enqueue(ifp, txr->br, m);
 837
 838         return (err);
 839 }
 840
 841 static int
 842 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 843 {
 844         struct adapter  *adapter = txr->adapter;
 845         struct mbuf     *next;
 846         int             err = 0, enq;
 847
 848         IGB_TX_LOCK_ASSERT(txr);
 849
 850         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) !=
 851             IFF_RUNNING || adapter->link_active == 0) {
 852                 if (m != NULL)
 853                         err = drbr_enqueue(ifp, txr->br, m);
 854                 return (err);
 855         }
 856
 857         enq = 0;
 858         if (m == NULL) {
 859                 next = drbr_dequeue(ifp, txr->br);
 860         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 861                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 862                         return (err);
 863                 next = drbr_dequeue(ifp, txr->br);
 864         } else
 865                 next = m;
 866         /* Process the queue */
 867         while (next != NULL) {
 868                 if ((err = igb_xmit(txr, &next)) != 0) {
 869                         if (next != NULL)
 870                                 err = drbr_enqueue(ifp, txr->br, next);
 871                         break;
 872                 }
 873                 enq++;
 874                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 875                 ETHER_BPF_MTAP(ifp, next);
 876                 if ((ifp->if_flags & IFF_RUNNING) == 0)
 877                         break;
 878                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
 879                         ifp->if_flags |= IFF_OACTIVE;
 880                         break;
 881                 }
 882                 next = drbr_dequeue(ifp, txr->br);
 883         }
 884         if (enq > 0) {
 885                 /* Set the watchdog */
 886                 txr->watchdog_check = TRUE;
 887         }
 888         return (err);
 889 }
 890
 891 /*
 892 ** Flush all ring buffers
 893 */
 894 static void
 895 igb_qflush(struct ifnet *ifp)
 896 {
 897         struct adapter  *adapter = ifp->if_softc;
 898         struct tx_ring  *txr = adapter->tx_rings;
 899         struct mbuf     *m;
 900
 901         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 902                 IGB_TX_LOCK(txr);
 903                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 904                         m_freem(m);
 905                 IGB_TX_UNLOCK(txr);
 906         }
 907         if_qflush(ifp);
 908 }
 909 #endif /* __FreeBSD_version >= 800000 */
 910
 911 /*********************************************************************
 912  *  Ioctl entry point
 913  *
 914  *  igb_ioctl is called when the user wants to configure the
 915  *  interface.
 916  *
 917  *  return 0 on success, positive on failure
 918  **********************************************************************/
 919
 920 static int
 921 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cred)
 922 {
 923         struct adapter  *adapter = ifp->if_softc;
 924         struct ifreq *ifr = (struct ifreq *)data;
 925 #ifdef INET
 926         struct ifaddr *ifa = (struct ifaddr *)data;
 927 #endif
 928         int error = 0;
 929
 930         if (adapter->in_detach)
 931                 return (error);
 932
 933         switch (command) {
 934         case SIOCSIFADDR:
 935 #ifdef INET
 936                 if (ifa->ifa_addr->sa_family == AF_INET) {
 937                         /*
 938                          * XXX
 939                          * Since resetting hardware takes a very long time
 940                          * and results in link renegotiation we only
 941                          * initialize the hardware only when it is absolutely
 942                          * required.
 943                          */
 944                         ifp->if_flags |= IFF_UP;
 945                         if (!(ifp->if_flags & IFF_RUNNING)) {
 946                                 IGB_CORE_LOCK(adapter);
 947                                 igb_init_locked(adapter);
 948                                 IGB_CORE_UNLOCK(adapter);
 949                         }
 950                         if (!(ifp->if_flags & IFF_NOARP))
 951                                 arp_ifinit(ifp, ifa);
 952                 } else
 953 #endif
 954                         error = ether_ioctl(ifp, command, data);
 955                 break;
 956         case SIOCSIFMTU:
 957             {
 958                 int max_frame_size;
 959
 960                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 961
 962                 IGB_CORE_LOCK(adapter);
 963                 max_frame_size = 9234;
 964                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 965                     ETHER_CRC_LEN) {
 966                         IGB_CORE_UNLOCK(adapter);
 967                         error = EINVAL;
 968                         break;
 969                 }
 970
 971                 ifp->if_mtu = ifr->ifr_mtu;
 972                 adapter->max_frame_size =
 973                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 974                 igb_init_locked(adapter);
 975                 IGB_CORE_UNLOCK(adapter);
 976                 break;
 977             }
 978         case SIOCSIFFLAGS:
 979                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 980                     SIOCSIFFLAGS (Set Interface Flags)");
 981                 IGB_CORE_LOCK(adapter);
 982                 if (ifp->if_flags & IFF_UP) {
 983                         if ((ifp->if_flags & IFF_RUNNING)) {
 984                                 if ((ifp->if_flags ^ adapter->if_flags) &
 985                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 986                                         igb_disable_promisc(adapter);
 987                                         igb_set_promisc(adapter);
 988                                 }
 989                         } else
 990                                 igb_init_locked(adapter);
 991                 } else
 992                         if (ifp->if_flags & IFF_RUNNING)
 993                                 igb_stop(adapter);
 994                 adapter->if_flags = ifp->if_flags;
 995                 IGB_CORE_UNLOCK(adapter);
 996                 break;
 997         case SIOCADDMULTI:
 998         case SIOCDELMULTI:
 999                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1000                 if (ifp->if_flags & IFF_RUNNING) {
1001                         IGB_CORE_LOCK(adapter);
1002                         igb_disable_intr(adapter);
1003                         igb_set_multi(adapter);
1004 #ifdef DEVICE_POLLING
1005                         if (!(ifp->if_capenable & IFCAP_POLLING))
1006 #endif
1007                                 igb_enable_intr(adapter);
1008                         IGB_CORE_UNLOCK(adapter);
1009                 }
1010                 break;
1011         case SIOCSIFMEDIA:
1012                 /* Check SOL/IDER usage */
1013                 IGB_CORE_LOCK(adapter);
1014                 if (e1000_check_reset_block(&adapter->hw)) {
1015                         IGB_CORE_UNLOCK(adapter);
1016                         device_printf(adapter->dev, "Media change is"
1017                             " blocked due to SOL/IDER session.\n");
1018                         break;
1019                 }
1020                 IGB_CORE_UNLOCK(adapter);
1021         case SIOCGIFMEDIA:
1022                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1023                     SIOCxIFMEDIA (Get/Set Interface Media)");
1024                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1025                 break;
1026         case SIOCSIFCAP:
1027             {
1028                 int mask, reinit;
1029
1030                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1031                 reinit = 0;
1032                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1033 #ifdef DEVICE_POLLING
1034                 if (mask & IFCAP_POLLING) {
1035                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1036                                 error = ether_poll_register(igb_poll, ifp);
1037                                 if (error)
1038                                         return (error);
1039                                 IGB_CORE_LOCK(adapter);
1040                                 igb_disable_intr(adapter);
1041                                 ifp->if_capenable |= IFCAP_POLLING;
1042                                 IGB_CORE_UNLOCK(adapter);
1043                         } else {
1044                                 error = ether_poll_deregister(ifp);
1045                                 /* Enable interrupt even in error case */
1046                                 IGB_CORE_LOCK(adapter);
1047                                 igb_enable_intr(adapter);
1048                                 ifp->if_capenable &= ~IFCAP_POLLING;
1049                                 IGB_CORE_UNLOCK(adapter);
1050                         }
1051                 }
1052 #endif
1053                 if (mask & IFCAP_HWCSUM) {
1054                         ifp->if_capenable ^= IFCAP_HWCSUM;
1055                         reinit = 1;
1056                 }
1057 #ifdef NET_TSO
1058                 if (mask & IFCAP_TSO4) {
1059                         ifp->if_capenable ^= IFCAP_TSO4;
1060                         reinit = 1;
1061                 }
1062 #endif
1063                 if (mask & IFCAP_VLAN_HWTAGGING) {
1064                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1065                         reinit = 1;
1066                 }
1067 #ifdef NET_LRO
1068                 if (mask & IFCAP_LRO) {
1069                         ifp->if_capenable ^= IFCAP_LRO;
1070                         reinit = 1;
1071                 }
1072 #endif
1073                 if (reinit && (ifp->if_flags & IFF_RUNNING))
1074                         igb_init(adapter);
1075 #if 0
1076                 VLAN_CAPABILITIES(ifp);
1077 #endif
1078                 break;
1079             }
1080
1081         default:
1082                 error = ether_ioctl(ifp, command, data);
1083                 break;
1084         }
1085
1086         return (error);
1087 }
1088
1089
1090 /*********************************************************************
1091  *  Init entry point
1092  *
1093  *  This routine is used in two ways. It is used by the stack as
1094  *  init entry point in network interface structure. It is also used
1095  *  by the driver as a hw/sw initialization routine to get to a
1096  *  consistent state.
1097  *
1098  *  return 0 on success, positive on failure
1099  **********************************************************************/
1100
1101 static void
1102 igb_init_locked(struct adapter *adapter)
1103 {
1104         struct ifnet    *ifp = adapter->ifp;
1105         device_t        dev = adapter->dev;
1106
1107         INIT_DEBUGOUT("igb_init: begin");
1108
1109         IGB_CORE_LOCK_ASSERT(adapter);
1110
1111         igb_disable_intr(adapter);
1112         callout_stop(&adapter->timer);
1113
1114         /* Get the latest mac address, User can use a LAA */
1115         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1116               ETHER_ADDR_LEN);
1117
1118         /* Put the address into the Receive Address Array */
1119         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1120
1121         igb_reset(adapter);
1122         igb_update_link_status(adapter);
1123
1124         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1125
1126         /* Set hardware offload abilities */
1127         ifp->if_hwassist = 0;
1128         if (ifp->if_capenable & IFCAP_TXCSUM) {
1129                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1130 #if __FreeBSD_version >= 800000
1131                 if (adapter->hw.mac.type == e1000_82576)
1132                         ifp->if_hwassist |= CSUM_SCTP;
1133 #endif
1134         }
1135
1136 #ifdef NET_TSO
1137         if (ifp->if_capenable & IFCAP_TSO4)
1138                 ifp->if_hwassist |= CSUM_TSO;
1139 #endif
1140
1141         /* Configure for OS presence */
1142         igb_init_manageability(adapter);
1143
1144         /* Prepare transmit descriptors and buffers */
1145         igb_setup_transmit_structures(adapter);
1146         igb_initialize_transmit_units(adapter);
1147
1148         /* Setup Multicast table */
1149         igb_set_multi(adapter);
1150
1151         /*
1152         ** Figure out the desired mbuf pool
1153         ** for doing jumbo/packetsplit
1154         */
1155         if (ifp->if_mtu > ETHERMTU)
1156                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1157         else
1158                 adapter->rx_mbuf_sz = MCLBYTES;
1159
1160         /* Prepare receive descriptors and buffers */
1161         if (igb_setup_receive_structures(adapter)) {
1162                 device_printf(dev, "Could not setup receive structures\n");
1163                 return;
1164         }
1165         igb_initialize_receive_units(adapter);
1166
1167         /* Don't lose promiscuous settings */
1168         igb_set_promisc(adapter);
1169
1170         ifp->if_flags |= IFF_RUNNING;
1171         ifp->if_flags &= ~IFF_OACTIVE;
1172
1173         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1174         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1175
1176         if (adapter->msix > 1) /* Set up queue routing */
1177                 igb_configure_queues(adapter);
1178
1179         /* Set up VLAN tag offload and filter */
1180         igb_setup_vlan_hw_support(adapter);
1181
1182         /* this clears any pending interrupts */
1183         E1000_READ_REG(&adapter->hw, E1000_ICR);
1184 #ifdef DEVICE_POLLING
1185         /*
1186          * Only enable interrupts if we are not polling, make sure
1187          * they are off otherwise.
1188          */
1189         if (ifp->if_capenable & IFCAP_POLLING)
1190                 igb_disable_intr(adapter);
1191         else
1192 #endif /* DEVICE_POLLING */
1193         {
1194         igb_enable_intr(adapter);
1195         E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1196         }
1197
1198         /* Don't reset the phy next time init gets called */
1199         adapter->hw.phy.reset_disable = TRUE;
1200 }
1201
1202 static void
1203 igb_init(void *arg)
1204 {
1205         struct adapter *adapter = arg;
1206
1207         IGB_CORE_LOCK(adapter);
1208         igb_init_locked(adapter);
1209         IGB_CORE_UNLOCK(adapter);
1210 }
1211
1212
1213 static void
1214 igb_handle_rxtx(void *context, int pending)
1215 {
1216         struct adapter  *adapter = context;
1217         struct tx_ring  *txr = adapter->tx_rings;
1218         struct rx_ring  *rxr = adapter->rx_rings;
1219         struct ifnet    *ifp;
1220
1221         ifp = adapter->ifp;
1222
1223         if (ifp->if_flags & IFF_RUNNING) {
1224                 if (igb_rxeof(rxr, adapter->rx_process_limit))
1225                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1226                 IGB_TX_LOCK(txr);
1227                 igb_txeof(txr);
1228
1229 #if __FreeBSD_version >= 800000
1230                 if (!drbr_empty(ifp, txr->br))
1231                         igb_mq_start_locked(ifp, txr, NULL);
1232 #else
1233                 if (!ifq_is_empty(&ifp->if_snd))
1234                         igb_start_locked(txr, ifp);
1235 #endif
1236                 IGB_TX_UNLOCK(txr);
1237         }
1238
1239         igb_enable_intr(adapter);
1240 }
1241
1242 static void
1243 igb_handle_que(void *context, int pending)
1244 {
1245         struct igb_queue *que = context;
1246         struct adapter *adapter = que->adapter;
1247         struct tx_ring *txr = que->txr;
1248         struct rx_ring *rxr = que->rxr;
1249         struct ifnet    *ifp = adapter->ifp;
1250         u32             loop = IGB_MAX_LOOP;
1251         bool            more;
1252
1253         /* RX first */
1254         do {
1255                 more = igb_rxeof(rxr, -1);
1256         } while (loop-- && more);
1257
1258         if (IGB_TX_TRYLOCK(txr)) {
1259                 loop = IGB_MAX_LOOP;
1260                 do {
1261                         more = igb_txeof(txr);
1262                 } while (loop-- && more);
1263 #if __FreeBSD_version >= 800000
1264                 igb_mq_start_locked(ifp, txr, NULL);
1265 #else
1266                 if (!ifq_is_empty(&ifp->if_snd))
1267                         igb_start_locked(txr, ifp);
1268 #endif
1269                 IGB_TX_UNLOCK(txr);
1270         }
1271
1272         /* Reenable this interrupt */
1273 #ifdef DEVICE_POLLING
1274         if (!(ifp->if_capenable & IFCAP_POLLING))
1275 #endif
1276         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1277 }
1278
1279 /* Deal with link in a sleepable context */
1280 static void
1281 igb_handle_link(void *context, int pending)
1282 {
1283         struct adapter *adapter = context;
1284
1285         adapter->hw.mac.get_link_status = 1;
1286         igb_update_link_status(adapter);
1287 }
1288
1289 /*********************************************************************
1290  *
1291  *  MSI/Legacy Deferred
1292  *  Interrupt Service routine
1293  *
1294  *********************************************************************/
1295 #define FILTER_STRAY
1296 #define FILTER_HANDLED
1297 static void
1298 igb_irq_fast(void *arg)
1299 {
1300         struct adapter  *adapter = arg;
1301         uint32_t        reg_icr;
1302
1303
1304         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1305
1306         /* Hot eject?  */
1307         if (reg_icr == 0xffffffff)
1308                 return FILTER_STRAY;
1309
1310         /* Definitely not our interrupt.  */
1311         if (reg_icr == 0x0)
1312                 return FILTER_STRAY;
1313
1314         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1315                 return FILTER_STRAY;
1316
1317         /*
1318          * Mask interrupts until the taskqueue is finished running.  This is
1319          * cheap, just assume that it is needed.  This also works around the
1320          * MSI message reordering errata on certain systems.
1321          */
1322         igb_disable_intr(adapter);
1323         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1324
1325         /* Link status change */
1326         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1327                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1328
1329         if (reg_icr & E1000_ICR_RXO)
1330                 adapter->rx_overruns++;
1331         return FILTER_HANDLED;
1332 }
1333
1334 #ifdef DEVICE_POLLING
1335 /*********************************************************************
1336  *
1337  *  Legacy polling routine
1338  *
1339  *********************************************************************/
1340 #if __FreeBSD_version >= 800000
1341 #define POLL_RETURN_COUNT(a) (a)
1342 static int
1343 #else
1344 #define POLL_RETURN_COUNT(a)
1345 static void
1346 #endif
1347 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1348 {
1349         struct adapter *adapter = ifp->if_softc;
1350         struct rx_ring  *rxr = adapter->rx_rings;
1351         struct tx_ring  *txr = adapter->tx_rings;
1352         u32             reg_icr, rx_done = 0;
1353         u32             loop = IGB_MAX_LOOP;
1354         bool            more;
1355
1356         IGB_CORE_LOCK(adapter);
1357         if ((ifp->if_flags & IFF_RUNNING) == 0) {
1358                 IGB_CORE_UNLOCK(adapter);
1359                 return POLL_RETURN_COUNT(rx_done);
1360         }
1361
1362         if (cmd == POLL_AND_CHECK_STATUS) {
1363                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1364                 /* Link status change */
1365                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1366                         taskqueue_enqueue(adapter->tq, &adapter->link_task);
1367
1368                 if (reg_icr & E1000_ICR_RXO)
1369                         adapter->rx_overruns++;
1370         }
1371         IGB_CORE_UNLOCK(adapter);
1372
1373         /* TODO: rx_count */
1374         rx_done = igb_rxeof(rxr, count) ? 1 : 0;
1375
1376         IGB_TX_LOCK(txr);
1377         do {
1378                 more = igb_txeof(txr);
1379         } while (loop-- && more);
1380 #if __FreeBSD_version >= 800000
1381         if (!drbr_empty(ifp, txr->br))
1382                 igb_mq_start_locked(ifp, txr, NULL);
1383 #else
1384         if (!ifq_is_empty(&ifp->if_snd))
1385                 igb_start_locked(txr, ifp);
1386 #endif
1387         IGB_TX_UNLOCK(txr);
1388         return POLL_RETURN_COUNT(rx_done);
1389 }
1390 #endif /* DEVICE_POLLING */
1391
1392 /*********************************************************************
1393  *
1394  *  MSIX TX Interrupt Service routine
1395  *
1396  **********************************************************************/
1397 static void
1398 igb_msix_que(void *arg)
1399 {
1400         struct igb_queue *que = arg;
1401         struct adapter *adapter = que->adapter;
1402         struct tx_ring *txr = que->txr;
1403         struct rx_ring *rxr = que->rxr;
1404         u32             newitr = 0;
1405         bool            more_tx, more_rx;
1406
1407         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1408         ++que->irqs;
1409
1410         IGB_TX_LOCK(txr);
1411         more_tx = igb_txeof(txr);
1412         IGB_TX_UNLOCK(txr);
1413
1414         more_rx = igb_rxeof(rxr, adapter->rx_process_limit);
1415
1416         if (igb_enable_aim == FALSE)
1417                 goto no_calc;
1418         /*
1419         ** Do Adaptive Interrupt Moderation:
1420         **  - Write out last calculated setting
1421         **  - Calculate based on average size over
1422         **    the last interval.
1423         */
1424         if (que->eitr_setting)
1425                 E1000_WRITE_REG(&adapter->hw,
1426                     E1000_EITR(que->msix), que->eitr_setting);
1427
1428         que->eitr_setting = 0;
1429
1430         /* Idle, do nothing */
1431         if ((txr->bytes == 0) && (rxr->bytes == 0))
1432                 goto no_calc;
1433
1434         /* Used half Default if sub-gig */
1435         if (adapter->link_speed != 1000)
1436                 newitr = IGB_DEFAULT_ITR / 2;
1437         else {
1438                 if ((txr->bytes) && (txr->packets))
1439                         newitr = txr->bytes/txr->packets;
1440                 if ((rxr->bytes) && (rxr->packets))
1441                         newitr = max(newitr,
1442                             (rxr->bytes / rxr->packets));
1443                 newitr += 24; /* account for hardware frame, crc */
1444                 /* set an upper boundary */
1445                 newitr = min(newitr, 3000);
1446                 /* Be nice to the mid range */
1447                 if ((newitr > 300) && (newitr < 1200))
1448                         newitr = (newitr / 3);
1449                 else
1450                         newitr = (newitr / 2);
1451         }
1452         newitr &= 0x7FFC;  /* Mask invalid bits */
1453         if (adapter->hw.mac.type == e1000_82575)
1454                 newitr |= newitr << 16;
1455         else
1456                 newitr |= 0x8000000;
1457
1458         /* save for next interrupt */
1459         que->eitr_setting = newitr;
1460
1461         /* Reset state */
1462         txr->bytes = 0;
1463         txr->packets = 0;
1464         rxr->bytes = 0;
1465         rxr->packets = 0;
1466
1467 no_calc:
1468         /* Schedule a clean task if needed*/
1469         if (more_tx || more_rx)
1470                 taskqueue_enqueue(que->tq, &que->que_task);
1471         else
1472                 /* Reenable this interrupt */
1473                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1474         return;
1475 }
1476
1477
1478 /*********************************************************************
1479  *
1480  *  MSIX Link Interrupt Service routine
1481  *
1482  **********************************************************************/
1483
1484 static void
1485 igb_msix_link(void *arg)
1486 {
1487         struct adapter  *adapter = arg;
1488         u32             icr;
1489
1490         ++adapter->link_irq;
1491         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1492         if (!(icr & E1000_ICR_LSC))
1493                 goto spurious;
1494         taskqueue_enqueue(adapter->tq, &adapter->link_task);
1495
1496 spurious:
1497         /* Rearm */
1498         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1499         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1500         return;
1501 }
1502
1503
1504 /*********************************************************************
1505  *
1506  *  Media Ioctl callback
1507  *
1508  *  This routine is called whenever the user queries the status of
1509  *  the interface using ifconfig.
1510  *
1511  **********************************************************************/
1512 static void
1513 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1514 {
1515         struct adapter *adapter = ifp->if_softc;
1516         u_char fiber_type = IFM_1000_SX;
1517
1518         INIT_DEBUGOUT("igb_media_status: begin");
1519
1520         IGB_CORE_LOCK(adapter);
1521         igb_update_link_status(adapter);
1522
1523         ifmr->ifm_status = IFM_AVALID;
1524         ifmr->ifm_active = IFM_ETHER;
1525
1526         if (!adapter->link_active) {
1527                 IGB_CORE_UNLOCK(adapter);
1528                 return;
1529         }
1530
1531         ifmr->ifm_status |= IFM_ACTIVE;
1532
1533         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1534             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
1535                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1536         else {
1537                 switch (adapter->link_speed) {
1538                 case 10:
1539                         ifmr->ifm_active |= IFM_10_T;
1540                         break;
1541                 case 100:
1542                         ifmr->ifm_active |= IFM_100_TX;
1543                         break;
1544                 case 1000:
1545                         ifmr->ifm_active |= IFM_1000_T;
1546                         break;
1547                 }
1548                 if (adapter->link_duplex == FULL_DUPLEX)
1549                         ifmr->ifm_active |= IFM_FDX;
1550                 else
1551                         ifmr->ifm_active |= IFM_HDX;
1552         }
1553         IGB_CORE_UNLOCK(adapter);
1554 }
1555
1556 /*********************************************************************
1557  *
1558  *  Media Ioctl callback
1559  *
1560  *  This routine is called when the user changes speed/duplex using
1561  *  media/mediopt option with ifconfig.
1562  *
1563  **********************************************************************/
1564 static int
1565 igb_media_change(struct ifnet *ifp)
1566 {
1567         struct adapter *adapter = ifp->if_softc;
1568         struct ifmedia  *ifm = &adapter->media;
1569
1570         INIT_DEBUGOUT("igb_media_change: begin");
1571
1572         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1573                 return (EINVAL);
1574
1575         IGB_CORE_LOCK(adapter);
1576         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1577         case IFM_AUTO:
1578                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1579                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1580                 break;
1581         case IFM_1000_LX:
1582         case IFM_1000_SX:
1583         case IFM_1000_T:
1584                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1585                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1586                 break;
1587         case IFM_100_TX:
1588                 adapter->hw.mac.autoneg = FALSE;
1589                 adapter->hw.phy.autoneg_advertised = 0;
1590                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1591                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1592                 else
1593                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1594                 break;
1595         case IFM_10_T:
1596                 adapter->hw.mac.autoneg = FALSE;
1597                 adapter->hw.phy.autoneg_advertised = 0;
1598                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1599                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1600                 else
1601                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1602                 break;
1603         default:
1604                 device_printf(adapter->dev, "Unsupported media type\n");
1605         }
1606
1607         /* As the speed/duplex settings my have changed we need to
1608          * reset the PHY.
1609          */
1610         adapter->hw.phy.reset_disable = FALSE;
1611
1612         igb_init_locked(adapter);
1613         IGB_CORE_UNLOCK(adapter);
1614
1615         return (0);
1616 }
1617
1618
1619 /*********************************************************************
1620  *
1621  *  This routine maps the mbufs to Advanced TX descriptors.
1622  *  used by the 82575 adapter.
1623  *
1624  **********************************************************************/
1625
1626 static int
1627 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1628 {
1629         struct adapter          *adapter = txr->adapter;
1630         bus_dma_segment_t       segs[IGB_MAX_SCATTER];
1631         bus_dmamap_t            map;
1632         struct igb_tx_buffer    *tx_buffer, *tx_buffer_mapped;
1633         union e1000_adv_tx_desc *txd = NULL;
1634         struct mbuf             *m_head;
1635         u32                     olinfo_status = 0, cmd_type_len = 0;
1636         int                     nsegs, i, j, error, first, last = 0;
1637         u32                     hdrlen = 0;
1638
1639         m_head = *m_headp;
1640
1641
1642         /* Set basic descriptor constants */
1643         cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1644         cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1645         if (m_head->m_flags & M_VLANTAG)
1646                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1647
1648         /*
1649          * Force a cleanup if number of TX descriptors
1650          * available hits the threshold
1651          */
1652         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) {
1653                 igb_txeof(txr);
1654                 /* Now do we at least have a minimal? */
1655                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
1656                         txr->no_desc_avail++;
1657                         return (ENOBUFS);
1658                 }
1659         }
1660
1661         /*
1662          * Map the packet for DMA.
1663          *
1664          * Capture the first descriptor index,
1665          * this descriptor will have the index
1666          * of the EOP which is the only one that
1667          * now gets a DONE bit writeback.
1668          */
1669         first = txr->next_avail_desc;
1670         tx_buffer = &txr->tx_buffers[first];
1671         tx_buffer_mapped = tx_buffer;
1672         map = tx_buffer->map;
1673
1674         error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1675             *m_headp, segs, IGB_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
1676
1677         if (error == EFBIG) {
1678                 struct mbuf *m;
1679
1680                 m = m_defrag(*m_headp, MB_DONTWAIT);
1681                 if (m == NULL) {
1682                         adapter->mbuf_defrag_failed++;
1683                         m_freem(*m_headp);
1684                         *m_headp = NULL;
1685                         return (ENOBUFS);
1686                 }
1687                 *m_headp = m;
1688
1689                 /* Try it again */
1690                 error = bus_dmamap_load_mbuf_segment(txr->txtag, map,
1691                     *m_headp, segs, IGB_MAX_SCATTER, &nsegs, BUS_DMA_NOWAIT);
1692
1693                 if (error == ENOMEM) {
1694                         adapter->no_tx_dma_setup++;
1695                         return (error);
1696                 } else if (error != 0) {
1697                         adapter->no_tx_dma_setup++;
1698                         m_freem(*m_headp);
1699                         *m_headp = NULL;
1700                         return (error);
1701                 }
1702         } else if (error == ENOMEM) {
1703                 adapter->no_tx_dma_setup++;
1704                 return (error);
1705         } else if (error != 0) {
1706                 adapter->no_tx_dma_setup++;
1707                 m_freem(*m_headp);
1708                 *m_headp = NULL;
1709                 return (error);
1710         }
1711
1712         /* Check again to be sure we have enough descriptors */
1713         if (nsegs > (txr->tx_avail - 2)) {
1714                 txr->no_desc_avail++;
1715                 bus_dmamap_unload(txr->txtag, map);
1716                 return (ENOBUFS);
1717         }
1718         m_head = *m_headp;
1719
1720         /*
1721          * Set up the context descriptor:
1722          * used when any hardware offload is done.
1723          * This includes CSUM, VLAN, and TSO. It
1724          * will use the first descriptor.
1725          */
1726 #ifdef NET_TSO
1727         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1728                 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1729                         cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1730                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1731                         olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1732                 } else
1733                         return (ENXIO);
1734         } else
1735 #endif
1736                if (igb_tx_ctx_setup(txr, m_head))
1737                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1738
1739         /* Calculate payload length */
1740         olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1741             << E1000_ADVTXD_PAYLEN_SHIFT);
1742
1743         /* 82575 needs the queue index added */
1744         if (adapter->hw.mac.type == e1000_82575)
1745                 olinfo_status |= txr->me << 4;
1746
1747         /* Set up our transmit descriptors */
1748         i = txr->next_avail_desc;
1749         for (j = 0; j < nsegs; j++) {
1750                 bus_size_t seg_len;
1751                 bus_addr_t seg_addr;
1752
1753                 tx_buffer = &txr->tx_buffers[i];
1754                 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
1755                 seg_addr = segs[j].ds_addr;
1756                 seg_len  = segs[j].ds_len;
1757
1758                 txd->read.buffer_addr = htole64(seg_addr);
1759                 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len);
1760                 txd->read.olinfo_status = htole32(olinfo_status);
1761                 last = i;
1762                 if (++i == adapter->num_tx_desc)
1763                         i = 0;
1764                 tx_buffer->m_head = NULL;
1765                 tx_buffer->next_eop = -1;
1766         }
1767
1768         txr->next_avail_desc = i;
1769         txr->tx_avail -= nsegs;
1770
1771         tx_buffer->m_head = m_head;
1772         tx_buffer_mapped->map = tx_buffer->map;
1773         tx_buffer->map = map;
1774         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1775
1776         /*
1777          * Last Descriptor of Packet
1778          * needs End Of Packet (EOP)
1779          * and Report Status (RS)
1780          */
1781         txd->read.cmd_type_len |=
1782             htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS);
1783         /*
1784          * Keep track in the first buffer which
1785          * descriptor will be written back
1786          */
1787         tx_buffer = &txr->tx_buffers[first];
1788         tx_buffer->next_eop = last;
1789         txr->watchdog_time = ticks;
1790
1791         /*
1792          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1793          * that this frame is available to transmit.
1794          */
1795         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1796             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1797         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1798         ++txr->tx_packets;
1799
1800         return (0);
1801
1802 }
1803
1804 static void
1805 igb_set_promisc(struct adapter *adapter)
1806 {
1807         struct ifnet    *ifp = adapter->ifp;
1808         uint32_t        reg_rctl;
1809
1810         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1811
1812         if (ifp->if_flags & IFF_PROMISC) {
1813                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1814                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1815         } else if (ifp->if_flags & IFF_ALLMULTI) {
1816                 reg_rctl |= E1000_RCTL_MPE;
1817                 reg_rctl &= ~E1000_RCTL_UPE;
1818                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1819         }
1820 }
1821
1822 static void
1823 igb_disable_promisc(struct adapter *adapter)
1824 {
1825         uint32_t        reg_rctl;
1826
1827         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1828
1829         reg_rctl &=  (~E1000_RCTL_UPE);
1830         reg_rctl &=  (~E1000_RCTL_MPE);
1831         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1832 }
1833
1834
1835 /*********************************************************************
1836  *  Multicast Update
1837  *
1838  *  This routine is called whenever multicast address list is updated.
1839  *
1840  **********************************************************************/
1841
1842 static void
1843 igb_set_multi(struct adapter *adapter)
1844 {
1845         struct ifnet    *ifp = adapter->ifp;
1846         struct ifmultiaddr *ifma;
1847         u32 reg_rctl = 0;
1848         u8  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN];
1849
1850         int mcnt = 0;
1851
1852         IOCTL_DEBUGOUT("igb_set_multi: begin");
1853
1854 #if 0
1855 #if __FreeBSD_version < 800000
1856         IF_ADDR_LOCK(ifp);
1857 #else
1858         if_maddr_rlock(ifp);
1859 #endif
1860 #endif
1861
1862         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1863                 if (ifma->ifma_addr->sa_family != AF_LINK)
1864                         continue;
1865
1866                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1867                         break;
1868
1869                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1870                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
1871                 mcnt++;
1872         }
1873 #if 0
1874 #if __FreeBSD_version < 800000
1875         IF_ADDR_UNLOCK(ifp);
1876 #else
1877         if_maddr_runlock(ifp);
1878 #endif
1879 #endif
1880
1881         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1882                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1883                 reg_rctl |= E1000_RCTL_MPE;
1884                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1885         } else
1886                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1887 }
1888
1889
1890 /*********************************************************************
1891  *  Timer routine:
1892  *      This routine checks for link status,
1893  *      updates statistics, and does the watchdog.
1894  *
1895  **********************************************************************/
1896
1897 static void
1898 igb_local_timer(void *arg)
1899 {
1900         struct adapter          *adapter = arg;
1901
1902         IGB_CORE_LOCK(adapter);
1903
1904         struct ifnet            *ifp = adapter->ifp;
1905         device_t                dev = adapter->dev;
1906         struct tx_ring          *txr = adapter->tx_rings;
1907
1908
1909         IGB_CORE_LOCK_ASSERT(adapter);
1910
1911         igb_update_link_status(adapter);
1912         igb_update_stats_counters(adapter);
1913
1914         if (igb_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1915                 igb_print_hw_stats(adapter);
1916
1917         /*
1918         ** Watchdog: check for time since any descriptor was cleaned
1919         */
1920         for (int i = 0; i < adapter->num_queues; i++, txr++) {
1921                 if (txr->watchdog_check == FALSE)
1922                         continue;
1923                 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG)
1924                         goto timeout;
1925         }
1926
1927         /* Trigger an RX interrupt on all queues */
1928 #ifdef DEVICE_POLLING
1929         if (!(ifp->if_capenable & IFCAP_POLLING))
1930 #endif
1931         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask);
1932         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1933         IGB_CORE_UNLOCK(adapter);
1934         return;
1935
1936 timeout:
1937         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1938         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1939             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
1940             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
1941         device_printf(dev,"TX(%d) desc avail = %d,"
1942             "Next TX to Clean = %d\n",
1943             txr->me, txr->tx_avail, txr->next_to_clean);
1944         adapter->ifp->if_flags &= ~IFF_RUNNING;
1945         adapter->watchdog_events++;
1946         igb_init_locked(adapter);
1947         IGB_CORE_UNLOCK(adapter);
1948 }
1949
1950 static void
1951 igb_update_link_status(struct adapter *adapter)
1952 {
1953         struct e1000_hw *hw = &adapter->hw;
1954         struct ifnet *ifp = adapter->ifp;
1955         device_t dev = adapter->dev;
1956         struct tx_ring *txr = adapter->tx_rings;
1957         u32 link_check = 0;
1958
1959         /* Get the cached link value or read for real */
1960         switch (hw->phy.media_type) {
1961         case e1000_media_type_copper:
1962                 if (hw->mac.get_link_status) {
1963                         /* Do the work to read phy */
1964                         e1000_check_for_link(hw);
1965                         link_check = !hw->mac.get_link_status;
1966                 } else
1967                         link_check = TRUE;
1968                 break;
1969         case e1000_media_type_fiber:
1970                 e1000_check_for_link(hw);
1971                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1972                                  E1000_STATUS_LU);
1973                 break;
1974         case e1000_media_type_internal_serdes:
1975                 e1000_check_for_link(hw);
1976                 link_check = adapter->hw.mac.serdes_has_link;
1977                 break;
1978         default:
1979         case e1000_media_type_unknown:
1980                 break;
1981         }
1982
1983         /* Now we check if a transition has happened */
1984         if (link_check && (adapter->link_active == 0)) {
1985                 e1000_get_speed_and_duplex(&adapter->hw,
1986                     &adapter->link_speed, &adapter->link_duplex);
1987                 if (bootverbose)
1988                         device_printf(dev, "Link is up %d Mbps %s\n",
1989                             adapter->link_speed,
1990                             ((adapter->link_duplex == FULL_DUPLEX) ?
1991                             "Full Duplex" : "Half Duplex"));
1992                 adapter->link_active = 1;
1993                 ifp->if_baudrate = adapter->link_speed * 1000000;
1994                 ifp->if_link_state = LINK_STATE_UP;
1995                 if_link_state_change(ifp);
1996         } else if (!link_check && (adapter->link_active == 1)) {
1997                 ifp->if_baudrate = adapter->link_speed = 0;
1998                 adapter->link_duplex = 0;
1999                 if (bootverbose)
2000                         device_printf(dev, "Link is Down\n");
2001                 adapter->link_active = 0;
2002                 ifp->if_link_state = LINK_STATE_DOWN;
2003                 if_link_state_change(ifp);
2004                 /* Turn off watchdogs */
2005                 for (int i = 0; i < adapter->num_queues; i++, txr++)
2006                         txr->watchdog_check = FALSE;
2007         }
2008 }
2009
2010 /*********************************************************************
2011  *
2012  *  This routine disables all traffic on the adapter by issuing a
2013  *  global reset on the MAC and deallocates TX/RX buffers.
2014  *
2015  **********************************************************************/
2016
2017 static void
2018 igb_stop(void *arg)
2019 {
2020         struct adapter  *adapter = arg;
2021         struct ifnet    *ifp = adapter->ifp;
2022         struct tx_ring *txr = adapter->tx_rings;
2023
2024         IGB_CORE_LOCK_ASSERT(adapter);
2025
2026         INIT_DEBUGOUT("igb_stop: begin");
2027
2028         igb_disable_intr(adapter);
2029
2030         callout_stop(&adapter->timer);
2031
2032         /* Tell the stack that the interface is no longer active */
2033         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
2034
2035         /* Unarm watchdog timer. */
2036         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2037                 IGB_TX_LOCK(txr);
2038                 txr->watchdog_check = FALSE;
2039                 IGB_TX_UNLOCK(txr);
2040         }
2041
2042         e1000_reset_hw(&adapter->hw);
2043         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2044 }
2045
2046
2047 /*********************************************************************
2048  *
2049  *  Determine hardware revision.
2050  *
2051  **********************************************************************/
2052 static void
2053 igb_identify_hardware(struct adapter *adapter)
2054 {
2055         device_t dev = adapter->dev;
2056
2057         /* Make sure our PCI config space has the necessary stuff set */
2058         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2059         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2060             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2061                 device_printf(dev, "Memory Access and/or Bus Master bits "
2062                     "were not set!\n");
2063                 adapter->hw.bus.pci_cmd_word |=
2064                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2065                 pci_write_config(dev, PCIR_COMMAND,
2066                     adapter->hw.bus.pci_cmd_word, 2);
2067         }
2068
2069         /* Save off the information about this board */
2070         adapter->hw.vendor_id = pci_get_vendor(dev);
2071         adapter->hw.device_id = pci_get_device(dev);
2072         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2073         adapter->hw.subsystem_vendor_id =
2074             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2075         adapter->hw.subsystem_device_id =
2076             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2077
2078         /* Do Shared Code Init and Setup */
2079         if (e1000_set_mac_type(&adapter->hw)) {
2080                 device_printf(dev, "Setup init failure\n");
2081                 return;
2082         }
2083 }
2084
2085 static int
2086 igb_allocate_pci_resources(struct adapter *adapter)
2087 {
2088         device_t        dev = adapter->dev;
2089         int             rid;
2090
2091         rid = PCIR_BAR(0);
2092         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2093             &rid, RF_ACTIVE);
2094         if (adapter->pci_mem == NULL) {
2095                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2096                 return (ENXIO);
2097         }
2098         adapter->osdep.mem_bus_space_tag =
2099             rman_get_bustag(adapter->pci_mem);
2100         adapter->osdep.mem_bus_space_handle =
2101             rman_get_bushandle(adapter->pci_mem);
2102         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2103
2104         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2105
2106         /* This will setup either MSI/X or MSI */
2107         adapter->msix = igb_setup_msix(adapter);
2108         adapter->hw.back = &adapter->osdep;
2109
2110         return (0);
2111 }
2112
2113 /*********************************************************************
2114  *
2115  *  Setup the Legacy or MSI Interrupt handler
2116  *
2117  **********************************************************************/
2118 static int
2119 igb_allocate_legacy(struct adapter *adapter)
2120 {
2121         device_t dev = adapter->dev;
2122         int error, rid = 0;
2123
2124         /* Turn off all interrupts */
2125         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2126
2127         /* MSI RID is 1 */
2128         if (adapter->msix == 1)
2129                 rid = 1;
2130
2131         /* We allocate a single interrupt resource */
2132         adapter->res = bus_alloc_resource_any(dev,
2133             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2134         if (adapter->res == NULL) {
2135                 device_printf(dev, "Unable to allocate bus resource: "
2136                     "interrupt\n");
2137                 return (ENXIO);
2138         }
2139
2140         /*
2141          * Try allocating a fast interrupt and the associated deferred
2142          * processing contexts.
2143          */
2144         TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
2145         /* Make tasklet for deferred link handling */
2146         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2147         adapter->tq = taskqueue_create("igb_taskq", M_NOWAIT,
2148             taskqueue_thread_enqueue, &adapter->tq);
2149         taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s taskq",
2150             device_get_nameunit(adapter->dev));
2151         if ((error = bus_setup_intr(dev, adapter->res,
2152             /*INTR_TYPE_NET |*/ INTR_MPSAFE, igb_irq_fast,
2153             adapter, &adapter->tag, NULL)) != 0) {
2154                 device_printf(dev, "Failed to register fast interrupt "
2155                             "handler: %d\n", error);
2156                 taskqueue_free(adapter->tq);
2157                 adapter->tq = NULL;
2158                 return (error);
2159         }
2160
2161         return (0);
2162 }
2163
2164
2165 /*********************************************************************
2166  *
2167  *  Setup the MSIX Queue Interrupt handlers:
2168  *
2169  **********************************************************************/
2170 static int
2171 igb_allocate_msix(struct adapter *adapter)
2172 {
2173         device_t                dev = adapter->dev;
2174         struct igb_queue        *que = adapter->queues;
2175         int                     error, rid, vector = 0;
2176
2177
2178         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2179                 rid = vector + 1;
2180                 que->res = bus_alloc_resource_any(dev,
2181                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2182                 if (que->res == NULL) {
2183                         device_printf(dev,
2184                             "Unable to allocate bus resource: "
2185                             "MSIX Queue Interrupt\n");
2186                         return (ENXIO);
2187                 }
2188                 error = bus_setup_intr(dev, que->res,
2189                     /*INTR_TYPE_NET |*/ INTR_MPSAFE,
2190                     igb_msix_que, que, &que->tag, NULL);
2191                 if (error) {
2192                         que->res = NULL;
2193                         device_printf(dev, "Failed to register Queue handler");
2194                         return (error);
2195                 }
2196                 que->msix = vector;
2197                 if (adapter->hw.mac.type == e1000_82575)
2198                         que->eims = E1000_EICR_TX_QUEUE0 << i;
2199                 else
2200                         que->eims = 1 << vector;
2201                 /*
2202                 ** Bind the msix vector, and thus the
2203                 ** rings to the corresponding cpu.
2204                 */
2205 #if 0
2206                 if (adapter->num_queues > 1)
2207                         bus_bind_intr(dev, que->res, i);
2208 #endif
2209                 /* Make tasklet for deferred handling */
2210                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2211                 que->tq = taskqueue_create("igb_que", M_NOWAIT,
2212                     taskqueue_thread_enqueue, &que->tq);
2213                 taskqueue_start_threads(&que->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s que",
2214                     device_get_nameunit(adapter->dev));
2215         }
2216
2217         /* And Link */
2218         rid = vector + 1;
2219         adapter->res = bus_alloc_resource_any(dev,
2220             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2221         if (adapter->res == NULL) {
2222                 device_printf(dev,
2223                     "Unable to allocate bus resource: "
2224                     "MSIX Link Interrupt\n");
2225                 return (ENXIO);
2226         }
2227         if ((error = bus_setup_intr(dev, adapter->res,
2228             /*INTR_TYPE_NET |*/ INTR_MPSAFE,
2229             igb_msix_link, adapter, &adapter->tag, NULL)) != 0) {
2230                 device_printf(dev, "Failed to register Link handler");
2231                 return (error);
2232         }
2233         adapter->linkvec = vector;
2234
2235         /* Make tasklet for deferred handling */
2236         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2237         adapter->tq = taskqueue_create("igb_link", M_NOWAIT,
2238             taskqueue_thread_enqueue, &adapter->tq);
2239         taskqueue_start_threads(&adapter->tq, 1, TDPRI_KERN_DAEMON /*PI_NET*/, -1, "%s link",
2240             device_get_nameunit(adapter->dev));
2241
2242         return (0);
2243 }
2244
2245
2246 static void
2247 igb_configure_queues(struct adapter *adapter)
2248 {
2249         struct  e1000_hw        *hw = &adapter->hw;
2250         struct  igb_queue       *que;
2251         u32                     tmp, ivar = 0;
2252         u32                     newitr = IGB_DEFAULT_ITR;
2253
2254         /* First turn on RSS capability */
2255         if (adapter->hw.mac.type > e1000_82575)
2256                 E1000_WRITE_REG(hw, E1000_GPIE,
2257                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2258                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
2259
2260         /* Turn on MSIX */
2261         switch (adapter->hw.mac.type) {
2262         case e1000_82580:
2263                 /* RX entries */
2264                 for (int i = 0; i < adapter->num_queues; i++) {
2265                         u32 index = i >> 1;
2266                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2267                         que = &adapter->queues[i];
2268                         if (i & 1) {
2269                                 ivar &= 0xFF00FFFF;
2270                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2271                         } else {
2272                                 ivar &= 0xFFFFFF00;
2273                                 ivar |= que->msix | E1000_IVAR_VALID;
2274                         }
2275                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2276                 }
2277                 /* TX entries */
2278                 for (int i = 0; i < adapter->num_queues; i++) {
2279                         u32 index = i >> 1;
2280                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2281                         que = &adapter->queues[i];
2282                         if (i & 1) {
2283                                 ivar &= 0x00FFFFFF;
2284                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2285                         } else {
2286                                 ivar &= 0xFFFF00FF;
2287                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2288                         }
2289                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2290                         adapter->eims_mask |= que->eims;
2291                 }
2292
2293                 /* And for the link interrupt */
2294                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2295                 adapter->link_mask = 1 << adapter->linkvec;
2296                 adapter->eims_mask |= adapter->link_mask;
2297                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2298                 break;
2299         case e1000_82576:
2300                 /* RX entries */
2301                 for (int i = 0; i < adapter->num_queues; i++) {
2302                         u32 index = i & 0x7; /* Each IVAR has two entries */
2303                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2304                         que = &adapter->queues[i];
2305                         if (i < 8) {
2306                                 ivar &= 0xFFFFFF00;
2307                                 ivar |= que->msix | E1000_IVAR_VALID;
2308                         } else {
2309                                 ivar &= 0xFF00FFFF;
2310                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2311                         }
2312                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2313                         adapter->eims_mask |= que->eims;
2314                 }
2315                 /* TX entries */
2316                 for (int i = 0; i < adapter->num_queues; i++) {
2317                         u32 index = i & 0x7; /* Each IVAR has two entries */
2318                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2319                         que = &adapter->queues[i];
2320                         if (i < 8) {
2321                                 ivar &= 0xFFFF00FF;
2322                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2323                         } else {
2324                                 ivar &= 0x00FFFFFF;
2325                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2326                         }
2327                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2328                         adapter->eims_mask |= que->eims;
2329                 }
2330
2331                 /* And for the link interrupt */
2332                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2333                 adapter->link_mask = 1 << adapter->linkvec;
2334                 adapter->eims_mask |= adapter->link_mask;
2335                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2336                 break;
2337
2338         case e1000_82575:
2339                 /* enable MSI-X support*/
2340                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2341                 tmp |= E1000_CTRL_EXT_PBA_CLR;
2342                 /* Auto-Mask interrupts upon ICR read. */
2343                 tmp |= E1000_CTRL_EXT_EIAME;
2344                 tmp |= E1000_CTRL_EXT_IRCA;
2345                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2346
2347                 /* Queues */
2348                 for (int i = 0; i < adapter->num_queues; i++) {
2349                         que = &adapter->queues[i];
2350                         tmp = E1000_EICR_RX_QUEUE0 << i;
2351                         tmp |= E1000_EICR_TX_QUEUE0 << i;
2352                         que->eims = tmp;
2353                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2354                             i, que->eims);
2355                         adapter->eims_mask |= que->eims;
2356                 }
2357
2358                 /* Link */
2359                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2360                     E1000_EIMS_OTHER);
2361                 adapter->link_mask |= E1000_EIMS_OTHER;
2362                 adapter->eims_mask |= adapter->link_mask;
2363         default:
2364                 break;
2365         }
2366
2367         /* Set the starting interrupt rate */
2368         if (hw->mac.type == e1000_82575)
2369                 newitr |= newitr << 16;
2370         else
2371                 newitr |= 0x8000000;
2372
2373         for (int i = 0; i < adapter->num_queues; i++) {
2374                 que = &adapter->queues[i];
2375                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2376         }
2377
2378         return;
2379 }
2380
2381
2382 static void
2383 igb_free_pci_resources(struct adapter *adapter)
2384 {
2385         struct          igb_queue *que = adapter->queues;
2386         device_t        dev = adapter->dev;
2387         int             rid;
2388
2389         /*
2390         ** There is a slight possibility of a failure mode
2391         ** in attach that will result in entering this function
2392         ** before interrupt resources have been initialized, and
2393         ** in that case we do not want to execute the loops below
2394         ** We can detect this reliably by the state of the adapter
2395         ** res pointer.
2396         */
2397         if (adapter->res == NULL)
2398                 goto mem;
2399
2400         /*
2401          * First release all the interrupt resources:
2402          */
2403         for (int i = 0; i < adapter->num_queues; i++, que++) {
2404                 rid = que->msix + 1;
2405                 if (que->tag != NULL) {
2406                         bus_teardown_intr(dev, que->res, que->tag);
2407                         que->tag = NULL;
2408                 }
2409                 if (que->res != NULL)
2410                         bus_release_resource(dev,
2411                             SYS_RES_IRQ, rid, que->res);
2412         }
2413
2414         /* Clean the Legacy or Link interrupt last */
2415         if (adapter->linkvec) /* we are doing MSIX */
2416                 rid = adapter->linkvec + 1;
2417         else
2418                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2419
2420         if (adapter->tag != NULL) {
2421                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2422                 adapter->tag = NULL;
2423         }
2424         if (adapter->res != NULL)
2425                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2426
2427 mem:
2428         if (adapter->msix)
2429                 pci_release_msi(dev);
2430
2431         if (adapter->msix_mem != NULL)
2432                 bus_release_resource(dev, SYS_RES_MEMORY,
2433                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2434
2435         if (adapter->pci_mem != NULL)
2436                 bus_release_resource(dev, SYS_RES_MEMORY,
2437                     PCIR_BAR(0), adapter->pci_mem);
2438
2439 }
2440
2441 /*
2442  * Setup Either MSI/X or MSI
2443  */
2444 static int
2445 igb_setup_msix(struct adapter *adapter)
2446 {
2447         device_t dev = adapter->dev;
2448         int rid, want, queues, msgs;
2449
2450         /* tuneable override */
2451         if (igb_enable_msix == 0)
2452                 goto msi;
2453
2454         /* First try MSI/X */
2455         rid = PCIR_BAR(IGB_MSIX_BAR);
2456         adapter->msix_mem = bus_alloc_resource_any(dev,
2457             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2458         if (!adapter->msix_mem) {
2459                 /* May not be enabled */
2460                 device_printf(adapter->dev,
2461                     "Unable to map MSIX table \n");
2462                 goto msi;
2463         }
2464
2465         msgs = pci_msix_count(dev);
2466         if (msgs == 0) { /* system has msix disabled */
2467                 bus_release_resource(dev, SYS_RES_MEMORY,
2468                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2469                 adapter->msix_mem = NULL;
2470                 goto msi;
2471         }
2472
2473         /* Figure out a reasonable auto config value */
2474         queues = (ncpus > (msgs-1)) ? (msgs-1) : ncpus;
2475
2476         /* Can have max of 4 queues on 82575 */
2477         if (adapter->hw.mac.type == e1000_82575) {
2478                 if (queues > 4)
2479                         queues = 4;
2480                 if (igb_num_queues > 4)
2481                         igb_num_queues = 4;
2482         }
2483
2484         if (igb_num_queues == 0)
2485                 igb_num_queues = queues;
2486
2487         /*
2488         ** One vector (RX/TX pair) per queue
2489         ** plus an additional for Link interrupt
2490         */
2491         want = igb_num_queues + 1;
2492         if (msgs >= want)
2493                 msgs = want;
2494         else {
2495                 device_printf(adapter->dev,
2496                     "MSIX Configuration Problem, "
2497                     "%d vectors configured, but %d queues wanted!\n",
2498                     msgs, want);
2499                 return (ENXIO);
2500         }
2501         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2502                 device_printf(adapter->dev,
2503                     "Using MSIX interrupts with %d vectors\n", msgs);
2504                 adapter->num_queues = igb_num_queues;
2505                 return (msgs);
2506         }
2507 msi:
2508         msgs = pci_msi_count(dev);
2509         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2510                 device_printf(adapter->dev,"Using MSI interrupt\n");
2511         return (msgs);
2512 }
2513
2514 /*********************************************************************
2515  *
2516  *  Set up an fresh starting state
2517  *
2518  **********************************************************************/
2519 static void
2520 igb_reset(struct adapter *adapter)
2521 {
2522         device_t        dev = adapter->dev;
2523         struct e1000_hw *hw = &adapter->hw;
2524         struct e1000_fc_info *fc = &hw->fc;
2525         struct ifnet    *ifp = adapter->ifp;
2526         u32             pba = 0;
2527         u16             hwm;
2528
2529         INIT_DEBUGOUT("igb_reset: begin");
2530
2531         /* Let the firmware know the OS is in control */
2532         igb_get_hw_control(adapter);
2533
2534         /*
2535          * Packet Buffer Allocation (PBA)
2536          * Writing PBA sets the receive portion of the buffer
2537          * the remainder is used for the transmit buffer.
2538          */
2539         switch (hw->mac.type) {
2540         case e1000_82575:
2541                 pba = E1000_PBA_32K;
2542                 break;
2543         case e1000_82576:
2544                 pba = E1000_PBA_64K;
2545                 break;
2546         case e1000_82580:
2547                 pba = E1000_PBA_35K;
2548         default:
2549                 break;
2550         }
2551
2552         /* Special needs in case of Jumbo frames */
2553         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
2554                 u32 tx_space, min_tx, min_rx;
2555                 pba = E1000_READ_REG(hw, E1000_PBA);
2556                 tx_space = pba >> 16;
2557                 pba &= 0xffff;
2558                 min_tx = (adapter->max_frame_size +
2559                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
2560                 min_tx = roundup2(min_tx, 1024);
2561                 min_tx >>= 10;
2562                 min_rx = adapter->max_frame_size;
2563                 min_rx = roundup2(min_rx, 1024);
2564                 min_rx >>= 10;
2565                 if (tx_space < min_tx &&
2566                     ((min_tx - tx_space) < pba)) {
2567                         pba = pba - (min_tx - tx_space);
2568                         /*
2569                          * if short on rx space, rx wins
2570                          * and must trump tx adjustment
2571                          */
2572                         if (pba < min_rx)
2573                                 pba = min_rx;
2574                 }
2575                 E1000_WRITE_REG(hw, E1000_PBA, pba);
2576         }
2577
2578         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
2579
2580         /*
2581          * These parameters control the automatic generation (Tx) and
2582          * response (Rx) to Ethernet PAUSE frames.
2583          * - High water mark should allow for at least two frames to be
2584          *   received after sending an XOFF.
2585          * - Low water mark works best when it is very near the high water mark.
2586          *   This allows the receiver to restart by sending XON when it has
2587          *   drained a bit.
2588          */
2589         hwm = min(((pba << 10) * 9 / 10),
2590             ((pba << 10) - 2 * adapter->max_frame_size));
2591
2592         if (hw->mac.type < e1000_82576) {
2593                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
2594                 fc->low_water = fc->high_water - 8;
2595         } else {
2596                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
2597                 fc->low_water = fc->high_water - 16;
2598         }
2599
2600         fc->pause_time = IGB_FC_PAUSE_TIME;
2601         fc->send_xon = TRUE;
2602
2603         /* Set Flow control, use the tunable location if sane */
2604         if ((igb_fc_setting >= 0) || (igb_fc_setting < 4))
2605                 fc->requested_mode = igb_fc_setting;
2606         else
2607                 fc->requested_mode = e1000_fc_none;
2608
2609         fc->current_mode = fc->requested_mode;
2610
2611         /* Issue a global reset */
2612         e1000_reset_hw(hw);
2613         E1000_WRITE_REG(hw, E1000_WUC, 0);
2614
2615         if (e1000_init_hw(hw) < 0)
2616                 device_printf(dev, "Hardware Initialization Failed\n");
2617
2618         if (hw->mac.type == e1000_82580) {
2619                 u32 reg;
2620
2621                 hwm = (pba << 10) - (2 * adapter->max_frame_size);
2622                 /*
2623                  * 0x80000000 - enable DMA COAL
2624                  * 0x10000000 - use L0s as low power
2625                  * 0x20000000 - use L1 as low power
2626                  * X << 16 - exit dma coal when rx data exceeds X kB
2627                  * Y - upper limit to stay in dma coal in units of 32usecs
2628                  */
2629                 E1000_WRITE_REG(hw, E1000_DMACR,
2630                     0xA0000006 | ((hwm << 6) & 0x00FF0000));
2631
2632                 /* set hwm to PBA -  2 * max frame size */
2633                 E1000_WRITE_REG(hw, E1000_FCRTC, hwm);
2634                 /*
2635                  * This sets the time to wait before requesting transition to
2636                  * low power state to number of usecs needed to receive 1 512
2637                  * byte frame at gigabit line rate
2638                  */
2639                 E1000_WRITE_REG(hw, E1000_DMCTLX, 4);
2640
2641                 /* free space in tx packet buffer to wake from DMA coal */
2642                 E1000_WRITE_REG(hw, E1000_DMCTXTH,
2643                     (20480 - (2 * adapter->max_frame_size)) >> 6);
2644
2645                 /* make low power state decision controlled by DMA coal */
2646                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2647                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2648                     reg | E1000_PCIEMISC_LX_DECISION);
2649         }
2650
2651         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2652         e1000_get_phy_info(hw);
2653         e1000_check_for_link(hw);
2654         return;
2655 }
2656
2657 /*********************************************************************
2658  *
2659  *  Setup networking device structure and register an interface.
2660  *
2661  **********************************************************************/
2662 static void
2663 igb_setup_interface(device_t dev, struct adapter *adapter)
2664 {
2665         struct ifnet   *ifp;
2666
2667         INIT_DEBUGOUT("igb_setup_interface: begin");
2668
2669         ifp = adapter->ifp = &adapter->arpcom.ac_if;
2670         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2671         ifp->if_mtu = ETHERMTU;
2672         ifp->if_init =  igb_init;
2673         ifp->if_softc = adapter;
2674         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2675         ifp->if_ioctl = igb_ioctl;
2676         ifp->if_start = igb_start;
2677 #if __FreeBSD_version >= 800000
2678         ifp->if_transmit = igb_mq_start;
2679         ifp->if_qflush = igb_qflush;
2680 #endif
2681         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
2682         ifq_set_ready(&ifp->if_snd);
2683
2684         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2685
2686         ifp->if_capabilities = ifp->if_capenable = 0;
2687
2688         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU;
2689 #ifdef NET_TSO
2690         ifp->if_capabilities |= IFCAP_TSO4;
2691 #endif
2692         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2693 #ifdef NET_LRO
2694         if (igb_header_split)
2695                 ifp->if_capabilities |= IFCAP_LRO;
2696 #endif
2697
2698         ifp->if_capenable = ifp->if_capabilities;
2699 #ifdef DEVICE_POLLING
2700         ifp->if_capabilities |= IFCAP_POLLING;
2701 #endif
2702
2703         /*
2704          * Tell the upper layer(s) we support long frames.
2705          */
2706         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2707         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2708         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2709
2710         /*
2711          * Specify the media types supported by this adapter and register
2712          * callbacks to update media and link information
2713          */
2714         ifmedia_init(&adapter->media, IFM_IMASK,
2715             igb_media_change, igb_media_status);
2716         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2717             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2718                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
2719                             0, NULL);
2720                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2721         } else {
2722                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2723                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2724                             0, NULL);
2725                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2726                             0, NULL);
2727                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2728                             0, NULL);
2729                 if (adapter->hw.phy.type != e1000_phy_ife) {
2730                         ifmedia_add(&adapter->media,
2731                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2732                         ifmedia_add(&adapter->media,
2733                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2734                 }
2735         }
2736         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2737         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2738 }
2739
2740
2741 /*
2742  * Manage DMA'able memory.
2743  */
2744 static void
2745 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2746 {
2747         if (error)
2748                 return;
2749         *(bus_addr_t *) arg = segs[0].ds_addr;
2750 }
2751
2752 static int
2753 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
2754         struct igb_dma_alloc *dma, int mapflags)
2755 {
2756         int error;
2757
2758         error = bus_dma_tag_create(NULL,                /* parent */
2759                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
2760                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2761                                 BUS_SPACE_MAXADDR,      /* highaddr */
2762                                 NULL, NULL,             /* filter, filterarg */
2763                                 size,                   /* maxsize */
2764                                 1,                      /* nsegments */
2765                                 size,                   /* maxsegsize */
2766                                 0,                      /* flags */
2767                                 &dma->dma_tag);
2768         if (error) {
2769                 device_printf(adapter->dev,
2770                     "%s: bus_dma_tag_create failed: %d\n",
2771                     __func__, error);
2772                 goto fail_0;
2773         }
2774
2775         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2776             BUS_DMA_NOWAIT, &dma->dma_map);
2777         if (error) {
2778                 device_printf(adapter->dev,
2779                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2780                     __func__, (uintmax_t)size, error);
2781                 goto fail_2;
2782         }
2783
2784         dma->dma_paddr = 0;
2785         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2786             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2787         if (error || dma->dma_paddr == 0) {
2788                 device_printf(adapter->dev,
2789                     "%s: bus_dmamap_load failed: %d\n",
2790                     __func__, error);
2791                 goto fail_3;
2792         }
2793
2794         return (0);
2795
2796 fail_3:
2797         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2798 fail_2:
2799         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2800         bus_dma_tag_destroy(dma->dma_tag);
2801 fail_0:
2802         dma->dma_map = NULL;
2803         dma->dma_tag = NULL;
2804
2805         return (error);
2806 }
2807
2808 static void
2809 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
2810 {
2811         if (dma->dma_tag == NULL)
2812                 return;
2813         if (dma->dma_map != NULL) {
2814                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2815                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2816                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2817                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2818                 dma->dma_map = NULL;
2819         }
2820         bus_dma_tag_destroy(dma->dma_tag);
2821         dma->dma_tag = NULL;
2822 }
2823
2824
2825 /*********************************************************************
2826  *
2827  *  Allocate memory for the transmit and receive rings, and then
2828  *  the descriptors associated with each, called only once at attach.
2829  *
2830  **********************************************************************/
2831 static int
2832 igb_allocate_queues(struct adapter *adapter)
2833 {
2834         device_t dev = adapter->dev;
2835         struct igb_queue        *que = NULL;
2836         struct tx_ring          *txr = NULL;
2837         struct rx_ring          *rxr = NULL;
2838         int rsize, tsize, error = E1000_SUCCESS;
2839         int txconf = 0, rxconf = 0;
2840
2841         /* First allocate the top level queue structs */
2842         if (!(adapter->queues =
2843             (struct igb_queue *) kmalloc(sizeof(struct igb_queue) *
2844             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2845                 device_printf(dev, "Unable to allocate queue memory\n");
2846                 error = ENOMEM;
2847                 goto fail;
2848         }
2849
2850         /* Next allocate the TX ring struct memory */
2851         if (!(adapter->tx_rings =
2852             (struct tx_ring *) kmalloc(sizeof(struct tx_ring) *
2853             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2854                 device_printf(dev, "Unable to allocate TX ring memory\n");
2855                 error = ENOMEM;
2856                 goto tx_fail;
2857         }
2858
2859         /* Now allocate the RX */
2860         if (!(adapter->rx_rings =
2861             (struct rx_ring *) kmalloc(sizeof(struct rx_ring) *
2862             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2863                 device_printf(dev, "Unable to allocate RX ring memory\n");
2864                 error = ENOMEM;
2865                 goto rx_fail;
2866         }
2867
2868         tsize = roundup2(adapter->num_tx_desc *
2869             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
2870         /*
2871          * Now set up the TX queues, txconf is needed to handle the
2872          * possibility that things fail midcourse and we need to
2873          * undo memory gracefully
2874          */
2875         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2876                 /* Set up some basics */
2877                 txr = &adapter->tx_rings[i];
2878                 txr->adapter = adapter;
2879                 txr->me = i;
2880
2881                 /* Initialize the TX lock */
2882                 ksnprintf(txr->spin_name, sizeof(txr->spin_name), "%s:tx(%d)",
2883                     device_get_nameunit(dev), txr->me);
2884
2885                 spin_init(&txr->tx_spin);
2886
2887                 if (igb_dma_malloc(adapter, tsize,
2888                         &txr->txdma, BUS_DMA_NOWAIT)) {
2889                         device_printf(dev,
2890                             "Unable to allocate TX Descriptor memory\n");
2891                         error = ENOMEM;
2892                         goto err_tx_desc;
2893                 }
2894                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
2895                 bzero((void *)txr->tx_base, tsize);
2896
2897                 /* Now allocate transmit buffers for the ring */
2898                 if (igb_allocate_transmit_buffers(txr)) {
2899                         device_printf(dev,
2900                             "Critical Failure setting up transmit buffers\n");
2901                         error = ENOMEM;
2902                         goto err_tx_desc;
2903                 }
2904 #if __FreeBSD_version >= 800000
2905                 /* Allocate a buf ring */
2906                 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
2907                     M_WAITOK, &txr->tx_mtx);
2908 #endif
2909         }
2910
2911         /*
2912          * Next the RX queues...
2913          */
2914         rsize = roundup2(adapter->num_rx_desc *
2915             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
2916         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2917                 rxr = &adapter->rx_rings[i];
2918                 rxr->adapter = adapter;
2919                 rxr->me = i;
2920
2921                 /* Initialize the RX lock */
2922                 ksnprintf(rxr->spin_name, sizeof(rxr->spin_name), "%s:rx(%d)",
2923                     device_get_nameunit(dev), txr->me);
2924
2925                 spin_init(&rxr->rx_spin);
2926
2927                 if (igb_dma_malloc(adapter, rsize,
2928                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2929                         device_printf(dev,
2930                             "Unable to allocate RxDescriptor memory\n");
2931                         error = ENOMEM;
2932                         goto err_rx_desc;
2933                 }
2934                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2935                 bzero((void *)rxr->rx_base, rsize);
2936
2937                 /* Allocate receive buffers for the ring*/
2938                 if (igb_allocate_receive_buffers(rxr)) {
2939                         device_printf(dev,
2940                             "Critical Failure setting up receive buffers\n");
2941                         error = ENOMEM;
2942                         goto err_rx_desc;
2943                 }
2944         }
2945
2946         /*
2947         ** Finally set up the queue holding structs
2948         */
2949         for (int i = 0; i < adapter->num_queues; i++) {
2950                 que = &adapter->queues[i];
2951                 que->adapter = adapter;
2952                 que->txr = &adapter->tx_rings[i];
2953                 que->rxr = &adapter->rx_rings[i];
2954         }
2955
2956         return (0);
2957
2958 err_rx_desc:
2959         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2960                 igb_dma_free(adapter, &rxr->rxdma);
2961 err_tx_desc:
2962         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2963                 igb_dma_free(adapter, &txr->txdma);
2964         kfree(adapter->rx_rings, M_DEVBUF);
2965 rx_fail:
2966 #if __FreeBSD_version >= 800000
2967         buf_ring_free(txr->br, M_DEVBUF);
2968 #endif
2969         kfree(adapter->tx_rings, M_DEVBUF);
2970 tx_fail:
2971         kfree(adapter->queues, M_DEVBUF);
2972 fail:
2973         return (error);
2974 }
2975
2976 /*********************************************************************
2977  *
2978  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2979  *  the information needed to transmit a packet on the wire. This is
2980  *  called only once at attach, setup is done every reset.
2981  *
2982  **********************************************************************/
2983 static int
2984 igb_allocate_transmit_buffers(struct tx_ring *txr)
2985 {
2986         struct adapter *adapter = txr->adapter;
2987         device_t dev = adapter->dev;
2988         struct igb_tx_buffer *txbuf;
2989         int error, i;
2990
2991         /*
2992          * Setup DMA descriptor areas.
2993          */
2994         if ((error = bus_dma_tag_create(NULL,
2995                                1, 0,                    /* alignment, bounds */
2996                                BUS_SPACE_MAXADDR,       /* lowaddr */
2997                                BUS_SPACE_MAXADDR,       /* highaddr */
2998                                NULL, NULL,              /* filter, filterarg */
2999                                IGB_TSO_SIZE,            /* maxsize */
3000                                IGB_MAX_SCATTER,         /* nsegments */
3001                                PAGE_SIZE,               /* maxsegsize */
3002                                0,                       /* flags */
3003                                &txr->txtag))) {
3004                 device_printf(dev,"Unable to allocate TX DMA tag\n");
3005                 goto fail;
3006         }
3007
3008         if (!(txr->tx_buffers =
3009             (struct igb_tx_buffer *) kmalloc(sizeof(struct igb_tx_buffer) *
3010             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
3011                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3012                 error = ENOMEM;
3013                 goto fail;
3014         }
3015
3016         /* Create the descriptor buffer dma maps */
3017         txbuf = txr->tx_buffers;
3018         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3019                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
3020                 if (error != 0) {
3021                         device_printf(dev, "Unable to create TX DMA map\n");
3022                         goto fail;
3023                 }
3024         }
3025
3026         return 0;
3027 fail:
3028         /* We free all, it handles case where we are in the middle */
3029         igb_free_transmit_structures(adapter);
3030         return (error);
3031 }
3032
3033 /*********************************************************************
3034  *
3035  *  Initialize a transmit ring.
3036  *
3037  **********************************************************************/
3038 static void
3039 igb_setup_transmit_ring(struct tx_ring *txr)
3040 {
3041         struct adapter *adapter = txr->adapter;
3042         struct igb_tx_buffer *txbuf;
3043         int i;
3044
3045         /* Clear the old descriptor contents */
3046         IGB_TX_LOCK(txr);
3047         bzero((void *)txr->tx_base,
3048               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3049         /* Reset indices */
3050         txr->next_avail_desc = 0;
3051         txr->next_to_clean = 0;
3052
3053         /* Free any existing tx buffers. */
3054         txbuf = txr->tx_buffers;
3055         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3056                 if (txbuf->m_head != NULL) {
3057                         bus_dmamap_sync(txr->txtag, txbuf->map,
3058                             BUS_DMASYNC_POSTWRITE);
3059                         bus_dmamap_unload(txr->txtag, txbuf->map);
3060                         m_freem(txbuf->m_head);
3061                         txbuf->m_head = NULL;
3062                 }
3063                 /* clear the watch index */
3064                 txbuf->next_eop = -1;
3065         }
3066
3067         /* Set number of descriptors available */
3068         txr->tx_avail = adapter->num_tx_desc;
3069
3070         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3071             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3072         IGB_TX_UNLOCK(txr);
3073 }
3074
3075 /*********************************************************************
3076  *
3077  *  Initialize all transmit rings.
3078  *
3079  **********************************************************************/
3080 static void
3081 igb_setup_transmit_structures(struct adapter *adapter)
3082 {
3083         struct tx_ring *txr = adapter->tx_rings;
3084
3085         for (int i = 0; i < adapter->num_queues; i++, txr++)
3086                 igb_setup_transmit_ring(txr);
3087
3088         return;
3089 }
3090
3091 /*********************************************************************
3092  *
3093  *  Enable transmit unit.
3094  *
3095  **********************************************************************/
3096 static void
3097 igb_initialize_transmit_units(struct adapter *adapter)
3098 {
3099         struct tx_ring  *txr = adapter->tx_rings;
3100         struct e1000_hw *hw = &adapter->hw;
3101         u32             tctl, txdctl;
3102
3103         INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3104
3105         /* Setup the Tx Descriptor Rings */
3106         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3107                 u64 bus_addr = txr->txdma.dma_paddr;
3108
3109                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3110                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3111                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3112                     (uint32_t)(bus_addr >> 32));
3113                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3114                     (uint32_t)bus_addr);
3115
3116                 /* Setup the HW Tx Head and Tail descriptor pointers */
3117                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3118                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3119
3120                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3121                     E1000_READ_REG(hw, E1000_TDBAL(i)),
3122                     E1000_READ_REG(hw, E1000_TDLEN(i)));
3123
3124                 txr->watchdog_check = FALSE;
3125
3126                 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
3127                 txdctl |= IGB_TX_PTHRESH;
3128                 txdctl |= IGB_TX_HTHRESH << 8;
3129                 txdctl |= IGB_TX_WTHRESH << 16;
3130                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3131                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3132         }
3133
3134         /* Program the Transmit Control Register */
3135         tctl = E1000_READ_REG(hw, E1000_TCTL);
3136         tctl &= ~E1000_TCTL_CT;
3137         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3138                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3139
3140         e1000_config_collision_dist(hw);
3141
3142         /* This write will effectively turn on the transmit unit. */
3143         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3144 }
3145
3146 /*********************************************************************
3147  *
3148  *  Free all transmit rings.
3149  *
3150  **********************************************************************/
3151 static void
3152 igb_free_transmit_structures(struct adapter *adapter)
3153 {
3154         struct tx_ring *txr = adapter->tx_rings;
3155
3156         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3157                 IGB_TX_LOCK(txr);
3158                 igb_free_transmit_buffers(txr);
3159                 igb_dma_free(adapter, &txr->txdma);
3160                 IGB_TX_UNLOCK(txr);
3161                 IGB_TX_LOCK_DESTROY(txr);
3162         }
3163         kfree(adapter->tx_rings, M_DEVBUF);
3164 }
3165
3166 /*********************************************************************
3167  *
3168  *  Free transmit ring related data structures.
3169  *
3170  **********************************************************************/
3171 static void
3172 igb_free_transmit_buffers(struct tx_ring *txr)
3173 {
3174         struct adapter *adapter = txr->adapter;
3175         struct igb_tx_buffer *tx_buffer;
3176         int             i;
3177
3178         INIT_DEBUGOUT("free_transmit_ring: begin");
3179
3180         if (txr->tx_buffers == NULL)
3181                 return;
3182
3183         tx_buffer = txr->tx_buffers;
3184         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3185                 if (tx_buffer->m_head != NULL) {
3186                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3187                             BUS_DMASYNC_POSTWRITE);
3188                         bus_dmamap_unload(txr->txtag,
3189                             tx_buffer->map);
3190                         m_freem(tx_buffer->m_head);
3191                         tx_buffer->m_head = NULL;
3192                         if (tx_buffer->map != NULL) {
3193                                 bus_dmamap_destroy(txr->txtag,
3194                                     tx_buffer->map);
3195                                 tx_buffer->map = NULL;
3196                         }
3197                 } else if (tx_buffer->map != NULL) {
3198                         bus_dmamap_unload(txr->txtag,
3199                             tx_buffer->map);
3200                         bus_dmamap_destroy(txr->txtag,
3201                             tx_buffer->map);
3202                         tx_buffer->map = NULL;
3203                 }
3204         }
3205 #if __FreeBSD_version >= 800000
3206         if (txr->br != NULL)
3207                 buf_ring_free(txr->br, M_DEVBUF);
3208 #endif
3209         if (txr->tx_buffers != NULL) {
3210                 kfree(txr->tx_buffers, M_DEVBUF);
3211                 txr->tx_buffers = NULL;
3212         }
3213         if (txr->txtag != NULL) {
3214                 bus_dma_tag_destroy(txr->txtag);
3215                 txr->txtag = NULL;
3216         }
3217         return;
3218 }
3219
3220 /**********************************************************************
3221  *
3222  *  Setup work for hardware segmentation offload (TSO)
3223  *
3224  **********************************************************************/
3225 #ifdef NET_TSO
3226 static boolean_t
3227 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
3228 {
3229         struct adapter *adapter = txr->adapter;
3230         struct e1000_adv_tx_context_desc *TXD;
3231         struct igb_tx_buffer        *tx_buffer;
3232         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3233         u32 mss_l4len_idx = 0;
3234         u16 vtag = 0;
3235         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3236         struct ether_vlan_header *eh;
3237         struct ip *ip;
3238         struct tcphdr *th;
3239
3240
3241         /*
3242          * Determine where frame payload starts.
3243          * Jump over vlan headers if already present
3244          */
3245         eh = mtod(mp, struct ether_vlan_header *);
3246         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3247                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3248         else
3249                 ehdrlen = ETHER_HDR_LEN;
3250
3251         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3252         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3253                 return FALSE;
3254
3255         /* Only supports IPV4 for now */
3256         ctxd = txr->next_avail_desc;
3257         tx_buffer = &txr->tx_buffers[ctxd];
3258         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3259
3260         ip = (struct ip *)(mp->m_data + ehdrlen);
3261         if (ip->ip_p != IPPROTO_TCP)
3262                 return FALSE;   /* 0 */
3263         ip->ip_sum = 0;
3264         ip_hlen = ip->ip_hl << 2;
3265         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3266         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3267             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3268         tcp_hlen = th->th_off << 2;
3269         /*
3270          * Calculate header length, this is used
3271          * in the transmit desc in igb_xmit
3272          */
3273         *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3274
3275         /* VLAN MACLEN IPLEN */
3276         if (mp->m_flags & M_VLANTAG) {
3277                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3278                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3279         }
3280
3281         vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
3282         vlan_macip_lens |= ip_hlen;
3283         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3284
3285         /* ADV DTYPE TUCMD */
3286         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3287         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3288         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3289         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3290
3291         /* MSS L4LEN IDX */
3292         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3293         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3294         /* 82575 needs the queue index added */
3295         if (adapter->hw.mac.type == e1000_82575)
3296                 mss_l4len_idx |= txr->me << 4;
3297         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3298
3299         TXD->seqnum_seed = htole32(0);
3300         tx_buffer->m_head = NULL;
3301         tx_buffer->next_eop = -1;
3302
3303         if (++ctxd == adapter->num_tx_desc)
3304                 ctxd = 0;
3305
3306         txr->tx_avail--;
3307         txr->next_avail_desc = ctxd;
3308         return TRUE;
3309 }
3310 #endif
3311
3312 /*********************************************************************
3313  *
3314  *  Context Descriptor setup for VLAN or CSUM
3315  *
3316  **********************************************************************/
3317
3318 static bool
3319 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3320 {
3321         struct adapter *adapter = txr->adapter;
3322         struct e1000_adv_tx_context_desc *TXD;
3323         struct igb_tx_buffer        *tx_buffer;
3324         u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx;
3325         struct ether_vlan_header *eh;
3326         struct ip *ip = NULL;
3327         struct ip6_hdr *ip6;
3328         int  ehdrlen, ctxd, ip_hlen = 0;
3329         u16     etype, vtag = 0;
3330         u8      ipproto = 0;
3331         bool    offload = TRUE;
3332
3333         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3334                 offload = FALSE;
3335
3336         vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0;
3337         ctxd = txr->next_avail_desc;
3338         tx_buffer = &txr->tx_buffers[ctxd];
3339         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3340
3341         /*
3342         ** In advanced descriptors the vlan tag must
3343         ** be placed into the context descriptor, thus
3344         ** we need to be here just for that setup.
3345         */
3346         if (mp->m_flags & M_VLANTAG) {
3347                 vtag = htole16(mp->m_pkthdr.ether_vlantag);
3348                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3349         } else if (offload == FALSE)
3350                 return FALSE;
3351
3352         /*
3353          * Determine where frame payload starts.
3354          * Jump over vlan headers if already present,
3355          * helpful for QinQ too.
3356          */
3357         eh = mtod(mp, struct ether_vlan_header *);
3358         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3359                 etype = ntohs(eh->evl_proto);
3360                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3361         } else {
3362                 etype = ntohs(eh->evl_encap_proto);
3363                 ehdrlen = ETHER_HDR_LEN;
3364         }
3365
3366         /* Set the ether header length */
3367         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3368
3369         switch (etype) {
3370                 case ETHERTYPE_IP:
3371                         ip = (struct ip *)(mp->m_data + ehdrlen);
3372                         ip_hlen = ip->ip_hl << 2;
3373                         if (mp->m_len < ehdrlen + ip_hlen) {
3374                                 offload = FALSE;
3375                                 break;
3376                         }
3377                         ipproto = ip->ip_p;
3378                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3379                         break;
3380                 case ETHERTYPE_IPV6:
3381                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3382                         ip_hlen = sizeof(struct ip6_hdr);
3383                         if (mp->m_len < ehdrlen + ip_hlen)
3384                                 return (FALSE);
3385                         ipproto = ip6->ip6_nxt;
3386                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3387                         break;
3388                 default:
3389                         offload = FALSE;
3390                         break;
3391         }
3392
3393         vlan_macip_lens |= ip_hlen;
3394         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3395
3396         switch (ipproto) {
3397                 case IPPROTO_TCP:
3398                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3399                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3400                         break;
3401                 case IPPROTO_UDP:
3402                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3403                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3404                         break;
3405 #if __FreeBSD_version >= 800000
3406                 case IPPROTO_SCTP:
3407                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3408                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3409                         break;
3410 #endif
3411                 default:
3412                         offload = FALSE;
3413                         break;
3414         }
3415
3416         /* 82575 needs the queue index added */
3417         if (adapter->hw.mac.type == e1000_82575)
3418                 mss_l4len_idx = txr->me << 4;
3419
3420         /* Now copy bits into descriptor */
3421         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3422         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3423         TXD->seqnum_seed = htole32(0);
3424         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3425
3426         tx_buffer->m_head = NULL;
3427         tx_buffer->next_eop = -1;
3428
3429         /* We've consumed the first desc, adjust counters */
3430         if (++ctxd == adapter->num_tx_desc)
3431                 ctxd = 0;
3432         txr->next_avail_desc = ctxd;
3433         --txr->tx_avail;
3434
3435         return (offload);
3436 }
3437
3438
3439 /**********************************************************************
3440  *
3441  *  Examine each tx_buffer in the used queue. If the hardware is done
3442  *  processing the packet then free associated resources. The
3443  *  tx_buffer is put back on the free queue.
3444  *
3445  *  TRUE return means there's work in the ring to clean, FALSE its empty.
3446  **********************************************************************/
3447 static bool
3448 igb_txeof(struct tx_ring *txr)
3449 {
3450         struct adapter  *adapter = txr->adapter;
3451         int first, last, done;
3452         struct igb_tx_buffer *tx_buffer;
3453         struct e1000_tx_desc   *tx_desc, *eop_desc;
3454         struct ifnet   *ifp = adapter->ifp;
3455
3456         IGB_TX_LOCK_ASSERT(txr);
3457
3458         if (txr->tx_avail == adapter->num_tx_desc)
3459                 return FALSE;
3460
3461         first = txr->next_to_clean;
3462         tx_desc = &txr->tx_base[first];
3463         tx_buffer = &txr->tx_buffers[first];
3464         last = tx_buffer->next_eop;
3465         eop_desc = &txr->tx_base[last];
3466
3467         /*
3468          * What this does is get the index of the
3469          * first descriptor AFTER the EOP of the
3470          * first packet, that way we can do the
3471          * simple comparison on the inner while loop.
3472          */
3473         if (++last == adapter->num_tx_desc)
3474                 last = 0;
3475         done = last;
3476
3477         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3478             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3479
3480         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3481                 /* We clean the range of the packet */
3482                 while (first != done) {
3483                         tx_desc->upper.data = 0;
3484                         tx_desc->lower.data = 0;
3485                         tx_desc->buffer_addr = 0;
3486                         ++txr->tx_avail;
3487
3488                         if (tx_buffer->m_head) {
3489                                 txr->bytes +=
3490                                     tx_buffer->m_head->m_pkthdr.len;
3491                                 bus_dmamap_sync(txr->txtag,
3492                                     tx_buffer->map,
3493                                     BUS_DMASYNC_POSTWRITE);
3494                                 bus_dmamap_unload(txr->txtag,
3495                                     tx_buffer->map);
3496
3497                                 m_freem(tx_buffer->m_head);
3498                                 tx_buffer->m_head = NULL;
3499                         }
3500                         tx_buffer->next_eop = -1;
3501                         txr->watchdog_time = ticks;
3502
3503                         if (++first == adapter->num_tx_desc)
3504                                 first = 0;
3505
3506                         tx_buffer = &txr->tx_buffers[first];
3507                         tx_desc = &txr->tx_base[first];
3508                 }
3509                 ++txr->packets;
3510                 ++ifp->if_opackets;
3511                 /* See if we can continue to the next packet */
3512                 last = tx_buffer->next_eop;
3513                 if (last != -1) {
3514                         eop_desc = &txr->tx_base[last];
3515                         /* Get new done point */
3516                         if (++last == adapter->num_tx_desc) last = 0;
3517                         done = last;
3518                 } else
3519                         break;
3520         }
3521         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3522             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3523
3524         txr->next_to_clean = first;
3525
3526         /*
3527          * If we have enough room, clear IFF_DRV_OACTIVE
3528          * to tell the stack that it is OK to send packets.
3529          */
3530         if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) {
3531                 ifp->if_flags &= ~IFF_OACTIVE;
3532                 /* All clean, turn off the watchdog */
3533                 if (txr->tx_avail == adapter->num_tx_desc) {
3534                         txr->watchdog_check = FALSE;
3535                         return FALSE;
3536                 }
3537         }
3538
3539         return (TRUE);
3540 }
3541
3542
3543 /*********************************************************************
3544  *
3545  *  Setup descriptor buffer(s) from system mbuf buffer pools.
3546  *              i - designates the ring index
3547  *              clean - tells the function whether to update
3548  *                      the header, the packet buffer, or both.
3549  *
3550  **********************************************************************/
3551 static int
3552 igb_get_buf(struct rx_ring *rxr, int i, u8 clean)
3553 {
3554         struct adapter          *adapter = rxr->adapter;
3555         struct igb_rx_buf       *rxbuf;
3556         struct mbuf             *mh, *mp;
3557         bus_dma_segment_t       hseg[1];
3558         bus_dma_segment_t       pseg[1];
3559         bus_dmamap_t            map;
3560         int                     nsegs, error;
3561
3562
3563         rxbuf = &rxr->rx_buffers[i];
3564         mh = mp = NULL;
3565         if ((clean & IGB_CLEAN_HEADER) != 0) {
3566                 mh = m_gethdr(MB_DONTWAIT, MT_DATA);
3567                 if (mh == NULL) {
3568                         adapter->mbuf_header_failed++;
3569                         return (ENOBUFS);
3570                 }
3571                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3572                 /*
3573                  * Because IGB_HDR_BUF size is less than MHLEN
3574                  * and we configure controller to split headers
3575                  * we can align mbuf on ETHER_ALIGN boundary.
3576                  */
3577                 m_adj(mh, ETHER_ALIGN);
3578                 error = bus_dmamap_load_mbuf_segment(rxr->rx_htag,
3579                     rxr->rx_hspare_map, mh, hseg, 1, &nsegs, BUS_DMA_NOWAIT);
3580                 if (error != 0) {
3581                         m_freem(mh);
3582                         return (error);
3583                 }
3584                 mh->m_flags &= ~M_PKTHDR;
3585         }
3586         if ((clean & IGB_CLEAN_PAYLOAD) != 0) {
3587                 mp = m_getl(adapter->rx_mbuf_sz,
3588                     MB_DONTWAIT, MT_DATA, M_PKTHDR, NULL);
3589 #if 0
3590                 mp = m_getjcl(MB_DONTWAIT, MT_DATA, M_PKTHDR,
3591                     adapter->rx_mbuf_sz);
3592 #endif
3593                 if (mp == NULL) {
3594                         if (mh != NULL) {
3595                                 adapter->mbuf_packet_failed++;
3596                                 bus_dmamap_unload(rxr->rx_htag,
3597                                     rxbuf->head_map);
3598                                 mh->m_flags |= M_PKTHDR;
3599                                 m_freem(mh);
3600                         }
3601                         return (ENOBUFS);
3602                 }
3603                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3604                 error = bus_dmamap_load_mbuf_segment(rxr->rx_ptag,
3605                     rxr->rx_pspare_map, mp, pseg, 1, &nsegs, BUS_DMA_NOWAIT);
3606                 if (error != 0) {
3607                         if (mh != NULL) {
3608                                 bus_dmamap_unload(rxr->rx_htag,
3609                                     rxbuf->head_map);
3610                                 mh->m_flags |= M_PKTHDR;
3611                                 m_freem(mh);
3612                         }
3613                         m_freem(mp);
3614                         return (error);
3615                 }
3616                 mp->m_flags &= ~M_PKTHDR;
3617         }
3618
3619         /* Loading new DMA maps complete, unload maps for received buffers. */
3620         if ((clean & IGB_CLEAN_HEADER) != 0 && rxbuf->m_head != NULL) {
3621                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3622                     BUS_DMASYNC_POSTREAD);
3623                 bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map);
3624         }
3625         if ((clean & IGB_CLEAN_PAYLOAD) != 0 && rxbuf->m_pack != NULL) {
3626                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3627                     BUS_DMASYNC_POSTREAD);
3628                 bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map);
3629         }
3630
3631         /* Reflect loaded dmamaps. */
3632         if ((clean & IGB_CLEAN_HEADER) != 0) {
3633                 map = rxbuf->head_map;
3634                 rxbuf->head_map = rxr->rx_hspare_map;
3635                 rxr->rx_hspare_map = map;
3636                 rxbuf->m_head = mh;
3637                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3638                     BUS_DMASYNC_PREREAD);
3639                 rxr->rx_base[i].read.hdr_addr = htole64(hseg[0].ds_addr);
3640         }
3641         if ((clean & IGB_CLEAN_PAYLOAD) != 0) {
3642                 map = rxbuf->pack_map;
3643                 rxbuf->pack_map = rxr->rx_pspare_map;
3644                 rxr->rx_pspare_map = map;
3645                 rxbuf->m_pack = mp;
3646                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3647                     BUS_DMASYNC_PREREAD);
3648                 rxr->rx_base[i].read.pkt_addr = htole64(pseg[0].ds_addr);
3649         }
3650
3651         return (0);
3652 }
3653
3654 /*********************************************************************
3655  *
3656  *  Allocate memory for rx_buffer structures. Since we use one
3657  *  rx_buffer per received packet, the maximum number of rx_buffer's
3658  *  that we'll need is equal to the number of receive descriptors
3659  *  that we've allocated.
3660  *
3661  **********************************************************************/
3662 static int
3663 igb_allocate_receive_buffers(struct rx_ring *rxr)
3664 {
3665         struct  adapter         *adapter = rxr->adapter;
3666         device_t                dev = adapter->dev;
3667         struct igb_rx_buf       *rxbuf;
3668         int                     i, bsize, error;
3669
3670         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
3671         if (!(rxr->rx_buffers =
3672             (struct igb_rx_buf *) kmalloc(bsize,
3673             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3674                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3675                 error = ENOMEM;
3676                 goto fail;
3677         }
3678
3679         if ((error = bus_dma_tag_create(NULL,
3680                                    1, 0,                /* alignment, bounds */
3681                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3682                                    BUS_SPACE_MAXADDR,   /* highaddr */
3683                                    NULL, NULL,          /* filter, filterarg */
3684                                    MSIZE,               /* maxsize */
3685                                    1,                   /* nsegments */
3686                                    MSIZE,               /* maxsegsize */
3687                                    0,                   /* flags */
3688                                    &rxr->rx_htag))) {
3689                 device_printf(dev, "Unable to create RX DMA tag\n");
3690                 goto fail;
3691         }
3692
3693         if ((error = bus_dma_tag_create(NULL,
3694                                    1, 0,                /* alignment, bounds */
3695                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3696                                    BUS_SPACE_MAXADDR,   /* highaddr */
3697                                    NULL, NULL,          /* filter, filterarg */
3698                                    MJUMPAGESIZE,        /* maxsize */
3699                                    1,                   /* nsegments */
3700                                    MJUMPAGESIZE,        /* maxsegsize */
3701                                    0,                   /* flags */
3702                                    &rxr->rx_ptag))) {
3703                 device_printf(dev, "Unable to create RX payload DMA tag\n");
3704                 goto fail;
3705         }
3706
3707         /* Create the spare maps (used by getbuf) */
3708         error = bus_dmamap_create(rxr->rx_htag, BUS_DMA_NOWAIT,
3709              &rxr->rx_hspare_map);
3710         if (error) {
3711                 device_printf(dev,
3712                     "%s: bus_dmamap_create header spare failed: %d\n",
3713                     __func__, error);
3714                 goto fail;
3715         }
3716         error = bus_dmamap_create(rxr->rx_ptag, BUS_DMA_NOWAIT,
3717              &rxr->rx_pspare_map);
3718         if (error) {
3719                 device_printf(dev,
3720                     "%s: bus_dmamap_create packet spare failed: %d\n",
3721                     __func__, error);
3722                 goto fail;
3723         }
3724
3725         for (i = 0; i < adapter->num_rx_desc; i++) {
3726                 rxbuf = &rxr->rx_buffers[i];
3727                 error = bus_dmamap_create(rxr->rx_htag,
3728                     BUS_DMA_NOWAIT, &rxbuf->head_map);
3729                 if (error) {
3730                         device_printf(dev,
3731                             "Unable to create RX head DMA maps\n");
3732                         goto fail;
3733                 }
3734                 error = bus_dmamap_create(rxr->rx_ptag,
3735                     BUS_DMA_NOWAIT, &rxbuf->pack_map);
3736                 if (error) {
3737                         device_printf(dev,
3738                             "Unable to create RX packet DMA maps\n");
3739                         goto fail;
3740                 }
3741         }
3742
3743         return (0);
3744
3745 fail:
3746         /* Frees all, but can handle partial completion */
3747         igb_free_receive_structures(adapter);
3748         return (error);
3749 }
3750
3751
3752 static void
3753 igb_free_receive_ring(struct rx_ring *rxr)
3754 {
3755         struct  adapter         *adapter;
3756         struct igb_rx_buf       *rxbuf;
3757         int i;
3758
3759         adapter = rxr->adapter;
3760         for (i = 0; i < adapter->num_rx_desc; i++) {
3761                 rxbuf = &rxr->rx_buffers[i];
3762                 if (rxbuf->m_head != NULL) {
3763                         bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3764                             BUS_DMASYNC_POSTREAD);
3765                         bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map);
3766                         rxbuf->m_head->m_flags |= M_PKTHDR;
3767                         m_freem(rxbuf->m_head);
3768                 }
3769                 if (rxbuf->m_pack != NULL) {
3770                         bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3771                             BUS_DMASYNC_POSTREAD);
3772                         bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map);
3773                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3774                         m_freem(rxbuf->m_pack);
3775                 }
3776                 rxbuf->m_head = NULL;
3777                 rxbuf->m_pack = NULL;
3778         }
3779 }
3780
3781
3782 /*********************************************************************
3783  *
3784  *  Initialize a receive ring and its buffers.
3785  *
3786  **********************************************************************/
3787 static int
3788 igb_setup_receive_ring(struct rx_ring *rxr)
3789 {
3790         struct  adapter         *adapter;
3791         struct  ifnet           *ifp;
3792         device_t                dev;
3793 #ifdef NET_LRO
3794         struct lro_ctrl         *lro = &rxr->lro;
3795 #endif
3796         int                     j, rsize, error = 0;
3797
3798         adapter = rxr->adapter;
3799         dev = adapter->dev;
3800         ifp = adapter->ifp;
3801
3802         /* Clear the ring contents */
3803         IGB_RX_LOCK(rxr);
3804         rsize = roundup2(adapter->num_rx_desc *
3805             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3806         bzero((void *)rxr->rx_base, rsize);
3807
3808         /*
3809         ** Free current RX buffer structures and their mbufs
3810         */
3811         igb_free_receive_ring(rxr);
3812
3813         /* Now replenish the ring mbufs */
3814         for (j = 0; j < adapter->num_rx_desc; j++) {
3815                 if ((error = igb_get_buf(rxr, j, IGB_CLEAN_BOTH)) != 0)
3816                         goto fail;
3817         }
3818
3819         /* Setup our descriptor indices */
3820         rxr->next_to_check = 0;
3821         rxr->last_cleaned = 0;
3822         rxr->lro_enabled = FALSE;
3823
3824         if (igb_header_split)
3825                 rxr->hdr_split = TRUE;
3826 #if NET_LRO
3827         else
3828                 ifp->if_capabilities &= ~IFCAP_LRO;
3829 #endif
3830
3831         rxr->fmp = NULL;
3832         rxr->lmp = NULL;
3833         rxr->discard = FALSE;
3834
3835         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3836             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3837
3838         /*
3839         ** Now set up the LRO interface, we
3840         ** also only do head split when LRO
3841         ** is enabled, since so often they
3842         ** are undesireable in similar setups.
3843         */
3844 #if NET_LRO
3845         if (ifp->if_capenable & IFCAP_LRO) {
3846                 int err = tcp_lro_init(lro);
3847                 if (err) {
3848                         device_printf(dev, "LRO Initialization failed!\n");
3849                         goto fail;
3850                 }
3851                 INIT_DEBUGOUT("RX LRO Initialized\n");
3852                 rxr->lro_enabled = TRUE;
3853                 lro->ifp = adapter->ifp;
3854         }
3855 #endif
3856
3857         IGB_RX_UNLOCK(rxr);
3858         return (0);
3859
3860 fail:
3861         igb_free_receive_ring(rxr);
3862         IGB_RX_UNLOCK(rxr);
3863         return (error);
3864 }
3865
3866 /*********************************************************************
3867  *
3868  *  Initialize all receive rings.
3869  *
3870  **********************************************************************/
3871 static int
3872 igb_setup_receive_structures(struct adapter *adapter)
3873 {
3874         struct rx_ring *rxr = adapter->rx_rings;
3875         int i, j;
3876
3877         for (i = 0; i < adapter->num_queues; i++, rxr++)
3878                 if (igb_setup_receive_ring(rxr))
3879                         goto fail;
3880
3881         return (0);
3882 fail:
3883         /*
3884          * Free RX buffers allocated so far, we will only handle
3885          * the rings that completed, the failing case will have
3886          * cleaned up for itself. The value of 'i' will be the
3887          * failed ring so we must pre-decrement it.
3888          */
3889         rxr = adapter->rx_rings;
3890         for (--i; i > 0; i--, rxr++) {
3891                 for (j = 0; j < adapter->num_rx_desc; j++)
3892                         igb_free_receive_ring(rxr);
3893         }
3894
3895         return (ENOBUFS);
3896 }
3897
3898 /*********************************************************************
3899  *
3900  *  Enable receive unit.
3901  *
3902  **********************************************************************/
3903 static void
3904 igb_initialize_receive_units(struct adapter *adapter)
3905 {
3906         struct rx_ring  *rxr = adapter->rx_rings;
3907         struct ifnet    *ifp = adapter->ifp;
3908         struct e1000_hw *hw = &adapter->hw;
3909         u32             rctl, rxcsum, psize, srrctl = 0;
3910
3911         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
3912
3913         /*
3914          * Make sure receives are disabled while setting
3915          * up the descriptor ring
3916          */
3917         rctl = E1000_READ_REG(hw, E1000_RCTL);
3918         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3919
3920         /*
3921         ** Set up for header split
3922         */
3923         if (rxr->hdr_split) {
3924                 /* Use a standard mbuf for the header */
3925                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3926                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3927         } else
3928                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3929
3930         /*
3931         ** Set up for jumbo frames
3932         */
3933         if (ifp->if_mtu > ETHERMTU) {
3934                 rctl |= E1000_RCTL_LPE;
3935                 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3936                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
3937
3938                 /* Set maximum packet len */
3939                 psize = adapter->max_frame_size;
3940                 /* are we on a vlan? */
3941                 if (adapter->ifp->if_vlantrunks != NULL)
3942                         psize += VLAN_TAG_SIZE;
3943                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
3944         } else {
3945                 rctl &= ~E1000_RCTL_LPE;
3946                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3947                 rctl |= E1000_RCTL_SZ_2048;
3948         }
3949
3950         /* Setup the Base and Length of the Rx Descriptor Rings */
3951         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3952                 u64 bus_addr = rxr->rxdma.dma_paddr;
3953                 u32 rxdctl;
3954
3955                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
3956                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3957                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
3958                     (uint32_t)(bus_addr >> 32));
3959                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
3960                     (uint32_t)bus_addr);
3961                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
3962                 /* Enable this Queue */
3963                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
3964                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3965                 rxdctl &= 0xFFF00000;
3966                 rxdctl |= IGB_RX_PTHRESH;
3967                 rxdctl |= IGB_RX_HTHRESH << 8;
3968                 rxdctl |= IGB_RX_WTHRESH << 16;
3969                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
3970         }
3971
3972         /*
3973         ** Setup for RX MultiQueue
3974         */
3975         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3976         if (adapter->num_queues >1) {
3977                 u32 random[10], mrqc, shift = 0;
3978                 union igb_reta {
3979                         u32 dword;
3980                         u8  bytes[4];
3981                 } reta;
3982
3983                 karc4rand(&random, sizeof(random));
3984                 if (adapter->hw.mac.type == e1000_82575)
3985                         shift = 6;
3986                 /* Warning FM follows */
3987                 for (int i = 0; i < 128; i++) {
3988                         reta.bytes[i & 3] =
3989                             (i % adapter->num_queues) << shift;
3990                         if ((i & 3) == 3)
3991                                 E1000_WRITE_REG(hw,
3992                                     E1000_RETA(i >> 2), reta.dword);
3993                 }
3994                 /* Now fill in hash table */
3995                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3996                 for (int i = 0; i < 10; i++)
3997                         E1000_WRITE_REG_ARRAY(hw,
3998                             E1000_RSSRK(0), i, random[i]);
3999
4000                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
4001                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
4002                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
4003                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
4004                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
4005                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
4006                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
4007                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
4008
4009                 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
4010
4011                 /*
4012                 ** NOTE: Receive Full-Packet Checksum Offload
4013                 ** is mutually exclusive with Multiqueue. However
4014                 ** this is not the same as TCP/IP checksums which
4015                 ** still work.
4016                 */
4017                 rxcsum |= E1000_RXCSUM_PCSD;
4018 #if __FreeBSD_version >= 800000
4019                 /* For SCTP Offload */
4020                 if ((hw->mac.type == e1000_82576)
4021                     && (ifp->if_capenable & IFCAP_RXCSUM))
4022                         rxcsum |= E1000_RXCSUM_CRCOFL;
4023 #endif
4024         } else {
4025                 /* Non RSS setup */
4026                 if (ifp->if_capenable & IFCAP_RXCSUM) {
4027                         rxcsum |= E1000_RXCSUM_IPPCSE;
4028 #if __FreeBSD_version >= 800000
4029                         if (adapter->hw.mac.type == e1000_82576)
4030                                 rxcsum |= E1000_RXCSUM_CRCOFL;
4031 #endif
4032                 } else
4033                         rxcsum &= ~E1000_RXCSUM_TUOFL;
4034         }
4035         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
4036
4037         /* Setup the Receive Control Register */
4038         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4039         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4040                    E1000_RCTL_RDMTS_HALF |
4041                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4042         /* Strip CRC bytes. */
4043         rctl |= E1000_RCTL_SECRC;
4044         /* Make sure VLAN Filters are off */
4045         rctl &= ~E1000_RCTL_VFE;
4046         /* Don't store bad packets */
4047         rctl &= ~E1000_RCTL_SBP;
4048
4049         /* Enable Receives */
4050         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4051
4052         /*
4053          * Setup the HW Rx Head and Tail Descriptor Pointers
4054          *   - needs to be after enable
4055          */
4056         for (int i = 0; i < adapter->num_queues; i++) {
4057                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4058                 E1000_WRITE_REG(hw, E1000_RDT(i),
4059                      adapter->num_rx_desc - 1);
4060         }
4061         return;
4062 }
4063
4064 /*********************************************************************
4065  *
4066  *  Free receive rings.
4067  *
4068  **********************************************************************/
4069 static void
4070 igb_free_receive_structures(struct adapter *adapter)
4071 {
4072         struct rx_ring *rxr = adapter->rx_rings;
4073
4074         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4075 #ifdef NET_LRO
4076                 struct lro_ctrl *lro = &rxr->lro;
4077 #endif
4078                 igb_free_receive_buffers(rxr);
4079 #ifdef NET_LRO
4080                 tcp_lro_free(lro);
4081 #endif
4082                 igb_dma_free(adapter, &rxr->rxdma);
4083         }
4084
4085         kfree(adapter->rx_rings, M_DEVBUF);
4086 }
4087
4088 /*********************************************************************
4089  *
4090  *  Free receive ring data structures.
4091  *
4092  **********************************************************************/
4093 static void
4094 igb_free_receive_buffers(struct rx_ring *rxr)
4095 {
4096         struct adapter          *adapter = rxr->adapter;
4097         struct igb_rx_buf       *rxbuf;
4098         int i;
4099
4100         INIT_DEBUGOUT("free_receive_structures: begin");
4101
4102         if (rxr->rx_hspare_map != NULL) {
4103                 bus_dmamap_destroy(rxr->rx_htag, rxr->rx_hspare_map);
4104                 rxr->rx_hspare_map = NULL;
4105         }
4106
4107         if (rxr->rx_hspare_map != NULL) {
4108                 bus_dmamap_destroy(rxr->rx_ptag, rxr->rx_pspare_map);
4109                 rxr->rx_pspare_map = NULL;
4110         }
4111
4112         /* Cleanup any existing buffers */
4113         if (rxr->rx_buffers != NULL) {
4114                 for (i = 0; i < adapter->num_rx_desc; i++) {
4115                         rxbuf = &rxr->rx_buffers[i];
4116                         if (rxbuf->m_head != NULL) {
4117                                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
4118                                     BUS_DMASYNC_POSTREAD);
4119                                 bus_dmamap_unload(rxr->rx_htag,
4120                                     rxbuf->head_map);
4121                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4122                                 m_freem(rxbuf->m_head);
4123                         }
4124                         if (rxbuf->m_pack != NULL) {
4125                                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
4126                                     BUS_DMASYNC_POSTREAD);
4127                                 bus_dmamap_unload(rxr->rx_ptag,
4128                                     rxbuf->pack_map);
4129                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4130                                 m_freem(rxbuf->m_pack);
4131                         }
4132                         rxbuf->m_head = NULL;
4133                         rxbuf->m_pack = NULL;
4134                         if (rxbuf->head_map != NULL) {
4135                                 bus_dmamap_destroy(rxr->rx_htag,
4136                                     rxbuf->head_map);
4137                                 rxbuf->head_map = NULL;
4138                         }
4139                         if (rxbuf->pack_map != NULL) {
4140                                 bus_dmamap_destroy(rxr->rx_ptag,
4141                                     rxbuf->pack_map);
4142                                 rxbuf->pack_map = NULL;
4143                         }
4144                 }
4145                 if (rxr->rx_buffers != NULL) {
4146                         kfree(rxr->rx_buffers, M_DEVBUF);
4147                         rxr->rx_buffers = NULL;
4148                 }
4149         }
4150
4151         if (rxr->rx_htag != NULL) {
4152                 bus_dma_tag_destroy(rxr->rx_htag);
4153                 rxr->rx_htag = NULL;
4154         }
4155         if (rxr->rx_ptag != NULL) {
4156                 bus_dma_tag_destroy(rxr->rx_ptag);
4157                 rxr->rx_ptag = NULL;
4158         }
4159 }
4160
4161 static __inline void
4162 igb_rx_discard(struct rx_ring *rxr, union e1000_adv_rx_desc *cur, int i)
4163 {
4164
4165         if (rxr->fmp != NULL) {
4166                 rxr->fmp->m_flags |= M_PKTHDR;
4167                 m_freem(rxr->fmp);
4168                 rxr->fmp = NULL;
4169                 rxr->lmp = NULL;
4170         }
4171 }
4172
4173 static __inline void
4174 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4175 {
4176
4177         /*
4178          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4179          * should be computed by hardware. Also it should not have VLAN tag in
4180          * ethernet header.
4181          */
4182 #ifdef NET_LRO
4183         if (rxr->lro_enabled &&
4184             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4185             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4186             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4187             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4188             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4189             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4190                 /*
4191                  * Send to the stack if:
4192                  **  - LRO not enabled, or
4193                  **  - no LRO resources, or
4194                  **  - lro enqueue fails
4195                  */
4196                 if (rxr->lro.lro_cnt != 0)
4197                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4198                                 return;
4199         }
4200 #endif
4201         (*ifp->if_input)(ifp, m);
4202 }
4203
4204 /*********************************************************************
4205  *
4206  *  This routine executes in interrupt context. It replenishes
4207  *  the mbufs in the descriptor and sends data which has been
4208  *  dma'ed into host memory to upper layer.
4209  *
4210  *  We loop at most count times if count is > 0, or until done if
4211  *  count < 0.
4212  *
4213  *  Return TRUE if more to clean, FALSE otherwise
4214  *********************************************************************/
4215 static bool
4216 igb_rxeof(struct rx_ring *rxr, int count)
4217 {
4218         struct adapter          *adapter = rxr->adapter;
4219         struct ifnet            *ifp = adapter->ifp;
4220 #ifdef NET_LRO
4221         struct lro_ctrl         *lro = &rxr->lro;
4222         struct lro_entry        *queued;
4223 #endif
4224         int                     i, prog = 0;
4225         u32                     ptype, staterr = 0;
4226         union e1000_adv_rx_desc *cur;
4227
4228         IGB_RX_LOCK(rxr);
4229
4230         /* Main clean loop */
4231         for (i = rxr->next_to_check; count > 0; prog++) {
4232                 struct mbuf *sendmp, *mh, *mp;
4233                 u16 hlen, plen, hdr, vtag;
4234                 bool eop = FALSE;
4235                 u8 dopayload;
4236
4237                 /* Sync the ring. */
4238                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4239                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4240                 cur = &rxr->rx_base[i];
4241                 staterr = le32toh(cur->wb.upper.status_error);
4242                 if ((staterr & E1000_RXD_STAT_DD) == 0)
4243                         break;
4244                 if ((ifp->if_flags & IFF_RUNNING) == 0)
4245                         break;
4246                 count--;
4247                 sendmp = mh = mp = NULL;
4248                 cur->wb.upper.status_error = 0;
4249                 plen = le16toh(cur->wb.upper.length);
4250                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4251                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4252                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4253
4254                 /* Make sure all segments of a bad packet are discarded */
4255                 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4256                     (rxr->discard)) {
4257                         ifp->if_ierrors++;
4258                         ++rxr->rx_discarded;
4259                         if (!eop) /* Catch subsequent segs */
4260                                 rxr->discard = TRUE;
4261                         else
4262                                 rxr->discard = FALSE;
4263                         igb_rx_discard(rxr, cur, i);
4264                         goto next_desc;
4265                 }
4266
4267                 /*
4268                 ** The way the hardware is configured to
4269                 ** split, it will ONLY use the header buffer
4270                 ** when header split is enabled, otherwise we
4271                 ** get normal behavior, ie, both header and
4272                 ** payload are DMA'd into the payload buffer.
4273                 **
4274                 ** The fmp test is to catch the case where a
4275                 ** packet spans multiple descriptors, in that
4276                 ** case only the first header is valid.
4277                 */
4278                 if (rxr->hdr_split && rxr->fmp == NULL) {
4279                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4280                             E1000_RXDADV_HDRBUFLEN_SHIFT;
4281                         if (hlen > IGB_HDR_BUF)
4282                                 hlen = IGB_HDR_BUF;
4283                         /* Handle the header mbuf */
4284                         mh = rxr->rx_buffers[i].m_head;
4285                         mh->m_len = hlen;
4286                         dopayload = IGB_CLEAN_HEADER;
4287                         /*
4288                         ** Get the payload length, this
4289                         ** could be zero if its a small
4290                         ** packet.
4291                         */
4292                         if (plen > 0) {
4293                                 mp = rxr->rx_buffers[i].m_pack;
4294                                 mp->m_len = plen;
4295                                 mh->m_next = mp;
4296                                 dopayload = IGB_CLEAN_BOTH;
4297                                 rxr->rx_split_packets++;
4298                         }
4299                 } else {
4300                         /*
4301                         ** Either no header split, or a
4302                         ** secondary piece of a fragmented
4303                         ** split packet.
4304                         */
4305                         mh = rxr->rx_buffers[i].m_pack;
4306                         mh->m_len = plen;
4307                         dopayload = IGB_CLEAN_PAYLOAD;
4308                 }
4309
4310                 /*
4311                 ** get_buf will overwrite the writeback
4312                 ** descriptor so save the VLAN tag now.
4313                 */
4314                 vtag = le16toh(cur->wb.upper.vlan);
4315                 if (igb_get_buf(rxr, i, dopayload) != 0) {
4316                         ifp->if_iqdrops++;
4317                         /*
4318                          * We've dropped a frame due to lack of resources
4319                          * so we should drop entire multi-segmented
4320                          * frames until we encounter EOP.
4321                          */
4322                         if ((staterr & E1000_RXD_STAT_EOP) != 0)
4323                                 rxr->discard = TRUE;
4324                         igb_rx_discard(rxr, cur, i);
4325                         goto next_desc;
4326                 }
4327
4328                 /* Initial frame - setup */
4329                 if (rxr->fmp == NULL) {
4330                         mh->m_pkthdr.len = mh->m_len;
4331                         /* Store the first mbuf */
4332                         rxr->fmp = mh;
4333                         rxr->lmp = mh;
4334                         if (mp != NULL) {
4335                                 /* Add payload if split */
4336                                 mh->m_pkthdr.len += mp->m_len;
4337                                 rxr->lmp = mh->m_next;
4338                         }
4339                 } else {
4340                         /* Chain mbuf's together */
4341                         rxr->lmp->m_next = mh;
4342                         rxr->lmp = rxr->lmp->m_next;
4343                         rxr->fmp->m_pkthdr.len += mh->m_len;
4344                 }
4345
4346                 if (eop) {
4347                         rxr->fmp->m_pkthdr.rcvif = ifp;
4348                         ifp->if_ipackets++;
4349                         rxr->rx_packets++;
4350                         /* capture data for AIM */
4351                         rxr->packets++;
4352                         rxr->bytes += rxr->fmp->m_pkthdr.len;
4353                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4354
4355                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4356                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
4357                         /* XXX igb(4) always strips VLAN. */
4358                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4359                             (staterr & E1000_RXD_STAT_VP) != 0) {
4360                                 rxr->fmp->m_pkthdr.ether_vlantag = vtag;
4361                                 rxr->fmp->m_flags |= M_VLANTAG;
4362                         }
4363 #if __FreeBSD_version >= 800000
4364                         rxr->fmp->m_pkthdr.flowid = curcpu;
4365                         rxr->fmp->m_flags |= M_FLOWID;
4366 #endif
4367                         sendmp = rxr->fmp;
4368                         /* Make sure to set M_PKTHDR. */
4369                         sendmp->m_flags |= M_PKTHDR;
4370                         rxr->fmp = NULL;
4371                         rxr->lmp = NULL;
4372                 }
4373
4374 next_desc:
4375                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4376                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4377
4378                 rxr->last_cleaned = i; /* For updating tail */
4379
4380                 /* Advance our pointers to the next descriptor. */
4381                 if (++i == adapter->num_rx_desc)
4382                         i = 0;
4383
4384                 /*
4385                 ** Note that we hold the RX lock thru
4386                 ** the following call so this ring's
4387                 ** next_to_check is not gonna change.
4388                 */
4389                 if (sendmp != NULL)
4390                         igb_rx_input(rxr, ifp, sendmp, ptype);
4391         }
4392
4393         if (prog == 0) {
4394                 IGB_RX_UNLOCK(rxr);
4395                 return (FALSE);
4396         }
4397
4398         rxr->next_to_check = i;
4399
4400         /* Advance the E1000's Receive Queue "Tail Pointer". */
4401         E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned);
4402
4403         /*
4404          * Flush any outstanding LRO work
4405          */
4406 #ifdef NET_LRO
4407         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4408                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4409                 tcp_lro_flush(lro, queued);
4410         }
4411 #endif
4412
4413         IGB_RX_UNLOCK(rxr);
4414
4415         /*
4416         ** We still have cleaning to do?
4417         ** Schedule another interrupt if so.
4418         */
4419         if ((staterr & E1000_RXD_STAT_DD) != 0)
4420                 return (TRUE);
4421
4422         return (FALSE);
4423 }
4424
4425 /*********************************************************************
4426  *
4427  *  Verify that the hardware indicated that the checksum is valid.
4428  *  Inform the stack about the status of checksum so that stack
4429  *  doesn't spend time verifying the checksum.
4430  *
4431  *********************************************************************/
4432 static void
4433 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
4434 {
4435         u16 status = (u16)staterr;
4436         u8  errors = (u8) (staterr >> 24);
4437         int sctp;
4438
4439         /* Ignore Checksum bit is set */
4440         if (status & E1000_RXD_STAT_IXSM) {
4441                 mp->m_pkthdr.csum_flags = 0;
4442                 return;
4443         }
4444
4445         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4446             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
4447                 sctp = 1;
4448         else
4449                 sctp = 0;
4450         if (status & E1000_RXD_STAT_IPCS) {
4451                 /* Did it pass? */
4452                 if (!(errors & E1000_RXD_ERR_IPE)) {
4453                         /* IP Checksum Good */
4454                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4455                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4456                 } else
4457                         mp->m_pkthdr.csum_flags = 0;
4458         }
4459
4460         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
4461                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4462 #if __FreeBSD_version >= 800000
4463                 if (sctp) /* reassign */
4464                         type = CSUM_SCTP_VALID;
4465 #endif
4466                 /* Did it pass? */
4467                 if (!(errors & E1000_RXD_ERR_TCPE)) {
4468                         mp->m_pkthdr.csum_flags |= type;
4469                         if (sctp == 0)
4470                                 mp->m_pkthdr.csum_data = htons(0xffff);
4471                 }
4472         }
4473         return;
4474 }
4475
4476 /*
4477  * This routine is run via an vlan
4478  * config EVENT
4479  */
4480 static void
4481 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4482 {
4483         struct adapter  *adapter = ifp->if_softc;
4484         u32             index, bit;
4485
4486         if (ifp->if_softc !=  arg)   /* Not our event */
4487                 return;
4488
4489         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4490                 return;
4491
4492         index = (vtag >> 5) & 0x7F;
4493         bit = vtag & 0x1F;
4494         igb_shadow_vfta[index] |= (1 << bit);
4495         ++adapter->num_vlans;
4496         /* Re-init to load the changes */
4497         igb_init(adapter);
4498 }
4499
4500 /*
4501  * This routine is run via an vlan
4502  * unconfig EVENT
4503  */
4504 static void
4505 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4506 {
4507         struct adapter  *adapter = ifp->if_softc;
4508         u32             index, bit;
4509
4510         if (ifp->if_softc !=  arg)
4511                 return;
4512
4513         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4514                 return;
4515
4516         index = (vtag >> 5) & 0x7F;
4517         bit = vtag & 0x1F;
4518         igb_shadow_vfta[index] &= ~(1 << bit);
4519         --adapter->num_vlans;
4520         /* Re-init to load the changes */
4521         igb_init(adapter);
4522 }
4523
4524 static void
4525 igb_setup_vlan_hw_support(struct adapter *adapter)
4526 {
4527         struct e1000_hw *hw = &adapter->hw;
4528         u32             reg;
4529
4530         /*
4531         ** We get here thru init_locked, meaning
4532         ** a soft reset, this has already cleared
4533         ** the VFTA and other state, so if there
4534         ** have been no vlan's registered do nothing.
4535         */
4536         if (adapter->num_vlans == 0)
4537                 return;
4538
4539         /*
4540         ** A soft reset zero's out the VFTA, so
4541         ** we need to repopulate it now.
4542         */
4543         for (int i = 0; i < IGB_VFTA_SIZE; i++)
4544                 if (igb_shadow_vfta[i] != 0)
4545                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4546                             i, igb_shadow_vfta[i]);
4547
4548         reg = E1000_READ_REG(hw, E1000_CTRL);
4549         reg |= E1000_CTRL_VME;
4550         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4551
4552         /* Enable the Filter Table */
4553         reg = E1000_READ_REG(hw, E1000_RCTL);
4554         reg &= ~E1000_RCTL_CFIEN;
4555         reg |= E1000_RCTL_VFE;
4556         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4557
4558         /* Update the frame size */
4559         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4560             adapter->max_frame_size + VLAN_TAG_SIZE);
4561 }
4562
4563 static void
4564 igb_enable_intr(struct adapter *adapter)
4565 {
4566         /* With RSS set up what to auto clear */
4567         if (adapter->msix_mem) {
4568                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
4569                     adapter->eims_mask);
4570                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
4571                     adapter->eims_mask);
4572                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS,
4573                     adapter->eims_mask);
4574                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4575                     E1000_IMS_LSC);
4576         } else {
4577                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4578                     IMS_ENABLE_MASK);
4579         }
4580         E1000_WRITE_FLUSH(&adapter->hw);
4581
4582         return;
4583 }
4584
4585 static void
4586 igb_disable_intr(struct adapter *adapter)
4587 {
4588         if (adapter->msix_mem) {
4589                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
4590                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
4591         }
4592         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
4593         E1000_WRITE_FLUSH(&adapter->hw);
4594         return;
4595 }
4596
4597 /*
4598  * Bit of a misnomer, what this really means is
4599  * to enable OS management of the system... aka
4600  * to disable special hardware management features
4601  */
4602 static void
4603 igb_init_manageability(struct adapter *adapter)
4604 {
4605         if (adapter->has_manage) {
4606                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4607                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4608
4609                 /* disable hardware interception of ARP */
4610                 manc &= ~(E1000_MANC_ARP_EN);
4611
4612                 /* enable receiving management packets to the host */
4613                 manc |= E1000_MANC_EN_MNG2HOST;
4614                 manc2h |= 1 << 5;  /* Mng Port 623 */
4615                 manc2h |= 1 << 6;  /* Mng Port 664 */
4616                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4617                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4618         }
4619 }
4620
4621 /*
4622  * Give control back to hardware management
4623  * controller if there is one.
4624  */
4625 static void
4626 igb_release_manageability(struct adapter *adapter)
4627 {
4628         if (adapter->has_manage) {
4629                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4630
4631                 /* re-enable hardware interception of ARP */
4632                 manc |= E1000_MANC_ARP_EN;
4633                 manc &= ~E1000_MANC_EN_MNG2HOST;
4634
4635                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4636         }
4637 }
4638
4639 /*
4640  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
4641  * For ASF and Pass Through versions of f/w this means that
4642  * the driver is loaded.
4643  *
4644  */
4645 static void
4646 igb_get_hw_control(struct adapter *adapter)
4647 {
4648         u32 ctrl_ext;
4649
4650         /* Let firmware know the driver has taken over */
4651         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4652         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4653             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4654 }
4655
4656 /*
4657  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
4658  * For ASF and Pass Through versions of f/w this means that the
4659  * driver is no longer loaded.
4660  *
4661  */
4662 static void
4663 igb_release_hw_control(struct adapter *adapter)
4664 {
4665         u32 ctrl_ext;
4666
4667         /* Let firmware taken over control of h/w */
4668         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4669         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4670             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4671 }
4672
4673 static int
4674 igb_is_valid_ether_addr(uint8_t *addr)
4675 {
4676         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4677
4678         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4679                 return (FALSE);
4680         }
4681
4682         return (TRUE);
4683 }
4684
4685
4686 /*
4687  * Enable PCI Wake On Lan capability
4688  */
4689 void
4690 igb_enable_wakeup(device_t dev)
4691 {
4692         u16     cap, status;
4693         u8      id;
4694
4695         /* First find the capabilities pointer*/
4696         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4697         /* Read the PM Capabilities */
4698         id = pci_read_config(dev, cap, 1);
4699         if (id != PCIY_PMG)     /* Something wrong */
4700                 return;
4701         /* OK, we have the power capabilities, so
4702            now get the status register */
4703         cap += PCIR_POWER_STATUS;
4704         status = pci_read_config(dev, cap, 2);
4705         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4706         pci_write_config(dev, cap, status, 2);
4707         return;
4708 }
4709
4710
4711 /**********************************************************************
4712  *
4713  *  Update the board statistics counters.
4714  *
4715  **********************************************************************/
4716 static void
4717 igb_update_stats_counters(struct adapter *adapter)
4718 {
4719         struct ifnet   *ifp;
4720
4721         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4722            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4723                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4724                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4725         }
4726         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4727         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4728         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4729         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4730
4731         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4732         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4733         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4734         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4735         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4736         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4737         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4738         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4739         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4740         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4741         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4742         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4743         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4744         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4745         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4746         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4747         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4748         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4749         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4750         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4751
4752         /* For the 64-bit byte counters the low dword must be read first. */
4753         /* Both registers clear on the read of the high dword */
4754
4755         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
4756         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
4757
4758         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4759         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4760         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4761         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4762         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4763
4764         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4765         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4766
4767         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4768         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4769         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4770         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4771         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4772         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4773         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4774         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4775         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4776         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4777
4778         adapter->stats.algnerrc +=
4779                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4780         adapter->stats.rxerrc +=
4781                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4782         adapter->stats.tncrs +=
4783                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4784         adapter->stats.cexterr +=
4785                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4786         adapter->stats.tsctc +=
4787                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4788         adapter->stats.tsctfc +=
4789                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4790         ifp = adapter->ifp;
4791
4792         ifp->if_collisions = adapter->stats.colc;
4793
4794         /* Rx Errors */
4795         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
4796             adapter->stats.crcerrs + adapter->stats.algnerrc +
4797             adapter->stats.ruc + adapter->stats.roc +
4798             adapter->stats.mpc + adapter->stats.cexterr;
4799
4800         /* Tx Errors */
4801         ifp->if_oerrors = adapter->stats.ecol +
4802             adapter->stats.latecol + adapter->watchdog_events;
4803 }
4804
4805
4806 /**********************************************************************
4807  *
4808  *  This routine is called only when igb_display_debug_stats is enabled.
4809  *  This routine provides a way to take a look at important statistics
4810  *  maintained by the driver and hardware.
4811  *
4812  **********************************************************************/
4813 static void
4814 igb_print_debug_info(struct adapter *adapter)
4815 {
4816         device_t dev = adapter->dev;
4817         struct igb_queue *que = adapter->queues;
4818         struct rx_ring *rxr = adapter->rx_rings;
4819         struct tx_ring *txr = adapter->tx_rings;
4820         uint8_t *hw_addr = adapter->hw.hw_addr;
4821
4822         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
4823         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
4824             E1000_READ_REG(&adapter->hw, E1000_CTRL),
4825             E1000_READ_REG(&adapter->hw, E1000_RCTL));
4826
4827 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
4828         device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n",
4829             E1000_READ_REG(&adapter->hw, E1000_IMS),
4830             E1000_READ_REG(&adapter->hw, E1000_EIMS));
4831 #endif
4832
4833         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4834             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4835             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4836         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4837             adapter->hw.fc.high_water,
4838             adapter->hw.fc.low_water);
4839
4840         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
4841                 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d  ", i,
4842                     E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
4843                     E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
4844                 device_printf(dev, "rdh = %d, rdt = %d\n",
4845                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4846                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4847                 device_printf(dev, "TX(%d) no descriptors avail event = %lld\n",
4848                     txr->me, (long long)txr->no_desc_avail);
4849                 device_printf(dev, "TX(%d) Packets sent = %lld\n",
4850                     txr->me, (long long)txr->tx_packets);
4851                 device_printf(dev, "RX(%d) Packets received = %lld  ",
4852                     rxr->me, (long long)rxr->rx_packets);
4853         }
4854
4855         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4856 #ifdef NET_LRO
4857                 struct lro_ctrl *lro = &rxr->lro;
4858 #endif
4859                 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i,
4860                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4861                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4862                 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
4863                     (long long)rxr->rx_packets);
4864                 device_printf(dev, " Split Packets = %lld ",
4865                     (long long)rxr->rx_split_packets);
4866                 device_printf(dev, " Byte count = %lld\n",
4867                     (long long)rxr->rx_bytes);
4868 #ifdef NET_LRO
4869                 device_printf(dev,"RX(%d) LRO Queued= %d  ",
4870                     i, lro->lro_queued);
4871                 device_printf(dev,"LRO Flushed= %d\n",lro->lro_flushed);
4872 #endif
4873         }
4874
4875         for (int i = 0; i < adapter->num_queues; i++, que++)
4876                 device_printf(dev,"QUE(%d) IRQs = %llx\n",
4877                     i, (long long)que->irqs);
4878
4879         device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq);
4880         device_printf(dev, "Mbuf defrag failed = %ld\n",
4881             adapter->mbuf_defrag_failed);
4882         device_printf(dev, "Std mbuf header failed = %ld\n",
4883             adapter->mbuf_header_failed);
4884         device_printf(dev, "Std mbuf packet failed = %ld\n",
4885             adapter->mbuf_packet_failed);
4886         device_printf(dev, "Driver dropped packets = %ld\n",
4887             adapter->dropped_pkts);
4888         device_printf(dev, "Driver tx dma failure in xmit = %ld\n",
4889                 adapter->no_tx_dma_setup);
4890 }
4891
4892 static void
4893 igb_print_hw_stats(struct adapter *adapter)
4894 {
4895         device_t dev = adapter->dev;
4896
4897         device_printf(dev, "Excessive collisions = %lld\n",
4898             (long long)adapter->stats.ecol);
4899 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
4900         device_printf(dev, "Symbol errors = %lld\n",
4901             (long long)adapter->stats.symerrs);
4902 #endif
4903         device_printf(dev, "Sequence errors = %lld\n",
4904             (long long)adapter->stats.sec);
4905         device_printf(dev, "Defer count = %lld\n",
4906             (long long)adapter->stats.dc);
4907         device_printf(dev, "Missed Packets = %lld\n",
4908             (long long)adapter->stats.mpc);
4909         device_printf(dev, "Receive No Buffers = %lld\n",
4910             (long long)adapter->stats.rnbc);
4911         /* RLEC is inaccurate on some hardware, calculate our own. */
4912         device_printf(dev, "Receive Length Errors = %lld\n",
4913             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
4914         device_printf(dev, "Receive errors = %lld\n",
4915             (long long)adapter->stats.rxerrc);
4916         device_printf(dev, "Crc errors = %lld\n",
4917             (long long)adapter->stats.crcerrs);
4918         device_printf(dev, "Alignment errors = %lld\n",
4919             (long long)adapter->stats.algnerrc);
4920         /* On 82575 these are collision counts */
4921         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
4922             (long long)adapter->stats.cexterr);
4923         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
4924         device_printf(dev, "watchdog timeouts = %ld\n",
4925             adapter->watchdog_events);
4926         device_printf(dev, "XON Rcvd = %lld\n",
4927             (long long)adapter->stats.xonrxc);
4928         device_printf(dev, "XON Xmtd = %lld\n",
4929             (long long)adapter->stats.xontxc);
4930         device_printf(dev, "XOFF Rcvd = %lld\n",
4931             (long long)adapter->stats.xoffrxc);
4932         device_printf(dev, "XOFF Xmtd = %lld\n",
4933             (long long)adapter->stats.xofftxc);
4934         device_printf(dev, "Good Packets Rcvd = %lld\n",
4935             (long long)adapter->stats.gprc);
4936         device_printf(dev, "Good Packets Xmtd = %lld\n",
4937             (long long)adapter->stats.gptc);
4938         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
4939             (long long)adapter->stats.tsctc);
4940         device_printf(dev, "TSO Contexts Failed = %lld\n",
4941             (long long)adapter->stats.tsctfc);
4942 }
4943
4944 /**********************************************************************
4945  *
4946  *  This routine provides a way to dump out the adapter eeprom,
4947  *  often a useful debug/service tool. This only dumps the first
4948  *  32 words, stuff that matters is in that extent.
4949  *
4950  **********************************************************************/
4951 static void
4952 igb_print_nvm_info(struct adapter *adapter)
4953 {
4954         u16     eeprom_data;
4955         int     i, j, row = 0;
4956
4957         /* Its a bit crude, but it gets the job done */
4958         kprintf("\nInterface EEPROM Dump:\n");
4959         kprintf("Offset\n0x0000  ");
4960         for (i = 0, j = 0; i < 32; i++, j++) {
4961                 if (j == 8) { /* Make the offset block */
4962                         j = 0; ++row;
4963                         kprintf("\n0x00%x0  ",row);
4964                 }
4965                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
4966                 kprintf("%04x ", eeprom_data);
4967         }
4968         kprintf("\n");
4969 }
4970
4971 static int
4972 igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
4973 {
4974         struct adapter *adapter;
4975         int error;
4976         int result;
4977
4978         result = -1;
4979         error = sysctl_handle_int(oidp, &result, 0, req);
4980
4981         if (error || !req->newptr)
4982                 return (error);
4983
4984         if (result == 1) {
4985                 adapter = (struct adapter *)arg1;
4986                 igb_print_debug_info(adapter);
4987         }
4988         /*
4989          * This value will cause a hex dump of the
4990          * first 32 16-bit words of the EEPROM to
4991          * the screen.
4992          */
4993         if (result == 2) {
4994                 adapter = (struct adapter *)arg1;
4995                 igb_print_nvm_info(adapter);
4996         }
4997
4998         return (error);
4999 }
5000
5001
5002 static int
5003 igb_sysctl_stats(SYSCTL_HANDLER_ARGS)
5004 {
5005         struct adapter *adapter;
5006         int error;
5007         int result;
5008
5009         result = -1;
5010         error = sysctl_handle_int(oidp, &result, 0, req);
5011
5012         if (error || !req->newptr)
5013                 return (error);
5014
5015         if (result == 1) {
5016                 adapter = (struct adapter *)arg1;
5017                 igb_print_hw_stats(adapter);
5018         }
5019
5020         return (error);
5021 }
5022
5023 static void
5024 igb_add_rx_process_limit(struct adapter *adapter, const char *name,
5025         const char *description, int *limit, int value)
5026 {
5027         *limit = value;
5028         SYSCTL_ADD_INT(&adapter->sysctl_ctx,
5029             SYSCTL_CHILDREN(adapter->sysctl_tree),
5030             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5031 }