sys/dev/netif/e1000/if_igb.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35
  36 #ifdef HAVE_KERNEL_OPTION_HEADERS
  37 #include "opt_device_polling.h"
  38 #include "opt_inet.h"
  39 #include "opt_altq.h"
  40 #endif
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #if __FreeBSD_version >= 800000
  45 #include <sys/buf_ring.h>
  46 #endif
  47 #include <sys/bus.h>
  48 #include <sys/endian.h>
  49 #include <sys/kernel.h>
  50 #include <sys/kthread.h>
  51 #include <sys/malloc.h>
  52 #include <sys/mbuf.h>
  53 #include <sys/module.h>
  54 #include <sys/rman.h>
  55 #include <sys/socket.h>
  56 #include <sys/sockio.h>
  57 #include <sys/sysctl.h>
  58 #include <sys/taskqueue.h>
  59 #include <sys/eventhandler.h>
  60 #include <sys/pcpu.h>
  61 #include <sys/smp.h>
  62 #include <machine/smp.h>
  63 #include <machine/bus.h>
  64 #include <machine/resource.h>
  65
  66 #ifdef IGB_IEEE1588
  67 #include <sys/ieee1588.h>
  68 #endif
  69
  70 #include <net/bpf.h>
  71 #include <net/ethernet.h>
  72 #include <net/if.h>
  73 #include <net/if_arp.h>
  74 #include <net/if_dl.h>
  75 #include <net/if_media.h>
  76
  77 #include <net/if_types.h>
  78 #include <net/if_vlan_var.h>
  79
  80 #include <netinet/in_systm.h>
  81 #include <netinet/in.h>
  82 #include <netinet/if_ether.h>
  83 #include <netinet/ip.h>
  84 #include <netinet/ip6.h>
  85 #include <netinet/tcp.h>
  86 #include <netinet/tcp_lro.h>
  87 #include <netinet/udp.h>
  88
  89 #include <machine/in_cksum.h>
  90 #include <dev/pci/pcivar.h>
  91 #include <dev/pci/pcireg.h>
  92
  93 #include "e1000_api.h"
  94 #include "e1000_82575.h"
  95 #include "if_igb.h"
  96
  97 /*********************************************************************
  98  *  Set this to one to display debug statistics
  99  *********************************************************************/
 100 int     igb_display_debug_stats = 0;
 101
 102 /*********************************************************************
 103  *  Driver version:
 104  *********************************************************************/
 105 char igb_driver_version[] = "version - 1.9.1";
 106
 107
 108 /*********************************************************************
 109  *  PCI Device ID Table
 110  *
 111  *  Used by probe to select devices to load on
 112  *  Last field stores an index into e1000_strings
 113  *  Last entry must be all 0s
 114  *
 115  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 116  *********************************************************************/
 117
 118 static igb_vendor_info_t igb_vendor_info_array[] =
 119 {
 120         { 0x8086, E1000_DEV_ID_82575EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES,
 122                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER,
 124                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82576,           PCI_ANY_ID, PCI_ANY_ID, 0},
 126         { 0x8086, E1000_DEV_ID_82576_NS,        PCI_ANY_ID, PCI_ANY_ID, 0},
 127         { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0},
 128         { 0x8086, E1000_DEV_ID_82576_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 129         { 0x8086, E1000_DEV_ID_82576_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 130         { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD,
 131                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 132         { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER,
 133                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82580_COPPER,    PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82580_FIBER,     PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_82580_SERDES,    PCI_ANY_ID, PCI_ANY_ID, 0},
 137         { 0x8086, E1000_DEV_ID_82580_SGMII,     PCI_ANY_ID, PCI_ANY_ID, 0},
 138         { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL,
 139                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 140         /* required last entry */
 141         { 0, 0, 0, 0, 0}
 142 };
 143
 144 /*********************************************************************
 145  *  Table of branding strings for all supported NICs.
 146  *********************************************************************/
 147
 148 static char *igb_strings[] = {
 149         "Intel(R) PRO/1000 Network Connection"
 150 };
 151
 152 /*********************************************************************
 153  *  Function prototypes
 154  *********************************************************************/
 155 static int      igb_probe(device_t);
 156 static int      igb_attach(device_t);
 157 static int      igb_detach(device_t);
 158 static int      igb_shutdown(device_t);
 159 static int      igb_suspend(device_t);
 160 static int      igb_resume(device_t);
 161 static void     igb_start(struct ifnet *);
 162 static void     igb_start_locked(struct tx_ring *, struct ifnet *ifp);
 163 #if __FreeBSD_version >= 800000
 164 static int      igb_mq_start(struct ifnet *, struct mbuf *);
 165 static int      igb_mq_start_locked(struct ifnet *,
 166                     struct tx_ring *, struct mbuf *);
 167 static void     igb_qflush(struct ifnet *);
 168 #endif
 169 static int      igb_ioctl(struct ifnet *, u_long, caddr_t);
 170 static void     igb_init(void *);
 171 static void     igb_init_locked(struct adapter *);
 172 static void     igb_stop(void *);
 173 static void     igb_media_status(struct ifnet *, struct ifmediareq *);
 174 static int      igb_media_change(struct ifnet *);
 175 static void     igb_identify_hardware(struct adapter *);
 176 static int      igb_allocate_pci_resources(struct adapter *);
 177 static int      igb_allocate_msix(struct adapter *);
 178 static int      igb_allocate_legacy(struct adapter *);
 179 static int      igb_setup_msix(struct adapter *);
 180 static void     igb_free_pci_resources(struct adapter *);
 181 static void     igb_local_timer(void *);
 182 static void     igb_reset(struct adapter *);
 183 static void     igb_setup_interface(device_t, struct adapter *);
 184 static int      igb_allocate_queues(struct adapter *);
 185 static void     igb_configure_queues(struct adapter *);
 186
 187 static int      igb_allocate_transmit_buffers(struct tx_ring *);
 188 static void     igb_setup_transmit_structures(struct adapter *);
 189 static void     igb_setup_transmit_ring(struct tx_ring *);
 190 static void     igb_initialize_transmit_units(struct adapter *);
 191 static void     igb_free_transmit_structures(struct adapter *);
 192 static void     igb_free_transmit_buffers(struct tx_ring *);
 193
 194 static int      igb_allocate_receive_buffers(struct rx_ring *);
 195 static int      igb_setup_receive_structures(struct adapter *);
 196 static int      igb_setup_receive_ring(struct rx_ring *);
 197 static void     igb_initialize_receive_units(struct adapter *);
 198 static void     igb_free_receive_structures(struct adapter *);
 199 static void     igb_free_receive_buffers(struct rx_ring *);
 200 static void     igb_free_receive_ring(struct rx_ring *);
 201
 202 static void     igb_enable_intr(struct adapter *);
 203 static void     igb_disable_intr(struct adapter *);
 204 static void     igb_update_stats_counters(struct adapter *);
 205 static bool     igb_txeof(struct tx_ring *);
 206
 207 static __inline void igb_rx_discard(struct rx_ring *,
 208                     union e1000_adv_rx_desc *, int);
 209 static __inline void igb_rx_input(struct rx_ring *,
 210                     struct ifnet *, struct mbuf *, u32);
 211
 212 static bool     igb_rxeof(struct rx_ring *, int);
 213 static void     igb_rx_checksum(u32, struct mbuf *, u32);
 214 static int      igb_tx_ctx_setup(struct tx_ring *, struct mbuf *);
 215 static bool     igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *);
 216 static void     igb_set_promisc(struct adapter *);
 217 static void     igb_disable_promisc(struct adapter *);
 218 static void     igb_set_multi(struct adapter *);
 219 static void     igb_print_hw_stats(struct adapter *);
 220 static void     igb_update_link_status(struct adapter *);
 221 static int      igb_get_buf(struct rx_ring *, int, u8);
 222
 223 static void     igb_register_vlan(void *, struct ifnet *, u16);
 224 static void     igb_unregister_vlan(void *, struct ifnet *, u16);
 225 static void     igb_setup_vlan_hw_support(struct adapter *);
 226
 227 static int      igb_xmit(struct tx_ring *, struct mbuf **);
 228 static int      igb_dma_malloc(struct adapter *, bus_size_t,
 229                     struct igb_dma_alloc *, int);
 230 static void     igb_dma_free(struct adapter *, struct igb_dma_alloc *);
 231 static void     igb_print_debug_info(struct adapter *);
 232 static void     igb_print_nvm_info(struct adapter *);
 233 static int      igb_is_valid_ether_addr(u8 *);
 234 static int      igb_sysctl_stats(SYSCTL_HANDLER_ARGS);
 235 static int      igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 236 /* Management and WOL Support */
 237 static void     igb_init_manageability(struct adapter *);
 238 static void     igb_release_manageability(struct adapter *);
 239 static void     igb_get_hw_control(struct adapter *);
 240 static void     igb_release_hw_control(struct adapter *);
 241 static void     igb_enable_wakeup(device_t);
 242
 243 static int      igb_irq_fast(void *);
 244 static void     igb_add_rx_process_limit(struct adapter *, const char *,
 245                     const char *, int *, int);
 246 static void     igb_handle_rxtx(void *context, int pending);
 247 static void     igb_handle_que(void *context, int pending);
 248 static void     igb_handle_link(void *context, int pending);
 249
 250 /* These are MSIX only irq handlers */
 251 static void     igb_msix_que(void *);
 252 static void     igb_msix_link(void *);
 253
 254 #ifdef DEVICE_POLLING
 255 static poll_handler_t igb_poll;
 256 #endif /* POLLING */
 257
 258 /*********************************************************************
 259  *  FreeBSD Device Interface Entry Points
 260  *********************************************************************/
 261
 262 static device_method_t igb_methods[] = {
 263         /* Device interface */
 264         DEVMETHOD(device_probe, igb_probe),
 265         DEVMETHOD(device_attach, igb_attach),
 266         DEVMETHOD(device_detach, igb_detach),
 267         DEVMETHOD(device_shutdown, igb_shutdown),
 268         DEVMETHOD(device_suspend, igb_suspend),
 269         DEVMETHOD(device_resume, igb_resume),
 270         {0, 0}
 271 };
 272
 273 static driver_t igb_driver = {
 274         "igb", igb_methods, sizeof(struct adapter),
 275 };
 276
 277 static devclass_t igb_devclass;
 278 DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0);
 279 MODULE_DEPEND(igb, pci, 1, 1, 1);
 280 MODULE_DEPEND(igb, ether, 1, 1, 1);
 281
 282 /*********************************************************************
 283  *  Tunable default values.
 284  *********************************************************************/
 285
 286 /* Descriptor defaults */
 287 static int igb_rxd = IGB_DEFAULT_RXD;
 288 static int igb_txd = IGB_DEFAULT_TXD;
 289 TUNABLE_INT("hw.igb.rxd", &igb_rxd);
 290 TUNABLE_INT("hw.igb.txd", &igb_txd);
 291
 292 /*
 293 ** AIM: Adaptive Interrupt Moderation
 294 ** which means that the interrupt rate
 295 ** is varied over time based on the
 296 ** traffic for that interrupt vector
 297 */
 298 static int igb_enable_aim = TRUE;
 299 TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim);
 300
 301 /*
 302  * MSIX should be the default for best performance,
 303  * but this allows it to be forced off for testing.
 304  */
 305 static int igb_enable_msix = 1;
 306 TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix);
 307
 308 /*
 309  * Header split has seemed to be beneficial in
 310  * many circumstances tested, however there have
 311  * been some stability issues, so the default is
 312  * off.
 313  */
 314 static bool igb_header_split = FALSE;
 315 TUNABLE_INT("hw.igb.hdr_split", &igb_header_split);
 316
 317 /*
 318 ** This will autoconfigure based on
 319 ** the number of CPUs if left at 0.
 320 */
 321 static int igb_num_queues = 0;
 322 TUNABLE_INT("hw.igb.num_queues", &igb_num_queues);
 323
 324 /* How many packets rxeof tries to clean at a time */
 325 static int igb_rx_process_limit = 100;
 326 TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit);
 327
 328 /* Flow control setting - default to FULL */
 329 static int igb_fc_setting = e1000_fc_full;
 330 TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting);
 331
 332 /*
 333 ** Shadow VFTA table, this is needed because
 334 ** the real filter table gets cleared during
 335 ** a soft reset and the driver needs to be able
 336 ** to repopulate it.
 337 */
 338 static u32 igb_shadow_vfta[IGB_VFTA_SIZE];
 339
 340
 341 /*********************************************************************
 342  *  Device identification routine
 343  *
 344  *  igb_probe determines if the driver should be loaded on
 345  *  adapter based on PCI vendor/device id of the adapter.
 346  *
 347  *  return BUS_PROBE_DEFAULT on success, positive on failure
 348  *********************************************************************/
 349
 350 static int
 351 igb_probe(device_t dev)
 352 {
 353         char            adapter_name[60];
 354         uint16_t        pci_vendor_id = 0;
 355         uint16_t        pci_device_id = 0;
 356         uint16_t        pci_subvendor_id = 0;
 357         uint16_t        pci_subdevice_id = 0;
 358         igb_vendor_info_t *ent;
 359
 360         INIT_DEBUGOUT("igb_probe: begin");
 361
 362         pci_vendor_id = pci_get_vendor(dev);
 363         if (pci_vendor_id != IGB_VENDOR_ID)
 364                 return (ENXIO);
 365
 366         pci_device_id = pci_get_device(dev);
 367         pci_subvendor_id = pci_get_subvendor(dev);
 368         pci_subdevice_id = pci_get_subdevice(dev);
 369
 370         ent = igb_vendor_info_array;
 371         while (ent->vendor_id != 0) {
 372                 if ((pci_vendor_id == ent->vendor_id) &&
 373                     (pci_device_id == ent->device_id) &&
 374
 375                     ((pci_subvendor_id == ent->subvendor_id) ||
 376                     (ent->subvendor_id == PCI_ANY_ID)) &&
 377
 378                     ((pci_subdevice_id == ent->subdevice_id) ||
 379                     (ent->subdevice_id == PCI_ANY_ID))) {
 380                         sprintf(adapter_name, "%s %s",
 381                                 igb_strings[ent->index],
 382                                 igb_driver_version);
 383                         device_set_desc_copy(dev, adapter_name);
 384                         return (BUS_PROBE_DEFAULT);
 385                 }
 386                 ent++;
 387         }
 388
 389         return (ENXIO);
 390 }
 391
 392 /*********************************************************************
 393  *  Device initialization routine
 394  *
 395  *  The attach entry point is called when the driver is being loaded.
 396  *  This routine identifies the type of hardware, allocates all resources
 397  *  and initializes the hardware.
 398  *
 399  *  return 0 on success, positive on failure
 400  *********************************************************************/
 401
 402 static int
 403 igb_attach(device_t dev)
 404 {
 405         struct adapter  *adapter;
 406         int             error = 0;
 407         u16             eeprom_data;
 408
 409         INIT_DEBUGOUT("igb_attach: begin");
 410
 411         adapter = device_get_softc(dev);
 412         adapter->dev = adapter->osdep.dev = dev;
 413         IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 414
 415         /* SYSCTL stuff */
 416         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 417             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 418             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 419             igb_sysctl_debug_info, "I", "Debug Information");
 420
 421         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 422             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 423             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 424             igb_sysctl_stats, "I", "Statistics");
 425
 426         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
 427             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
 428             OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW,
 429             &igb_fc_setting, 0, "Flow Control");
 430
 431         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
 432             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 433             OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW,
 434             &igb_enable_aim, 1, "Interrupt Moderation");
 435
 436         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 437
 438         /* Determine hardware and mac info */
 439         igb_identify_hardware(adapter);
 440
 441         /* Setup PCI resources */
 442         if (igb_allocate_pci_resources(adapter)) {
 443                 device_printf(dev, "Allocation of PCI resources failed\n");
 444                 error = ENXIO;
 445                 goto err_pci;
 446         }
 447
 448         /* Do Shared Code initialization */
 449         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 450                 device_printf(dev, "Setup of Shared code failed\n");
 451                 error = ENXIO;
 452                 goto err_pci;
 453         }
 454
 455         e1000_get_bus_info(&adapter->hw);
 456
 457         /* Sysctls for limiting the amount of work done in the taskqueue */
 458         igb_add_rx_process_limit(adapter, "rx_processing_limit",
 459             "max number of rx packets to process", &adapter->rx_process_limit,
 460             igb_rx_process_limit);
 461
 462         /*
 463          * Validate number of transmit and receive descriptors. It
 464          * must not exceed hardware maximum, and must be multiple
 465          * of E1000_DBA_ALIGN.
 466          */
 467         if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 ||
 468             (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) {
 469                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 470                     IGB_DEFAULT_TXD, igb_txd);
 471                 adapter->num_tx_desc = IGB_DEFAULT_TXD;
 472         } else
 473                 adapter->num_tx_desc = igb_txd;
 474         if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 ||
 475             (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) {
 476                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 477                     IGB_DEFAULT_RXD, igb_rxd);
 478                 adapter->num_rx_desc = IGB_DEFAULT_RXD;
 479         } else
 480                 adapter->num_rx_desc = igb_rxd;
 481
 482         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 483         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 484         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 485
 486         /* Copper options */
 487         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 488                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 489                 adapter->hw.phy.disable_polarity_correction = FALSE;
 490                 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE;
 491         }
 492
 493         /*
 494          * Set the frame limits assuming
 495          * standard ethernet sized frames.
 496          */
 497         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 498         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 499
 500         /*
 501         ** Allocate and Setup Queues
 502         */
 503         if (igb_allocate_queues(adapter)) {
 504                 error = ENOMEM;
 505                 goto err_pci;
 506         }
 507
 508         /*
 509         ** Start from a known state, this is
 510         ** important in reading the nvm and
 511         ** mac from that.
 512         */
 513         e1000_reset_hw(&adapter->hw);
 514
 515         /* Make sure we have a good EEPROM before we read from it */
 516         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 517                 /*
 518                 ** Some PCI-E parts fail the first check due to
 519                 ** the link being in sleep state, call it again,
 520                 ** if it fails a second time its a real issue.
 521                 */
 522                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 523                         device_printf(dev,
 524                             "The EEPROM Checksum Is Not Valid\n");
 525                         error = EIO;
 526                         goto err_late;
 527                 }
 528         }
 529
 530         /*
 531         ** Copy the permanent MAC address out of the EEPROM
 532         */
 533         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 534                 device_printf(dev, "EEPROM read error while reading MAC"
 535                     " address\n");
 536                 error = EIO;
 537                 goto err_late;
 538         }
 539         /* Check its sanity */
 540         if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) {
 541                 device_printf(dev, "Invalid MAC address\n");
 542                 error = EIO;
 543                 goto err_late;
 544         }
 545
 546         /*
 547         ** Configure Interrupts
 548         */
 549         if ((adapter->msix > 1) && (igb_enable_msix))
 550                 error = igb_allocate_msix(adapter);
 551         else /* MSI or Legacy */
 552                 error = igb_allocate_legacy(adapter);
 553         if (error)
 554                 goto err_late;
 555
 556         /* Setup OS specific network interface */
 557         igb_setup_interface(dev, adapter);
 558
 559         /* Now get a good starting state */
 560         igb_reset(adapter);
 561
 562         /* Initialize statistics */
 563         igb_update_stats_counters(adapter);
 564
 565         adapter->hw.mac.get_link_status = 1;
 566         igb_update_link_status(adapter);
 567
 568         /* Indicate SOL/IDER usage */
 569         if (e1000_check_reset_block(&adapter->hw))
 570                 device_printf(dev,
 571                     "PHY reset is blocked due to SOL/IDER session.\n");
 572
 573         /* Determine if we have to control management hardware */
 574         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
 575
 576         /*
 577          * Setup Wake-on-Lan
 578          */
 579         /* APME bit in EEPROM is mapped to WUC.APME */
 580         eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME;
 581         if (eeprom_data)
 582                 adapter->wol = E1000_WUFC_MAG;
 583
 584         /* Register for VLAN events */
 585         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 586              igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 587         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 588              igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 589
 590         /* Tell the stack that the interface is not active */
 591         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 592
 593         INIT_DEBUGOUT("igb_attach: end");
 594
 595         return (0);
 596
 597 err_late:
 598         igb_free_transmit_structures(adapter);
 599         igb_free_receive_structures(adapter);
 600         igb_release_hw_control(adapter);
 601 err_pci:
 602         igb_free_pci_resources(adapter);
 603         IGB_CORE_LOCK_DESTROY(adapter);
 604
 605         return (error);
 606 }
 607
 608 /*********************************************************************
 609  *  Device removal routine
 610  *
 611  *  The detach entry point is called when the driver is being removed.
 612  *  This routine stops the adapter and deallocates all the resources
 613  *  that were allocated for driver operation.
 614  *
 615  *  return 0 on success, positive on failure
 616  *********************************************************************/
 617
 618 static int
 619 igb_detach(device_t dev)
 620 {
 621         struct adapter  *adapter = device_get_softc(dev);
 622         struct ifnet    *ifp = adapter->ifp;
 623
 624         INIT_DEBUGOUT("igb_detach: begin");
 625
 626         /* Make sure VLANS are not using driver */
 627         if (adapter->ifp->if_vlantrunk != NULL) {
 628                 device_printf(dev,"Vlan in use, detach first\n");
 629                 return (EBUSY);
 630         }
 631
 632 #ifdef DEVICE_POLLING
 633         if (ifp->if_capenable & IFCAP_POLLING)
 634                 ether_poll_deregister(ifp);
 635 #endif
 636
 637         IGB_CORE_LOCK(adapter);
 638         adapter->in_detach = 1;
 639         igb_stop(adapter);
 640         IGB_CORE_UNLOCK(adapter);
 641
 642         e1000_phy_hw_reset(&adapter->hw);
 643
 644         /* Give control back to firmware */
 645         igb_release_manageability(adapter);
 646         igb_release_hw_control(adapter);
 647
 648         if (adapter->wol) {
 649                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 650                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 651                 igb_enable_wakeup(dev);
 652         }
 653
 654         /* Unregister VLAN events */
 655         if (adapter->vlan_attach != NULL)
 656                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 657         if (adapter->vlan_detach != NULL)
 658                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 659
 660         ether_ifdetach(adapter->ifp);
 661
 662         //callout_drain(&adapter->timer);
 663         callout_stop(&adapter->timer);
 664
 665         igb_free_pci_resources(adapter);
 666         bus_generic_detach(dev);
 667         if_free(ifp);
 668
 669         igb_free_transmit_structures(adapter);
 670         igb_free_receive_structures(adapter);
 671
 672         IGB_CORE_LOCK_DESTROY(adapter);
 673
 674         return (0);
 675 }
 676
 677 /*********************************************************************
 678  *
 679  *  Shutdown entry point
 680  *
 681  **********************************************************************/
 682
 683 static int
 684 igb_shutdown(device_t dev)
 685 {
 686         return igb_suspend(dev);
 687 }
 688
 689 /*
 690  * Suspend/resume device methods.
 691  */
 692 static int
 693 igb_suspend(device_t dev)
 694 {
 695         struct adapter *adapter = device_get_softc(dev);
 696
 697         IGB_CORE_LOCK(adapter);
 698
 699         igb_stop(adapter);
 700
 701         igb_release_manageability(adapter);
 702         igb_release_hw_control(adapter);
 703
 704         if (adapter->wol) {
 705                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 706                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 707                 igb_enable_wakeup(dev);
 708         }
 709
 710         IGB_CORE_UNLOCK(adapter);
 711
 712         return bus_generic_suspend(dev);
 713 }
 714
 715 static int
 716 igb_resume(device_t dev)
 717 {
 718         struct adapter *adapter = device_get_softc(dev);
 719         struct ifnet *ifp = adapter->ifp;
 720
 721         IGB_CORE_LOCK(adapter);
 722         igb_init_locked(adapter);
 723         igb_init_manageability(adapter);
 724
 725         if ((ifp->if_flags & IFF_UP) &&
 726             (ifp->if_drv_flags & IFF_DRV_RUNNING))
 727                 igb_start(ifp);
 728
 729         IGB_CORE_UNLOCK(adapter);
 730
 731         return bus_generic_resume(dev);
 732 }
 733
 734
 735 /*********************************************************************
 736  *  Transmit entry point
 737  *
 738  *  igb_start is called by the stack to initiate a transmit.
 739  *  The driver will remain in this routine as long as there are
 740  *  packets to transmit and transmit resources are available.
 741  *  In case resources are not available stack is notified and
 742  *  the packet is requeued.
 743  **********************************************************************/
 744
 745 static void
 746 igb_start_locked(struct tx_ring *txr, struct ifnet *ifp)
 747 {
 748         struct adapter  *adapter = ifp->if_softc;
 749         struct mbuf     *m_head;
 750
 751         IGB_TX_LOCK_ASSERT(txr);
 752
 753         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 754             IFF_DRV_RUNNING)
 755                 return;
 756         if (!adapter->link_active)
 757                 return;
 758
 759         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
 760
 761                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
 762                 if (m_head == NULL)
 763                         break;
 764                 /*
 765                  *  Encapsulation can modify our pointer, and or make it
 766                  *  NULL on failure.  In that event, we can't requeue.
 767                  */
 768                 if (igb_xmit(txr, &m_head)) {
 769                         if (m_head == NULL)
 770                                 break;
 771                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 772                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
 773                         break;
 774                 }
 775
 776                 /* Send a copy of the frame to the BPF listener */
 777                 ETHER_BPF_MTAP(ifp, m_head);
 778
 779                 /* Set watchdog on */
 780                 txr->watchdog_check = TRUE;
 781         }
 782 }
 783
 784 /*
 785  * Legacy TX driver routine, called from the
 786  * stack, always uses tx[0], and spins for it.
 787  * Should not be used with multiqueue tx
 788  */
 789 static void
 790 igb_start(struct ifnet *ifp)
 791 {
 792         struct adapter  *adapter = ifp->if_softc;
 793         struct tx_ring  *txr = adapter->tx_rings;
 794
 795         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 796                 IGB_TX_LOCK(txr);
 797                 igb_start_locked(txr, ifp);
 798                 IGB_TX_UNLOCK(txr);
 799         }
 800         return;
 801 }
 802
 803 #if __FreeBSD_version >= 800000
 804 /*
 805 ** Multiqueue Transmit driver
 806 **
 807 */
 808 static int
 809 igb_mq_start(struct ifnet *ifp, struct mbuf *m)
 810 {
 811         struct adapter  *adapter = ifp->if_softc;
 812         struct tx_ring  *txr;
 813         int             i = 0, err = 0;
 814
 815         /* Which queue to use */
 816         if ((m->m_flags & M_FLOWID) != 0)
 817                 i = m->m_pkthdr.flowid % adapter->num_queues;
 818         txr = &adapter->tx_rings[i];
 819
 820         if (IGB_TX_TRYLOCK(txr)) {
 821                 err = igb_mq_start_locked(ifp, txr, m);
 822                 IGB_TX_UNLOCK(txr);
 823         } else
 824                 err = drbr_enqueue(ifp, txr->br, m);
 825
 826         return (err);
 827 }
 828
 829 static int
 830 igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m)
 831 {
 832         struct adapter  *adapter = txr->adapter;
 833         struct mbuf     *next;
 834         int             err = 0, enq;
 835
 836         IGB_TX_LOCK_ASSERT(txr);
 837
 838         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 839             IFF_DRV_RUNNING || adapter->link_active == 0) {
 840                 if (m != NULL)
 841                         err = drbr_enqueue(ifp, txr->br, m);
 842                 return (err);
 843         }
 844
 845         enq = 0;
 846         if (m == NULL) {
 847                 next = drbr_dequeue(ifp, txr->br);
 848         } else if (drbr_needs_enqueue(ifp, txr->br)) {
 849                 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0)
 850                         return (err);
 851                 next = drbr_dequeue(ifp, txr->br);
 852         } else
 853                 next = m;
 854         /* Process the queue */
 855         while (next != NULL) {
 856                 if ((err = igb_xmit(txr, &next)) != 0) {
 857                         if (next != NULL)
 858                                 err = drbr_enqueue(ifp, txr->br, next);
 859                         break;
 860                 }
 861                 enq++;
 862                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 863                 ETHER_BPF_MTAP(ifp, next);
 864                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 865                         break;
 866                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
 867                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 868                         break;
 869                 }
 870                 next = drbr_dequeue(ifp, txr->br);
 871         }
 872         if (enq > 0) {
 873                 /* Set the watchdog */
 874                 txr->watchdog_check = TRUE;
 875         }
 876         return (err);
 877 }
 878
 879 /*
 880 ** Flush all ring buffers
 881 */
 882 static void
 883 igb_qflush(struct ifnet *ifp)
 884 {
 885         struct adapter  *adapter = ifp->if_softc;
 886         struct tx_ring  *txr = adapter->tx_rings;
 887         struct mbuf     *m;
 888
 889         for (int i = 0; i < adapter->num_queues; i++, txr++) {
 890                 IGB_TX_LOCK(txr);
 891                 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
 892                         m_freem(m);
 893                 IGB_TX_UNLOCK(txr);
 894         }
 895         if_qflush(ifp);
 896 }
 897 #endif /* __FreeBSD_version >= 800000 */
 898
 899 /*********************************************************************
 900  *  Ioctl entry point
 901  *
 902  *  igb_ioctl is called when the user wants to configure the
 903  *  interface.
 904  *
 905  *  return 0 on success, positive on failure
 906  **********************************************************************/
 907
 908 static int
 909 igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 910 {
 911         struct adapter  *adapter = ifp->if_softc;
 912         struct ifreq *ifr = (struct ifreq *)data;
 913 #ifdef INET
 914         struct ifaddr *ifa = (struct ifaddr *)data;
 915 #endif
 916         int error = 0;
 917
 918         if (adapter->in_detach)
 919                 return (error);
 920
 921         switch (command) {
 922         case SIOCSIFADDR:
 923 #ifdef INET
 924                 if (ifa->ifa_addr->sa_family == AF_INET) {
 925                         /*
 926                          * XXX
 927                          * Since resetting hardware takes a very long time
 928                          * and results in link renegotiation we only
 929                          * initialize the hardware only when it is absolutely
 930                          * required.
 931                          */
 932                         ifp->if_flags |= IFF_UP;
 933                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 934                                 IGB_CORE_LOCK(adapter);
 935                                 igb_init_locked(adapter);
 936                                 IGB_CORE_UNLOCK(adapter);
 937                         }
 938                         if (!(ifp->if_flags & IFF_NOARP))
 939                                 arp_ifinit(ifp, ifa);
 940                 } else
 941 #endif
 942                         error = ether_ioctl(ifp, command, data);
 943                 break;
 944         case SIOCSIFMTU:
 945             {
 946                 int max_frame_size;
 947
 948                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 949
 950                 IGB_CORE_LOCK(adapter);
 951                 max_frame_size = 9234;
 952                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
 953                     ETHER_CRC_LEN) {
 954                         IGB_CORE_UNLOCK(adapter);
 955                         error = EINVAL;
 956                         break;
 957                 }
 958
 959                 ifp->if_mtu = ifr->ifr_mtu;
 960                 adapter->max_frame_size =
 961                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 962                 igb_init_locked(adapter);
 963                 IGB_CORE_UNLOCK(adapter);
 964                 break;
 965             }
 966         case SIOCSIFFLAGS:
 967                 IOCTL_DEBUGOUT("ioctl rcv'd:\
 968                     SIOCSIFFLAGS (Set Interface Flags)");
 969                 IGB_CORE_LOCK(adapter);
 970                 if (ifp->if_flags & IFF_UP) {
 971                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 972                                 if ((ifp->if_flags ^ adapter->if_flags) &
 973                                     (IFF_PROMISC | IFF_ALLMULTI)) {
 974                                         igb_disable_promisc(adapter);
 975                                         igb_set_promisc(adapter);
 976                                 }
 977                         } else
 978                                 igb_init_locked(adapter);
 979                 } else
 980                         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 981                                 igb_stop(adapter);
 982                 adapter->if_flags = ifp->if_flags;
 983                 IGB_CORE_UNLOCK(adapter);
 984                 break;
 985         case SIOCADDMULTI:
 986         case SIOCDELMULTI:
 987                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 988                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 989                         IGB_CORE_LOCK(adapter);
 990                         igb_disable_intr(adapter);
 991                         igb_set_multi(adapter);
 992 #ifdef DEVICE_POLLING
 993                         if (!(ifp->if_capenable & IFCAP_POLLING))
 994 #endif
 995                                 igb_enable_intr(adapter);
 996                         IGB_CORE_UNLOCK(adapter);
 997                 }
 998                 break;
 999         case SIOCSIFMEDIA:
1000                 /* Check SOL/IDER usage */
1001                 IGB_CORE_LOCK(adapter);
1002                 if (e1000_check_reset_block(&adapter->hw)) {
1003                         IGB_CORE_UNLOCK(adapter);
1004                         device_printf(adapter->dev, "Media change is"
1005                             " blocked due to SOL/IDER session.\n");
1006                         break;
1007                 }
1008                 IGB_CORE_UNLOCK(adapter);
1009         case SIOCGIFMEDIA:
1010                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1011                     SIOCxIFMEDIA (Get/Set Interface Media)");
1012                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1013                 break;
1014         case SIOCSIFCAP:
1015             {
1016                 int mask, reinit;
1017
1018                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1019                 reinit = 0;
1020                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1021 #ifdef DEVICE_POLLING
1022                 if (mask & IFCAP_POLLING) {
1023                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1024                                 error = ether_poll_register(igb_poll, ifp);
1025                                 if (error)
1026                                         return (error);
1027                                 IGB_CORE_LOCK(adapter);
1028                                 igb_disable_intr(adapter);
1029                                 ifp->if_capenable |= IFCAP_POLLING;
1030                                 IGB_CORE_UNLOCK(adapter);
1031                         } else {
1032                                 error = ether_poll_deregister(ifp);
1033                                 /* Enable interrupt even in error case */
1034                                 IGB_CORE_LOCK(adapter);
1035                                 igb_enable_intr(adapter);
1036                                 ifp->if_capenable &= ~IFCAP_POLLING;
1037                                 IGB_CORE_UNLOCK(adapter);
1038                         }
1039                 }
1040 #endif
1041                 if (mask & IFCAP_HWCSUM) {
1042                         ifp->if_capenable ^= IFCAP_HWCSUM;
1043                         reinit = 1;
1044                 }
1045                 if (mask & IFCAP_TSO4) {
1046                         ifp->if_capenable ^= IFCAP_TSO4;
1047                         reinit = 1;
1048                 }
1049                 if (mask & IFCAP_VLAN_HWTAGGING) {
1050                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1051                         reinit = 1;
1052                 }
1053                 if (mask & IFCAP_LRO) {
1054                         ifp->if_capenable ^= IFCAP_LRO;
1055                         reinit = 1;
1056                 }
1057                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1058                         igb_init(adapter);
1059                 VLAN_CAPABILITIES(ifp);
1060                 break;
1061             }
1062
1063         default:
1064                 error = ether_ioctl(ifp, command, data);
1065                 break;
1066         }
1067
1068         return (error);
1069 }
1070
1071
1072 /*********************************************************************
1073  *  Init entry point
1074  *
1075  *  This routine is used in two ways. It is used by the stack as
1076  *  init entry point in network interface structure. It is also used
1077  *  by the driver as a hw/sw initialization routine to get to a
1078  *  consistent state.
1079  *
1080  *  return 0 on success, positive on failure
1081  **********************************************************************/
1082
1083 static void
1084 igb_init_locked(struct adapter *adapter)
1085 {
1086         struct ifnet    *ifp = adapter->ifp;
1087         device_t        dev = adapter->dev;
1088
1089         INIT_DEBUGOUT("igb_init: begin");
1090
1091         IGB_CORE_LOCK_ASSERT(adapter);
1092
1093         igb_disable_intr(adapter);
1094         callout_stop(&adapter->timer);
1095
1096         /* Get the latest mac address, User can use a LAA */
1097         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1098               ETHER_ADDR_LEN);
1099
1100         /* Put the address into the Receive Address Array */
1101         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1102
1103         igb_reset(adapter);
1104         igb_update_link_status(adapter);
1105
1106         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1107
1108         /* Set hardware offload abilities */
1109         ifp->if_hwassist = 0;
1110         if (ifp->if_capenable & IFCAP_TXCSUM) {
1111                 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1112 #if __FreeBSD_version >= 800000
1113                 if (adapter->hw.mac.type == e1000_82576)
1114                         ifp->if_hwassist |= CSUM_SCTP;
1115 #endif
1116         }
1117
1118         if (ifp->if_capenable & IFCAP_TSO4)
1119                 ifp->if_hwassist |= CSUM_TSO;
1120
1121         /* Configure for OS presence */
1122         igb_init_manageability(adapter);
1123
1124         /* Prepare transmit descriptors and buffers */
1125         igb_setup_transmit_structures(adapter);
1126         igb_initialize_transmit_units(adapter);
1127
1128         /* Setup Multicast table */
1129         igb_set_multi(adapter);
1130
1131         /*
1132         ** Figure out the desired mbuf pool
1133         ** for doing jumbo/packetsplit
1134         */
1135         if (ifp->if_mtu > ETHERMTU)
1136                 adapter->rx_mbuf_sz = MJUMPAGESIZE;
1137         else
1138                 adapter->rx_mbuf_sz = MCLBYTES;
1139
1140         /* Prepare receive descriptors and buffers */
1141         if (igb_setup_receive_structures(adapter)) {
1142                 device_printf(dev, "Could not setup receive structures\n");
1143                 return;
1144         }
1145         igb_initialize_receive_units(adapter);
1146
1147         /* Don't lose promiscuous settings */
1148         igb_set_promisc(adapter);
1149
1150         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1151         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1152
1153         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1154         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1155
1156         if (adapter->msix > 1) /* Set up queue routing */
1157                 igb_configure_queues(adapter);
1158
1159         /* Set up VLAN tag offload and filter */
1160         igb_setup_vlan_hw_support(adapter);
1161
1162         /* this clears any pending interrupts */
1163         E1000_READ_REG(&adapter->hw, E1000_ICR);
1164 #ifdef DEVICE_POLLING
1165         /*
1166          * Only enable interrupts if we are not polling, make sure
1167          * they are off otherwise.
1168          */
1169         if (ifp->if_capenable & IFCAP_POLLING)
1170                 igb_disable_intr(adapter);
1171         else
1172 #endif /* DEVICE_POLLING */
1173         {
1174         igb_enable_intr(adapter);
1175         E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC);
1176         }
1177
1178         /* Don't reset the phy next time init gets called */
1179         adapter->hw.phy.reset_disable = TRUE;
1180 }
1181
1182 static void
1183 igb_init(void *arg)
1184 {
1185         struct adapter *adapter = arg;
1186
1187         IGB_CORE_LOCK(adapter);
1188         igb_init_locked(adapter);
1189         IGB_CORE_UNLOCK(adapter);
1190 }
1191
1192
1193 static void
1194 igb_handle_rxtx(void *context, int pending)
1195 {
1196         struct adapter  *adapter = context;
1197         struct tx_ring  *txr = adapter->tx_rings;
1198         struct rx_ring  *rxr = adapter->rx_rings;
1199         struct ifnet    *ifp;
1200
1201         ifp = adapter->ifp;
1202
1203         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1204                 if (igb_rxeof(rxr, adapter->rx_process_limit))
1205                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1206                 IGB_TX_LOCK(txr);
1207                 igb_txeof(txr);
1208
1209 #if __FreeBSD_version >= 800000
1210                 if (!drbr_empty(ifp, txr->br))
1211                         igb_mq_start_locked(ifp, txr, NULL);
1212 #else
1213                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1214                         igb_start_locked(txr, ifp);
1215 #endif
1216                 IGB_TX_UNLOCK(txr);
1217         }
1218
1219         igb_enable_intr(adapter);
1220 }
1221
1222 static void
1223 igb_handle_que(void *context, int pending)
1224 {
1225         struct igb_queue *que = context;
1226         struct adapter *adapter = que->adapter;
1227         struct tx_ring *txr = que->txr;
1228         struct rx_ring *rxr = que->rxr;
1229         struct ifnet    *ifp = adapter->ifp;
1230         u32             loop = IGB_MAX_LOOP;
1231         bool            more;
1232
1233         /* RX first */
1234         do {
1235                 more = igb_rxeof(rxr, -1);
1236         } while (loop-- && more);
1237
1238         if (IGB_TX_TRYLOCK(txr)) {
1239                 loop = IGB_MAX_LOOP;
1240                 do {
1241                         more = igb_txeof(txr);
1242                 } while (loop-- && more);
1243 #if __FreeBSD_version >= 800000
1244                 igb_mq_start_locked(ifp, txr, NULL);
1245 #else
1246                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1247                         igb_start_locked(txr, ifp);
1248 #endif
1249                 IGB_TX_UNLOCK(txr);
1250         }
1251
1252         /* Reenable this interrupt */
1253 #ifdef DEVICE_POLLING
1254         if (!(ifp->if_capenable & IFCAP_POLLING))
1255 #endif
1256         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1257 }
1258
1259 /* Deal with link in a sleepable context */
1260 static void
1261 igb_handle_link(void *context, int pending)
1262 {
1263         struct adapter *adapter = context;
1264
1265         adapter->hw.mac.get_link_status = 1;
1266         igb_update_link_status(adapter);
1267 }
1268
1269 /*********************************************************************
1270  *
1271  *  MSI/Legacy Deferred
1272  *  Interrupt Service routine
1273  *
1274  *********************************************************************/
1275 static int
1276 igb_irq_fast(void *arg)
1277 {
1278         struct adapter  *adapter = arg;
1279         uint32_t        reg_icr;
1280
1281
1282         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1283
1284         /* Hot eject?  */
1285         if (reg_icr == 0xffffffff)
1286                 return FILTER_STRAY;
1287
1288         /* Definitely not our interrupt.  */
1289         if (reg_icr == 0x0)
1290                 return FILTER_STRAY;
1291
1292         if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1293                 return FILTER_STRAY;
1294
1295         /*
1296          * Mask interrupts until the taskqueue is finished running.  This is
1297          * cheap, just assume that it is needed.  This also works around the
1298          * MSI message reordering errata on certain systems.
1299          */
1300         igb_disable_intr(adapter);
1301         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1302
1303         /* Link status change */
1304         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1305                 taskqueue_enqueue(adapter->tq, &adapter->link_task);
1306
1307         if (reg_icr & E1000_ICR_RXO)
1308                 adapter->rx_overruns++;
1309         return FILTER_HANDLED;
1310 }
1311
1312 #ifdef DEVICE_POLLING
1313 /*********************************************************************
1314  *
1315  *  Legacy polling routine
1316  *
1317  *********************************************************************/
1318 #if __FreeBSD_version >= 800000
1319 #define POLL_RETURN_COUNT(a) (a)
1320 static int
1321 #else
1322 #define POLL_RETURN_COUNT(a)
1323 static void
1324 #endif
1325 igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1326 {
1327         struct adapter *adapter = ifp->if_softc;
1328         struct rx_ring  *rxr = adapter->rx_rings;
1329         struct tx_ring  *txr = adapter->tx_rings;
1330         u32             reg_icr, rx_done = 0;
1331         u32             loop = IGB_MAX_LOOP;
1332         bool            more;
1333
1334         IGB_CORE_LOCK(adapter);
1335         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1336                 IGB_CORE_UNLOCK(adapter);
1337                 return POLL_RETURN_COUNT(rx_done);
1338         }
1339
1340         if (cmd == POLL_AND_CHECK_STATUS) {
1341                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1342                 /* Link status change */
1343                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1344                         taskqueue_enqueue(adapter->tq, &adapter->link_task);
1345
1346                 if (reg_icr & E1000_ICR_RXO)
1347                         adapter->rx_overruns++;
1348         }
1349         IGB_CORE_UNLOCK(adapter);
1350
1351         /* TODO: rx_count */
1352         rx_done = igb_rxeof(rxr, count) ? 1 : 0;
1353
1354         IGB_TX_LOCK(txr);
1355         do {
1356                 more = igb_txeof(txr);
1357         } while (loop-- && more);
1358 #if __FreeBSD_version >= 800000
1359         if (!drbr_empty(ifp, txr->br))
1360                 igb_mq_start_locked(ifp, txr, NULL);
1361 #else
1362         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1363                 igb_start_locked(txr, ifp);
1364 #endif
1365         IGB_TX_UNLOCK(txr);
1366         return POLL_RETURN_COUNT(rx_done);
1367 }
1368 #endif /* DEVICE_POLLING */
1369
1370 /*********************************************************************
1371  *
1372  *  MSIX TX Interrupt Service routine
1373  *
1374  **********************************************************************/
1375 static void
1376 igb_msix_que(void *arg)
1377 {
1378         struct igb_queue *que = arg;
1379         struct adapter *adapter = que->adapter;
1380         struct tx_ring *txr = que->txr;
1381         struct rx_ring *rxr = que->rxr;
1382         u32             newitr = 0;
1383         bool            more_tx, more_rx;
1384
1385         E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims);
1386         ++que->irqs;
1387
1388         IGB_TX_LOCK(txr);
1389         more_tx = igb_txeof(txr);
1390         IGB_TX_UNLOCK(txr);
1391
1392         more_rx = igb_rxeof(rxr, adapter->rx_process_limit);
1393
1394         if (igb_enable_aim == FALSE)
1395                 goto no_calc;
1396         /*
1397         ** Do Adaptive Interrupt Moderation:
1398         **  - Write out last calculated setting
1399         **  - Calculate based on average size over
1400         **    the last interval.
1401         */
1402         if (que->eitr_setting)
1403                 E1000_WRITE_REG(&adapter->hw,
1404                     E1000_EITR(que->msix), que->eitr_setting);
1405
1406         que->eitr_setting = 0;
1407
1408         /* Idle, do nothing */
1409         if ((txr->bytes == 0) && (rxr->bytes == 0))
1410                 goto no_calc;
1411
1412         /* Used half Default if sub-gig */
1413         if (adapter->link_speed != 1000)
1414                 newitr = IGB_DEFAULT_ITR / 2;
1415         else {
1416                 if ((txr->bytes) && (txr->packets))
1417                         newitr = txr->bytes/txr->packets;
1418                 if ((rxr->bytes) && (rxr->packets))
1419                         newitr = max(newitr,
1420                             (rxr->bytes / rxr->packets));
1421                 newitr += 24; /* account for hardware frame, crc */
1422                 /* set an upper boundary */
1423                 newitr = min(newitr, 3000);
1424                 /* Be nice to the mid range */
1425                 if ((newitr > 300) && (newitr < 1200))
1426                         newitr = (newitr / 3);
1427                 else
1428                         newitr = (newitr / 2);
1429         }
1430         newitr &= 0x7FFC;  /* Mask invalid bits */
1431         if (adapter->hw.mac.type == e1000_82575)
1432                 newitr |= newitr << 16;
1433         else
1434                 newitr |= 0x8000000;
1435
1436         /* save for next interrupt */
1437         que->eitr_setting = newitr;
1438
1439         /* Reset state */
1440         txr->bytes = 0;
1441         txr->packets = 0;
1442         rxr->bytes = 0;
1443         rxr->packets = 0;
1444
1445 no_calc:
1446         /* Schedule a clean task if needed*/
1447         if (more_tx || more_rx)
1448                 taskqueue_enqueue(que->tq, &que->que_task);
1449         else
1450                 /* Reenable this interrupt */
1451                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims);
1452         return;
1453 }
1454
1455
1456 /*********************************************************************
1457  *
1458  *  MSIX Link Interrupt Service routine
1459  *
1460  **********************************************************************/
1461
1462 static void
1463 igb_msix_link(void *arg)
1464 {
1465         struct adapter  *adapter = arg;
1466         u32             icr;
1467
1468         ++adapter->link_irq;
1469         icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1470         if (!(icr & E1000_ICR_LSC))
1471                 goto spurious;
1472         taskqueue_enqueue(adapter->tq, &adapter->link_task);
1473
1474 spurious:
1475         /* Rearm */
1476         E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC);
1477         E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask);
1478         return;
1479 }
1480
1481
1482 /*********************************************************************
1483  *
1484  *  Media Ioctl callback
1485  *
1486  *  This routine is called whenever the user queries the status of
1487  *  the interface using ifconfig.
1488  *
1489  **********************************************************************/
1490 static void
1491 igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1492 {
1493         struct adapter *adapter = ifp->if_softc;
1494         u_char fiber_type = IFM_1000_SX;
1495
1496         INIT_DEBUGOUT("igb_media_status: begin");
1497
1498         IGB_CORE_LOCK(adapter);
1499         igb_update_link_status(adapter);
1500
1501         ifmr->ifm_status = IFM_AVALID;
1502         ifmr->ifm_active = IFM_ETHER;
1503
1504         if (!adapter->link_active) {
1505                 IGB_CORE_UNLOCK(adapter);
1506                 return;
1507         }
1508
1509         ifmr->ifm_status |= IFM_ACTIVE;
1510
1511         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1512             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes))
1513                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1514         else {
1515                 switch (adapter->link_speed) {
1516                 case 10:
1517                         ifmr->ifm_active |= IFM_10_T;
1518                         break;
1519                 case 100:
1520                         ifmr->ifm_active |= IFM_100_TX;
1521                         break;
1522                 case 1000:
1523                         ifmr->ifm_active |= IFM_1000_T;
1524                         break;
1525                 }
1526                 if (adapter->link_duplex == FULL_DUPLEX)
1527                         ifmr->ifm_active |= IFM_FDX;
1528                 else
1529                         ifmr->ifm_active |= IFM_HDX;
1530         }
1531         IGB_CORE_UNLOCK(adapter);
1532 }
1533
1534 /*********************************************************************
1535  *
1536  *  Media Ioctl callback
1537  *
1538  *  This routine is called when the user changes speed/duplex using
1539  *  media/mediopt option with ifconfig.
1540  *
1541  **********************************************************************/
1542 static int
1543 igb_media_change(struct ifnet *ifp)
1544 {
1545         struct adapter *adapter = ifp->if_softc;
1546         struct ifmedia  *ifm = &adapter->media;
1547
1548         INIT_DEBUGOUT("igb_media_change: begin");
1549
1550         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1551                 return (EINVAL);
1552
1553         IGB_CORE_LOCK(adapter);
1554         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1555         case IFM_AUTO:
1556                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1557                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1558                 break;
1559         case IFM_1000_LX:
1560         case IFM_1000_SX:
1561         case IFM_1000_T:
1562                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1563                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1564                 break;
1565         case IFM_100_TX:
1566                 adapter->hw.mac.autoneg = FALSE;
1567                 adapter->hw.phy.autoneg_advertised = 0;
1568                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1569                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1570                 else
1571                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1572                 break;
1573         case IFM_10_T:
1574                 adapter->hw.mac.autoneg = FALSE;
1575                 adapter->hw.phy.autoneg_advertised = 0;
1576                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1577                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1578                 else
1579                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1580                 break;
1581         default:
1582                 device_printf(adapter->dev, "Unsupported media type\n");
1583         }
1584
1585         /* As the speed/duplex settings my have changed we need to
1586          * reset the PHY.
1587          */
1588         adapter->hw.phy.reset_disable = FALSE;
1589
1590         igb_init_locked(adapter);
1591         IGB_CORE_UNLOCK(adapter);
1592
1593         return (0);
1594 }
1595
1596
1597 /*********************************************************************
1598  *
1599  *  This routine maps the mbufs to Advanced TX descriptors.
1600  *  used by the 82575 adapter.
1601  *
1602  **********************************************************************/
1603
1604 static int
1605 igb_xmit(struct tx_ring *txr, struct mbuf **m_headp)
1606 {
1607         struct adapter          *adapter = txr->adapter;
1608         bus_dma_segment_t       segs[IGB_MAX_SCATTER];
1609         bus_dmamap_t            map;
1610         struct igb_tx_buffer    *tx_buffer, *tx_buffer_mapped;
1611         union e1000_adv_tx_desc *txd = NULL;
1612         struct mbuf             *m_head;
1613         u32                     olinfo_status = 0, cmd_type_len = 0;
1614         int                     nsegs, i, j, error, first, last = 0;
1615         u32                     hdrlen = 0;
1616
1617         m_head = *m_headp;
1618
1619
1620         /* Set basic descriptor constants */
1621         cmd_type_len |= E1000_ADVTXD_DTYP_DATA;
1622         cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT;
1623         if (m_head->m_flags & M_VLANTAG)
1624                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
1625
1626         /*
1627          * Force a cleanup if number of TX descriptors
1628          * available hits the threshold
1629          */
1630         if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) {
1631                 igb_txeof(txr);
1632                 /* Now do we at least have a minimal? */
1633                 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) {
1634                         txr->no_desc_avail++;
1635                         return (ENOBUFS);
1636                 }
1637         }
1638
1639         /*
1640          * Map the packet for DMA.
1641          *
1642          * Capture the first descriptor index,
1643          * this descriptor will have the index
1644          * of the EOP which is the only one that
1645          * now gets a DONE bit writeback.
1646          */
1647         first = txr->next_avail_desc;
1648         tx_buffer = &txr->tx_buffers[first];
1649         tx_buffer_mapped = tx_buffer;
1650         map = tx_buffer->map;
1651
1652         error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1653             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1654
1655         if (error == EFBIG) {
1656                 struct mbuf *m;
1657
1658                 m = m_defrag(*m_headp, M_DONTWAIT);
1659                 if (m == NULL) {
1660                         adapter->mbuf_defrag_failed++;
1661                         m_freem(*m_headp);
1662                         *m_headp = NULL;
1663                         return (ENOBUFS);
1664                 }
1665                 *m_headp = m;
1666
1667                 /* Try it again */
1668                 error = bus_dmamap_load_mbuf_sg(txr->txtag, map,
1669                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1670
1671                 if (error == ENOMEM) {
1672                         adapter->no_tx_dma_setup++;
1673                         return (error);
1674                 } else if (error != 0) {
1675                         adapter->no_tx_dma_setup++;
1676                         m_freem(*m_headp);
1677                         *m_headp = NULL;
1678                         return (error);
1679                 }
1680         } else if (error == ENOMEM) {
1681                 adapter->no_tx_dma_setup++;
1682                 return (error);
1683         } else if (error != 0) {
1684                 adapter->no_tx_dma_setup++;
1685                 m_freem(*m_headp);
1686                 *m_headp = NULL;
1687                 return (error);
1688         }
1689
1690         /* Check again to be sure we have enough descriptors */
1691         if (nsegs > (txr->tx_avail - 2)) {
1692                 txr->no_desc_avail++;
1693                 bus_dmamap_unload(txr->txtag, map);
1694                 return (ENOBUFS);
1695         }
1696         m_head = *m_headp;
1697
1698         /*
1699          * Set up the context descriptor:
1700          * used when any hardware offload is done.
1701          * This includes CSUM, VLAN, and TSO. It
1702          * will use the first descriptor.
1703          */
1704         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1705                 if (igb_tso_setup(txr, m_head, &hdrlen)) {
1706                         cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
1707                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
1708                         olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1709                 } else
1710                         return (ENXIO);
1711         } else if (igb_tx_ctx_setup(txr, m_head))
1712                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
1713
1714         /* Calculate payload length */
1715         olinfo_status |= ((m_head->m_pkthdr.len - hdrlen)
1716             << E1000_ADVTXD_PAYLEN_SHIFT);
1717
1718         /* 82575 needs the queue index added */
1719         if (adapter->hw.mac.type == e1000_82575)
1720                 olinfo_status |= txr->me << 4;
1721
1722         /* Set up our transmit descriptors */
1723         i = txr->next_avail_desc;
1724         for (j = 0; j < nsegs; j++) {
1725                 bus_size_t seg_len;
1726                 bus_addr_t seg_addr;
1727
1728                 tx_buffer = &txr->tx_buffers[i];
1729                 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i];
1730                 seg_addr = segs[j].ds_addr;
1731                 seg_len  = segs[j].ds_len;
1732
1733                 txd->read.buffer_addr = htole64(seg_addr);
1734                 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len);
1735                 txd->read.olinfo_status = htole32(olinfo_status);
1736                 last = i;
1737                 if (++i == adapter->num_tx_desc)
1738                         i = 0;
1739                 tx_buffer->m_head = NULL;
1740                 tx_buffer->next_eop = -1;
1741         }
1742
1743         txr->next_avail_desc = i;
1744         txr->tx_avail -= nsegs;
1745
1746         tx_buffer->m_head = m_head;
1747         tx_buffer_mapped->map = tx_buffer->map;
1748         tx_buffer->map = map;
1749         bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE);
1750
1751         /*
1752          * Last Descriptor of Packet
1753          * needs End Of Packet (EOP)
1754          * and Report Status (RS)
1755          */
1756         txd->read.cmd_type_len |=
1757             htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS);
1758         /*
1759          * Keep track in the first buffer which
1760          * descriptor will be written back
1761          */
1762         tx_buffer = &txr->tx_buffers[first];
1763         tx_buffer->next_eop = last;
1764         txr->watchdog_time = ticks;
1765
1766         /*
1767          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
1768          * that this frame is available to transmit.
1769          */
1770         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
1771             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1772         E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i);
1773         ++txr->tx_packets;
1774
1775         return (0);
1776
1777 }
1778
1779 static void
1780 igb_set_promisc(struct adapter *adapter)
1781 {
1782         struct ifnet    *ifp = adapter->ifp;
1783         uint32_t        reg_rctl;
1784
1785         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1786
1787         if (ifp->if_flags & IFF_PROMISC) {
1788                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1789                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1790         } else if (ifp->if_flags & IFF_ALLMULTI) {
1791                 reg_rctl |= E1000_RCTL_MPE;
1792                 reg_rctl &= ~E1000_RCTL_UPE;
1793                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1794         }
1795 }
1796
1797 static void
1798 igb_disable_promisc(struct adapter *adapter)
1799 {
1800         uint32_t        reg_rctl;
1801
1802         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1803
1804         reg_rctl &=  (~E1000_RCTL_UPE);
1805         reg_rctl &=  (~E1000_RCTL_MPE);
1806         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1807 }
1808
1809
1810 /*********************************************************************
1811  *  Multicast Update
1812  *
1813  *  This routine is called whenever multicast address list is updated.
1814  *
1815  **********************************************************************/
1816
1817 static void
1818 igb_set_multi(struct adapter *adapter)
1819 {
1820         struct ifnet    *ifp = adapter->ifp;
1821         struct ifmultiaddr *ifma;
1822         u32 reg_rctl = 0;
1823         u8  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN];
1824
1825         int mcnt = 0;
1826
1827         IOCTL_DEBUGOUT("igb_set_multi: begin");
1828
1829 #if __FreeBSD_version < 800000
1830         IF_ADDR_LOCK(ifp);
1831 #else
1832         if_maddr_rlock(ifp);
1833 #endif
1834         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1835                 if (ifma->ifma_addr->sa_family != AF_LINK)
1836                         continue;
1837
1838                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1839                         break;
1840
1841                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1842                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
1843                 mcnt++;
1844         }
1845 #if __FreeBSD_version < 800000
1846         IF_ADDR_UNLOCK(ifp);
1847 #else
1848         if_maddr_runlock(ifp);
1849 #endif
1850
1851         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1852                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1853                 reg_rctl |= E1000_RCTL_MPE;
1854                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1855         } else
1856                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1857 }
1858
1859
1860 /*********************************************************************
1861  *  Timer routine:
1862  *      This routine checks for link status,
1863  *      updates statistics, and does the watchdog.
1864  *
1865  **********************************************************************/
1866
1867 static void
1868 igb_local_timer(void *arg)
1869 {
1870         struct adapter          *adapter = arg;
1871         struct ifnet            *ifp = adapter->ifp;
1872         device_t                dev = adapter->dev;
1873         struct tx_ring          *txr = adapter->tx_rings;
1874
1875
1876         IGB_CORE_LOCK_ASSERT(adapter);
1877
1878         igb_update_link_status(adapter);
1879         igb_update_stats_counters(adapter);
1880
1881         if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
1882                 igb_print_hw_stats(adapter);
1883
1884         /*
1885         ** Watchdog: check for time since any descriptor was cleaned
1886         */
1887         for (int i = 0; i < adapter->num_queues; i++, txr++) {
1888                 if (txr->watchdog_check == FALSE)
1889                         continue;
1890                 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG)
1891                         goto timeout;
1892         }
1893
1894         /* Trigger an RX interrupt on all queues */
1895 #ifdef DEVICE_POLLING
1896         if (!(ifp->if_capenable & IFCAP_POLLING))
1897 #endif
1898         E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask);
1899         callout_reset(&adapter->timer, hz, igb_local_timer, adapter);
1900         return;
1901
1902 timeout:
1903         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
1904         device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me,
1905             E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)),
1906             E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me)));
1907         device_printf(dev,"TX(%d) desc avail = %d,"
1908             "Next TX to Clean = %d\n",
1909             txr->me, txr->tx_avail, txr->next_to_clean);
1910         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1911         adapter->watchdog_events++;
1912         igb_init_locked(adapter);
1913 }
1914
1915 static void
1916 igb_update_link_status(struct adapter *adapter)
1917 {
1918         struct e1000_hw *hw = &adapter->hw;
1919         struct ifnet *ifp = adapter->ifp;
1920         device_t dev = adapter->dev;
1921         struct tx_ring *txr = adapter->tx_rings;
1922         u32 link_check = 0;
1923
1924         /* Get the cached link value or read for real */
1925         switch (hw->phy.media_type) {
1926         case e1000_media_type_copper:
1927                 if (hw->mac.get_link_status) {
1928                         /* Do the work to read phy */
1929                         e1000_check_for_link(hw);
1930                         link_check = !hw->mac.get_link_status;
1931                 } else
1932                         link_check = TRUE;
1933                 break;
1934         case e1000_media_type_fiber:
1935                 e1000_check_for_link(hw);
1936                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
1937                                  E1000_STATUS_LU);
1938                 break;
1939         case e1000_media_type_internal_serdes:
1940                 e1000_check_for_link(hw);
1941                 link_check = adapter->hw.mac.serdes_has_link;
1942                 break;
1943         default:
1944         case e1000_media_type_unknown:
1945                 break;
1946         }
1947
1948         /* Now we check if a transition has happened */
1949         if (link_check && (adapter->link_active == 0)) {
1950                 e1000_get_speed_and_duplex(&adapter->hw,
1951                     &adapter->link_speed, &adapter->link_duplex);
1952                 if (bootverbose)
1953                         device_printf(dev, "Link is up %d Mbps %s\n",
1954                             adapter->link_speed,
1955                             ((adapter->link_duplex == FULL_DUPLEX) ?
1956                             "Full Duplex" : "Half Duplex"));
1957                 adapter->link_active = 1;
1958                 ifp->if_baudrate = adapter->link_speed * 1000000;
1959                 if_link_state_change(ifp, LINK_STATE_UP);
1960         } else if (!link_check && (adapter->link_active == 1)) {
1961                 ifp->if_baudrate = adapter->link_speed = 0;
1962                 adapter->link_duplex = 0;
1963                 if (bootverbose)
1964                         device_printf(dev, "Link is Down\n");
1965                 adapter->link_active = 0;
1966                 if_link_state_change(ifp, LINK_STATE_DOWN);
1967                 /* Turn off watchdogs */
1968                 for (int i = 0; i < adapter->num_queues; i++, txr++)
1969                         txr->watchdog_check = FALSE;
1970         }
1971 }
1972
1973 /*********************************************************************
1974  *
1975  *  This routine disables all traffic on the adapter by issuing a
1976  *  global reset on the MAC and deallocates TX/RX buffers.
1977  *
1978  **********************************************************************/
1979
1980 static void
1981 igb_stop(void *arg)
1982 {
1983         struct adapter  *adapter = arg;
1984         struct ifnet    *ifp = adapter->ifp;
1985         struct tx_ring *txr = adapter->tx_rings;
1986
1987         IGB_CORE_LOCK_ASSERT(adapter);
1988
1989         INIT_DEBUGOUT("igb_stop: begin");
1990
1991         igb_disable_intr(adapter);
1992
1993         callout_stop(&adapter->timer);
1994
1995         /* Tell the stack that the interface is no longer active */
1996         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1997
1998         /* Unarm watchdog timer. */
1999         for (int i = 0; i < adapter->num_queues; i++, txr++) {
2000                 IGB_TX_LOCK(txr);
2001                 txr->watchdog_check = FALSE;
2002                 IGB_TX_UNLOCK(txr);
2003         }
2004
2005         e1000_reset_hw(&adapter->hw);
2006         E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2007 }
2008
2009
2010 /*********************************************************************
2011  *
2012  *  Determine hardware revision.
2013  *
2014  **********************************************************************/
2015 static void
2016 igb_identify_hardware(struct adapter *adapter)
2017 {
2018         device_t dev = adapter->dev;
2019
2020         /* Make sure our PCI config space has the necessary stuff set */
2021         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2022         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2023             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2024                 device_printf(dev, "Memory Access and/or Bus Master bits "
2025                     "were not set!\n");
2026                 adapter->hw.bus.pci_cmd_word |=
2027                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2028                 pci_write_config(dev, PCIR_COMMAND,
2029                     adapter->hw.bus.pci_cmd_word, 2);
2030         }
2031
2032         /* Save off the information about this board */
2033         adapter->hw.vendor_id = pci_get_vendor(dev);
2034         adapter->hw.device_id = pci_get_device(dev);
2035         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2036         adapter->hw.subsystem_vendor_id =
2037             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2038         adapter->hw.subsystem_device_id =
2039             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2040
2041         /* Do Shared Code Init and Setup */
2042         if (e1000_set_mac_type(&adapter->hw)) {
2043                 device_printf(dev, "Setup init failure\n");
2044                 return;
2045         }
2046 }
2047
2048 static int
2049 igb_allocate_pci_resources(struct adapter *adapter)
2050 {
2051         device_t        dev = adapter->dev;
2052         int             rid;
2053
2054         rid = PCIR_BAR(0);
2055         adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2056             &rid, RF_ACTIVE);
2057         if (adapter->pci_mem == NULL) {
2058                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2059                 return (ENXIO);
2060         }
2061         adapter->osdep.mem_bus_space_tag =
2062             rman_get_bustag(adapter->pci_mem);
2063         adapter->osdep.mem_bus_space_handle =
2064             rman_get_bushandle(adapter->pci_mem);
2065         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2066
2067         adapter->num_queues = 1; /* Defaults for Legacy or MSI */
2068
2069         /* This will setup either MSI/X or MSI */
2070         adapter->msix = igb_setup_msix(adapter);
2071         adapter->hw.back = &adapter->osdep;
2072
2073         return (0);
2074 }
2075
2076 /*********************************************************************
2077  *
2078  *  Setup the Legacy or MSI Interrupt handler
2079  *
2080  **********************************************************************/
2081 static int
2082 igb_allocate_legacy(struct adapter *adapter)
2083 {
2084         device_t dev = adapter->dev;
2085         int error, rid = 0;
2086
2087         /* Turn off all interrupts */
2088         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2089
2090         /* MSI RID is 1 */
2091         if (adapter->msix == 1)
2092                 rid = 1;
2093
2094         /* We allocate a single interrupt resource */
2095         adapter->res = bus_alloc_resource_any(dev,
2096             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2097         if (adapter->res == NULL) {
2098                 device_printf(dev, "Unable to allocate bus resource: "
2099                     "interrupt\n");
2100                 return (ENXIO);
2101         }
2102
2103         /*
2104          * Try allocating a fast interrupt and the associated deferred
2105          * processing contexts.
2106          */
2107         TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter);
2108         /* Make tasklet for deferred link handling */
2109         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2110         adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT,
2111             taskqueue_thread_enqueue, &adapter->tq);
2112         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2113             device_get_nameunit(adapter->dev));
2114         if ((error = bus_setup_intr(dev, adapter->res,
2115             INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL,
2116             adapter, &adapter->tag)) != 0) {
2117                 device_printf(dev, "Failed to register fast interrupt "
2118                             "handler: %d\n", error);
2119                 taskqueue_free(adapter->tq);
2120                 adapter->tq = NULL;
2121                 return (error);
2122         }
2123
2124         return (0);
2125 }
2126
2127
2128 /*********************************************************************
2129  *
2130  *  Setup the MSIX Queue Interrupt handlers:
2131  *
2132  **********************************************************************/
2133 static int
2134 igb_allocate_msix(struct adapter *adapter)
2135 {
2136         device_t                dev = adapter->dev;
2137         struct igb_queue        *que = adapter->queues;
2138         int                     error, rid, vector = 0;
2139
2140
2141         for (int i = 0; i < adapter->num_queues; i++, vector++, que++) {
2142                 rid = vector +1;
2143                 que->res = bus_alloc_resource_any(dev,
2144                     SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2145                 if (que->res == NULL) {
2146                         device_printf(dev,
2147                             "Unable to allocate bus resource: "
2148                             "MSIX Queue Interrupt\n");
2149                         return (ENXIO);
2150                 }
2151                 error = bus_setup_intr(dev, que->res,
2152                     INTR_TYPE_NET | INTR_MPSAFE, NULL,
2153                     igb_msix_que, que, &que->tag);
2154                 if (error) {
2155                         que->res = NULL;
2156                         device_printf(dev, "Failed to register Queue handler");
2157                         return (error);
2158                 }
2159                 que->msix = vector;
2160                 if (adapter->hw.mac.type == e1000_82575)
2161                         que->eims = E1000_EICR_TX_QUEUE0 << i;
2162                 else
2163                         que->eims = 1 << vector;
2164                 /*
2165                 ** Bind the msix vector, and thus the
2166                 ** rings to the corresponding cpu.
2167                 */
2168                 if (adapter->num_queues > 1)
2169                         bus_bind_intr(dev, que->res, i);
2170                 /* Make tasklet for deferred handling */
2171                 TASK_INIT(&que->que_task, 0, igb_handle_que, que);
2172                 que->tq = taskqueue_create_fast("igb_que", M_NOWAIT,
2173                     taskqueue_thread_enqueue, &que->tq);
2174                 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
2175                     device_get_nameunit(adapter->dev));
2176         }
2177
2178         /* And Link */
2179         rid = vector + 1;
2180         adapter->res = bus_alloc_resource_any(dev,
2181             SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2182         if (adapter->res == NULL) {
2183                 device_printf(dev,
2184                     "Unable to allocate bus resource: "
2185                     "MSIX Link Interrupt\n");
2186                 return (ENXIO);
2187         }
2188         if ((error = bus_setup_intr(dev, adapter->res,
2189             INTR_TYPE_NET | INTR_MPSAFE, NULL,
2190             igb_msix_link, adapter, &adapter->tag)) != 0) {
2191                 device_printf(dev, "Failed to register Link handler");
2192                 return (error);
2193         }
2194         adapter->linkvec = vector;
2195
2196         /* Make tasklet for deferred handling */
2197         TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter);
2198         adapter->tq = taskqueue_create_fast("igb_link", M_NOWAIT,
2199             taskqueue_thread_enqueue, &adapter->tq);
2200         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s link",
2201             device_get_nameunit(adapter->dev));
2202
2203         return (0);
2204 }
2205
2206
2207 static void
2208 igb_configure_queues(struct adapter *adapter)
2209 {
2210         struct  e1000_hw        *hw = &adapter->hw;
2211         struct  igb_queue       *que;
2212         u32                     tmp, ivar = 0;
2213         u32                     newitr = IGB_DEFAULT_ITR;
2214
2215         /* First turn on RSS capability */
2216         if (adapter->hw.mac.type > e1000_82575)
2217                 E1000_WRITE_REG(hw, E1000_GPIE,
2218                     E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME |
2219                     E1000_GPIE_PBA | E1000_GPIE_NSICR);
2220
2221         /* Turn on MSIX */
2222         switch (adapter->hw.mac.type) {
2223         case e1000_82580:
2224                 /* RX entries */
2225                 for (int i = 0; i < adapter->num_queues; i++) {
2226                         u32 index = i >> 1;
2227                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2228                         que = &adapter->queues[i];
2229                         if (i & 1) {
2230                                 ivar &= 0xFF00FFFF;
2231                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2232                         } else {
2233                                 ivar &= 0xFFFFFF00;
2234                                 ivar |= que->msix | E1000_IVAR_VALID;
2235                         }
2236                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2237                 }
2238                 /* TX entries */
2239                 for (int i = 0; i < adapter->num_queues; i++) {
2240                         u32 index = i >> 1;
2241                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2242                         que = &adapter->queues[i];
2243                         if (i & 1) {
2244                                 ivar &= 0x00FFFFFF;
2245                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2246                         } else {
2247                                 ivar &= 0xFFFF00FF;
2248                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2249                         }
2250                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2251                         adapter->eims_mask |= que->eims;
2252                 }
2253
2254                 /* And for the link interrupt */
2255                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2256                 adapter->link_mask = 1 << adapter->linkvec;
2257                 adapter->eims_mask |= adapter->link_mask;
2258                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2259                 break;
2260         case e1000_82576:
2261                 /* RX entries */
2262                 for (int i = 0; i < adapter->num_queues; i++) {
2263                         u32 index = i & 0x7; /* Each IVAR has two entries */
2264                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2265                         que = &adapter->queues[i];
2266                         if (i < 8) {
2267                                 ivar &= 0xFFFFFF00;
2268                                 ivar |= que->msix | E1000_IVAR_VALID;
2269                         } else {
2270                                 ivar &= 0xFF00FFFF;
2271                                 ivar |= (que->msix | E1000_IVAR_VALID) << 16;
2272                         }
2273                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2274                         adapter->eims_mask |= que->eims;
2275                 }
2276                 /* TX entries */
2277                 for (int i = 0; i < adapter->num_queues; i++) {
2278                         u32 index = i & 0x7; /* Each IVAR has two entries */
2279                         ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index);
2280                         que = &adapter->queues[i];
2281                         if (i < 8) {
2282                                 ivar &= 0xFFFF00FF;
2283                                 ivar |= (que->msix | E1000_IVAR_VALID) << 8;
2284                         } else {
2285                                 ivar &= 0x00FFFFFF;
2286                                 ivar |= (que->msix | E1000_IVAR_VALID) << 24;
2287                         }
2288                         E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar);
2289                         adapter->eims_mask |= que->eims;
2290                 }
2291
2292                 /* And for the link interrupt */
2293                 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8;
2294                 adapter->link_mask = 1 << adapter->linkvec;
2295                 adapter->eims_mask |= adapter->link_mask;
2296                 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar);
2297                 break;
2298
2299         case e1000_82575:
2300                 /* enable MSI-X support*/
2301                 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT);
2302                 tmp |= E1000_CTRL_EXT_PBA_CLR;
2303                 /* Auto-Mask interrupts upon ICR read. */
2304                 tmp |= E1000_CTRL_EXT_EIAME;
2305                 tmp |= E1000_CTRL_EXT_IRCA;
2306                 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp);
2307
2308                 /* Queues */
2309                 for (int i = 0; i < adapter->num_queues; i++) {
2310                         que = &adapter->queues[i];
2311                         tmp = E1000_EICR_RX_QUEUE0 << i;
2312                         tmp |= E1000_EICR_TX_QUEUE0 << i;
2313                         que->eims = tmp;
2314                         E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0),
2315                             i, que->eims);
2316                         adapter->eims_mask |= que->eims;
2317                 }
2318
2319                 /* Link */
2320                 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec),
2321                     E1000_EIMS_OTHER);
2322                 adapter->link_mask |= E1000_EIMS_OTHER;
2323                 adapter->eims_mask |= adapter->link_mask;
2324         default:
2325                 break;
2326         }
2327
2328         /* Set the starting interrupt rate */
2329         if (hw->mac.type == e1000_82575)
2330                 newitr |= newitr << 16;
2331         else
2332                 newitr |= 0x8000000;
2333
2334         for (int i = 0; i < adapter->num_queues; i++) {
2335                 que = &adapter->queues[i];
2336                 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr);
2337         }
2338
2339         return;
2340 }
2341
2342
2343 static void
2344 igb_free_pci_resources(struct adapter *adapter)
2345 {
2346         struct          igb_queue *que = adapter->queues;
2347         device_t        dev = adapter->dev;
2348         int             rid;
2349
2350         /*
2351         ** There is a slight possibility of a failure mode
2352         ** in attach that will result in entering this function
2353         ** before interrupt resources have been initialized, and
2354         ** in that case we do not want to execute the loops below
2355         ** We can detect this reliably by the state of the adapter
2356         ** res pointer.
2357         */
2358         if (adapter->res == NULL)
2359                 goto mem;
2360
2361         /*
2362          * First release all the interrupt resources:
2363          */
2364         for (int i = 0; i < adapter->num_queues; i++, que++) {
2365                 rid = que->msix + 1;
2366                 if (que->tag != NULL) {
2367                         bus_teardown_intr(dev, que->res, que->tag);
2368                         que->tag = NULL;
2369                 }
2370                 if (que->res != NULL)
2371                         bus_release_resource(dev,
2372                             SYS_RES_IRQ, rid, que->res);
2373         }
2374
2375         /* Clean the Legacy or Link interrupt last */
2376         if (adapter->linkvec) /* we are doing MSIX */
2377                 rid = adapter->linkvec + 1;
2378         else
2379                 (adapter->msix != 0) ? (rid = 1):(rid = 0);
2380
2381         if (adapter->tag != NULL) {
2382                 bus_teardown_intr(dev, adapter->res, adapter->tag);
2383                 adapter->tag = NULL;
2384         }
2385         if (adapter->res != NULL)
2386                 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res);
2387
2388 mem:
2389         if (adapter->msix)
2390                 pci_release_msi(dev);
2391
2392         if (adapter->msix_mem != NULL)
2393                 bus_release_resource(dev, SYS_RES_MEMORY,
2394                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2395
2396         if (adapter->pci_mem != NULL)
2397                 bus_release_resource(dev, SYS_RES_MEMORY,
2398                     PCIR_BAR(0), adapter->pci_mem);
2399
2400 }
2401
2402 /*
2403  * Setup Either MSI/X or MSI
2404  */
2405 static int
2406 igb_setup_msix(struct adapter *adapter)
2407 {
2408         device_t dev = adapter->dev;
2409         int rid, want, queues, msgs;
2410
2411         /* tuneable override */
2412         if (igb_enable_msix == 0)
2413                 goto msi;
2414
2415         /* First try MSI/X */
2416         rid = PCIR_BAR(IGB_MSIX_BAR);
2417         adapter->msix_mem = bus_alloc_resource_any(dev,
2418             SYS_RES_MEMORY, &rid, RF_ACTIVE);
2419         if (!adapter->msix_mem) {
2420                 /* May not be enabled */
2421                 device_printf(adapter->dev,
2422                     "Unable to map MSIX table \n");
2423                 goto msi;
2424         }
2425
2426         msgs = pci_msix_count(dev);
2427         if (msgs == 0) { /* system has msix disabled */
2428                 bus_release_resource(dev, SYS_RES_MEMORY,
2429                     PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem);
2430                 adapter->msix_mem = NULL;
2431                 goto msi;
2432         }
2433
2434         /* Figure out a reasonable auto config value */
2435         queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
2436
2437         /* Can have max of 4 queues on 82575 */
2438         if (adapter->hw.mac.type == e1000_82575) {
2439                 if (queues > 4)
2440                         queues = 4;
2441                 if (igb_num_queues > 4)
2442                         igb_num_queues = 4;
2443         }
2444
2445         if (igb_num_queues == 0)
2446                 igb_num_queues = queues;
2447
2448         /*
2449         ** One vector (RX/TX pair) per queue
2450         ** plus an additional for Link interrupt
2451         */
2452         want = igb_num_queues + 1;
2453         if (msgs >= want)
2454                 msgs = want;
2455         else {
2456                 device_printf(adapter->dev,
2457                     "MSIX Configuration Problem, "
2458                     "%d vectors configured, but %d queues wanted!\n",
2459                     msgs, want);
2460                 return (ENXIO);
2461         }
2462         if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) {
2463                 device_printf(adapter->dev,
2464                     "Using MSIX interrupts with %d vectors\n", msgs);
2465                 adapter->num_queues = igb_num_queues;
2466                 return (msgs);
2467         }
2468 msi:
2469         msgs = pci_msi_count(dev);
2470         if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0)
2471                 device_printf(adapter->dev,"Using MSI interrupt\n");
2472         return (msgs);
2473 }
2474
2475 /*********************************************************************
2476  *
2477  *  Set up an fresh starting state
2478  *
2479  **********************************************************************/
2480 static void
2481 igb_reset(struct adapter *adapter)
2482 {
2483         device_t        dev = adapter->dev;
2484         struct e1000_hw *hw = &adapter->hw;
2485         struct e1000_fc_info *fc = &hw->fc;
2486         struct ifnet    *ifp = adapter->ifp;
2487         u32             pba = 0;
2488         u16             hwm;
2489
2490         INIT_DEBUGOUT("igb_reset: begin");
2491
2492         /* Let the firmware know the OS is in control */
2493         igb_get_hw_control(adapter);
2494
2495         /*
2496          * Packet Buffer Allocation (PBA)
2497          * Writing PBA sets the receive portion of the buffer
2498          * the remainder is used for the transmit buffer.
2499          */
2500         switch (hw->mac.type) {
2501         case e1000_82575:
2502                 pba = E1000_PBA_32K;
2503                 break;
2504         case e1000_82576:
2505                 pba = E1000_PBA_64K;
2506                 break;
2507         case e1000_82580:
2508                 pba = E1000_PBA_35K;
2509         default:
2510                 break;
2511         }
2512
2513         /* Special needs in case of Jumbo frames */
2514         if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) {
2515                 u32 tx_space, min_tx, min_rx;
2516                 pba = E1000_READ_REG(hw, E1000_PBA);
2517                 tx_space = pba >> 16;
2518                 pba &= 0xffff;
2519                 min_tx = (adapter->max_frame_size +
2520                     sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2;
2521                 min_tx = roundup2(min_tx, 1024);
2522                 min_tx >>= 10;
2523                 min_rx = adapter->max_frame_size;
2524                 min_rx = roundup2(min_rx, 1024);
2525                 min_rx >>= 10;
2526                 if (tx_space < min_tx &&
2527                     ((min_tx - tx_space) < pba)) {
2528                         pba = pba - (min_tx - tx_space);
2529                         /*
2530                          * if short on rx space, rx wins
2531                          * and must trump tx adjustment
2532                          */
2533                         if (pba < min_rx)
2534                                 pba = min_rx;
2535                 }
2536                 E1000_WRITE_REG(hw, E1000_PBA, pba);
2537         }
2538
2539         INIT_DEBUGOUT1("igb_init: pba=%dK",pba);
2540
2541         /*
2542          * These parameters control the automatic generation (Tx) and
2543          * response (Rx) to Ethernet PAUSE frames.
2544          * - High water mark should allow for at least two frames to be
2545          *   received after sending an XOFF.
2546          * - Low water mark works best when it is very near the high water mark.
2547          *   This allows the receiver to restart by sending XON when it has
2548          *   drained a bit.
2549          */
2550         hwm = min(((pba << 10) * 9 / 10),
2551             ((pba << 10) - 2 * adapter->max_frame_size));
2552
2553         if (hw->mac.type < e1000_82576) {
2554                 fc->high_water = hwm & 0xFFF8;  /* 8-byte granularity */
2555                 fc->low_water = fc->high_water - 8;
2556         } else {
2557                 fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
2558                 fc->low_water = fc->high_water - 16;
2559         }
2560
2561         fc->pause_time = IGB_FC_PAUSE_TIME;
2562         fc->send_xon = TRUE;
2563
2564         /* Set Flow control, use the tunable location if sane */
2565         if ((igb_fc_setting >= 0) || (igb_fc_setting < 4))
2566                 fc->requested_mode = igb_fc_setting;
2567         else
2568                 fc->requested_mode = e1000_fc_none;
2569
2570         fc->current_mode = fc->requested_mode;
2571
2572         /* Issue a global reset */
2573         e1000_reset_hw(hw);
2574         E1000_WRITE_REG(hw, E1000_WUC, 0);
2575
2576         if (e1000_init_hw(hw) < 0)
2577                 device_printf(dev, "Hardware Initialization Failed\n");
2578
2579         if (hw->mac.type == e1000_82580) {
2580                 u32 reg;
2581
2582                 hwm = (pba << 10) - (2 * adapter->max_frame_size);
2583                 /*
2584                  * 0x80000000 - enable DMA COAL
2585                  * 0x10000000 - use L0s as low power
2586                  * 0x20000000 - use L1 as low power
2587                  * X << 16 - exit dma coal when rx data exceeds X kB
2588                  * Y - upper limit to stay in dma coal in units of 32usecs
2589                  */
2590                 E1000_WRITE_REG(hw, E1000_DMACR,
2591                     0xA0000006 | ((hwm << 6) & 0x00FF0000));
2592
2593                 /* set hwm to PBA -  2 * max frame size */
2594                 E1000_WRITE_REG(hw, E1000_FCRTC, hwm);
2595                 /*
2596                  * This sets the time to wait before requesting transition to
2597                  * low power state to number of usecs needed to receive 1 512
2598                  * byte frame at gigabit line rate
2599                  */
2600                 E1000_WRITE_REG(hw, E1000_DMCTLX, 4);
2601
2602                 /* free space in tx packet buffer to wake from DMA coal */
2603                 E1000_WRITE_REG(hw, E1000_DMCTXTH,
2604                     (20480 - (2 * adapter->max_frame_size)) >> 6);
2605
2606                 /* make low power state decision controlled by DMA coal */
2607                 reg = E1000_READ_REG(hw, E1000_PCIEMISC);
2608                 E1000_WRITE_REG(hw, E1000_PCIEMISC,
2609                     reg | E1000_PCIEMISC_LX_DECISION);
2610         }
2611
2612         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2613         e1000_get_phy_info(hw);
2614         e1000_check_for_link(hw);
2615         return;
2616 }
2617
2618 /*********************************************************************
2619  *
2620  *  Setup networking device structure and register an interface.
2621  *
2622  **********************************************************************/
2623 static void
2624 igb_setup_interface(device_t dev, struct adapter *adapter)
2625 {
2626         struct ifnet   *ifp;
2627
2628         INIT_DEBUGOUT("igb_setup_interface: begin");
2629
2630         ifp = adapter->ifp = if_alloc(IFT_ETHER);
2631         if (ifp == NULL)
2632                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
2633         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2634         ifp->if_mtu = ETHERMTU;
2635         ifp->if_init =  igb_init;
2636         ifp->if_softc = adapter;
2637         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2638         ifp->if_ioctl = igb_ioctl;
2639         ifp->if_start = igb_start;
2640 #if __FreeBSD_version >= 800000
2641         ifp->if_transmit = igb_mq_start;
2642         ifp->if_qflush = igb_qflush;
2643 #endif
2644         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2645         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2646         IFQ_SET_READY(&ifp->if_snd);
2647
2648         ether_ifattach(ifp, adapter->hw.mac.addr);
2649
2650         ifp->if_capabilities = ifp->if_capenable = 0;
2651
2652         ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU;
2653         ifp->if_capabilities |= IFCAP_TSO4;
2654         ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2655         if (igb_header_split)
2656                 ifp->if_capabilities |= IFCAP_LRO;
2657
2658         ifp->if_capenable = ifp->if_capabilities;
2659 #ifdef DEVICE_POLLING
2660         ifp->if_capabilities |= IFCAP_POLLING;
2661 #endif
2662
2663         /*
2664          * Tell the upper layer(s) we support long frames.
2665          */
2666         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2667         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2668         ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2669
2670         /*
2671          * Specify the media types supported by this adapter and register
2672          * callbacks to update media and link information
2673          */
2674         ifmedia_init(&adapter->media, IFM_IMASK,
2675             igb_media_change, igb_media_status);
2676         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
2677             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
2678                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
2679                             0, NULL);
2680                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2681         } else {
2682                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2683                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2684                             0, NULL);
2685                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2686                             0, NULL);
2687                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2688                             0, NULL);
2689                 if (adapter->hw.phy.type != e1000_phy_ife) {
2690                         ifmedia_add(&adapter->media,
2691                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2692                         ifmedia_add(&adapter->media,
2693                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2694                 }
2695         }
2696         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2697         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2698 }
2699
2700
2701 /*
2702  * Manage DMA'able memory.
2703  */
2704 static void
2705 igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2706 {
2707         if (error)
2708                 return;
2709         *(bus_addr_t *) arg = segs[0].ds_addr;
2710 }
2711
2712 static int
2713 igb_dma_malloc(struct adapter *adapter, bus_size_t size,
2714         struct igb_dma_alloc *dma, int mapflags)
2715 {
2716         int error;
2717
2718         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
2719                                 IGB_DBA_ALIGN, 0,       /* alignment, bounds */
2720                                 BUS_SPACE_MAXADDR,      /* lowaddr */
2721                                 BUS_SPACE_MAXADDR,      /* highaddr */
2722                                 NULL, NULL,             /* filter, filterarg */
2723                                 size,                   /* maxsize */
2724                                 1,                      /* nsegments */
2725                                 size,                   /* maxsegsize */
2726                                 0,                      /* flags */
2727                                 NULL,                   /* lockfunc */
2728                                 NULL,                   /* lockarg */
2729                                 &dma->dma_tag);
2730         if (error) {
2731                 device_printf(adapter->dev,
2732                     "%s: bus_dma_tag_create failed: %d\n",
2733                     __func__, error);
2734                 goto fail_0;
2735         }
2736
2737         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2738             BUS_DMA_NOWAIT, &dma->dma_map);
2739         if (error) {
2740                 device_printf(adapter->dev,
2741                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2742                     __func__, (uintmax_t)size, error);
2743                 goto fail_2;
2744         }
2745
2746         dma->dma_paddr = 0;
2747         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2748             size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2749         if (error || dma->dma_paddr == 0) {
2750                 device_printf(adapter->dev,
2751                     "%s: bus_dmamap_load failed: %d\n",
2752                     __func__, error);
2753                 goto fail_3;
2754         }
2755
2756         return (0);
2757
2758 fail_3:
2759         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2760 fail_2:
2761         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2762         bus_dma_tag_destroy(dma->dma_tag);
2763 fail_0:
2764         dma->dma_map = NULL;
2765         dma->dma_tag = NULL;
2766
2767         return (error);
2768 }
2769
2770 static void
2771 igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma)
2772 {
2773         if (dma->dma_tag == NULL)
2774                 return;
2775         if (dma->dma_map != NULL) {
2776                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2777                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2778                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2779                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2780                 dma->dma_map = NULL;
2781         }
2782         bus_dma_tag_destroy(dma->dma_tag);
2783         dma->dma_tag = NULL;
2784 }
2785
2786
2787 /*********************************************************************
2788  *
2789  *  Allocate memory for the transmit and receive rings, and then
2790  *  the descriptors associated with each, called only once at attach.
2791  *
2792  **********************************************************************/
2793 static int
2794 igb_allocate_queues(struct adapter *adapter)
2795 {
2796         device_t dev = adapter->dev;
2797         struct igb_queue        *que = NULL;
2798         struct tx_ring          *txr = NULL;
2799         struct rx_ring          *rxr = NULL;
2800         int rsize, tsize, error = E1000_SUCCESS;
2801         int txconf = 0, rxconf = 0;
2802
2803         /* First allocate the top level queue structs */
2804         if (!(adapter->queues =
2805             (struct igb_queue *) malloc(sizeof(struct igb_queue) *
2806             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2807                 device_printf(dev, "Unable to allocate queue memory\n");
2808                 error = ENOMEM;
2809                 goto fail;
2810         }
2811
2812         /* Next allocate the TX ring struct memory */
2813         if (!(adapter->tx_rings =
2814             (struct tx_ring *) malloc(sizeof(struct tx_ring) *
2815             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2816                 device_printf(dev, "Unable to allocate TX ring memory\n");
2817                 error = ENOMEM;
2818                 goto tx_fail;
2819         }
2820
2821         /* Now allocate the RX */
2822         if (!(adapter->rx_rings =
2823             (struct rx_ring *) malloc(sizeof(struct rx_ring) *
2824             adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2825                 device_printf(dev, "Unable to allocate RX ring memory\n");
2826                 error = ENOMEM;
2827                 goto rx_fail;
2828         }
2829
2830         tsize = roundup2(adapter->num_tx_desc *
2831             sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN);
2832         /*
2833          * Now set up the TX queues, txconf is needed to handle the
2834          * possibility that things fail midcourse and we need to
2835          * undo memory gracefully
2836          */
2837         for (int i = 0; i < adapter->num_queues; i++, txconf++) {
2838                 /* Set up some basics */
2839                 txr = &adapter->tx_rings[i];
2840                 txr->adapter = adapter;
2841                 txr->me = i;
2842
2843                 /* Initialize the TX lock */
2844                 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
2845                     device_get_nameunit(dev), txr->me);
2846                 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF);
2847
2848                 if (igb_dma_malloc(adapter, tsize,
2849                         &txr->txdma, BUS_DMA_NOWAIT)) {
2850                         device_printf(dev,
2851                             "Unable to allocate TX Descriptor memory\n");
2852                         error = ENOMEM;
2853                         goto err_tx_desc;
2854                 }
2855                 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr;
2856                 bzero((void *)txr->tx_base, tsize);
2857
2858                 /* Now allocate transmit buffers for the ring */
2859                 if (igb_allocate_transmit_buffers(txr)) {
2860                         device_printf(dev,
2861                             "Critical Failure setting up transmit buffers\n");
2862                         error = ENOMEM;
2863                         goto err_tx_desc;
2864                 }
2865 #if __FreeBSD_version >= 800000
2866                 /* Allocate a buf ring */
2867                 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF,
2868                     M_WAITOK, &txr->tx_mtx);
2869 #endif
2870         }
2871
2872         /*
2873          * Next the RX queues...
2874          */
2875         rsize = roundup2(adapter->num_rx_desc *
2876             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
2877         for (int i = 0; i < adapter->num_queues; i++, rxconf++) {
2878                 rxr = &adapter->rx_rings[i];
2879                 rxr->adapter = adapter;
2880                 rxr->me = i;
2881
2882                 /* Initialize the RX lock */
2883                 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
2884                     device_get_nameunit(dev), txr->me);
2885                 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF);
2886
2887                 if (igb_dma_malloc(adapter, rsize,
2888                         &rxr->rxdma, BUS_DMA_NOWAIT)) {
2889                         device_printf(dev,
2890                             "Unable to allocate RxDescriptor memory\n");
2891                         error = ENOMEM;
2892                         goto err_rx_desc;
2893                 }
2894                 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2895                 bzero((void *)rxr->rx_base, rsize);
2896
2897                 /* Allocate receive buffers for the ring*/
2898                 if (igb_allocate_receive_buffers(rxr)) {
2899                         device_printf(dev,
2900                             "Critical Failure setting up receive buffers\n");
2901                         error = ENOMEM;
2902                         goto err_rx_desc;
2903                 }
2904         }
2905
2906         /*
2907         ** Finally set up the queue holding structs
2908         */
2909         for (int i = 0; i < adapter->num_queues; i++) {
2910                 que = &adapter->queues[i];
2911                 que->adapter = adapter;
2912                 que->txr = &adapter->tx_rings[i];
2913                 que->rxr = &adapter->rx_rings[i];
2914         }
2915
2916         return (0);
2917
2918 err_rx_desc:
2919         for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--)
2920                 igb_dma_free(adapter, &rxr->rxdma);
2921 err_tx_desc:
2922         for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--)
2923                 igb_dma_free(adapter, &txr->txdma);
2924         free(adapter->rx_rings, M_DEVBUF);
2925 rx_fail:
2926 #if __FreeBSD_version >= 800000
2927         buf_ring_free(txr->br, M_DEVBUF);
2928 #endif
2929         free(adapter->tx_rings, M_DEVBUF);
2930 tx_fail:
2931         free(adapter->queues, M_DEVBUF);
2932 fail:
2933         return (error);
2934 }
2935
2936 /*********************************************************************
2937  *
2938  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2939  *  the information needed to transmit a packet on the wire. This is
2940  *  called only once at attach, setup is done every reset.
2941  *
2942  **********************************************************************/
2943 static int
2944 igb_allocate_transmit_buffers(struct tx_ring *txr)
2945 {
2946         struct adapter *adapter = txr->adapter;
2947         device_t dev = adapter->dev;
2948         struct igb_tx_buffer *txbuf;
2949         int error, i;
2950
2951         /*
2952          * Setup DMA descriptor areas.
2953          */
2954         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
2955                                1, 0,                    /* alignment, bounds */
2956                                BUS_SPACE_MAXADDR,       /* lowaddr */
2957                                BUS_SPACE_MAXADDR,       /* highaddr */
2958                                NULL, NULL,              /* filter, filterarg */
2959                                IGB_TSO_SIZE,            /* maxsize */
2960                                IGB_MAX_SCATTER,         /* nsegments */
2961                                PAGE_SIZE,               /* maxsegsize */
2962                                0,                       /* flags */
2963                                NULL,                    /* lockfunc */
2964                                NULL,                    /* lockfuncarg */
2965                                &txr->txtag))) {
2966                 device_printf(dev,"Unable to allocate TX DMA tag\n");
2967                 goto fail;
2968         }
2969
2970         if (!(txr->tx_buffers =
2971             (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) *
2972             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2973                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
2974                 error = ENOMEM;
2975                 goto fail;
2976         }
2977
2978         /* Create the descriptor buffer dma maps */
2979         txbuf = txr->tx_buffers;
2980         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
2981                 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map);
2982                 if (error != 0) {
2983                         device_printf(dev, "Unable to create TX DMA map\n");
2984                         goto fail;
2985                 }
2986         }
2987
2988         return 0;
2989 fail:
2990         /* We free all, it handles case where we are in the middle */
2991         igb_free_transmit_structures(adapter);
2992         return (error);
2993 }
2994
2995 /*********************************************************************
2996  *
2997  *  Initialize a transmit ring.
2998  *
2999  **********************************************************************/
3000 static void
3001 igb_setup_transmit_ring(struct tx_ring *txr)
3002 {
3003         struct adapter *adapter = txr->adapter;
3004         struct igb_tx_buffer *txbuf;
3005         int i;
3006
3007         /* Clear the old descriptor contents */
3008         IGB_TX_LOCK(txr);
3009         bzero((void *)txr->tx_base,
3010               (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc);
3011         /* Reset indices */
3012         txr->next_avail_desc = 0;
3013         txr->next_to_clean = 0;
3014
3015         /* Free any existing tx buffers. */
3016         txbuf = txr->tx_buffers;
3017         for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) {
3018                 if (txbuf->m_head != NULL) {
3019                         bus_dmamap_sync(txr->txtag, txbuf->map,
3020                             BUS_DMASYNC_POSTWRITE);
3021                         bus_dmamap_unload(txr->txtag, txbuf->map);
3022                         m_freem(txbuf->m_head);
3023                         txbuf->m_head = NULL;
3024                 }
3025                 /* clear the watch index */
3026                 txbuf->next_eop = -1;
3027         }
3028
3029         /* Set number of descriptors available */
3030         txr->tx_avail = adapter->num_tx_desc;
3031
3032         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3033             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3034         IGB_TX_UNLOCK(txr);
3035 }
3036
3037 /*********************************************************************
3038  *
3039  *  Initialize all transmit rings.
3040  *
3041  **********************************************************************/
3042 static void
3043 igb_setup_transmit_structures(struct adapter *adapter)
3044 {
3045         struct tx_ring *txr = adapter->tx_rings;
3046
3047         for (int i = 0; i < adapter->num_queues; i++, txr++)
3048                 igb_setup_transmit_ring(txr);
3049
3050         return;
3051 }
3052
3053 /*********************************************************************
3054  *
3055  *  Enable transmit unit.
3056  *
3057  **********************************************************************/
3058 static void
3059 igb_initialize_transmit_units(struct adapter *adapter)
3060 {
3061         struct tx_ring  *txr = adapter->tx_rings;
3062         struct e1000_hw *hw = &adapter->hw;
3063         u32             tctl, txdctl;
3064
3065          INIT_DEBUGOUT("igb_initialize_transmit_units: begin");
3066
3067         /* Setup the Tx Descriptor Rings */
3068         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3069                 u64 bus_addr = txr->txdma.dma_paddr;
3070
3071                 E1000_WRITE_REG(hw, E1000_TDLEN(i),
3072                     adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3073                 E1000_WRITE_REG(hw, E1000_TDBAH(i),
3074                     (uint32_t)(bus_addr >> 32));
3075                 E1000_WRITE_REG(hw, E1000_TDBAL(i),
3076                     (uint32_t)bus_addr);
3077
3078                 /* Setup the HW Tx Head and Tail descriptor pointers */
3079                 E1000_WRITE_REG(hw, E1000_TDT(i), 0);
3080                 E1000_WRITE_REG(hw, E1000_TDH(i), 0);
3081
3082                 HW_DEBUGOUT2("Base = %x, Length = %x\n",
3083                     E1000_READ_REG(hw, E1000_TDBAL(i)),
3084                     E1000_READ_REG(hw, E1000_TDLEN(i)));
3085
3086                 txr->watchdog_check = FALSE;
3087
3088                 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i));
3089                 txdctl |= IGB_TX_PTHRESH;
3090                 txdctl |= IGB_TX_HTHRESH << 8;
3091                 txdctl |= IGB_TX_WTHRESH << 16;
3092                 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
3093                 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl);
3094         }
3095
3096         /* Program the Transmit Control Register */
3097         tctl = E1000_READ_REG(hw, E1000_TCTL);
3098         tctl &= ~E1000_TCTL_CT;
3099         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3100                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3101
3102         e1000_config_collision_dist(hw);
3103
3104         /* This write will effectively turn on the transmit unit. */
3105         E1000_WRITE_REG(hw, E1000_TCTL, tctl);
3106 }
3107
3108 /*********************************************************************
3109  *
3110  *  Free all transmit rings.
3111  *
3112  **********************************************************************/
3113 static void
3114 igb_free_transmit_structures(struct adapter *adapter)
3115 {
3116         struct tx_ring *txr = adapter->tx_rings;
3117
3118         for (int i = 0; i < adapter->num_queues; i++, txr++) {
3119                 IGB_TX_LOCK(txr);
3120                 igb_free_transmit_buffers(txr);
3121                 igb_dma_free(adapter, &txr->txdma);
3122                 IGB_TX_UNLOCK(txr);
3123                 IGB_TX_LOCK_DESTROY(txr);
3124         }
3125         free(adapter->tx_rings, M_DEVBUF);
3126 }
3127
3128 /*********************************************************************
3129  *
3130  *  Free transmit ring related data structures.
3131  *
3132  **********************************************************************/
3133 static void
3134 igb_free_transmit_buffers(struct tx_ring *txr)
3135 {
3136         struct adapter *adapter = txr->adapter;
3137         struct igb_tx_buffer *tx_buffer;
3138         int             i;
3139
3140         INIT_DEBUGOUT("free_transmit_ring: begin");
3141
3142         if (txr->tx_buffers == NULL)
3143                 return;
3144
3145         tx_buffer = txr->tx_buffers;
3146         for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3147                 if (tx_buffer->m_head != NULL) {
3148                         bus_dmamap_sync(txr->txtag, tx_buffer->map,
3149                             BUS_DMASYNC_POSTWRITE);
3150                         bus_dmamap_unload(txr->txtag,
3151                             tx_buffer->map);
3152                         m_freem(tx_buffer->m_head);
3153                         tx_buffer->m_head = NULL;
3154                         if (tx_buffer->map != NULL) {
3155                                 bus_dmamap_destroy(txr->txtag,
3156                                     tx_buffer->map);
3157                                 tx_buffer->map = NULL;
3158                         }
3159                 } else if (tx_buffer->map != NULL) {
3160                         bus_dmamap_unload(txr->txtag,
3161                             tx_buffer->map);
3162                         bus_dmamap_destroy(txr->txtag,
3163                             tx_buffer->map);
3164                         tx_buffer->map = NULL;
3165                 }
3166         }
3167 #if __FreeBSD_version >= 800000
3168         if (txr->br != NULL)
3169                 buf_ring_free(txr->br, M_DEVBUF);
3170 #endif
3171         if (txr->tx_buffers != NULL) {
3172                 free(txr->tx_buffers, M_DEVBUF);
3173                 txr->tx_buffers = NULL;
3174         }
3175         if (txr->txtag != NULL) {
3176                 bus_dma_tag_destroy(txr->txtag);
3177                 txr->txtag = NULL;
3178         }
3179         return;
3180 }
3181
3182 /**********************************************************************
3183  *
3184  *  Setup work for hardware segmentation offload (TSO)
3185  *
3186  **********************************************************************/
3187 static boolean_t
3188 igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen)
3189 {
3190         struct adapter *adapter = txr->adapter;
3191         struct e1000_adv_tx_context_desc *TXD;
3192         struct igb_tx_buffer        *tx_buffer;
3193         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
3194         u32 mss_l4len_idx = 0;
3195         u16 vtag = 0;
3196         int ctxd, ehdrlen, ip_hlen, tcp_hlen;
3197         struct ether_vlan_header *eh;
3198         struct ip *ip;
3199         struct tcphdr *th;
3200
3201
3202         /*
3203          * Determine where frame payload starts.
3204          * Jump over vlan headers if already present
3205          */
3206         eh = mtod(mp, struct ether_vlan_header *);
3207         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN))
3208                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3209         else
3210                 ehdrlen = ETHER_HDR_LEN;
3211
3212         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3213         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3214                 return FALSE;
3215
3216         /* Only supports IPV4 for now */
3217         ctxd = txr->next_avail_desc;
3218         tx_buffer = &txr->tx_buffers[ctxd];
3219         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3220
3221         ip = (struct ip *)(mp->m_data + ehdrlen);
3222         if (ip->ip_p != IPPROTO_TCP)
3223                 return FALSE;   /* 0 */
3224         ip->ip_sum = 0;
3225         ip_hlen = ip->ip_hl << 2;
3226         th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3227         th->th_sum = in_pseudo(ip->ip_src.s_addr,
3228             ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3229         tcp_hlen = th->th_off << 2;
3230         /*
3231          * Calculate header length, this is used
3232          * in the transmit desc in igb_xmit
3233          */
3234         *hdrlen = ehdrlen + ip_hlen + tcp_hlen;
3235
3236         /* VLAN MACLEN IPLEN */
3237         if (mp->m_flags & M_VLANTAG) {
3238                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3239                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3240         }
3241
3242         vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT);
3243         vlan_macip_lens |= ip_hlen;
3244         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3245
3246         /* ADV DTYPE TUCMD */
3247         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3248         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3249         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3250         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3251
3252         /* MSS L4LEN IDX */
3253         mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT);
3254         mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT);
3255         /* 82575 needs the queue index added */
3256         if (adapter->hw.mac.type == e1000_82575)
3257                 mss_l4len_idx |= txr->me << 4;
3258         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3259
3260         TXD->seqnum_seed = htole32(0);
3261         tx_buffer->m_head = NULL;
3262         tx_buffer->next_eop = -1;
3263
3264         if (++ctxd == adapter->num_tx_desc)
3265                 ctxd = 0;
3266
3267         txr->tx_avail--;
3268         txr->next_avail_desc = ctxd;
3269         return TRUE;
3270 }
3271
3272
3273 /*********************************************************************
3274  *
3275  *  Context Descriptor setup for VLAN or CSUM
3276  *
3277  **********************************************************************/
3278
3279 static bool
3280 igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp)
3281 {
3282         struct adapter *adapter = txr->adapter;
3283         struct e1000_adv_tx_context_desc *TXD;
3284         struct igb_tx_buffer        *tx_buffer;
3285         u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx;
3286         struct ether_vlan_header *eh;
3287         struct ip *ip = NULL;
3288         struct ip6_hdr *ip6;
3289         int  ehdrlen, ctxd, ip_hlen = 0;
3290         u16     etype, vtag = 0;
3291         u8      ipproto = 0;
3292         bool    offload = TRUE;
3293
3294         if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0)
3295                 offload = FALSE;
3296
3297         vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0;
3298         ctxd = txr->next_avail_desc;
3299         tx_buffer = &txr->tx_buffers[ctxd];
3300         TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd];
3301
3302         /*
3303         ** In advanced descriptors the vlan tag must
3304         ** be placed into the context descriptor, thus
3305         ** we need to be here just for that setup.
3306         */
3307         if (mp->m_flags & M_VLANTAG) {
3308                 vtag = htole16(mp->m_pkthdr.ether_vtag);
3309                 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT);
3310         } else if (offload == FALSE)
3311                 return FALSE;
3312
3313         /*
3314          * Determine where frame payload starts.
3315          * Jump over vlan headers if already present,
3316          * helpful for QinQ too.
3317          */
3318         eh = mtod(mp, struct ether_vlan_header *);
3319         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3320                 etype = ntohs(eh->evl_proto);
3321                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3322         } else {
3323                 etype = ntohs(eh->evl_encap_proto);
3324                 ehdrlen = ETHER_HDR_LEN;
3325         }
3326
3327         /* Set the ether header length */
3328         vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT;
3329
3330         switch (etype) {
3331                 case ETHERTYPE_IP:
3332                         ip = (struct ip *)(mp->m_data + ehdrlen);
3333                         ip_hlen = ip->ip_hl << 2;
3334                         if (mp->m_len < ehdrlen + ip_hlen) {
3335                                 offload = FALSE;
3336                                 break;
3337                         }
3338                         ipproto = ip->ip_p;
3339                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4;
3340                         break;
3341                 case ETHERTYPE_IPV6:
3342                         ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3343                         ip_hlen = sizeof(struct ip6_hdr);
3344                         if (mp->m_len < ehdrlen + ip_hlen)
3345                                 return (FALSE);
3346                         ipproto = ip6->ip6_nxt;
3347                         type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6;
3348                         break;
3349                 default:
3350                         offload = FALSE;
3351                         break;
3352         }
3353
3354         vlan_macip_lens |= ip_hlen;
3355         type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3356
3357         switch (ipproto) {
3358                 case IPPROTO_TCP:
3359                         if (mp->m_pkthdr.csum_flags & CSUM_TCP)
3360                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP;
3361                         break;
3362                 case IPPROTO_UDP:
3363                         if (mp->m_pkthdr.csum_flags & CSUM_UDP)
3364                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP;
3365                         break;
3366 #if __FreeBSD_version >= 800000
3367                 case IPPROTO_SCTP:
3368                         if (mp->m_pkthdr.csum_flags & CSUM_SCTP)
3369                                 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3370                         break;
3371 #endif
3372                 default:
3373                         offload = FALSE;
3374                         break;
3375         }
3376
3377         /* 82575 needs the queue index added */
3378         if (adapter->hw.mac.type == e1000_82575)
3379                 mss_l4len_idx = txr->me << 4;
3380
3381         /* Now copy bits into descriptor */
3382         TXD->vlan_macip_lens |= htole32(vlan_macip_lens);
3383         TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl);
3384         TXD->seqnum_seed = htole32(0);
3385         TXD->mss_l4len_idx = htole32(mss_l4len_idx);
3386
3387         tx_buffer->m_head = NULL;
3388         tx_buffer->next_eop = -1;
3389
3390         /* We've consumed the first desc, adjust counters */
3391         if (++ctxd == adapter->num_tx_desc)
3392                 ctxd = 0;
3393         txr->next_avail_desc = ctxd;
3394         --txr->tx_avail;
3395
3396         return (offload);
3397 }
3398
3399
3400 /**********************************************************************
3401  *
3402  *  Examine each tx_buffer in the used queue. If the hardware is done
3403  *  processing the packet then free associated resources. The
3404  *  tx_buffer is put back on the free queue.
3405  *
3406  *  TRUE return means there's work in the ring to clean, FALSE its empty.
3407  **********************************************************************/
3408 static bool
3409 igb_txeof(struct tx_ring *txr)
3410 {
3411         struct adapter  *adapter = txr->adapter;
3412         int first, last, done;
3413         struct igb_tx_buffer *tx_buffer;
3414         struct e1000_tx_desc   *tx_desc, *eop_desc;
3415         struct ifnet   *ifp = adapter->ifp;
3416
3417         IGB_TX_LOCK_ASSERT(txr);
3418
3419         if (txr->tx_avail == adapter->num_tx_desc)
3420                 return FALSE;
3421
3422         first = txr->next_to_clean;
3423         tx_desc = &txr->tx_base[first];
3424         tx_buffer = &txr->tx_buffers[first];
3425         last = tx_buffer->next_eop;
3426         eop_desc = &txr->tx_base[last];
3427
3428         /*
3429          * What this does is get the index of the
3430          * first descriptor AFTER the EOP of the
3431          * first packet, that way we can do the
3432          * simple comparison on the inner while loop.
3433          */
3434         if (++last == adapter->num_tx_desc)
3435                 last = 0;
3436         done = last;
3437
3438         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3439             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3440
3441         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3442                 /* We clean the range of the packet */
3443                 while (first != done) {
3444                         tx_desc->upper.data = 0;
3445                         tx_desc->lower.data = 0;
3446                         tx_desc->buffer_addr = 0;
3447                         ++txr->tx_avail;
3448
3449                         if (tx_buffer->m_head) {
3450                                 txr->bytes +=
3451                                     tx_buffer->m_head->m_pkthdr.len;
3452                                 bus_dmamap_sync(txr->txtag,
3453                                     tx_buffer->map,
3454                                     BUS_DMASYNC_POSTWRITE);
3455                                 bus_dmamap_unload(txr->txtag,
3456                                     tx_buffer->map);
3457
3458                                 m_freem(tx_buffer->m_head);
3459                                 tx_buffer->m_head = NULL;
3460                         }
3461                         tx_buffer->next_eop = -1;
3462                         txr->watchdog_time = ticks;
3463
3464                         if (++first == adapter->num_tx_desc)
3465                                 first = 0;
3466
3467                         tx_buffer = &txr->tx_buffers[first];
3468                         tx_desc = &txr->tx_base[first];
3469                 }
3470                 ++txr->packets;
3471                 ++ifp->if_opackets;
3472                 /* See if we can continue to the next packet */
3473                 last = tx_buffer->next_eop;
3474                 if (last != -1) {
3475                         eop_desc = &txr->tx_base[last];
3476                         /* Get new done point */
3477                         if (++last == adapter->num_tx_desc) last = 0;
3478                         done = last;
3479                 } else
3480                         break;
3481         }
3482         bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
3483             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3484
3485         txr->next_to_clean = first;
3486
3487         /*
3488          * If we have enough room, clear IFF_DRV_OACTIVE
3489          * to tell the stack that it is OK to send packets.
3490          */
3491         if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) {
3492                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3493                 /* All clean, turn off the watchdog */
3494                 if (txr->tx_avail == adapter->num_tx_desc) {
3495                         txr->watchdog_check = FALSE;
3496                         return FALSE;
3497                 }
3498         }
3499
3500         return (TRUE);
3501 }
3502
3503
3504 /*********************************************************************
3505  *
3506  *  Setup descriptor buffer(s) from system mbuf buffer pools.
3507  *              i - designates the ring index
3508  *              clean - tells the function whether to update
3509  *                      the header, the packet buffer, or both.
3510  *
3511  **********************************************************************/
3512 static int
3513 igb_get_buf(struct rx_ring *rxr, int i, u8 clean)
3514 {
3515         struct adapter          *adapter = rxr->adapter;
3516         struct igb_rx_buf       *rxbuf;
3517         struct mbuf             *mh, *mp;
3518         bus_dma_segment_t       hseg[1];
3519         bus_dma_segment_t       pseg[1];
3520         bus_dmamap_t            map;
3521         int                     nsegs, error;
3522
3523
3524         rxbuf = &rxr->rx_buffers[i];
3525         mh = mp = NULL;
3526         if ((clean & IGB_CLEAN_HEADER) != 0) {
3527                 mh = m_gethdr(M_DONTWAIT, MT_DATA);
3528                 if (mh == NULL) {
3529                         adapter->mbuf_header_failed++;
3530                         return (ENOBUFS);
3531                 }
3532                 mh->m_pkthdr.len = mh->m_len = MHLEN;
3533                 /*
3534                  * Because IGB_HDR_BUF size is less than MHLEN
3535                  * and we configure controller to split headers
3536                  * we can align mbuf on ETHER_ALIGN boundary.
3537                  */
3538                 m_adj(mh, ETHER_ALIGN);
3539                 error = bus_dmamap_load_mbuf_sg(rxr->rx_htag,
3540                     rxr->rx_hspare_map, mh, hseg, &nsegs, 0);
3541                 if (error != 0) {
3542                         m_freem(mh);
3543                         return (error);
3544                 }
3545                 mh->m_flags &= ~M_PKTHDR;
3546         }
3547         if ((clean & IGB_CLEAN_PAYLOAD) != 0) {
3548                 mp = m_getjcl(M_DONTWAIT, MT_DATA, M_PKTHDR,
3549                     adapter->rx_mbuf_sz);
3550                 if (mp == NULL) {
3551                         if (mh != NULL) {
3552                                 adapter->mbuf_packet_failed++;
3553                                 bus_dmamap_unload(rxr->rx_htag,
3554                                     rxbuf->head_map);
3555                                 mh->m_flags |= M_PKTHDR;
3556                                 m_freem(mh);
3557                         }
3558                         return (ENOBUFS);
3559                 }
3560                 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz;
3561                 error = bus_dmamap_load_mbuf_sg(rxr->rx_ptag,
3562                     rxr->rx_pspare_map, mp, pseg, &nsegs, 0);
3563                 if (error != 0) {
3564                         if (mh != NULL) {
3565                                 bus_dmamap_unload(rxr->rx_htag,
3566                                     rxbuf->head_map);
3567                                 mh->m_flags |= M_PKTHDR;
3568                                 m_freem(mh);
3569                         }
3570                         m_freem(mp);
3571                         return (error);
3572                 }
3573                 mp->m_flags &= ~M_PKTHDR;
3574         }
3575
3576         /* Loading new DMA maps complete, unload maps for received buffers. */
3577         if ((clean & IGB_CLEAN_HEADER) != 0 && rxbuf->m_head != NULL) {
3578                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3579                     BUS_DMASYNC_POSTREAD);
3580                 bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map);
3581         }
3582         if ((clean & IGB_CLEAN_PAYLOAD) != 0 && rxbuf->m_pack != NULL) {
3583                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3584                     BUS_DMASYNC_POSTREAD);
3585                 bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map);
3586         }
3587
3588         /* Reflect loaded dmamaps. */
3589         if ((clean & IGB_CLEAN_HEADER) != 0) {
3590                 map = rxbuf->head_map;
3591                 rxbuf->head_map = rxr->rx_hspare_map;
3592                 rxr->rx_hspare_map = map;
3593                 rxbuf->m_head = mh;
3594                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3595                     BUS_DMASYNC_PREREAD);
3596                 rxr->rx_base[i].read.hdr_addr = htole64(hseg[0].ds_addr);
3597         }
3598         if ((clean & IGB_CLEAN_PAYLOAD) != 0) {
3599                 map = rxbuf->pack_map;
3600                 rxbuf->pack_map = rxr->rx_pspare_map;
3601                 rxr->rx_pspare_map = map;
3602                 rxbuf->m_pack = mp;
3603                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3604                     BUS_DMASYNC_PREREAD);
3605                 rxr->rx_base[i].read.pkt_addr = htole64(pseg[0].ds_addr);
3606         }
3607
3608         return (0);
3609 }
3610
3611 /*********************************************************************
3612  *
3613  *  Allocate memory for rx_buffer structures. Since we use one
3614  *  rx_buffer per received packet, the maximum number of rx_buffer's
3615  *  that we'll need is equal to the number of receive descriptors
3616  *  that we've allocated.
3617  *
3618  **********************************************************************/
3619 static int
3620 igb_allocate_receive_buffers(struct rx_ring *rxr)
3621 {
3622         struct  adapter         *adapter = rxr->adapter;
3623         device_t                dev = adapter->dev;
3624         struct igb_rx_buf       *rxbuf;
3625         int                     i, bsize, error;
3626
3627         bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc;
3628         if (!(rxr->rx_buffers =
3629             (struct igb_rx_buf *) malloc(bsize,
3630             M_DEVBUF, M_NOWAIT | M_ZERO))) {
3631                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
3632                 error = ENOMEM;
3633                 goto fail;
3634         }
3635
3636         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3637                                    1, 0,                /* alignment, bounds */
3638                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3639                                    BUS_SPACE_MAXADDR,   /* highaddr */
3640                                    NULL, NULL,          /* filter, filterarg */
3641                                    MSIZE,               /* maxsize */
3642                                    1,                   /* nsegments */
3643                                    MSIZE,               /* maxsegsize */
3644                                    0,                   /* flags */
3645                                    NULL,                /* lockfunc */
3646                                    NULL,                /* lockfuncarg */
3647                                    &rxr->rx_htag))) {
3648                 device_printf(dev, "Unable to create RX DMA tag\n");
3649                 goto fail;
3650         }
3651
3652         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev),
3653                                    1, 0,                /* alignment, bounds */
3654                                    BUS_SPACE_MAXADDR,   /* lowaddr */
3655                                    BUS_SPACE_MAXADDR,   /* highaddr */
3656                                    NULL, NULL,          /* filter, filterarg */
3657                                    MJUMPAGESIZE,        /* maxsize */
3658                                    1,                   /* nsegments */
3659                                    MJUMPAGESIZE,        /* maxsegsize */
3660                                    0,                   /* flags */
3661                                    NULL,                /* lockfunc */
3662                                    NULL,                /* lockfuncarg */
3663                                    &rxr->rx_ptag))) {
3664                 device_printf(dev, "Unable to create RX payload DMA tag\n");
3665                 goto fail;
3666         }
3667
3668         /* Create the spare maps (used by getbuf) */
3669         error = bus_dmamap_create(rxr->rx_htag, BUS_DMA_NOWAIT,
3670              &rxr->rx_hspare_map);
3671         if (error) {
3672                 device_printf(dev,
3673                     "%s: bus_dmamap_create header spare failed: %d\n",
3674                     __func__, error);
3675                 goto fail;
3676         }
3677         error = bus_dmamap_create(rxr->rx_ptag, BUS_DMA_NOWAIT,
3678              &rxr->rx_pspare_map);
3679         if (error) {
3680                 device_printf(dev,
3681                     "%s: bus_dmamap_create packet spare failed: %d\n",
3682                     __func__, error);
3683                 goto fail;
3684         }
3685
3686         for (i = 0; i < adapter->num_rx_desc; i++) {
3687                 rxbuf = &rxr->rx_buffers[i];
3688                 error = bus_dmamap_create(rxr->rx_htag,
3689                     BUS_DMA_NOWAIT, &rxbuf->head_map);
3690                 if (error) {
3691                         device_printf(dev,
3692                             "Unable to create RX head DMA maps\n");
3693                         goto fail;
3694                 }
3695                 error = bus_dmamap_create(rxr->rx_ptag,
3696                     BUS_DMA_NOWAIT, &rxbuf->pack_map);
3697                 if (error) {
3698                         device_printf(dev,
3699                             "Unable to create RX packet DMA maps\n");
3700                         goto fail;
3701                 }
3702         }
3703
3704         return (0);
3705
3706 fail:
3707         /* Frees all, but can handle partial completion */
3708         igb_free_receive_structures(adapter);
3709         return (error);
3710 }
3711
3712
3713 static void
3714 igb_free_receive_ring(struct rx_ring *rxr)
3715 {
3716         struct  adapter         *adapter;
3717         struct igb_rx_buf       *rxbuf;
3718         int i;
3719
3720         adapter = rxr->adapter;
3721         for (i = 0; i < adapter->num_rx_desc; i++) {
3722                 rxbuf = &rxr->rx_buffers[i];
3723                 if (rxbuf->m_head != NULL) {
3724                         bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
3725                             BUS_DMASYNC_POSTREAD);
3726                         bus_dmamap_unload(rxr->rx_htag, rxbuf->head_map);
3727                         rxbuf->m_head->m_flags |= M_PKTHDR;
3728                         m_freem(rxbuf->m_head);
3729                 }
3730                 if (rxbuf->m_pack != NULL) {
3731                         bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
3732                             BUS_DMASYNC_POSTREAD);
3733                         bus_dmamap_unload(rxr->rx_ptag, rxbuf->pack_map);
3734                         rxbuf->m_pack->m_flags |= M_PKTHDR;
3735                         m_freem(rxbuf->m_pack);
3736                 }
3737                 rxbuf->m_head = NULL;
3738                 rxbuf->m_pack = NULL;
3739         }
3740 }
3741
3742
3743 /*********************************************************************
3744  *
3745  *  Initialize a receive ring and its buffers.
3746  *
3747  **********************************************************************/
3748 static int
3749 igb_setup_receive_ring(struct rx_ring *rxr)
3750 {
3751         struct  adapter         *adapter;
3752         struct  ifnet           *ifp;
3753         device_t                dev;
3754         struct lro_ctrl         *lro = &rxr->lro;
3755         int                     j, rsize, error = 0;
3756
3757         adapter = rxr->adapter;
3758         dev = adapter->dev;
3759         ifp = adapter->ifp;
3760
3761         /* Clear the ring contents */
3762         IGB_RX_LOCK(rxr);
3763         rsize = roundup2(adapter->num_rx_desc *
3764             sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN);
3765         bzero((void *)rxr->rx_base, rsize);
3766
3767         /*
3768         ** Free current RX buffer structures and their mbufs
3769         */
3770         igb_free_receive_ring(rxr);
3771
3772         /* Now replenish the ring mbufs */
3773         for (j = 0; j < adapter->num_rx_desc; j++) {
3774                 if ((error = igb_get_buf(rxr, j, IGB_CLEAN_BOTH)) != 0)
3775                         goto fail;
3776         }
3777
3778         /* Setup our descriptor indices */
3779         rxr->next_to_check = 0;
3780         rxr->last_cleaned = 0;
3781         rxr->lro_enabled = FALSE;
3782
3783         if (igb_header_split)
3784                 rxr->hdr_split = TRUE;
3785         else
3786                 ifp->if_capabilities &= ~IFCAP_LRO;
3787
3788         rxr->fmp = NULL;
3789         rxr->lmp = NULL;
3790         rxr->discard = FALSE;
3791
3792         bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3793             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3794
3795         /*
3796         ** Now set up the LRO interface, we
3797         ** also only do head split when LRO
3798         ** is enabled, since so often they
3799         ** are undesireable in similar setups.
3800         */
3801         if (ifp->if_capenable & IFCAP_LRO) {
3802                 int err = tcp_lro_init(lro);
3803                 if (err) {
3804                         device_printf(dev, "LRO Initialization failed!\n");
3805                         goto fail;
3806                 }
3807                 INIT_DEBUGOUT("RX LRO Initialized\n");
3808                 rxr->lro_enabled = TRUE;
3809                 lro->ifp = adapter->ifp;
3810         }
3811
3812         IGB_RX_UNLOCK(rxr);
3813         return (0);
3814
3815 fail:
3816         igb_free_receive_ring(rxr);
3817         IGB_RX_UNLOCK(rxr);
3818         return (error);
3819 }
3820
3821 /*********************************************************************
3822  *
3823  *  Initialize all receive rings.
3824  *
3825  **********************************************************************/
3826 static int
3827 igb_setup_receive_structures(struct adapter *adapter)
3828 {
3829         struct rx_ring *rxr = adapter->rx_rings;
3830         int i, j;
3831
3832         for (i = 0; i < adapter->num_queues; i++, rxr++)
3833                 if (igb_setup_receive_ring(rxr))
3834                         goto fail;
3835
3836         return (0);
3837 fail:
3838         /*
3839          * Free RX buffers allocated so far, we will only handle
3840          * the rings that completed, the failing case will have
3841          * cleaned up for itself. The value of 'i' will be the
3842          * failed ring so we must pre-decrement it.
3843          */
3844         rxr = adapter->rx_rings;
3845         for (--i; i > 0; i--, rxr++) {
3846                 for (j = 0; j < adapter->num_rx_desc; j++)
3847                         igb_free_receive_ring(rxr);
3848         }
3849
3850         return (ENOBUFS);
3851 }
3852
3853 /*********************************************************************
3854  *
3855  *  Enable receive unit.
3856  *
3857  **********************************************************************/
3858 static void
3859 igb_initialize_receive_units(struct adapter *adapter)
3860 {
3861         struct rx_ring  *rxr = adapter->rx_rings;
3862         struct ifnet    *ifp = adapter->ifp;
3863         struct e1000_hw *hw = &adapter->hw;
3864         u32             rctl, rxcsum, psize, srrctl = 0;
3865
3866         INIT_DEBUGOUT("igb_initialize_receive_unit: begin");
3867
3868         /*
3869          * Make sure receives are disabled while setting
3870          * up the descriptor ring
3871          */
3872         rctl = E1000_READ_REG(hw, E1000_RCTL);
3873         E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3874
3875         /*
3876         ** Set up for header split
3877         */
3878         if (rxr->hdr_split) {
3879                 /* Use a standard mbuf for the header */
3880                 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3881                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3882         } else
3883                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3884
3885         /*
3886         ** Set up for jumbo frames
3887         */
3888         if (ifp->if_mtu > ETHERMTU) {
3889                 rctl |= E1000_RCTL_LPE;
3890                 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3891                 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX;
3892
3893                 /* Set maximum packet len */
3894                 psize = adapter->max_frame_size;
3895                 /* are we on a vlan? */
3896                 if (adapter->ifp->if_vlantrunk != NULL)
3897                         psize += VLAN_TAG_SIZE;
3898                 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize);
3899         } else {
3900                 rctl &= ~E1000_RCTL_LPE;
3901                 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3902                 rctl |= E1000_RCTL_SZ_2048;
3903         }
3904
3905         /* Setup the Base and Length of the Rx Descriptor Rings */
3906         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
3907                 u64 bus_addr = rxr->rxdma.dma_paddr;
3908                 u32 rxdctl;
3909
3910                 E1000_WRITE_REG(hw, E1000_RDLEN(i),
3911                     adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3912                 E1000_WRITE_REG(hw, E1000_RDBAH(i),
3913                     (uint32_t)(bus_addr >> 32));
3914                 E1000_WRITE_REG(hw, E1000_RDBAL(i),
3915                     (uint32_t)bus_addr);
3916                 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl);
3917                 /* Enable this Queue */
3918                 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i));
3919                 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3920                 rxdctl &= 0xFFF00000;
3921                 rxdctl |= IGB_RX_PTHRESH;
3922                 rxdctl |= IGB_RX_HTHRESH << 8;
3923                 rxdctl |= IGB_RX_WTHRESH << 16;
3924                 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl);
3925         }
3926
3927         /*
3928         ** Setup for RX MultiQueue
3929         */
3930         rxcsum = E1000_READ_REG(hw, E1000_RXCSUM);
3931         if (adapter->num_queues >1) {
3932                 u32 random[10], mrqc, shift = 0;
3933                 union igb_reta {
3934                         u32 dword;
3935                         u8  bytes[4];
3936                 } reta;
3937
3938                 arc4rand(&random, sizeof(random), 0);
3939                 if (adapter->hw.mac.type == e1000_82575)
3940                         shift = 6;
3941                 /* Warning FM follows */
3942                 for (int i = 0; i < 128; i++) {
3943                         reta.bytes[i & 3] =
3944                             (i % adapter->num_queues) << shift;
3945                         if ((i & 3) == 3)
3946                                 E1000_WRITE_REG(hw,
3947                                     E1000_RETA(i >> 2), reta.dword);
3948                 }
3949                 /* Now fill in hash table */
3950                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
3951                 for (int i = 0; i < 10; i++)
3952                         E1000_WRITE_REG_ARRAY(hw,
3953                             E1000_RSSRK(0), i, random[i]);
3954
3955                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
3956                     E1000_MRQC_RSS_FIELD_IPV4_TCP);
3957                 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
3958                     E1000_MRQC_RSS_FIELD_IPV6_TCP);
3959                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP |
3960                     E1000_MRQC_RSS_FIELD_IPV6_UDP);
3961                 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
3962                     E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3963
3964                 E1000_WRITE_REG(hw, E1000_MRQC, mrqc);
3965
3966                 /*
3967                 ** NOTE: Receive Full-Packet Checksum Offload
3968                 ** is mutually exclusive with Multiqueue. However
3969                 ** this is not the same as TCP/IP checksums which
3970                 ** still work.
3971                 */
3972                 rxcsum |= E1000_RXCSUM_PCSD;
3973 #if __FreeBSD_version >= 800000
3974                 /* For SCTP Offload */
3975                 if ((hw->mac.type == e1000_82576)
3976                     && (ifp->if_capenable & IFCAP_RXCSUM))
3977                         rxcsum |= E1000_RXCSUM_CRCOFL;
3978 #endif
3979         } else {
3980                 /* Non RSS setup */
3981                 if (ifp->if_capenable & IFCAP_RXCSUM) {
3982                         rxcsum |= E1000_RXCSUM_IPPCSE;
3983 #if __FreeBSD_version >= 800000
3984                         if (adapter->hw.mac.type == e1000_82576)
3985                                 rxcsum |= E1000_RXCSUM_CRCOFL;
3986 #endif
3987                 } else
3988                         rxcsum &= ~E1000_RXCSUM_TUOFL;
3989         }
3990         E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum);
3991
3992         /* Setup the Receive Control Register */
3993         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3994         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3995                    E1000_RCTL_RDMTS_HALF |
3996                    (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3997         /* Strip CRC bytes. */
3998         rctl |= E1000_RCTL_SECRC;
3999         /* Make sure VLAN Filters are off */
4000         rctl &= ~E1000_RCTL_VFE;
4001         /* Don't store bad packets */
4002         rctl &= ~E1000_RCTL_SBP;
4003
4004         /* Enable Receives */
4005         E1000_WRITE_REG(hw, E1000_RCTL, rctl);
4006
4007         /*
4008          * Setup the HW Rx Head and Tail Descriptor Pointers
4009          *   - needs to be after enable
4010          */
4011         for (int i = 0; i < adapter->num_queues; i++) {
4012                 E1000_WRITE_REG(hw, E1000_RDH(i), 0);
4013                 E1000_WRITE_REG(hw, E1000_RDT(i),
4014                      adapter->num_rx_desc - 1);
4015         }
4016         return;
4017 }
4018
4019 /*********************************************************************
4020  *
4021  *  Free receive rings.
4022  *
4023  **********************************************************************/
4024 static void
4025 igb_free_receive_structures(struct adapter *adapter)
4026 {
4027         struct rx_ring *rxr = adapter->rx_rings;
4028
4029         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4030                 struct lro_ctrl *lro = &rxr->lro;
4031                 igb_free_receive_buffers(rxr);
4032                 tcp_lro_free(lro);
4033                 igb_dma_free(adapter, &rxr->rxdma);
4034         }
4035
4036         free(adapter->rx_rings, M_DEVBUF);
4037 }
4038
4039 /*********************************************************************
4040  *
4041  *  Free receive ring data structures.
4042  *
4043  **********************************************************************/
4044 static void
4045 igb_free_receive_buffers(struct rx_ring *rxr)
4046 {
4047         struct adapter          *adapter = rxr->adapter;
4048         struct igb_rx_buf       *rxbuf;
4049         int i;
4050
4051         INIT_DEBUGOUT("free_receive_structures: begin");
4052
4053         if (rxr->rx_hspare_map != NULL) {
4054                 bus_dmamap_destroy(rxr->rx_htag, rxr->rx_hspare_map);
4055                 rxr->rx_hspare_map = NULL;
4056         }
4057
4058         if (rxr->rx_hspare_map != NULL) {
4059                 bus_dmamap_destroy(rxr->rx_ptag, rxr->rx_pspare_map);
4060                 rxr->rx_pspare_map = NULL;
4061         }
4062
4063         /* Cleanup any existing buffers */
4064         if (rxr->rx_buffers != NULL) {
4065                 for (i = 0; i < adapter->num_rx_desc; i++) {
4066                         rxbuf = &rxr->rx_buffers[i];
4067                         if (rxbuf->m_head != NULL) {
4068                                 bus_dmamap_sync(rxr->rx_htag, rxbuf->head_map,
4069                                     BUS_DMASYNC_POSTREAD);
4070                                 bus_dmamap_unload(rxr->rx_htag,
4071                                     rxbuf->head_map);
4072                                 rxbuf->m_head->m_flags |= M_PKTHDR;
4073                                 m_freem(rxbuf->m_head);
4074                         }
4075                         if (rxbuf->m_pack != NULL) {
4076                                 bus_dmamap_sync(rxr->rx_ptag, rxbuf->pack_map,
4077                                     BUS_DMASYNC_POSTREAD);
4078                                 bus_dmamap_unload(rxr->rx_ptag,
4079                                     rxbuf->pack_map);
4080                                 rxbuf->m_pack->m_flags |= M_PKTHDR;
4081                                 m_freem(rxbuf->m_pack);
4082                         }
4083                         rxbuf->m_head = NULL;
4084                         rxbuf->m_pack = NULL;
4085                         if (rxbuf->head_map != NULL) {
4086                                 bus_dmamap_destroy(rxr->rx_htag,
4087                                     rxbuf->head_map);
4088                                 rxbuf->head_map = NULL;
4089                         }
4090                         if (rxbuf->pack_map != NULL) {
4091                                 bus_dmamap_destroy(rxr->rx_ptag,
4092                                     rxbuf->pack_map);
4093                                 rxbuf->pack_map = NULL;
4094                         }
4095                 }
4096                 if (rxr->rx_buffers != NULL) {
4097                         free(rxr->rx_buffers, M_DEVBUF);
4098                         rxr->rx_buffers = NULL;
4099                 }
4100         }
4101
4102         if (rxr->rx_htag != NULL) {
4103                 bus_dma_tag_destroy(rxr->rx_htag);
4104                 rxr->rx_htag = NULL;
4105         }
4106         if (rxr->rx_ptag != NULL) {
4107                 bus_dma_tag_destroy(rxr->rx_ptag);
4108                 rxr->rx_ptag = NULL;
4109         }
4110 }
4111
4112 static __inline void
4113 igb_rx_discard(struct rx_ring *rxr, union e1000_adv_rx_desc *cur, int i)
4114 {
4115
4116         if (rxr->fmp != NULL) {
4117                 rxr->fmp->m_flags |= M_PKTHDR;
4118                 m_freem(rxr->fmp);
4119                 rxr->fmp = NULL;
4120                 rxr->lmp = NULL;
4121         }
4122 }
4123
4124 static __inline void
4125 igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype)
4126 {
4127
4128         /*
4129          * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet
4130          * should be computed by hardware. Also it should not have VLAN tag in
4131          * ethernet header.
4132          */
4133         if (rxr->lro_enabled &&
4134             (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4135             (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4136             (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) ==
4137             (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) &&
4138             (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) ==
4139             (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) {
4140                 /*
4141                  * Send to the stack if:
4142                  **  - LRO not enabled, or
4143                  **  - no LRO resources, or
4144                  **  - lro enqueue fails
4145                  */
4146                 if (rxr->lro.lro_cnt != 0)
4147                         if (tcp_lro_rx(&rxr->lro, m, 0) == 0)
4148                                 return;
4149         }
4150         (*ifp->if_input)(ifp, m);
4151 }
4152
4153 /*********************************************************************
4154  *
4155  *  This routine executes in interrupt context. It replenishes
4156  *  the mbufs in the descriptor and sends data which has been
4157  *  dma'ed into host memory to upper layer.
4158  *
4159  *  We loop at most count times if count is > 0, or until done if
4160  *  count < 0.
4161  *
4162  *  Return TRUE if more to clean, FALSE otherwise
4163  *********************************************************************/
4164 static bool
4165 igb_rxeof(struct rx_ring *rxr, int count)
4166 {
4167         struct adapter          *adapter = rxr->adapter;
4168         struct ifnet            *ifp = adapter->ifp;
4169         struct lro_ctrl         *lro = &rxr->lro;
4170         struct lro_entry        *queued;
4171         int                     i, prog = 0;
4172         u32                     ptype, staterr = 0;
4173         union e1000_adv_rx_desc *cur;
4174
4175         IGB_RX_LOCK(rxr);
4176
4177         /* Main clean loop */
4178         for (i = rxr->next_to_check; count > 0; prog++) {
4179                 struct mbuf *sendmp, *mh, *mp;
4180                 u16 hlen, plen, hdr, vtag;
4181                 bool eop = FALSE;
4182                 u8 dopayload;
4183
4184                 /* Sync the ring. */
4185                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4186                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
4187                 cur = &rxr->rx_base[i];
4188                 staterr = le32toh(cur->wb.upper.status_error);
4189                 if ((staterr & E1000_RXD_STAT_DD) == 0)
4190                         break;
4191                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
4192                         break;
4193                 count--;
4194                 sendmp = mh = mp = NULL;
4195                 cur->wb.upper.status_error = 0;
4196                 plen = le16toh(cur->wb.upper.length);
4197                 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK;
4198                 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info);
4199                 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP);
4200
4201                 /* Make sure all segments of a bad packet are discarded */
4202                 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) ||
4203                     (rxr->discard)) {
4204                         ifp->if_ierrors++;
4205                         ++rxr->rx_discarded;
4206                         if (!eop) /* Catch subsequent segs */
4207                                 rxr->discard = TRUE;
4208                         else
4209                                 rxr->discard = FALSE;
4210                         igb_rx_discard(rxr, cur, i);
4211                         goto next_desc;
4212                 }
4213
4214                 /*
4215                 ** The way the hardware is configured to
4216                 ** split, it will ONLY use the header buffer
4217                 ** when header split is enabled, otherwise we
4218                 ** get normal behavior, ie, both header and
4219                 ** payload are DMA'd into the payload buffer.
4220                 **
4221                 ** The fmp test is to catch the case where a
4222                 ** packet spans multiple descriptors, in that
4223                 ** case only the first header is valid.
4224                 */
4225                 if (rxr->hdr_split && rxr->fmp == NULL) {
4226                         hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >>
4227                             E1000_RXDADV_HDRBUFLEN_SHIFT;
4228                         if (hlen > IGB_HDR_BUF)
4229                                 hlen = IGB_HDR_BUF;
4230                         /* Handle the header mbuf */
4231                         mh = rxr->rx_buffers[i].m_head;
4232                         mh->m_len = hlen;
4233                         dopayload = IGB_CLEAN_HEADER;
4234                         /*
4235                         ** Get the payload length, this
4236                         ** could be zero if its a small
4237                         ** packet.
4238                         */
4239                         if (plen > 0) {
4240                                 mp = rxr->rx_buffers[i].m_pack;
4241                                 mp->m_len = plen;
4242                                 mh->m_next = mp;
4243                                 dopayload = IGB_CLEAN_BOTH;
4244                                 rxr->rx_split_packets++;
4245                         }
4246                 } else {
4247                         /*
4248                         ** Either no header split, or a
4249                         ** secondary piece of a fragmented
4250                         ** split packet.
4251                         */
4252                         mh = rxr->rx_buffers[i].m_pack;
4253                         mh->m_len = plen;
4254                         dopayload = IGB_CLEAN_PAYLOAD;
4255                 }
4256
4257                 /*
4258                 ** get_buf will overwrite the writeback
4259                 ** descriptor so save the VLAN tag now.
4260                 */
4261                 vtag = le16toh(cur->wb.upper.vlan);
4262                 if (igb_get_buf(rxr, i, dopayload) != 0) {
4263                         ifp->if_iqdrops++;
4264                         /*
4265                          * We've dropped a frame due to lack of resources
4266                          * so we should drop entire multi-segmented
4267                          * frames until we encounter EOP.
4268                          */
4269                         if ((staterr & E1000_RXD_STAT_EOP) != 0)
4270                                 rxr->discard = TRUE;
4271                         igb_rx_discard(rxr, cur, i);
4272                         goto next_desc;
4273                 }
4274
4275                 /* Initial frame - setup */
4276                 if (rxr->fmp == NULL) {
4277                         mh->m_pkthdr.len = mh->m_len;
4278                         /* Store the first mbuf */
4279                         rxr->fmp = mh;
4280                         rxr->lmp = mh;
4281                         if (mp != NULL) {
4282                                 /* Add payload if split */
4283                                 mh->m_pkthdr.len += mp->m_len;
4284                                 rxr->lmp = mh->m_next;
4285                         }
4286                 } else {
4287                         /* Chain mbuf's together */
4288                         rxr->lmp->m_next = mh;
4289                         rxr->lmp = rxr->lmp->m_next;
4290                         rxr->fmp->m_pkthdr.len += mh->m_len;
4291                 }
4292
4293                 if (eop) {
4294                         rxr->fmp->m_pkthdr.rcvif = ifp;
4295                         ifp->if_ipackets++;
4296                         rxr->rx_packets++;
4297                         /* capture data for AIM */
4298                         rxr->packets++;
4299                         rxr->bytes += rxr->fmp->m_pkthdr.len;
4300                         rxr->rx_bytes += rxr->fmp->m_pkthdr.len;
4301
4302                         if ((ifp->if_capenable & IFCAP_RXCSUM) != 0)
4303                                 igb_rx_checksum(staterr, rxr->fmp, ptype);
4304                         /* XXX igb(4) always strips VLAN. */
4305                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 &&
4306                             (staterr & E1000_RXD_STAT_VP) != 0) {
4307                                 rxr->fmp->m_pkthdr.ether_vtag = vtag;
4308                                 rxr->fmp->m_flags |= M_VLANTAG;
4309                         }
4310 #if __FreeBSD_version >= 800000
4311                         rxr->fmp->m_pkthdr.flowid = curcpu;
4312                         rxr->fmp->m_flags |= M_FLOWID;
4313 #endif
4314                         sendmp = rxr->fmp;
4315                         /* Make sure to set M_PKTHDR. */
4316                         sendmp->m_flags |= M_PKTHDR;
4317                         rxr->fmp = NULL;
4318                         rxr->lmp = NULL;
4319                 }
4320
4321 next_desc:
4322                 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
4323                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4324
4325                 rxr->last_cleaned = i; /* For updating tail */
4326
4327                 /* Advance our pointers to the next descriptor. */
4328                 if (++i == adapter->num_rx_desc)
4329                         i = 0;
4330
4331                 /*
4332                 ** Note that we hold the RX lock thru
4333                 ** the following call so this ring's
4334                 ** next_to_check is not gonna change.
4335                 */
4336                 if (sendmp != NULL)
4337                         igb_rx_input(rxr, ifp, sendmp, ptype);
4338         }
4339
4340         if (prog == 0) {
4341                 IGB_RX_UNLOCK(rxr);
4342                 return (FALSE);
4343         }
4344
4345         rxr->next_to_check = i;
4346
4347         /* Advance the E1000's Receive Queue "Tail Pointer". */
4348         E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned);
4349
4350         /*
4351          * Flush any outstanding LRO work
4352          */
4353         while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) {
4354                 SLIST_REMOVE_HEAD(&lro->lro_active, next);
4355                 tcp_lro_flush(lro, queued);
4356         }
4357
4358         IGB_RX_UNLOCK(rxr);
4359
4360         /*
4361         ** We still have cleaning to do?
4362         ** Schedule another interrupt if so.
4363         */
4364         if ((staterr & E1000_RXD_STAT_DD) != 0)
4365                 return (TRUE);
4366
4367         return (FALSE);
4368 }
4369
4370 /*********************************************************************
4371  *
4372  *  Verify that the hardware indicated that the checksum is valid.
4373  *  Inform the stack about the status of checksum so that stack
4374  *  doesn't spend time verifying the checksum.
4375  *
4376  *********************************************************************/
4377 static void
4378 igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype)
4379 {
4380         u16 status = (u16)staterr;
4381         u8  errors = (u8) (staterr >> 24);
4382         int sctp;
4383
4384         /* Ignore Checksum bit is set */
4385         if (status & E1000_RXD_STAT_IXSM) {
4386                 mp->m_pkthdr.csum_flags = 0;
4387                 return;
4388         }
4389
4390         if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 &&
4391             (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0)
4392                 sctp = 1;
4393         else
4394                 sctp = 0;
4395         if (status & E1000_RXD_STAT_IPCS) {
4396                 /* Did it pass? */
4397                 if (!(errors & E1000_RXD_ERR_IPE)) {
4398                         /* IP Checksum Good */
4399                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4400                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4401                 } else
4402                         mp->m_pkthdr.csum_flags = 0;
4403         }
4404
4405         if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) {
4406                 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4407 #if __FreeBSD_version >= 800000
4408                 if (sctp) /* reassign */
4409                         type = CSUM_SCTP_VALID;
4410 #endif
4411                 /* Did it pass? */
4412                 if (!(errors & E1000_RXD_ERR_TCPE)) {
4413                         mp->m_pkthdr.csum_flags |= type;
4414                         if (sctp == 0)
4415                                 mp->m_pkthdr.csum_data = htons(0xffff);
4416                 }
4417         }
4418         return;
4419 }
4420
4421 /*
4422  * This routine is run via an vlan
4423  * config EVENT
4424  */
4425 static void
4426 igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4427 {
4428         struct adapter  *adapter = ifp->if_softc;
4429         u32             index, bit;
4430
4431         if (ifp->if_softc !=  arg)   /* Not our event */
4432                 return;
4433
4434         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4435                 return;
4436
4437         index = (vtag >> 5) & 0x7F;
4438         bit = vtag & 0x1F;
4439         igb_shadow_vfta[index] |= (1 << bit);
4440         ++adapter->num_vlans;
4441         /* Re-init to load the changes */
4442         igb_init(adapter);
4443 }
4444
4445 /*
4446  * This routine is run via an vlan
4447  * unconfig EVENT
4448  */
4449 static void
4450 igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4451 {
4452         struct adapter  *adapter = ifp->if_softc;
4453         u32             index, bit;
4454
4455         if (ifp->if_softc !=  arg)
4456                 return;
4457
4458         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4459                 return;
4460
4461         index = (vtag >> 5) & 0x7F;
4462         bit = vtag & 0x1F;
4463         igb_shadow_vfta[index] &= ~(1 << bit);
4464         --adapter->num_vlans;
4465         /* Re-init to load the changes */
4466         igb_init(adapter);
4467 }
4468
4469 static void
4470 igb_setup_vlan_hw_support(struct adapter *adapter)
4471 {
4472         struct e1000_hw *hw = &adapter->hw;
4473         u32             reg;
4474
4475         /*
4476         ** We get here thru init_locked, meaning
4477         ** a soft reset, this has already cleared
4478         ** the VFTA and other state, so if there
4479         ** have been no vlan's registered do nothing.
4480         */
4481         if (adapter->num_vlans == 0)
4482                 return;
4483
4484         /*
4485         ** A soft reset zero's out the VFTA, so
4486         ** we need to repopulate it now.
4487         */
4488         for (int i = 0; i < IGB_VFTA_SIZE; i++)
4489                 if (igb_shadow_vfta[i] != 0)
4490                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4491                             i, igb_shadow_vfta[i]);
4492
4493         reg = E1000_READ_REG(hw, E1000_CTRL);
4494         reg |= E1000_CTRL_VME;
4495         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4496
4497         /* Enable the Filter Table */
4498         reg = E1000_READ_REG(hw, E1000_RCTL);
4499         reg &= ~E1000_RCTL_CFIEN;
4500         reg |= E1000_RCTL_VFE;
4501         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4502
4503         /* Update the frame size */
4504         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4505             adapter->max_frame_size + VLAN_TAG_SIZE);
4506 }
4507
4508 static void
4509 igb_enable_intr(struct adapter *adapter)
4510 {
4511         /* With RSS set up what to auto clear */
4512         if (adapter->msix_mem) {
4513                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC,
4514                     adapter->eims_mask);
4515                 E1000_WRITE_REG(&adapter->hw, E1000_EIAM,
4516                     adapter->eims_mask);
4517                 E1000_WRITE_REG(&adapter->hw, E1000_EIMS,
4518                     adapter->eims_mask);
4519                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4520                     E1000_IMS_LSC);
4521         } else {
4522                 E1000_WRITE_REG(&adapter->hw, E1000_IMS,
4523                     IMS_ENABLE_MASK);
4524         }
4525         E1000_WRITE_FLUSH(&adapter->hw);
4526
4527         return;
4528 }
4529
4530 static void
4531 igb_disable_intr(struct adapter *adapter)
4532 {
4533         if (adapter->msix_mem) {
4534                 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0);
4535                 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0);
4536         }
4537         E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0);
4538         E1000_WRITE_FLUSH(&adapter->hw);
4539         return;
4540 }
4541
4542 /*
4543  * Bit of a misnomer, what this really means is
4544  * to enable OS management of the system... aka
4545  * to disable special hardware management features
4546  */
4547 static void
4548 igb_init_manageability(struct adapter *adapter)
4549 {
4550         if (adapter->has_manage) {
4551                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4552                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4553
4554                 /* disable hardware interception of ARP */
4555                 manc &= ~(E1000_MANC_ARP_EN);
4556
4557                 /* enable receiving management packets to the host */
4558                 manc |= E1000_MANC_EN_MNG2HOST;
4559                 manc2h |= 1 << 5;  /* Mng Port 623 */
4560                 manc2h |= 1 << 6;  /* Mng Port 664 */
4561                 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4562                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4563         }
4564 }
4565
4566 /*
4567  * Give control back to hardware management
4568  * controller if there is one.
4569  */
4570 static void
4571 igb_release_manageability(struct adapter *adapter)
4572 {
4573         if (adapter->has_manage) {
4574                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4575
4576                 /* re-enable hardware interception of ARP */
4577                 manc |= E1000_MANC_ARP_EN;
4578                 manc &= ~E1000_MANC_EN_MNG2HOST;
4579
4580                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4581         }
4582 }
4583
4584 /*
4585  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
4586  * For ASF and Pass Through versions of f/w this means that
4587  * the driver is loaded.
4588  *
4589  */
4590 static void
4591 igb_get_hw_control(struct adapter *adapter)
4592 {
4593         u32 ctrl_ext;
4594
4595         /* Let firmware know the driver has taken over */
4596         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4597         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4598             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4599 }
4600
4601 /*
4602  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
4603  * For ASF and Pass Through versions of f/w this means that the
4604  * driver is no longer loaded.
4605  *
4606  */
4607 static void
4608 igb_release_hw_control(struct adapter *adapter)
4609 {
4610         u32 ctrl_ext;
4611
4612         /* Let firmware taken over control of h/w */
4613         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4614         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4615             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4616 }
4617
4618 static int
4619 igb_is_valid_ether_addr(uint8_t *addr)
4620 {
4621         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4622
4623         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4624                 return (FALSE);
4625         }
4626
4627         return (TRUE);
4628 }
4629
4630
4631 /*
4632  * Enable PCI Wake On Lan capability
4633  */
4634 void
4635 igb_enable_wakeup(device_t dev)
4636 {
4637         u16     cap, status;
4638         u8      id;
4639
4640         /* First find the capabilities pointer*/
4641         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4642         /* Read the PM Capabilities */
4643         id = pci_read_config(dev, cap, 1);
4644         if (id != PCIY_PMG)     /* Something wrong */
4645                 return;
4646         /* OK, we have the power capabilities, so
4647            now get the status register */
4648         cap += PCIR_POWER_STATUS;
4649         status = pci_read_config(dev, cap, 2);
4650         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4651         pci_write_config(dev, cap, status, 2);
4652         return;
4653 }
4654
4655
4656 /**********************************************************************
4657  *
4658  *  Update the board statistics counters.
4659  *
4660  **********************************************************************/
4661 static void
4662 igb_update_stats_counters(struct adapter *adapter)
4663 {
4664         struct ifnet   *ifp;
4665
4666         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4667            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4668                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4669                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4670         }
4671         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4672         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4673         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4674         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4675
4676         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4677         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4678         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4679         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4680         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4681         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4682         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4683         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4684         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4685         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4686         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4687         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4688         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4689         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4690         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4691         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4692         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4693         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4694         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4695         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4696
4697         /* For the 64-bit byte counters the low dword must be read first. */
4698         /* Both registers clear on the read of the high dword */
4699
4700         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
4701         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
4702
4703         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4704         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4705         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4706         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4707         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4708
4709         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4710         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4711
4712         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
4713         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
4714         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
4715         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
4716         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
4717         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
4718         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
4719         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
4720         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
4721         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
4722
4723         adapter->stats.algnerrc +=
4724                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
4725         adapter->stats.rxerrc +=
4726                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
4727         adapter->stats.tncrs +=
4728                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
4729         adapter->stats.cexterr +=
4730                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
4731         adapter->stats.tsctc +=
4732                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
4733         adapter->stats.tsctfc +=
4734                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
4735         ifp = adapter->ifp;
4736
4737         ifp->if_collisions = adapter->stats.colc;
4738
4739         /* Rx Errors */
4740         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
4741             adapter->stats.crcerrs + adapter->stats.algnerrc +
4742             adapter->stats.ruc + adapter->stats.roc +
4743             adapter->stats.mpc + adapter->stats.cexterr;
4744
4745         /* Tx Errors */
4746         ifp->if_oerrors = adapter->stats.ecol +
4747             adapter->stats.latecol + adapter->watchdog_events;
4748 }
4749
4750
4751 /**********************************************************************
4752  *
4753  *  This routine is called only when igb_display_debug_stats is enabled.
4754  *  This routine provides a way to take a look at important statistics
4755  *  maintained by the driver and hardware.
4756  *
4757  **********************************************************************/
4758 static void
4759 igb_print_debug_info(struct adapter *adapter)
4760 {
4761         device_t dev = adapter->dev;
4762         struct igb_queue *que = adapter->queues;
4763         struct rx_ring *rxr = adapter->rx_rings;
4764         struct tx_ring *txr = adapter->tx_rings;
4765         uint8_t *hw_addr = adapter->hw.hw_addr;
4766
4767         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
4768         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
4769             E1000_READ_REG(&adapter->hw, E1000_CTRL),
4770             E1000_READ_REG(&adapter->hw, E1000_RCTL));
4771
4772 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
4773         device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n",
4774             E1000_READ_REG(&adapter->hw, E1000_IMS),
4775             E1000_READ_REG(&adapter->hw, E1000_EIMS));
4776 #endif
4777
4778         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
4779             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
4780             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
4781         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
4782             adapter->hw.fc.high_water,
4783             adapter->hw.fc.low_water);
4784
4785         for (int i = 0; i < adapter->num_queues; i++, rxr++, txr++) {
4786                 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d  ", i,
4787                     E1000_READ_REG(&adapter->hw, E1000_TDH(i)),
4788                     E1000_READ_REG(&adapter->hw, E1000_TDT(i)));
4789                 device_printf(dev, "rdh = %d, rdt = %d\n",
4790                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4791                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4792                 device_printf(dev, "TX(%d) no descriptors avail event = %lld\n",
4793                     txr->me, (long long)txr->no_desc_avail);
4794                 device_printf(dev, "TX(%d) Packets sent = %lld\n",
4795                     txr->me, (long long)txr->tx_packets);
4796                 device_printf(dev, "RX(%d) Packets received = %lld  ",
4797                     rxr->me, (long long)rxr->rx_packets);
4798         }
4799
4800         for (int i = 0; i < adapter->num_queues; i++, rxr++) {
4801                 struct lro_ctrl *lro = &rxr->lro;
4802                 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i,
4803                     E1000_READ_REG(&adapter->hw, E1000_RDH(i)),
4804                     E1000_READ_REG(&adapter->hw, E1000_RDT(i)));
4805                 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me,
4806                     (long long)rxr->rx_packets);
4807                 device_printf(dev, " Split Packets = %lld ",
4808                     (long long)rxr->rx_split_packets);
4809                 device_printf(dev, " Byte count = %lld\n",
4810                     (long long)rxr->rx_bytes);
4811                 device_printf(dev,"RX(%d) LRO Queued= %d  ",
4812                     i, lro->lro_queued);
4813                 device_printf(dev,"LRO Flushed= %d\n",lro->lro_flushed);
4814         }
4815
4816         for (int i = 0; i < adapter->num_queues; i++, que++)
4817                 device_printf(dev,"QUE(%d) IRQs = %llx\n",
4818                     i, (long long)que->irqs);
4819
4820         device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq);
4821         device_printf(dev, "Mbuf defrag failed = %ld\n",
4822             adapter->mbuf_defrag_failed);
4823         device_printf(dev, "Std mbuf header failed = %ld\n",
4824             adapter->mbuf_header_failed);
4825         device_printf(dev, "Std mbuf packet failed = %ld\n",
4826             adapter->mbuf_packet_failed);
4827         device_printf(dev, "Driver dropped packets = %ld\n",
4828             adapter->dropped_pkts);
4829         device_printf(dev, "Driver tx dma failure in xmit = %ld\n",
4830                 adapter->no_tx_dma_setup);
4831 }
4832
4833 static void
4834 igb_print_hw_stats(struct adapter *adapter)
4835 {
4836         device_t dev = adapter->dev;
4837
4838         device_printf(dev, "Excessive collisions = %lld\n",
4839             (long long)adapter->stats.ecol);
4840 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
4841         device_printf(dev, "Symbol errors = %lld\n",
4842             (long long)adapter->stats.symerrs);
4843 #endif
4844         device_printf(dev, "Sequence errors = %lld\n",
4845             (long long)adapter->stats.sec);
4846         device_printf(dev, "Defer count = %lld\n",
4847             (long long)adapter->stats.dc);
4848         device_printf(dev, "Missed Packets = %lld\n",
4849             (long long)adapter->stats.mpc);
4850         device_printf(dev, "Receive No Buffers = %lld\n",
4851             (long long)adapter->stats.rnbc);
4852         /* RLEC is inaccurate on some hardware, calculate our own. */
4853         device_printf(dev, "Receive Length Errors = %lld\n",
4854             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
4855         device_printf(dev, "Receive errors = %lld\n",
4856             (long long)adapter->stats.rxerrc);
4857         device_printf(dev, "Crc errors = %lld\n",
4858             (long long)adapter->stats.crcerrs);
4859         device_printf(dev, "Alignment errors = %lld\n",
4860             (long long)adapter->stats.algnerrc);
4861         /* On 82575 these are collision counts */
4862         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
4863             (long long)adapter->stats.cexterr);
4864         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
4865         device_printf(dev, "watchdog timeouts = %ld\n",
4866             adapter->watchdog_events);
4867         device_printf(dev, "XON Rcvd = %lld\n",
4868             (long long)adapter->stats.xonrxc);
4869         device_printf(dev, "XON Xmtd = %lld\n",
4870             (long long)adapter->stats.xontxc);
4871         device_printf(dev, "XOFF Rcvd = %lld\n",
4872             (long long)adapter->stats.xoffrxc);
4873         device_printf(dev, "XOFF Xmtd = %lld\n",
4874             (long long)adapter->stats.xofftxc);
4875         device_printf(dev, "Good Packets Rcvd = %lld\n",
4876             (long long)adapter->stats.gprc);
4877         device_printf(dev, "Good Packets Xmtd = %lld\n",
4878             (long long)adapter->stats.gptc);
4879         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
4880             (long long)adapter->stats.tsctc);
4881         device_printf(dev, "TSO Contexts Failed = %lld\n",
4882             (long long)adapter->stats.tsctfc);
4883 }
4884
4885 /**********************************************************************
4886  *
4887  *  This routine provides a way to dump out the adapter eeprom,
4888  *  often a useful debug/service tool. This only dumps the first
4889  *  32 words, stuff that matters is in that extent.
4890  *
4891  **********************************************************************/
4892 static void
4893 igb_print_nvm_info(struct adapter *adapter)
4894 {
4895         u16     eeprom_data;
4896         int     i, j, row = 0;
4897
4898         /* Its a bit crude, but it gets the job done */
4899         printf("\nInterface EEPROM Dump:\n");
4900         printf("Offset\n0x0000  ");
4901         for (i = 0, j = 0; i < 32; i++, j++) {
4902                 if (j == 8) { /* Make the offset block */
4903                         j = 0; ++row;
4904                         printf("\n0x00%x0  ",row);
4905                 }
4906                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
4907                 printf("%04x ", eeprom_data);
4908         }
4909         printf("\n");
4910 }
4911
4912 static int
4913 igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
4914 {
4915         struct adapter *adapter;
4916         int error;
4917         int result;
4918
4919         result = -1;
4920         error = sysctl_handle_int(oidp, &result, 0, req);
4921
4922         if (error || !req->newptr)
4923                 return (error);
4924
4925         if (result == 1) {
4926                 adapter = (struct adapter *)arg1;
4927                 igb_print_debug_info(adapter);
4928         }
4929         /*
4930          * This value will cause a hex dump of the
4931          * first 32 16-bit words of the EEPROM to
4932          * the screen.
4933          */
4934         if (result == 2) {
4935                 adapter = (struct adapter *)arg1;
4936                 igb_print_nvm_info(adapter);
4937         }
4938
4939         return (error);
4940 }
4941
4942
4943 static int
4944 igb_sysctl_stats(SYSCTL_HANDLER_ARGS)
4945 {
4946         struct adapter *adapter;
4947         int error;
4948         int result;
4949
4950         result = -1;
4951         error = sysctl_handle_int(oidp, &result, 0, req);
4952
4953         if (error || !req->newptr)
4954                 return (error);
4955
4956         if (result == 1) {
4957                 adapter = (struct adapter *)arg1;
4958                 igb_print_hw_stats(adapter);
4959         }
4960
4961         return (error);
4962 }
4963
4964 static void
4965 igb_add_rx_process_limit(struct adapter *adapter, const char *name,
4966         const char *description, int *limit, int value)
4967 {
4968         *limit = value;
4969         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
4970             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
4971             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
4972 }