sys/dev/netif/em/if_em.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4
   5 Copyright (c) 2001-2003, Intel Corporation
   6 All rights reserved.
   7
   8 Redistribution and use in source and binary forms, with or without
   9 modification, are permitted provided that the following conditions are met:
  10
  11  1. Redistributions of source code must retain the above copyright notice,
  12     this list of conditions and the following disclaimer.
  13
  14  2. Redistributions in binary form must reproduce the above copyright
  15     notice, this list of conditions and the following disclaimer in the
  16     documentation and/or other materials provided with the distribution.
  17
  18  3. Neither the name of the Intel Corporation nor the names of its
  19     contributors may be used to endorse or promote products derived from
  20     this software without specific prior written permission.
  21
  22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32 POSSIBILITY OF SUCH DAMAGE.
  33
  34 ***************************************************************************/
  35
  36 /*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.15 2003/06/09 22:10:15 pdeuskar Exp $*/
  37 /*$DragonFly: src/sys/dev/netif/em/if_em.c,v 1.17 2004/06/05 13:09:00 joerg Exp $*/
  38
  39 #include <dev/netif/em/if_em.h>
  40
  41 /*********************************************************************
  42  *  Set this to one to display debug statistics
  43  *********************************************************************/
  44 int             em_display_debug_stats = 0;
  45
  46 /*********************************************************************
  47  *  Driver version
  48  *********************************************************************/
  49
  50 char em_driver_version[] = "1.7.25";
  51
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into em_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static em_vendor_info_t em_vendor_info_array[] =
  64 {
  65         /* Intel(R) PRO/1000 Network Connection */
  66         { 0x8086, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0},
  67         { 0x8086, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0},
  68         { 0x8086, 0x1004, PCI_ANY_ID, PCI_ANY_ID, 0},
  69         { 0x8086, 0x1008, PCI_ANY_ID, PCI_ANY_ID, 0},
  70         { 0x8086, 0x1009, PCI_ANY_ID, PCI_ANY_ID, 0},
  71         { 0x8086, 0x100C, PCI_ANY_ID, PCI_ANY_ID, 0},
  72         { 0x8086, 0x100D, PCI_ANY_ID, PCI_ANY_ID, 0},
  73         { 0x8086, 0x100E, PCI_ANY_ID, PCI_ANY_ID, 0},
  74         { 0x8086, 0x100F, PCI_ANY_ID, PCI_ANY_ID, 0},
  75         { 0x8086, 0x1010, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         { 0x8086, 0x1011, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         { 0x8086, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0},
  78         { 0x8086, 0x1013, PCI_ANY_ID, PCI_ANY_ID, 0},
  79         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
  80         { 0x8086, 0x1015, PCI_ANY_ID, PCI_ANY_ID, 0},
  81         { 0x8086, 0x1016, PCI_ANY_ID, PCI_ANY_ID, 0},
  82         { 0x8086, 0x1017, PCI_ANY_ID, PCI_ANY_ID, 0},
  83         { 0x8086, 0x1018, PCI_ANY_ID, PCI_ANY_ID, 0},
  84         { 0x8086, 0x1019, PCI_ANY_ID, PCI_ANY_ID, 0},
  85         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
  86         { 0x8086, 0x101D, PCI_ANY_ID, PCI_ANY_ID, 0},
  87         { 0x8086, 0x101E, PCI_ANY_ID, PCI_ANY_ID, 0},
  88         { 0x8086, 0x1026, PCI_ANY_ID, PCI_ANY_ID, 0},
  89         { 0x8086, 0x1027, PCI_ANY_ID, PCI_ANY_ID, 0},
  90         { 0x8086, 0x1028, PCI_ANY_ID, PCI_ANY_ID, 0},
  91         { 0x8086, 0x1075, PCI_ANY_ID, PCI_ANY_ID, 0},
  92         { 0x8086, 0x1076, PCI_ANY_ID, PCI_ANY_ID, 0},
  93         { 0x8086, 0x1077, PCI_ANY_ID, PCI_ANY_ID, 0},
  94         { 0x8086, 0x1078, PCI_ANY_ID, PCI_ANY_ID, 0},
  95         { 0x8086, 0x1079, PCI_ANY_ID, PCI_ANY_ID, 0},
  96         { 0x8086, 0x107A, PCI_ANY_ID, PCI_ANY_ID, 0},
  97         { 0x8086, 0x107B, PCI_ANY_ID, PCI_ANY_ID, 0},
  98         /* required last entry */
  99         { 0, 0, 0, 0, 0}
 100 };
 101
 102 /*********************************************************************
 103  *  Table of branding strings for all supported NICs.
 104  *********************************************************************/
 105
 106 static const char *em_strings[] = {
 107         "Intel(R) PRO/1000 Network Connection"
 108 };
 109
 110 /*********************************************************************
 111  *  Function prototypes
 112  *********************************************************************/
 113 static int      em_probe(device_t);
 114 static int      em_attach(device_t);
 115 static int      em_detach(device_t);
 116 static int      em_shutdown(device_t);
 117 static void     em_intr(void *);
 118 static void     em_start(struct ifnet *);
 119 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 120 static void     em_watchdog(struct ifnet *);
 121 static void     em_init(void *);
 122 static void     em_stop(void *);
 123 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 124 static int      em_media_change(struct ifnet *);
 125 static void     em_identify_hardware(struct adapter *);
 126 static void     em_local_timer(void *);
 127 static int      em_hardware_init(struct adapter *);
 128 static void     em_setup_interface(device_t, struct adapter *);
 129 static int      em_setup_transmit_structures(struct adapter *);
 130 static void     em_initialize_transmit_unit(struct adapter *);
 131 static int      em_setup_receive_structures(struct adapter *);
 132 static void     em_initialize_receive_unit(struct adapter *);
 133 static void     em_enable_intr(struct adapter *);
 134 static void     em_disable_intr(struct adapter *);
 135 static void     em_free_transmit_structures(struct adapter *);
 136 static void     em_free_receive_structures(struct adapter *);
 137 static void     em_update_stats_counters(struct adapter *);
 138 static void     em_clean_transmit_interrupts(struct adapter *);
 139 static int      em_allocate_receive_structures(struct adapter *);
 140 static int      em_allocate_transmit_structures(struct adapter *);
 141 static void     em_process_receive_interrupts(struct adapter *, int);
 142 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 143                                     struct mbuf *);
 144 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 145                                            uint32_t *, uint32_t *);
 146 static void     em_set_promisc(struct adapter *);
 147 static void     em_disable_promisc(struct adapter *);
 148 static void     em_set_multi(struct adapter *);
 149 static void     em_print_hw_stats(struct adapter *);
 150 static void     em_print_link_status(struct adapter *);
 151 static int      em_get_buf(int i, struct adapter *, struct mbuf *);
 152 static void     em_enable_vlans(struct adapter *);
 153 static int      em_encap(struct adapter *, struct mbuf *);
 154 static void     em_smartspeed(struct adapter *);
 155 static int      em_82547_fifo_workaround(struct adapter *, int);
 156 static void     em_82547_update_fifo_head(struct adapter *, int);
 157 static int      em_82547_tx_fifo_reset(struct adapter *);
 158 static void     em_82547_move_tail(void *arg);
 159 static int      em_dma_malloc(struct adapter *, bus_size_t,
 160                               struct em_dma_alloc *, int);
 161 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 162 static void     em_print_debug_info(struct adapter *);
 163 static int      em_is_valid_ether_addr(uint8_t *);
 164 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 165 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 166 static uint32_t em_fill_descriptors(uint64_t address, uint32_t length,
 167                                    PDESC_ARRAY desc_array);
 168 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 169 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 170                                         const char *,
 171                                         struct em_int_delay_info *, int, int);
 172
 173 /*********************************************************************
 174  *  FreeBSD Device Interface Entry Points
 175  *********************************************************************/
 176
 177 static device_method_t em_methods[] = {
 178         /* Device interface */
 179         DEVMETHOD(device_probe, em_probe),
 180         DEVMETHOD(device_attach, em_attach),
 181         DEVMETHOD(device_detach, em_detach),
 182         DEVMETHOD(device_shutdown, em_shutdown),
 183         {0, 0}
 184 };
 185
 186 static driver_t em_driver = {
 187         "em", em_methods, sizeof(struct adapter),
 188 };
 189
 190 static devclass_t em_devclass;
 191
 192 DECLARE_DUMMY_MODULE(if_em);
 193 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 194
 195 /*********************************************************************
 196  *  Tunable default values.
 197  *********************************************************************/
 198
 199 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 200 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 201
 202 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 203 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 204 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 205 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 206
 207 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 208 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 209 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 210 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 211
 212 /*********************************************************************
 213  *  Device identification routine
 214  *
 215  *  em_probe determines if the driver should be loaded on
 216  *  adapter based on PCI vendor/device id of the adapter.
 217  *
 218  *  return 0 on success, positive on failure
 219  *********************************************************************/
 220
 221 static int
 222 em_probe(device_t dev)
 223 {
 224         em_vendor_info_t *ent;
 225
 226         uint16_t pci_vendor_id = 0;
 227         uint16_t pci_device_id = 0;
 228         uint16_t pci_subvendor_id = 0;
 229         uint16_t pci_subdevice_id = 0;
 230         char adapter_name[60];
 231
 232         INIT_DEBUGOUT("em_probe: begin");
 233
 234         pci_vendor_id = pci_get_vendor(dev);
 235         if (pci_vendor_id != EM_VENDOR_ID)
 236                 return(ENXIO);
 237
 238         pci_device_id = pci_get_device(dev);
 239         pci_subvendor_id = pci_get_subvendor(dev);
 240         pci_subdevice_id = pci_get_subdevice(dev);
 241
 242         ent = em_vendor_info_array;
 243         while (ent->vendor_id != 0) {
 244                 if ((pci_vendor_id == ent->vendor_id) &&
 245                     (pci_device_id == ent->device_id) &&
 246
 247                     ((pci_subvendor_id == ent->subvendor_id) ||
 248                      (ent->subvendor_id == PCI_ANY_ID)) &&
 249
 250                     ((pci_subdevice_id == ent->subdevice_id) ||
 251                      (ent->subdevice_id == PCI_ANY_ID))) {
 252                         snprintf(adapter_name, sizeof(adapter_name),
 253                                  "%s, Version - %s",  em_strings[ent->index],
 254                                  em_driver_version);
 255                         device_set_desc_copy(dev, adapter_name);
 256                         return(0);
 257                 }
 258                 ent++;
 259         }
 260
 261         return(ENXIO);
 262 }
 263
 264 /*********************************************************************
 265  *  Device initialization routine
 266  *
 267  *  The attach entry point is called when the driver is being loaded.
 268  *  This routine identifies the type of hardware, allocates all resources
 269  *  and initializes the hardware.
 270  *
 271  *  return 0 on success, positive on failure
 272  *********************************************************************/
 273
 274 static int
 275 em_attach(device_t dev)
 276 {
 277         struct adapter *adapter;
 278         int tsize, rsize;
 279         int i, val, rid;
 280         int error = 0;
 281
 282         INIT_DEBUGOUT("em_attach: begin");
 283
 284         adapter = device_get_softc(dev);
 285
 286         bzero(adapter, sizeof(struct adapter));
 287
 288         callout_init(&adapter->timer);
 289         callout_init(&adapter->tx_fifo_timer);
 290
 291         adapter->dev = dev;
 292         adapter->osdep.dev = dev;
 293
 294         /* SYSCTL stuff */
 295         sysctl_ctx_init(&adapter->sysctl_ctx);
 296         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 297                                                SYSCTL_STATIC_CHILDREN(_hw),
 298                                                OID_AUTO,
 299                                                device_get_nameunit(dev),
 300                                                CTLFLAG_RD,
 301                                                0, "");
 302
 303         if (adapter->sysctl_tree == NULL) {
 304                 error = EIO;
 305                 goto fail;
 306         }
 307
 308         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 309                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 310                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 311                         (void *)adapter, 0,
 312                         em_sysctl_debug_info, "I", "Debug Information");
 313
 314         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 315                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 316                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 317                         (void *)adapter, 0,
 318                         em_sysctl_stats, "I", "Statistics");
 319
 320         /* Determine hardware revision */
 321         em_identify_hardware(adapter);
 322
 323         /* Set up some sysctls for the tunable interrupt delays */
 324         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 325                                 "receive interrupt delay in usecs",
 326                                 &adapter->rx_int_delay,
 327                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 328                                 em_rx_int_delay_dflt);
 329         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 330                                 "transmit interrupt delay in usecs",
 331                                 &adapter->tx_int_delay,
 332                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 333                                 em_tx_int_delay_dflt);
 334         if (adapter->hw.mac_type >= em_82540) {
 335                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 336                                         "receive interrupt delay limit in usecs",
 337                                         &adapter->rx_abs_int_delay,
 338                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 339                                         em_rx_abs_int_delay_dflt);
 340                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 341                                         "transmit interrupt delay limit in usecs",
 342                                         &adapter->tx_abs_int_delay,
 343                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 344                                         em_tx_abs_int_delay_dflt);
 345         }
 346
 347         /* Parameters (to be read from user) */
 348         adapter->num_tx_desc = EM_MAX_TXD;
 349         adapter->num_rx_desc = EM_MAX_RXD;
 350         adapter->hw.autoneg = DO_AUTO_NEG;
 351         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 352         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 353         adapter->hw.tbi_compatibility_en = TRUE;
 354         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 355
 356         /*
 357          * These parameters control the automatic generation(Tx) and
 358          * response(Rx) to Ethernet PAUSE frames.
 359          */
 360         adapter->hw.fc_high_water = FC_DEFAULT_HI_THRESH;
 361         adapter->hw.fc_low_water  = FC_DEFAULT_LO_THRESH;
 362         adapter->hw.fc_pause_time = FC_DEFAULT_TX_TIMER;
 363         adapter->hw.fc_send_xon   = TRUE;
 364         adapter->hw.fc = em_fc_full;
 365
 366         adapter->hw.phy_init_script = 1;
 367         adapter->hw.phy_reset_disable = FALSE;
 368
 369 #ifndef EM_MASTER_SLAVE
 370         adapter->hw.master_slave = em_ms_hw_default;
 371 #else
 372         adapter->hw.master_slave = EM_MASTER_SLAVE;
 373 #endif
 374
 375         /*
 376          * Set the max frame size assuming standard ethernet
 377          * sized frames
 378          */
 379         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 380
 381         adapter->hw.min_frame_size =
 382             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 383
 384         /*
 385          * This controls when hardware reports transmit completion
 386          * status.
 387          */
 388         adapter->hw.report_tx_early = 1;
 389
 390         rid = EM_MMBA;
 391         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 392                                                      &rid, RF_ACTIVE);
 393         if (!(adapter->res_memory)) {
 394                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 395                 error = ENXIO;
 396                 goto fail;
 397         }
 398         adapter->osdep.mem_bus_space_tag =
 399             rman_get_bustag(adapter->res_memory);
 400         adapter->osdep.mem_bus_space_handle =
 401             rman_get_bushandle(adapter->res_memory);
 402         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
 403
 404         if (adapter->hw.mac_type > em_82543) {
 405                 /* Figure our where our IO BAR is ? */
 406                 rid = EM_MMBA;
 407                 for (i = 0; i < 5; i++) {
 408                         val = pci_read_config(dev, rid, 4);
 409                         if (val & 0x00000001) {
 410                                 adapter->io_rid = rid;
 411                                 break;
 412                         }
 413                         rid += 4;
 414                 }
 415
 416                 adapter->res_ioport = bus_alloc_resource_any(dev,
 417                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
 418                 if (!(adapter->res_ioport)) {
 419                         device_printf(dev, "Unable to allocate bus resource: ioport\n");
 420                         error = ENXIO;
 421                         goto fail;
 422                 }
 423
 424                 adapter->hw.reg_io_tag = rman_get_bustag(adapter->res_ioport);
 425                 adapter->hw.reg_io_handle = rman_get_bushandle(adapter->res_ioport);
 426         }
 427
 428         rid = 0x0;
 429         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 430             &rid, RF_SHAREABLE | RF_ACTIVE);
 431         if (!(adapter->res_interrupt)) {
 432                 device_printf(dev, "Unable to allocate bus resource: interrupt\n");
 433                 error = ENXIO;
 434                 goto fail;
 435         }
 436
 437         adapter->hw.back = &adapter->osdep;
 438
 439         /* Initialize eeprom parameters */
 440         em_init_eeprom_params(&adapter->hw);
 441
 442         tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc);
 443
 444         /* Allocate Transmit Descriptor ring */
 445         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_WAITOK)) {
 446                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 447                 error = ENOMEM;
 448                 goto fail;
 449         }
 450         adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr;
 451
 452         rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc);
 453
 454         /* Allocate Receive Descriptor ring */
 455         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 456                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 457                 error = ENOMEM;
 458                 goto fail;
 459         }
 460         adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr;
 461
 462         /* Initialize the hardware */
 463         if (em_hardware_init(adapter)) {
 464                 device_printf(dev, "Unable to initialize the hardware\n");
 465                 error = EIO;
 466                 goto fail;
 467         }
 468
 469         /* Copy the permanent MAC address out of the EEPROM */
 470         if (em_read_mac_addr(&adapter->hw) < 0) {
 471                 device_printf(dev, "EEPROM read error while reading mac address\n");
 472                 error = EIO;
 473                 goto fail;
 474         }
 475
 476         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 477                 device_printf(dev, "Invalid mac address\n");
 478                 error = EIO;
 479                 goto fail;
 480         }
 481
 482         /* Setup OS specific network interface */
 483         em_setup_interface(dev, adapter);
 484
 485         /* Initialize statistics */
 486         em_clear_hw_cntrs(&adapter->hw);
 487         em_update_stats_counters(adapter);
 488         adapter->hw.get_link_status = 1;
 489         em_check_for_link(&adapter->hw);
 490
 491         /* Print the link status */
 492         if (adapter->link_active == 1) {
 493                 em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
 494                                         &adapter->link_duplex);
 495                 device_printf(dev, "Speed: %d Mbps, Duplex: %s\n",
 496                     adapter->link_speed,
 497                     adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
 498         } else
 499                 device_printf(dev, "Speed: N/A, Duplex:N/A\n");
 500
 501         /* Identify 82544 on PCIX */
 502         em_get_bus_info(&adapter->hw);
 503         if (adapter->hw.bus_type == em_bus_type_pcix &&
 504             adapter->hw.mac_type == em_82544)
 505                 adapter->pcix_82544 = TRUE;
 506         else
 507                 adapter->pcix_82544 = FALSE;
 508
 509         error = bus_setup_intr(dev, adapter->res_interrupt, INTR_TYPE_NET,
 510                            (void (*)(void *)) em_intr, adapter,
 511                            &adapter->int_handler_tag);
 512         if (error) {
 513                 device_printf(dev, "Error registering interrupt handler!\n");
 514                 ether_ifdetach(&adapter->interface_data.ac_if);
 515                 goto fail;
 516         }
 517
 518         INIT_DEBUGOUT("em_attach: end");
 519         return(0);
 520
 521 fail:
 522         em_detach(dev);
 523         return(error);
 524 }
 525
 526 /*********************************************************************
 527  *  Device removal routine
 528  *
 529  *  The detach entry point is called when the driver is being removed.
 530  *  This routine stops the adapter and deallocates all the resources
 531  *  that were allocated for driver operation.
 532  *
 533  *  return 0 on success, positive on failure
 534  *********************************************************************/
 535
 536 static int
 537 em_detach(device_t dev)
 538 {
 539         struct adapter * adapter = device_get_softc(dev);
 540         int s;
 541
 542         INIT_DEBUGOUT("em_detach: begin");
 543         s = splimp();
 544
 545         adapter->in_detach = 1;
 546
 547         if (device_is_attached(dev)) {
 548                 em_stop(adapter);
 549                 em_phy_hw_reset(&adapter->hw);
 550                 ether_ifdetach(&adapter->interface_data.ac_if);
 551         }
 552         bus_generic_detach(dev);
 553
 554         if (adapter->res_interrupt != NULL) {
 555                 bus_teardown_intr(dev, adapter->res_interrupt,
 556                                   adapter->int_handler_tag);
 557                 bus_release_resource(dev, SYS_RES_IRQ, 0,
 558                                      adapter->res_interrupt);
 559         }
 560         if (adapter->res_memory != NULL) {
 561                 bus_release_resource(dev, SYS_RES_MEMORY, EM_MMBA,
 562                                      adapter->res_memory);
 563         }
 564
 565         if (adapter->res_ioport != NULL) {
 566                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
 567                                      adapter->res_ioport);
 568         }
 569
 570         /* Free Transmit Descriptor ring */
 571         if (adapter->tx_desc_base != NULL) {
 572                 em_dma_free(adapter, &adapter->txdma);
 573                 adapter->tx_desc_base = NULL;
 574         }
 575
 576         /* Free Receive Descriptor ring */
 577         if (adapter->rx_desc_base != NULL) {
 578                 em_dma_free(adapter, &adapter->rxdma);
 579                 adapter->rx_desc_base = NULL;
 580         }
 581
 582         adapter->sysctl_tree = NULL;
 583         sysctl_ctx_free(&adapter->sysctl_ctx);
 584
 585         splx(s);
 586         return(0);
 587 }
 588
 589 /*********************************************************************
 590  *
 591  *  Shutdown entry point
 592  *
 593  **********************************************************************/
 594
 595 static int
 596 em_shutdown(device_t dev)
 597 {
 598         struct adapter *adapter = device_get_softc(dev);
 599         em_stop(adapter);
 600         return(0);
 601 }
 602
 603 /*********************************************************************
 604  *  Transmit entry point
 605  *
 606  *  em_start is called by the stack to initiate a transmit.
 607  *  The driver will remain in this routine as long as there are
 608  *  packets to transmit and transmit resources are available.
 609  *  In case resources are not available stack is notified and
 610  *  the packet is requeued.
 611  **********************************************************************/
 612
 613 static void
 614 em_start(struct ifnet *ifp)
 615 {
 616         int s;
 617         struct mbuf *m_head;
 618         struct adapter *adapter = ifp->if_softc;
 619
 620         if (!adapter->link_active)
 621                 return;
 622
 623         s = splimp();
 624         while (ifp->if_snd.ifq_head != NULL) {
 625                 IF_DEQUEUE(&ifp->if_snd, m_head);
 626
 627                 if (m_head == NULL)
 628                         break;
 629
 630                 if (em_encap(adapter, m_head)) {
 631                         ifp->if_flags |= IFF_OACTIVE;
 632                         IF_PREPEND(&ifp->if_snd, m_head);
 633                         break;
 634                 }
 635
 636                 /* Send a copy of the frame to the BPF listener */
 637                 BPF_MTAP(ifp, m_head);
 638
 639                 /* Set timeout in case hardware has problems transmitting */
 640                 ifp->if_timer = EM_TX_TIMEOUT;
 641         }
 642         splx(s);
 643 }
 644
 645 /*********************************************************************
 646  *  Ioctl entry point
 647  *
 648  *  em_ioctl is called when the user wants to configure the
 649  *  interface.
 650  *
 651  *  return 0 on success, positive on failure
 652  **********************************************************************/
 653
 654 static int
 655 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 656 {
 657         int s, mask, error = 0;
 658         struct ifreq *ifr = (struct ifreq *) data;
 659         struct adapter *adapter = ifp->if_softc;
 660
 661         s = splimp();
 662
 663         if (adapter->in_detach)
 664                 goto out;
 665
 666         switch (command) {
 667         case SIOCSIFADDR:
 668         case SIOCGIFADDR:
 669                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 670                 ether_ioctl(ifp, command, data);
 671                 break;
 672         case SIOCSIFMTU:
 673                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 674                 if (ifr->ifr_mtu > MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 675                         error = EINVAL;
 676                 } else {
 677                         ifp->if_mtu = ifr->ifr_mtu;
 678                         adapter->hw.max_frame_size =
 679                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 680                         em_init(adapter);
 681                 }
 682                 break;
 683         case SIOCSIFFLAGS:
 684                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 685                 if (ifp->if_flags & IFF_UP) {
 686                         if (!(ifp->if_flags & IFF_RUNNING))
 687                                 em_init(adapter);
 688                         em_disable_promisc(adapter);
 689                         em_set_promisc(adapter);
 690                 } else {
 691                         if (ifp->if_flags & IFF_RUNNING)
 692                                 em_stop(adapter);
 693                 }
 694                 break;
 695         case SIOCADDMULTI:
 696         case SIOCDELMULTI:
 697                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 698                 if (ifp->if_flags & IFF_RUNNING) {
 699                         em_disable_intr(adapter);
 700                         em_set_multi(adapter);
 701                         if (adapter->hw.mac_type == em_82542_rev2_0)
 702                                 em_initialize_receive_unit(adapter);
 703 #ifdef DEVICE_POLLING
 704                         if (!(ifp->if_flags & IFF_POLLING))
 705 #endif
 706                                 em_enable_intr(adapter);
 707                 }
 708                 break;
 709         case SIOCSIFMEDIA:
 710         case SIOCGIFMEDIA:
 711                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 712                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 713                 break;
 714         case SIOCSIFCAP:
 715                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 716                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 717                 if (mask & IFCAP_HWCSUM) {
 718                         if (IFCAP_HWCSUM & ifp->if_capenable)
 719                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 720                         else
 721                                 ifp->if_capenable |= IFCAP_HWCSUM;
 722                         if (ifp->if_flags & IFF_RUNNING)
 723                                 em_init(adapter);
 724                 }
 725                 break;
 726         default:
 727                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%x)\n", (int)command);
 728                 error = EINVAL;
 729         }
 730
 731 out:
 732         splx(s);
 733         return(error);
 734 }
 735
 736 /*********************************************************************
 737  *  Watchdog entry point
 738  *
 739  *  This routine is called whenever hardware quits transmitting.
 740  *
 741  **********************************************************************/
 742
 743 static void
 744 em_watchdog(struct ifnet *ifp)
 745 {
 746         struct adapter * adapter;
 747         adapter = ifp->if_softc;
 748
 749         /* If we are in this routine because of pause frames, then
 750          * don't reset the hardware.
 751          */
 752         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 753                 ifp->if_timer = EM_TX_TIMEOUT;
 754                 return;
 755         }
 756
 757         if (em_check_for_link(&adapter->hw))
 758                 if_printf(ifp, "watchdog timeout -- resetting\n");
 759
 760         ifp->if_flags &= ~IFF_RUNNING;
 761
 762         em_init(adapter);
 763
 764         ifp->if_oerrors++;
 765 }
 766
 767 /*********************************************************************
 768  *  Init entry point
 769  *
 770  *  This routine is used in two ways. It is used by the stack as
 771  *  init entry point in network interface structure. It is also used
 772  *  by the driver as a hw/sw initialization routine to get to a
 773  *  consistent state.
 774  *
 775  *  return 0 on success, positive on failure
 776  **********************************************************************/
 777
 778 static void
 779 em_init(void *arg)
 780 {
 781         int s;
 782         struct adapter *adapter = arg;
 783         struct ifnet *ifp = &adapter->interface_data.ac_if;
 784
 785         INIT_DEBUGOUT("em_init: begin");
 786
 787         s = splimp();
 788
 789         em_stop(adapter);
 790
 791         /* Get the latest mac address, User can use a LAA */
 792         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
 793               ETHER_ADDR_LEN);
 794
 795         /* Initialize the hardware */
 796         if (em_hardware_init(adapter)) {
 797                 if_printf(ifp, "Unable to initialize the hardware\n");
 798                 splx(s);
 799                 return;
 800         }
 801
 802         em_enable_vlans(adapter);
 803
 804         /* Prepare transmit descriptors and buffers */
 805         if (em_setup_transmit_structures(adapter)) {
 806                 if_printf(ifp, "Could not setup transmit structures\n");
 807                 em_stop(adapter);
 808                 splx(s);
 809                 return;
 810         }
 811         em_initialize_transmit_unit(adapter);
 812
 813         /* Setup Multicast table */
 814         em_set_multi(adapter);
 815
 816         /* Prepare receive descriptors and buffers */
 817         if (em_setup_receive_structures(adapter)) {
 818                 if_printf(ifp, "Could not setup receive structures\n");
 819                 em_stop(adapter);
 820                 splx(s);
 821                 return;
 822         }
 823         em_initialize_receive_unit(adapter);
 824
 825         /* Don't loose promiscuous settings */
 826         em_set_promisc(adapter);
 827
 828         ifp->if_flags |= IFF_RUNNING;
 829         ifp->if_flags &= ~IFF_OACTIVE;
 830
 831         if (adapter->hw.mac_type >= em_82543) {
 832                 if (ifp->if_capenable & IFCAP_TXCSUM)
 833                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
 834                 else
 835                         ifp->if_hwassist = 0;
 836         }
 837
 838         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 839         em_clear_hw_cntrs(&adapter->hw);
 840 #ifdef DEVICE_POLLING
 841         /*
 842          * Only enable interrupts if we are not polling, make sure
 843          * they are off otherwise.
 844          */
 845         if (ifp->if_flags & IFF_POLLING)
 846                 em_disable_intr(adapter);
 847         else
 848 #endif /* DEVICE_POLLING */
 849                 em_enable_intr(adapter);
 850
 851         /* Don't reset the phy next time init gets called */
 852         adapter->hw.phy_reset_disable = TRUE;
 853
 854         splx(s);
 855 }
 856
 857 #ifdef DEVICE_POLLING
 858 static poll_handler_t em_poll;
 859
 860 static void
 861 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 862 {
 863         struct adapter *adapter = ifp->if_softc;
 864         uint32_t reg_icr;
 865
 866         if (cmd == POLL_DEREGISTER) {       /* final call, enable interrupts */
 867                 em_enable_intr(adapter);
 868                 return;
 869         }
 870         if (cmd == POLL_AND_CHECK_STATUS) {
 871                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 872                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 873                         callout_stop(&adapter->timer);
 874                         adapter->hw.get_link_status = 1;
 875                         em_check_for_link(&adapter->hw);
 876                         em_print_link_status(adapter);
 877                         callout_reset(&adapter->timer, 2*hz, em_local_timer,
 878                                       adapter);
 879                 }
 880         }
 881         if (ifp->if_flags & IFF_RUNNING) {
 882                 em_process_receive_interrupts(adapter, count);
 883                 em_clean_transmit_interrupts(adapter);
 884         }
 885
 886         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 887                 em_start(ifp);
 888 }
 889 #endif /* DEVICE_POLLING */
 890
 891 /*********************************************************************
 892  *
 893  *  Interrupt Service routine
 894  *
 895  **********************************************************************/
 896 static void
 897 em_intr(void *arg)
 898 {
 899         uint32_t loop_cnt = EM_MAX_INTR;
 900         uint32_t reg_icr;
 901         struct ifnet *ifp;
 902         struct adapter *adapter = arg;
 903
 904         ifp = &adapter->interface_data.ac_if;
 905
 906 #ifdef DEVICE_POLLING
 907         if (ifp->if_flags & IFF_POLLING)
 908                 return;
 909
 910         if (ether_poll_register(em_poll, ifp)) {
 911                 em_disable_intr(adapter);
 912                 em_poll(ifp, 0, 1);
 913                 return;
 914         }
 915 #endif /* DEVICE_POLLING */
 916
 917         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 918         if (!reg_icr)
 919                 return;
 920
 921         /* Link status change */
 922         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 923                 callout_stop(&adapter->timer);
 924                 adapter->hw.get_link_status = 1;
 925                 em_check_for_link(&adapter->hw);
 926                 em_print_link_status(adapter);
 927                 callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 928         }
 929
 930         while (loop_cnt > 0) {
 931                 if (ifp->if_flags & IFF_RUNNING) {
 932                         em_process_receive_interrupts(adapter, -1);
 933                         em_clean_transmit_interrupts(adapter);
 934                 }
 935                 loop_cnt--;
 936         }
 937
 938         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 939                 em_start(ifp);
 940 }
 941
 942 /*********************************************************************
 943  *
 944  *  Media Ioctl callback
 945  *
 946  *  This routine is called whenever the user queries the status of
 947  *  the interface using ifconfig.
 948  *
 949  **********************************************************************/
 950 static void
 951 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 952 {
 953         struct adapter * adapter = ifp->if_softc;
 954
 955         INIT_DEBUGOUT("em_media_status: begin");
 956
 957         em_check_for_link(&adapter->hw);
 958         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
 959                 if (adapter->link_active == 0) {
 960                         em_get_speed_and_duplex(&adapter->hw,
 961                                                 &adapter->link_speed,
 962                                                 &adapter->link_duplex);
 963                         adapter->link_active = 1;
 964                 }
 965         } else {
 966                 if (adapter->link_active == 1) {
 967                         adapter->link_speed = 0;
 968                         adapter->link_duplex = 0;
 969                         adapter->link_active = 0;
 970                 }
 971         }
 972
 973         ifmr->ifm_status = IFM_AVALID;
 974         ifmr->ifm_active = IFM_ETHER;
 975
 976         if (!adapter->link_active)
 977                 return;
 978
 979         ifmr->ifm_status |= IFM_ACTIVE;
 980
 981         if (adapter->hw.media_type == em_media_type_fiber) {
 982                 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 983         } else {
 984                 switch (adapter->link_speed) {
 985                 case 10:
 986                         ifmr->ifm_active |= IFM_10_T;
 987                         break;
 988                 case 100:
 989                         ifmr->ifm_active |= IFM_100_TX;
 990                         break;
 991                 case 1000:
 992                         ifmr->ifm_active |= IFM_1000_TX;
 993                         break;
 994                 }
 995                 if (adapter->link_duplex == FULL_DUPLEX)
 996                         ifmr->ifm_active |= IFM_FDX;
 997                 else
 998                         ifmr->ifm_active |= IFM_HDX;
 999         }
1000 }
1001
1002 /*********************************************************************
1003  *
1004  *  Media Ioctl callback
1005  *
1006  *  This routine is called when the user changes speed/duplex using
1007  *  media/mediopt option with ifconfig.
1008  *
1009  **********************************************************************/
1010 static int
1011 em_media_change(struct ifnet *ifp)
1012 {
1013         struct adapter * adapter = ifp->if_softc;
1014         struct ifmedia  *ifm = &adapter->media;
1015
1016         INIT_DEBUGOUT("em_media_change: begin");
1017
1018         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1019                 return(EINVAL);
1020
1021         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1022         case IFM_AUTO:
1023                 adapter->hw.autoneg = DO_AUTO_NEG;
1024                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1025                 break;
1026         case IFM_1000_SX:
1027         case IFM_1000_TX:
1028                 adapter->hw.autoneg = DO_AUTO_NEG;
1029                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1030                 break;
1031         case IFM_100_TX:
1032                 adapter->hw.autoneg = FALSE;
1033                 adapter->hw.autoneg_advertised = 0;
1034                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1035                         adapter->hw.forced_speed_duplex = em_100_full;
1036                 else
1037                         adapter->hw.forced_speed_duplex = em_100_half;
1038                 break;
1039         case IFM_10_T:
1040                 adapter->hw.autoneg = FALSE;
1041                 adapter->hw.autoneg_advertised = 0;
1042                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1043                         adapter->hw.forced_speed_duplex = em_10_full;
1044                 else
1045                         adapter->hw.forced_speed_duplex = em_10_half;
1046                 break;
1047         default:
1048                 if_printf(ifp, "Unsupported media type\n");
1049         }
1050         /*
1051          * As the speed/duplex settings may have changed we need to
1052          * reset the PHY.
1053          */
1054         adapter->hw.phy_reset_disable = FALSE;
1055
1056         em_init(adapter);
1057
1058         return(0);
1059 }
1060
1061 static void
1062 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1063          int error)
1064 {
1065         struct em_q *q = arg;
1066
1067         if (error)
1068                 return;
1069         KASSERT(nsegs <= EM_MAX_SCATTER,
1070                 ("Too many DMA segments returned when mapping tx packet"));
1071         q->nsegs = nsegs;
1072         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1073 }
1074
1075 #define EM_FIFO_HDR              0x10
1076 #define EM_82547_PKT_THRESH      0x3e0
1077 #define EM_82547_TX_FIFO_SIZE    0x2800
1078 #define EM_82547_TX_FIFO_BEGIN   0xf00
1079 /*********************************************************************
1080  *
1081  *  This routine maps the mbufs to tx descriptors.
1082  *
1083  *  return 0 on success, positive on failure
1084  **********************************************************************/
1085 static int
1086 em_encap(struct adapter *adapter, struct mbuf *m_head)
1087 {
1088         uint32_t txd_upper;
1089         uint32_t txd_lower, txd_used = 0, txd_saved = 0;
1090         int i, j, error;
1091         uint64_t address;
1092
1093         /* For 82544 Workaround */
1094         DESC_ARRAY desc_array;
1095         uint32_t array_elements;
1096         uint32_t counter;
1097
1098 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1099         struct ifvlan *ifv = NULL;
1100 #else
1101         struct m_tag *mtag;
1102 #endif
1103         struct em_q q;
1104         struct em_buffer *tx_buffer = NULL;
1105         struct em_tx_desc *current_tx_desc = NULL;
1106         struct ifnet *ifp = &adapter->interface_data.ac_if;
1107
1108         /*
1109          * Force a cleanup if number of TX descriptors
1110          * available hits the threshold
1111          */
1112         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1113                 em_clean_transmit_interrupts(adapter);
1114                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1115                         adapter->no_tx_desc_avail1++;
1116                         return(ENOBUFS);
1117                 }
1118         }
1119         /*
1120          * Map the packet for DMA.
1121          */
1122         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q.map)) {
1123                 adapter->no_tx_map_avail++;
1124                 return(ENOMEM);
1125         }
1126         error = bus_dmamap_load_mbuf(adapter->txtag, q.map, m_head, em_tx_cb,
1127                                      &q, BUS_DMA_NOWAIT);
1128         if (error != 0) {
1129                 adapter->no_tx_dma_setup++;
1130                 bus_dmamap_destroy(adapter->txtag, q.map);
1131                 return(error);
1132         }
1133         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1134
1135         if (q.nsegs > adapter->num_tx_desc_avail) {
1136                 adapter->no_tx_desc_avail2++;
1137                 bus_dmamap_destroy(adapter->txtag, q.map);
1138                 return(ENOBUFS);
1139         }
1140
1141         if (ifp->if_hwassist > 0) {
1142                 em_transmit_checksum_setup(adapter,  m_head,
1143                                            &txd_upper, &txd_lower);
1144         }
1145         else
1146                 txd_upper = txd_lower = 0;
1147
1148         /* Find out if we are in vlan mode */
1149 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1150         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1151             m_head->m_pkthdr.rcvif != NULL &&
1152             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1153                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1154 #else
1155         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1156 #endif
1157
1158         i = adapter->next_avail_tx_desc;
1159         if (adapter->pcix_82544) {
1160                 txd_saved = i;
1161                 txd_used = 0;
1162         }
1163         for (j = 0; j < q.nsegs; j++) {
1164                 /* If adapter is 82544 and on PCIX bus */
1165                 if(adapter->pcix_82544) {
1166                         array_elements = 0;
1167                         address = htole64(q.segs[j].ds_addr);
1168                         /*
1169                          * Check the Address and Length combination and
1170                          * split the data accordingly
1171                          */
1172                         array_elements = em_fill_descriptors(address,
1173                                                              htole32(q.segs[j].ds_len),
1174                                                              &desc_array);
1175                         for (counter = 0; counter < array_elements; counter++) {
1176                                 if (txd_used == adapter->num_tx_desc_avail) {
1177                                         adapter->next_avail_tx_desc = txd_saved;
1178                                         adapter->no_tx_desc_avail2++;
1179                                         bus_dmamap_destroy(adapter->txtag, q.map);
1180                                         return(ENOBUFS);
1181                                 }
1182                                 tx_buffer = &adapter->tx_buffer_area[i];
1183                                 current_tx_desc = &adapter->tx_desc_base[i];
1184                                 current_tx_desc->buffer_addr = htole64(
1185                                 desc_array.descriptor[counter].address);
1186                                 current_tx_desc->lower.data = htole32(
1187                                 (adapter->txd_cmd | txd_lower |
1188                                 (uint16_t)desc_array.descriptor[counter].length));
1189                                 current_tx_desc->upper.data = htole32((txd_upper));
1190                                 if (++i == adapter->num_tx_desc)
1191                                         i = 0;
1192
1193                                 tx_buffer->m_head = NULL;
1194                                 txd_used++;
1195                         }
1196                 } else {
1197                         tx_buffer = &adapter->tx_buffer_area[i];
1198                         current_tx_desc = &adapter->tx_desc_base[i];
1199
1200                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1201                         current_tx_desc->lower.data = htole32(
1202                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1203                         current_tx_desc->upper.data = htole32(txd_upper);
1204
1205                         if (++i == adapter->num_tx_desc)
1206                                 i = 0;
1207
1208                         tx_buffer->m_head = NULL;
1209                 }
1210         }
1211
1212         adapter->next_avail_tx_desc = i;
1213         if (adapter->pcix_82544)
1214                 adapter->num_tx_desc_avail -= txd_used;
1215         else
1216                 adapter->num_tx_desc_avail -= q.nsegs;
1217
1218 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1219         if (ifv != NULL) {
1220                 /* Set the vlan id */
1221                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1222 #else
1223         if (mtag != NULL) {
1224                 /* Set the vlan id */
1225                 current_tx_desc->upper.fields.special = htole16(VLAN_TAG_VALUE(mtag));
1226 #endif
1227
1228                 /* Tell hardware to add tag */
1229                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1230         }
1231
1232         tx_buffer->m_head = m_head;
1233         tx_buffer->map = q.map;
1234         bus_dmamap_sync(adapter->txtag, q.map, BUS_DMASYNC_PREWRITE);
1235
1236         /*
1237          * Last Descriptor of Packet needs End Of Packet (EOP)
1238          */
1239         current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1240
1241         /*
1242          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1243          * that this frame is available to transmit.
1244          */
1245         if (adapter->hw.mac_type == em_82547 &&
1246             adapter->link_duplex == HALF_DUPLEX) {
1247                 em_82547_move_tail(adapter);
1248         } else {
1249                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1250                 if (adapter->hw.mac_type == em_82547) {
1251                         em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len);
1252                 }
1253         }
1254
1255         return(0);
1256 }
1257
1258 /*********************************************************************
1259  *
1260  * 82547 workaround to avoid controller hang in half-duplex environment.
1261  * The workaround is to avoid queuing a large packet that would span
1262  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1263  * in this case. We do that only when FIFO is quiescent.
1264  *
1265  **********************************************************************/
1266 static void
1267 em_82547_move_tail(void *arg)
1268 {
1269         int s;
1270         struct adapter *adapter = arg;
1271         uint16_t hw_tdt;
1272         uint16_t sw_tdt;
1273         struct em_tx_desc *tx_desc;
1274         uint16_t length = 0;
1275         boolean_t eop = 0;
1276
1277         s = splimp();
1278         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1279         sw_tdt = adapter->next_avail_tx_desc;
1280
1281         while (hw_tdt != sw_tdt) {
1282                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1283                 length += tx_desc->lower.flags.length;
1284                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1285                 if(++hw_tdt == adapter->num_tx_desc)
1286                         hw_tdt = 0;
1287
1288                 if(eop) {
1289                         if (em_82547_fifo_workaround(adapter, length)) {
1290                                 adapter->tx_fifo_wrk++;
1291                                 callout_reset(&adapter->tx_fifo_timer, 1,
1292                                         em_82547_move_tail, adapter);
1293                                 break;
1294                         }
1295                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1296                         em_82547_update_fifo_head(adapter, length);
1297                         length = 0;
1298                 }
1299         }
1300         splx(s);
1301 }
1302
1303 static int
1304 em_82547_fifo_workaround(struct adapter *adapter, int len)
1305 {
1306         int fifo_space, fifo_pkt_len;
1307
1308         fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1309
1310         if (adapter->link_duplex == HALF_DUPLEX) {
1311                 fifo_space = EM_82547_TX_FIFO_SIZE - adapter->tx_fifo_head;
1312
1313                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1314                         if (em_82547_tx_fifo_reset(adapter))
1315                                 return(0);
1316                         else
1317                                 return(1);
1318                 }
1319         }
1320
1321         return(0);
1322 }
1323
1324 static void
1325 em_82547_update_fifo_head(struct adapter *adapter, int len)
1326 {
1327         int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1328
1329         /* tx_fifo_head is always 16 byte aligned */
1330         adapter->tx_fifo_head += fifo_pkt_len;
1331         if (adapter->tx_fifo_head >= EM_82547_TX_FIFO_SIZE)
1332                 adapter->tx_fifo_head -= EM_82547_TX_FIFO_SIZE;
1333 }
1334
1335 static int
1336 em_82547_tx_fifo_reset(struct adapter *adapter)
1337 {
1338         uint32_t tctl;
1339
1340         if ( (E1000_READ_REG(&adapter->hw, TDT) ==
1341               E1000_READ_REG(&adapter->hw, TDH)) &&
1342              (E1000_READ_REG(&adapter->hw, TDFT) ==
1343               E1000_READ_REG(&adapter->hw, TDFH)) &&
1344              (E1000_READ_REG(&adapter->hw, TDFTS) ==
1345               E1000_READ_REG(&adapter->hw, TDFHS)) &&
1346              (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1347
1348                 /* Disable TX unit */
1349                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1350                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1351
1352                 /* Reset FIFO pointers */
1353                 E1000_WRITE_REG(&adapter->hw, TDFT, EM_82547_TX_FIFO_BEGIN);
1354                 E1000_WRITE_REG(&adapter->hw, TDFH, EM_82547_TX_FIFO_BEGIN);
1355                 E1000_WRITE_REG(&adapter->hw, TDFTS, EM_82547_TX_FIFO_BEGIN);
1356                 E1000_WRITE_REG(&adapter->hw, TDFHS, EM_82547_TX_FIFO_BEGIN);
1357
1358                 /* Re-enable TX unit */
1359                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1360                 E1000_WRITE_FLUSH(&adapter->hw);
1361
1362                 adapter->tx_fifo_head = 0;
1363                 adapter->tx_fifo_reset++;
1364
1365                 return(TRUE);
1366         }
1367         else {
1368                 return(FALSE);
1369         }
1370 }
1371
1372 static void
1373 em_set_promisc(struct adapter *adapter)
1374 {
1375         uint32_t reg_rctl;
1376         struct ifnet *ifp = &adapter->interface_data.ac_if;
1377
1378         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1379
1380         if (ifp->if_flags & IFF_PROMISC) {
1381                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1382                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1383         } else if (ifp->if_flags & IFF_ALLMULTI) {
1384                 reg_rctl |= E1000_RCTL_MPE;
1385                 reg_rctl &= ~E1000_RCTL_UPE;
1386                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1387         }
1388 }
1389
1390 static void
1391 em_disable_promisc(struct adapter *adapter)
1392 {
1393         uint32_t reg_rctl;
1394
1395         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1396
1397         reg_rctl &=  (~E1000_RCTL_UPE);
1398         reg_rctl &=  (~E1000_RCTL_MPE);
1399         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1400 }
1401
1402 /*********************************************************************
1403  *  Multicast Update
1404  *
1405  *  This routine is called whenever multicast address list is updated.
1406  *
1407  **********************************************************************/
1408
1409 static void
1410 em_set_multi(struct adapter *adapter)
1411 {
1412         uint32_t reg_rctl = 0;
1413         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1414         struct ifmultiaddr *ifma;
1415         int mcnt = 0;
1416         struct ifnet *ifp = &adapter->interface_data.ac_if;
1417
1418         IOCTL_DEBUGOUT("em_set_multi: begin");
1419
1420         if (adapter->hw.mac_type == em_82542_rev2_0) {
1421                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1422                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1423                         em_pci_clear_mwi(&adapter->hw);
1424                 reg_rctl |= E1000_RCTL_RST;
1425                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1426                 msec_delay(5);
1427         }
1428
1429         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1430                 if (ifma->ifma_addr->sa_family != AF_LINK)
1431                         continue;
1432
1433                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1434                         break;
1435
1436                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1437                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1438                 mcnt++;
1439         }
1440
1441         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1442                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1443                 reg_rctl |= E1000_RCTL_MPE;
1444                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1445         } else
1446                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1447
1448         if (adapter->hw.mac_type == em_82542_rev2_0) {
1449                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1450                 reg_rctl &= ~E1000_RCTL_RST;
1451                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1452                 msec_delay(5);
1453                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1454                         em_pci_set_mwi(&adapter->hw);
1455         }
1456 }
1457
1458 /*********************************************************************
1459  *  Timer routine
1460  *
1461  *  This routine checks for link status and updates statistics.
1462  *
1463  **********************************************************************/
1464
1465 static void
1466 em_local_timer(void *arg)
1467 {
1468         int s;
1469         struct ifnet *ifp;
1470         struct adapter *adapter = arg;
1471         ifp = &adapter->interface_data.ac_if;
1472
1473         s = splimp();
1474
1475         em_check_for_link(&adapter->hw);
1476         em_print_link_status(adapter);
1477         em_update_stats_counters(adapter);
1478         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1479                 em_print_hw_stats(adapter);
1480         em_smartspeed(adapter);
1481
1482         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
1483
1484         splx(s);
1485 }
1486
1487 static void
1488 em_print_link_status(struct adapter *adapter)
1489 {
1490         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1491                 if (adapter->link_active == 0) {
1492                         em_get_speed_and_duplex(&adapter->hw,
1493                                                 &adapter->link_speed,
1494                                                 &adapter->link_duplex);
1495                         device_printf(adapter->dev, "Link is up %d Mbps %s\n",
1496                                adapter->link_speed,
1497                                ((adapter->link_duplex == FULL_DUPLEX) ?
1498                                 "Full Duplex" : "Half Duplex"));
1499                         adapter->link_active = 1;
1500                         adapter->smartspeed = 0;
1501                 }
1502         } else {
1503                 if (adapter->link_active == 1) {
1504                         adapter->link_speed = 0;
1505                         adapter->link_duplex = 0;
1506                         device_printf(adapter->dev, "Link is Down\n");
1507                         adapter->link_active = 0;
1508                 }
1509         }
1510 }
1511
1512 /*********************************************************************
1513  *
1514  *  This routine disables all traffic on the adapter by issuing a
1515  *  global reset on the MAC and deallocates TX/RX buffers.
1516  *
1517  **********************************************************************/
1518
1519 static void
1520 em_stop(void *arg)
1521 {
1522         struct ifnet   *ifp;
1523         struct adapter * adapter = arg;
1524         ifp = &adapter->interface_data.ac_if;
1525
1526         INIT_DEBUGOUT("em_stop: begin");
1527         em_disable_intr(adapter);
1528         em_reset_hw(&adapter->hw);
1529         callout_stop(&adapter->timer);
1530         callout_stop(&adapter->tx_fifo_timer);
1531         em_free_transmit_structures(adapter);
1532         em_free_receive_structures(adapter);
1533
1534         /* Tell the stack that the interface is no longer active */
1535         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1536         ifp->if_timer = 0;
1537 }
1538
1539 /*********************************************************************
1540  *
1541  *  Determine hardware revision.
1542  *
1543  **********************************************************************/
1544 static void
1545 em_identify_hardware(struct adapter * adapter)
1546 {
1547         device_t dev = adapter->dev;
1548
1549         /* Make sure our PCI config space has the necessary stuff set */
1550         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1551         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1552               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1553                 device_printf(dev, "Memory Access and/or Bus Master bits were not set!\n");
1554                 adapter->hw.pci_cmd_word |=
1555                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1556                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1557         }
1558
1559         /* Save off the information about this board */
1560         adapter->hw.vendor_id = pci_get_vendor(dev);
1561         adapter->hw.device_id = pci_get_device(dev);
1562         adapter->hw.revision_id = pci_get_revid(dev);
1563         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1564         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1565
1566         /* Identify the MAC */
1567         if (em_set_mac_type(&adapter->hw))
1568                 device_printf(dev, "Unknown MAC Type\n");
1569
1570         if (adapter->hw.mac_type == em_82541 ||
1571             adapter->hw.mac_type == em_82541_rev_2 ||
1572             adapter->hw.mac_type == em_82547 ||
1573             adapter->hw.mac_type == em_82547_rev_2)
1574                 adapter->hw.phy_init_script = TRUE;
1575 }
1576
1577 /*********************************************************************
1578  *
1579  *  Initialize the hardware to a configuration as specified by the
1580  *  adapter structure. The controller is reset, the EEPROM is
1581  *  verified, the MAC address is set, then the shared initialization
1582  *  routines are called.
1583  *
1584  **********************************************************************/
1585 static int
1586 em_hardware_init(struct adapter *adapter)
1587 {
1588         INIT_DEBUGOUT("em_hardware_init: begin");
1589         /* Issue a global reset */
1590         em_reset_hw(&adapter->hw);
1591
1592         /* When hardware is reset, fifo_head is also reset */
1593         adapter->tx_fifo_head = 0;
1594
1595         /* Make sure we have a good EEPROM before we read from it */
1596         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
1597                 device_printf(adapter->dev, "The EEPROM Checksum Is Not Valid\n");
1598                 return(EIO);
1599         }
1600
1601         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
1602                 device_printf(adapter->dev, "EEPROM read error while reading part number\n");
1603                 return(EIO);
1604         }
1605
1606         if (em_init_hw(&adapter->hw) < 0) {
1607                 device_printf(adapter->dev, "Hardware Initialization Failed");
1608                 return(EIO);
1609         }
1610
1611         em_check_for_link(&adapter->hw);
1612         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)
1613                 adapter->link_active = 1;
1614         else
1615                 adapter->link_active = 0;
1616
1617         if (adapter->link_active) {
1618                 em_get_speed_and_duplex(&adapter->hw,
1619                                         &adapter->link_speed,
1620                                         &adapter->link_duplex);
1621         } else {
1622                 adapter->link_speed = 0;
1623                 adapter->link_duplex = 0;
1624         }
1625
1626         return(0);
1627 }
1628
1629 /*********************************************************************
1630  *
1631  *  Setup networking device structure and register an interface.
1632  *
1633  **********************************************************************/
1634 static void
1635 em_setup_interface(device_t dev, struct adapter *adapter)
1636 {
1637         struct ifnet   *ifp;
1638         INIT_DEBUGOUT("em_setup_interface: begin");
1639
1640         ifp = &adapter->interface_data.ac_if;
1641         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1642         ifp->if_mtu = ETHERMTU;
1643         ifp->if_output = ether_output;
1644         ifp->if_baudrate = 1000000000;
1645         ifp->if_init =  em_init;
1646         ifp->if_softc = adapter;
1647         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1648         ifp->if_ioctl = em_ioctl;
1649         ifp->if_start = em_start;
1650         ifp->if_watchdog = em_watchdog;
1651         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1;
1652
1653         ether_ifattach(ifp, adapter->hw.mac_addr);
1654
1655         if (adapter->hw.mac_type >= em_82543) {
1656                 ifp->if_capabilities = IFCAP_HWCSUM;
1657                 ifp->if_capenable = ifp->if_capabilities;
1658         }
1659
1660         /*
1661          * Tell the upper layer(s) we support long frames.
1662          */
1663         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1664 #if defined(__FreeBSD__) && __FreeBSD_version >= 500000
1665         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1666 #endif
1667
1668         /*
1669          * Specify the media types supported by this adapter and register
1670          * callbacks to update media and link information
1671          */
1672         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
1673                      em_media_status);
1674         if (adapter->hw.media_type == em_media_type_fiber) {
1675                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1676                             0, NULL);
1677                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1678                             0, NULL);
1679         } else {
1680                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1681                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1682                             0, NULL);
1683                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
1684                             0, NULL);
1685                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1686                             0, NULL);
1687                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX | IFM_FDX,
1688                             0, NULL);
1689                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX, 0, NULL);
1690         }
1691         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1692         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1693 }
1694
1695 /*********************************************************************
1696  *
1697  *  Workaround for SmartSpeed on 82541 and 82547 controllers
1698  *
1699  **********************************************************************/
1700 static void
1701 em_smartspeed(struct adapter *adapter)
1702 {
1703         uint16_t phy_tmp;
1704
1705         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
1706             !adapter->hw.autoneg ||
1707             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
1708                 return;
1709
1710         if (adapter->smartspeed == 0) {
1711                 /*
1712                  * If Master/Slave config fault is asserted twice,
1713                  * we assume back-to-back.
1714                  */
1715                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1716                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
1717                         return;
1718                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1719                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
1720                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL,
1721                                         &phy_tmp);
1722                         if (phy_tmp & CR_1000T_MS_ENABLE) {
1723                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
1724                                 em_write_phy_reg(&adapter->hw,
1725                                                  PHY_1000T_CTRL, phy_tmp);
1726                                 adapter->smartspeed++;
1727                                 if (adapter->hw.autoneg &&
1728                                     !em_phy_setup_autoneg(&adapter->hw) &&
1729                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
1730                                                      &phy_tmp)) {
1731                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1732                                                     MII_CR_RESTART_AUTO_NEG);
1733                                         em_write_phy_reg(&adapter->hw,
1734                                                          PHY_CTRL, phy_tmp);
1735                                 }
1736                         }
1737                 }
1738                 return;
1739         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
1740                 /* If still no link, perhaps using 2/3 pair cable */
1741                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
1742                 phy_tmp |= CR_1000T_MS_ENABLE;
1743                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
1744                 if (adapter->hw.autoneg &&
1745                     !em_phy_setup_autoneg(&adapter->hw) &&
1746                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
1747                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1748                                     MII_CR_RESTART_AUTO_NEG);
1749                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
1750                 }
1751         }
1752         /* Restart process after EM_SMARTSPEED_MAX iterations */
1753         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
1754                 adapter->smartspeed = 0;
1755 }
1756
1757 /*
1758  * Manage DMA'able memory.
1759  */
1760 static void
1761 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1762 {
1763         if (error)
1764                 return;
1765         *(bus_addr_t*) arg = segs->ds_addr;
1766 }
1767
1768 static int
1769 em_dma_malloc(struct adapter *adapter, bus_size_t size,
1770               struct em_dma_alloc *dma, int mapflags)
1771 {
1772         int r;
1773         device_t dev = adapter->dev;
1774
1775         r = bus_dma_tag_create(NULL,                    /* parent */
1776                                PAGE_SIZE, 0,            /* alignment, bounds */
1777                                BUS_SPACE_MAXADDR,       /* lowaddr */
1778                                BUS_SPACE_MAXADDR,       /* highaddr */
1779                                NULL, NULL,              /* filter, filterarg */
1780                                size,                    /* maxsize */
1781                                1,                       /* nsegments */
1782                                size,                    /* maxsegsize */
1783                                BUS_DMA_ALLOCNOW,        /* flags */
1784                                &dma->dma_tag);
1785         if (r != 0) {
1786                 device_printf(dev, "em_dma_malloc: bus_dma_tag_create failed; "
1787                               "error %u\n", r);
1788                 goto fail_0;
1789         }
1790
1791         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1792         if (r != 0) {
1793                 device_printf(dev, "em_dma_malloc: bus_dmamap_create failed; "
1794                               "error %u\n", r);
1795                 goto fail_1;
1796         }
1797
1798         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1799                              BUS_DMA_NOWAIT, &dma->dma_map);
1800         if (r != 0) {
1801                 device_printf(dev, "em_dma_malloc: bus_dmammem_alloc failed; "
1802                               "size %llu, error %d\n", (uintmax_t)size, r);
1803                 goto fail_2;
1804         }
1805
1806         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1807                             size,
1808                             em_dmamap_cb,
1809                             &dma->dma_paddr,
1810                             mapflags | BUS_DMA_NOWAIT);
1811         if (r != 0) {
1812                 device_printf(dev, "em_dma_malloc: bus_dmamap_load failed; "
1813                               "error %u\n", r);
1814                 goto fail_3;
1815         }
1816
1817         dma->dma_size = size;
1818         return(0);
1819
1820 fail_3:
1821         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1822 fail_2:
1823         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1824 fail_1:
1825         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1826         bus_dma_tag_destroy(dma->dma_tag);
1827 fail_0:
1828         dma->dma_map = NULL;
1829         dma->dma_tag = NULL;
1830         return(r);
1831 }
1832
1833 static void
1834 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
1835 {
1836         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1837         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1838         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1839         bus_dma_tag_destroy(dma->dma_tag);
1840 }
1841
1842 /*********************************************************************
1843  *
1844  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1845  *  the information needed to transmit a packet on the wire.
1846  *
1847  **********************************************************************/
1848 static int
1849 em_allocate_transmit_structures(struct adapter * adapter)
1850 {
1851         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
1852             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
1853         if (adapter->tx_buffer_area == NULL) {
1854                 device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
1855                 return(ENOMEM);
1856         }
1857
1858         return(0);
1859 }
1860
1861 /*********************************************************************
1862  *
1863  *  Allocate and initialize transmit structures.
1864  *
1865  **********************************************************************/
1866 static int
1867 em_setup_transmit_structures(struct adapter * adapter)
1868 {
1869         /*
1870          * Setup DMA descriptor areas.
1871          */
1872         if (bus_dma_tag_create(NULL,                    /* parent */
1873                                PAGE_SIZE, 0,            /* alignment, bounds */
1874                                BUS_SPACE_MAXADDR,       /* lowaddr */
1875                                BUS_SPACE_MAXADDR,       /* highaddr */
1876                                NULL, NULL,              /* filter, filterarg */
1877                                MCLBYTES * 8,            /* maxsize */
1878                                EM_MAX_SCATTER,          /* nsegments */
1879                                MCLBYTES * 8,            /* maxsegsize */
1880                                BUS_DMA_ALLOCNOW,        /* flags */
1881                                &adapter->txtag)) {
1882                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
1883                 return(ENOMEM);
1884         }
1885
1886         if (em_allocate_transmit_structures(adapter))
1887                 return(ENOMEM);
1888
1889         bzero((void *) adapter->tx_desc_base,
1890               (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
1891
1892         adapter->next_avail_tx_desc = 0;
1893         adapter->oldest_used_tx_desc = 0;
1894
1895         /* Set number of descriptors available */
1896         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1897
1898         /* Set checksum context */
1899         adapter->active_checksum_context = OFFLOAD_NONE;
1900
1901         return(0);
1902 }
1903
1904 /*********************************************************************
1905  *
1906  *  Enable transmit unit.
1907  *
1908  **********************************************************************/
1909 static void
1910 em_initialize_transmit_unit(struct adapter * adapter)
1911 {
1912         uint32_t reg_tctl;
1913         uint32_t reg_tipg = 0;
1914         uint64_t bus_addr;
1915
1916         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
1917
1918         /* Setup the Base and Length of the Tx Descriptor Ring */
1919         bus_addr = adapter->txdma.dma_paddr;
1920         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
1921         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
1922         E1000_WRITE_REG(&adapter->hw, TDLEN,
1923                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
1924
1925         /* Setup the HW Tx Head and Tail descriptor pointers */
1926         E1000_WRITE_REG(&adapter->hw, TDH, 0);
1927         E1000_WRITE_REG(&adapter->hw, TDT, 0);
1928
1929         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1930                      E1000_READ_REG(&adapter->hw, TDBAL),
1931                      E1000_READ_REG(&adapter->hw, TDLEN));
1932
1933         /* Set the default values for the Tx Inter Packet Gap timer */
1934         switch (adapter->hw.mac_type) {
1935         case em_82542_rev2_0:
1936         case em_82542_rev2_1:
1937                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
1938                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1939                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1940                 break;
1941         default:
1942                 if (adapter->hw.media_type == em_media_type_fiber)
1943                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1944                 else
1945                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1946                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1947                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1948         }
1949
1950         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
1951         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
1952         if (adapter->hw.mac_type >= em_82540)
1953                 E1000_WRITE_REG(&adapter->hw, TADV,
1954                                 adapter->tx_abs_int_delay.value);
1955
1956         /* Program the Transmit Control Register */
1957         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
1958                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1959         if (adapter->link_duplex == 1)
1960                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1961         else
1962                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1963         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
1964
1965         /* Setup Transmit Descriptor Settings for this adapter */
1966         adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
1967
1968         if (adapter->tx_int_delay.value > 0)
1969                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1970 }
1971
1972 /*********************************************************************
1973  *
1974  *  Free all transmit related data structures.
1975  *
1976  **********************************************************************/
1977 static void
1978 em_free_transmit_structures(struct adapter * adapter)
1979 {
1980         struct em_buffer *tx_buffer;
1981         int i;
1982
1983         INIT_DEBUGOUT("free_transmit_structures: begin");
1984
1985         if (adapter->tx_buffer_area != NULL) {
1986                 tx_buffer = adapter->tx_buffer_area;
1987                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
1988                         if (tx_buffer->m_head != NULL) {
1989                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1990                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
1991                                 m_freem(tx_buffer->m_head);
1992                         }
1993                         tx_buffer->m_head = NULL;
1994                 }
1995         }
1996         if (adapter->tx_buffer_area != NULL) {
1997                 free(adapter->tx_buffer_area, M_DEVBUF);
1998                 adapter->tx_buffer_area = NULL;
1999         }
2000         if (adapter->txtag != NULL) {
2001                 bus_dma_tag_destroy(adapter->txtag);
2002                 adapter->txtag = NULL;
2003         }
2004 }
2005
2006 /*********************************************************************
2007  *
2008  *  The offload context needs to be set when we transfer the first
2009  *  packet of a particular protocol (TCP/UDP). We change the
2010  *  context only if the protocol type changes.
2011  *
2012  **********************************************************************/
2013 static void
2014 em_transmit_checksum_setup(struct adapter * adapter,
2015                            struct mbuf *mp,
2016                            uint32_t *txd_upper,
2017                            uint32_t *txd_lower)
2018 {
2019         struct em_context_desc *TXD;
2020         struct em_buffer *tx_buffer;
2021         int curr_txd;
2022
2023         if (mp->m_pkthdr.csum_flags) {
2024                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2025                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2026                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2027                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2028                                 return;
2029                         else
2030                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2031                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2032                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2033                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2034                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2035                                 return;
2036                         else
2037                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2038                 } else {
2039                         *txd_upper = 0;
2040                         *txd_lower = 0;
2041                         return;
2042                 }
2043         } else {
2044                 *txd_upper = 0;
2045                 *txd_lower = 0;
2046                 return;
2047         }
2048
2049         /* If we reach this point, the checksum offload context
2050          * needs to be reset.
2051          */
2052         curr_txd = adapter->next_avail_tx_desc;
2053         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2054         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2055
2056         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2057         TXD->lower_setup.ip_fields.ipcso =
2058             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2059         TXD->lower_setup.ip_fields.ipcse =
2060             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2061
2062         TXD->upper_setup.tcp_fields.tucss =
2063             ETHER_HDR_LEN + sizeof(struct ip);
2064         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2065
2066         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2067                 TXD->upper_setup.tcp_fields.tucso =
2068                     ETHER_HDR_LEN + sizeof(struct ip) +
2069                     offsetof(struct tcphdr, th_sum);
2070         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2071                 TXD->upper_setup.tcp_fields.tucso =
2072                         ETHER_HDR_LEN + sizeof(struct ip) +
2073                         offsetof(struct udphdr, uh_sum);
2074         }
2075
2076         TXD->tcp_seg_setup.data = htole32(0);
2077         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2078
2079         tx_buffer->m_head = NULL;
2080
2081         if (++curr_txd == adapter->num_tx_desc)
2082                 curr_txd = 0;
2083
2084         adapter->num_tx_desc_avail--;
2085         adapter->next_avail_tx_desc = curr_txd;
2086 }
2087
2088 /**********************************************************************
2089  *
2090  *  Examine each tx_buffer in the used queue. If the hardware is done
2091  *  processing the packet then free associated resources. The
2092  *  tx_buffer is put back on the free queue.
2093  *
2094  **********************************************************************/
2095 static void
2096 em_clean_transmit_interrupts(struct adapter *adapter)
2097 {
2098         int s;
2099         int i, num_avail;
2100         struct em_buffer *tx_buffer;
2101         struct em_tx_desc *tx_desc;
2102         struct ifnet *ifp = &adapter->interface_data.ac_if;
2103
2104         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2105                 return;
2106
2107         s = splimp();
2108 #ifdef DBG_STATS
2109         adapter->clean_tx_interrupts++;
2110 #endif
2111         num_avail = adapter->num_tx_desc_avail;
2112         i = adapter->oldest_used_tx_desc;
2113
2114         tx_buffer = &adapter->tx_buffer_area[i];
2115         tx_desc = &adapter->tx_desc_base[i];
2116
2117         while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2118                 tx_desc->upper.data = 0;
2119                 num_avail++;
2120
2121                 if (tx_buffer->m_head) {
2122                         ifp->if_opackets++;
2123                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2124                                         BUS_DMASYNC_POSTWRITE);
2125                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2126                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2127
2128                         m_freem(tx_buffer->m_head);
2129                         tx_buffer->m_head = NULL;
2130                 }
2131
2132                 if (++i == adapter->num_tx_desc)
2133                         i = 0;
2134
2135                 tx_buffer = &adapter->tx_buffer_area[i];
2136                 tx_desc = &adapter->tx_desc_base[i];
2137         }
2138
2139         adapter->oldest_used_tx_desc = i;
2140
2141         /*
2142          * If we have enough room, clear IFF_OACTIVE to tell the stack
2143          * that it is OK to send packets.
2144          * If there are no pending descriptors, clear the timeout. Otherwise,
2145          * if some descriptors have been freed, restart the timeout.
2146          */
2147         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2148                 ifp->if_flags &= ~IFF_OACTIVE;
2149                 if (num_avail == adapter->num_tx_desc)
2150                         ifp->if_timer = 0;
2151                 else if (num_avail == adapter->num_tx_desc_avail)
2152                         ifp->if_timer = EM_TX_TIMEOUT;
2153         }
2154         adapter->num_tx_desc_avail = num_avail;
2155         splx(s);
2156 }
2157
2158 /*********************************************************************
2159  *
2160  *  Get a buffer from system mbuf buffer pool.
2161  *
2162  **********************************************************************/
2163 static int
2164 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp)
2165 {
2166         struct mbuf *mp = nmp;
2167         struct em_buffer *rx_buffer;
2168         struct ifnet *ifp;
2169         bus_addr_t paddr;
2170         int error;
2171
2172         ifp = &adapter->interface_data.ac_if;
2173
2174         if (mp == NULL) {
2175                 mp = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR);
2176                 if (mp == NULL) {
2177                         adapter->mbuf_cluster_failed++;
2178                         return(ENOBUFS);
2179                 }
2180                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2181         } else {
2182                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2183                 mp->m_data = mp->m_ext.ext_buf;
2184                 mp->m_next = NULL;
2185         }
2186         if (ifp->if_mtu <= ETHERMTU)
2187                 m_adj(mp, ETHER_ALIGN);
2188
2189         rx_buffer = &adapter->rx_buffer_area[i];
2190
2191         /*
2192          * Using memory from the mbuf cluster pool, invoke the
2193          * bus_dma machinery to arrange the memory mapping.
2194          */
2195         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2196                                 mtod(mp, void *), mp->m_len,
2197                                 em_dmamap_cb, &paddr, 0);
2198         if (error) {
2199                 m_free(mp);
2200                 return(error);
2201         }
2202         rx_buffer->m_head = mp;
2203         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2204         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2205
2206         return(0);
2207 }
2208
2209 /*********************************************************************
2210  *
2211  *  Allocate memory for rx_buffer structures. Since we use one
2212  *  rx_buffer per received packet, the maximum number of rx_buffer's
2213  *  that we'll need is equal to the number of receive descriptors
2214  *  that we've allocated.
2215  *
2216  **********************************************************************/
2217 static int
2218 em_allocate_receive_structures(struct adapter *adapter)
2219 {
2220         int i, error, size;
2221         struct em_buffer *rx_buffer;
2222
2223         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2224         adapter->rx_buffer_area = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2225
2226         error = bus_dma_tag_create(NULL,                /* parent */
2227                                    PAGE_SIZE, 0,        /* alignment, bounds */
2228                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2229                                    BUS_SPACE_MAXADDR,   /* highaddr */
2230                                    NULL, NULL,          /* filter, filterarg */
2231                                    MCLBYTES,            /* maxsize */
2232                                    1,                   /* nsegments */
2233                                    MCLBYTES,            /* maxsegsize */
2234                                    BUS_DMA_ALLOCNOW,    /* flags */
2235                                    &adapter->rxtag);
2236         if (error != 0) {
2237                 device_printf(adapter->dev, "em_allocate_receive_structures: "
2238                               "bus_dma_tag_create failed; error %u\n", error);
2239                 goto fail_0;
2240         }
2241
2242         rx_buffer = adapter->rx_buffer_area;
2243         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2244                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2245                                           &rx_buffer->map);
2246                 if (error != 0) {
2247                         device_printf(adapter->dev,
2248                                       "em_allocate_receive_structures: "
2249                                       "bus_dmamap_create failed; error %u\n",
2250                                       error);
2251                         goto fail_1;
2252                 }
2253         }
2254
2255         for (i = 0; i < adapter->num_rx_desc; i++) {
2256                 error = em_get_buf(i, adapter, NULL);
2257                 if (error != 0) {
2258                         adapter->rx_buffer_area[i].m_head = NULL;
2259                         adapter->rx_desc_base[i].buffer_addr = 0;
2260                         return(error);
2261                 }
2262         }
2263
2264         return(0);
2265
2266 fail_1:
2267         bus_dma_tag_destroy(adapter->rxtag);
2268 fail_0:
2269         adapter->rxtag = NULL;
2270         free(adapter->rx_buffer_area, M_DEVBUF);
2271         adapter->rx_buffer_area = NULL;
2272         return(error);
2273 }
2274
2275 /*********************************************************************
2276  *
2277  *  Allocate and initialize receive structures.
2278  *
2279  **********************************************************************/
2280 static int
2281 em_setup_receive_structures(struct adapter *adapter)
2282 {
2283         bzero((void *) adapter->rx_desc_base,
2284               (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2285
2286         if (em_allocate_receive_structures(adapter))
2287                 return(ENOMEM);
2288
2289         /* Setup our descriptor pointers */
2290         adapter->next_rx_desc_to_check = 0;
2291         return(0);
2292 }
2293
2294 /*********************************************************************
2295  *
2296  *  Enable receive unit.
2297  *
2298  **********************************************************************/
2299 static void
2300 em_initialize_receive_unit(struct adapter *adapter)
2301 {
2302         uint32_t reg_rctl;
2303         uint32_t reg_rxcsum;
2304         struct ifnet *ifp;
2305         uint64_t bus_addr;
2306
2307         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2308
2309         ifp = &adapter->interface_data.ac_if;
2310
2311         /* Make sure receives are disabled while setting up the descriptor ring */
2312         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2313
2314         /* Set the Receive Delay Timer Register */
2315         E1000_WRITE_REG(&adapter->hw, RDTR,
2316                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2317
2318         if(adapter->hw.mac_type >= em_82540) {
2319                 E1000_WRITE_REG(&adapter->hw, RADV,
2320                                 adapter->rx_abs_int_delay.value);
2321
2322                 /* Set the interrupt throttling rate.  Value is calculated
2323                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
2324                  */
2325 #define MAX_INTS_PER_SEC        8000
2326 #define DEFAULT_ITR             1000000000/(MAX_INTS_PER_SEC * 256)
2327                 E1000_WRITE_REG(&adapter->hw, ITR, DEFAULT_ITR);
2328         }
2329
2330         /* Setup the Base and Length of the Rx Descriptor Ring */
2331         bus_addr = adapter->rxdma.dma_paddr;
2332         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2333         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2334         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2335                         sizeof(struct em_rx_desc));
2336
2337         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2338         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2339         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2340
2341         /* Setup the Receive Control Register */
2342         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2343                    E1000_RCTL_RDMTS_HALF |
2344                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2345
2346         if (adapter->hw.tbi_compatibility_on == TRUE)
2347                 reg_rctl |= E1000_RCTL_SBP;
2348
2349         switch (adapter->rx_buffer_len) {
2350         default:
2351         case EM_RXBUFFER_2048:
2352                 reg_rctl |= E1000_RCTL_SZ_2048;
2353                 break;
2354         case EM_RXBUFFER_4096:
2355                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2356                 break;
2357         case EM_RXBUFFER_8192:
2358                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2359                 break;
2360         case EM_RXBUFFER_16384:
2361                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2362                 break;
2363         }
2364
2365         if (ifp->if_mtu > ETHERMTU)
2366                 reg_rctl |= E1000_RCTL_LPE;
2367
2368         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2369         if ((adapter->hw.mac_type >= em_82543) &&
2370             (ifp->if_capenable & IFCAP_RXCSUM)) {
2371                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2372                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2373                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2374         }
2375
2376         /* Enable Receives */
2377         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2378 }
2379
2380 /*********************************************************************
2381  *
2382  *  Free receive related data structures.
2383  *
2384  **********************************************************************/
2385 static void
2386 em_free_receive_structures(struct adapter *adapter)
2387 {
2388         struct em_buffer *rx_buffer;
2389         int i;
2390
2391         INIT_DEBUGOUT("free_receive_structures: begin");
2392
2393         if (adapter->rx_buffer_area != NULL) {
2394                 rx_buffer = adapter->rx_buffer_area;
2395                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2396                         if (rx_buffer->map != NULL) {
2397                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2398                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2399                         }
2400                         if (rx_buffer->m_head != NULL)
2401                                 m_freem(rx_buffer->m_head);
2402                         rx_buffer->m_head = NULL;
2403                 }
2404         }
2405         if (adapter->rx_buffer_area != NULL) {
2406                 free(adapter->rx_buffer_area, M_DEVBUF);
2407                 adapter->rx_buffer_area = NULL;
2408         }
2409         if (adapter->rxtag != NULL) {
2410                 bus_dma_tag_destroy(adapter->rxtag);
2411                 adapter->rxtag = NULL;
2412         }
2413 }
2414
2415 /*********************************************************************
2416  *
2417  *  This routine executes in interrupt context. It replenishes
2418  *  the mbufs in the descriptor and sends data which has been
2419  *  dma'ed into host memory to upper layer.
2420  *
2421  *  We loop at most count times if count is > 0, or until done if
2422  *  count < 0.
2423  *
2424  *********************************************************************/
2425 static void
2426 em_process_receive_interrupts(struct adapter *adapter, int count)
2427 {
2428         struct ifnet *ifp;
2429         struct mbuf *mp;
2430 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2431         struct ether_header *eh;
2432 #endif
2433         uint8_t accept_frame = 0;
2434         uint8_t eop = 0;
2435         uint16_t len, desc_len, prev_len_adj;
2436         int i;
2437
2438         /* Pointer to the receive descriptor being examined. */
2439         struct em_rx_desc *current_desc;
2440
2441         ifp = &adapter->interface_data.ac_if;
2442         i = adapter->next_rx_desc_to_check;
2443         current_desc = &adapter->rx_desc_base[i];
2444
2445         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
2446 #ifdef DBG_STATS
2447                 adapter->no_pkts_avail++;
2448 #endif
2449                 return;
2450         }
2451         while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0)) {
2452                 mp = adapter->rx_buffer_area[i].m_head;
2453                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2454                                 BUS_DMASYNC_POSTREAD);
2455
2456                 accept_frame = 1;
2457                 prev_len_adj = 0;
2458                 desc_len = le16toh(current_desc->length);
2459                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2460                         count--;
2461                         eop = 1;
2462                         if (desc_len < ETHER_CRC_LEN) {
2463                                 len = 0;
2464                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2465                         }
2466                         else {
2467                                 len = desc_len - ETHER_CRC_LEN;
2468                         }
2469                 } else {
2470                         eop = 0;
2471                         len = desc_len;
2472                 }
2473
2474                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2475                         uint8_t last_byte;
2476                         uint32_t pkt_len = desc_len;
2477
2478                         if (adapter->fmp != NULL)
2479                                 pkt_len += adapter->fmp->m_pkthdr.len;
2480
2481                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
2482
2483                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
2484                                        current_desc->errors,
2485                                        pkt_len, last_byte)) {
2486                                 em_tbi_adjust_stats(&adapter->hw,
2487                                                     &adapter->stats,
2488                                                     pkt_len,
2489                                                     adapter->hw.mac_addr);
2490                                 if (len > 0)
2491                                         len--;
2492                         }
2493                         else {
2494                                 accept_frame = 0;
2495                         }
2496                 }
2497
2498                 if (accept_frame) {
2499                         if (em_get_buf(i, adapter, NULL) == ENOBUFS) {
2500                                 adapter->dropped_pkts++;
2501                                 em_get_buf(i, adapter, mp);
2502                                 if (adapter->fmp != NULL)
2503                                         m_freem(adapter->fmp);
2504                                 adapter->fmp = NULL;
2505                                 adapter->lmp = NULL;
2506                                 break;
2507                         }
2508
2509                         /* Assign correct length to the current fragment */
2510                         mp->m_len = len;
2511
2512                         if (adapter->fmp == NULL) {
2513                                 mp->m_pkthdr.len = len;
2514                                 adapter->fmp = mp;       /* Store the first mbuf */
2515                                 adapter->lmp = mp;
2516                         } else {
2517                                 /* Chain mbuf's together */
2518                                 mp->m_flags &= ~M_PKTHDR;
2519                                 /*
2520                                  * Adjust length of previous mbuf in chain if we
2521                                  * received less than 4 bytes in the last descriptor.
2522                                  */
2523                                 if (prev_len_adj > 0) {
2524                                         adapter->lmp->m_len -= prev_len_adj;
2525                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
2526                                 }
2527                                 adapter->lmp->m_next = mp;
2528                                 adapter->lmp = adapter->lmp->m_next;
2529                                 adapter->fmp->m_pkthdr.len += len;
2530                         }
2531
2532                         if (eop) {
2533                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2534                                 ifp->if_ipackets++;
2535
2536 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2537                                 eh = mtod(adapter->fmp, struct ether_header *);
2538                                 /* Remove ethernet header from mbuf */
2539                                 m_adj(adapter->fmp, sizeof(struct ether_header));
2540                                 em_receive_checksum(adapter, current_desc,
2541                                                     adapter->fmp);
2542                                 if (current_desc->status & E1000_RXD_STAT_VP)
2543                                         VLAN_INPUT_TAG(eh, adapter->fmp,
2544                                                        (current_desc->special &
2545                                                         E1000_RXD_SPC_VLAN_MASK));
2546                                 else
2547                                         ether_input(ifp, eh, adapter->fmp);
2548 #else
2549                                 em_receive_checksum(adapter, current_desc,
2550                                                     adapter->fmp);
2551                                 if (current_desc->status & E1000_RXD_STAT_VP)
2552                                         VLAN_INPUT_TAG(ifp, adapter->fmp,
2553                                                        (current_desc->special &
2554                                                         E1000_RXD_SPC_VLAN_MASK),
2555                                                        adapter->fmp = NULL);
2556
2557                                 if (adapter->fmp != NULL)
2558                                         (*ifp->if_input)(ifp, adapter->fmp);
2559 #endif
2560                                 adapter->fmp = NULL;
2561                                 adapter->lmp = NULL;
2562                         }
2563                 } else {
2564                         adapter->dropped_pkts++;
2565                         em_get_buf(i, adapter, mp);
2566                         if (adapter->fmp != NULL)
2567                                 m_freem(adapter->fmp);
2568                         adapter->fmp = NULL;
2569                         adapter->lmp = NULL;
2570                 }
2571
2572                 /* Zero out the receive descriptors status  */
2573                 current_desc->status = 0;
2574
2575                 /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
2576                 E1000_WRITE_REG(&adapter->hw, RDT, i);
2577
2578                 /* Advance our pointers to the next descriptor */
2579                 if (++i == adapter->num_rx_desc) {
2580                         i = 0;
2581                         current_desc = adapter->rx_desc_base;
2582                 } else
2583                         current_desc++;
2584         }
2585         adapter->next_rx_desc_to_check = i;
2586 }
2587
2588 /*********************************************************************
2589  *
2590  *  Verify that the hardware indicated that the checksum is valid.
2591  *  Inform the stack about the status of checksum so that stack
2592  *  doesn't spend time verifying the checksum.
2593  *
2594  *********************************************************************/
2595 static void
2596 em_receive_checksum(struct adapter *adapter,
2597                     struct em_rx_desc *rx_desc,
2598                     struct mbuf *mp)
2599 {
2600         /* 82543 or newer only */
2601         if ((adapter->hw.mac_type < em_82543) ||
2602             /* Ignore Checksum bit is set */
2603             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
2604                 mp->m_pkthdr.csum_flags = 0;
2605                 return;
2606         }
2607
2608         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
2609                 /* Did it pass? */
2610                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
2611                         /* IP Checksum Good */
2612                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2613                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2614                 } else {
2615                         mp->m_pkthdr.csum_flags = 0;
2616                 }
2617         }
2618
2619         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
2620                 /* Did it pass? */
2621                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
2622                         mp->m_pkthdr.csum_flags |=
2623                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2624                         mp->m_pkthdr.csum_data = htons(0xffff);
2625                 }
2626         }
2627 }
2628
2629
2630 static void
2631 em_enable_vlans(struct adapter *adapter)
2632 {
2633         uint32_t ctrl;
2634
2635         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
2636
2637         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
2638         ctrl |= E1000_CTRL_VME;
2639         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
2640 }
2641
2642 static void
2643 em_enable_intr(struct adapter *adapter)
2644 {
2645         E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
2646 }
2647
2648 static void
2649 em_disable_intr(struct adapter *adapter)
2650 {
2651         E1000_WRITE_REG(&adapter->hw, IMC,
2652                         (0xffffffff & ~E1000_IMC_RXSEQ));
2653 }
2654
2655 static int
2656 em_is_valid_ether_addr(uint8_t *addr)
2657 {
2658         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2659
2660         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN)))
2661                 return(FALSE);
2662         else
2663                 return(TRUE);
2664 }
2665
2666 void
2667 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2668 {
2669         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
2670 }
2671
2672 void
2673 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2674 {
2675         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
2676 }
2677
2678 void
2679 em_pci_set_mwi(struct em_hw *hw)
2680 {
2681         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2682                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
2683 }
2684
2685 void
2686 em_pci_clear_mwi(struct em_hw *hw)
2687 {
2688         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2689                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
2690 }
2691
2692 uint32_t
2693 em_read_reg_io(struct em_hw *hw, uint32_t offset)
2694 {
2695         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2696         return(bus_space_read_4(hw->reg_io_tag, hw->reg_io_handle, 4));
2697 }
2698
2699 void
2700 em_write_reg_io(struct em_hw *hw, uint32_t offset, uint32_t value)
2701 {
2702         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2703         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 4, value);
2704 }
2705
2706 /*********************************************************************
2707  * 82544 Coexistence issue workaround.
2708  *    There are 2 issues.
2709  *      1. Transmit Hang issue.
2710  *    To detect this issue, following equation can be used...
2711  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2712  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
2713  *
2714  *      2. DAC issue.
2715  *    To detect this issue, following equation can be used...
2716  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2717  *          If SUM[3:0] is in between 9 to c, we will have this issue.
2718  *
2719  *
2720  *    WORKAROUND:
2721  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
2722  *          9,a,b,c (DAC)
2723  *
2724 *************************************************************************/
2725 static uint32_t
2726 em_fill_descriptors(uint64_t address, uint32_t length, PDESC_ARRAY desc_array)
2727 {
2728         /* Since issue is sensitive to length and address.*/
2729         /* Let us first check the address...*/
2730         uint32_t safe_terminator;
2731         if (length <= 4) {
2732                 desc_array->descriptor[0].address = address;
2733                 desc_array->descriptor[0].length = length;
2734                 desc_array->elements = 1;
2735                 return(desc_array->elements);
2736         }
2737         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
2738         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
2739         if (safe_terminator == 0 ||
2740             (safe_terminator > 4 && safe_terminator < 9) ||
2741             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
2742                 desc_array->descriptor[0].address = address;
2743                 desc_array->descriptor[0].length = length;
2744                 desc_array->elements = 1;
2745                 return(desc_array->elements);
2746         }
2747
2748         desc_array->descriptor[0].address = address;
2749         desc_array->descriptor[0].length = length - 4;
2750         desc_array->descriptor[1].address = address + (length - 4);
2751         desc_array->descriptor[1].length = 4;
2752         desc_array->elements = 2;
2753         return(desc_array->elements);
2754 }
2755
2756 /**********************************************************************
2757  *
2758  *  Update the board statistics counters.
2759  *
2760  **********************************************************************/
2761 static void
2762 em_update_stats_counters(struct adapter *adapter)
2763 {
2764         struct ifnet   *ifp;
2765
2766         if (adapter->hw.media_type == em_media_type_copper ||
2767             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
2768                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
2769                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
2770         }
2771         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
2772         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
2773         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
2774         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
2775
2776         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
2777         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
2778         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
2779         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
2780         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
2781         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
2782         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
2783         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
2784         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
2785         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
2786         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
2787         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
2788         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
2789         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
2790         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
2791         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
2792         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
2793         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
2794         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
2795         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
2796
2797         /* For the 64-bit byte counters the low dword must be read first. */
2798         /* Both registers clear on the read of the high dword */
2799
2800         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
2801         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
2802         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
2803         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
2804
2805         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
2806         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
2807         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
2808         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
2809         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
2810
2811         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
2812         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
2813         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
2814         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
2815
2816         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
2817         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
2818         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
2819         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
2820         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
2821         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
2822         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
2823         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
2824         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
2825         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
2826
2827         if (adapter->hw.mac_type >= em_82543) {
2828                 adapter->stats.algnerrc +=
2829                     E1000_READ_REG(&adapter->hw, ALGNERRC);
2830                 adapter->stats.rxerrc +=
2831                     E1000_READ_REG(&adapter->hw, RXERRC);
2832                 adapter->stats.tncrs +=
2833                     E1000_READ_REG(&adapter->hw, TNCRS);
2834                 adapter->stats.cexterr +=
2835                     E1000_READ_REG(&adapter->hw, CEXTERR);
2836                 adapter->stats.tsctc +=
2837                     E1000_READ_REG(&adapter->hw, TSCTC);
2838                 adapter->stats.tsctfc +=
2839                     E1000_READ_REG(&adapter->hw, TSCTFC);
2840         }
2841         ifp = &adapter->interface_data.ac_if;
2842
2843         /* Fill out the OS statistics structure */
2844         ifp->if_ibytes = adapter->stats.gorcl;
2845         ifp->if_obytes = adapter->stats.gotcl;
2846         ifp->if_imcasts = adapter->stats.mprc;
2847         ifp->if_collisions = adapter->stats.colc;
2848
2849         /* Rx Errors */
2850         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
2851             adapter->stats.crcerrs + adapter->stats.algnerrc +
2852             adapter->stats.rlec + adapter->stats.rnbc +
2853             adapter->stats.mpc + adapter->stats.cexterr;
2854
2855         /* Tx Errors */
2856         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol;
2857 }
2858
2859
2860 /**********************************************************************
2861  *
2862  *  This routine is called only when em_display_debug_stats is enabled.
2863  *  This routine provides a way to take a look at important statistics
2864  *  maintained by the driver and hardware.
2865  *
2866  **********************************************************************/
2867 static void
2868 em_print_debug_info(struct adapter *adapter)
2869 {
2870         device_t dev= adapter->dev;
2871         uint8_t *hw_addr = adapter->hw.hw_addr;
2872
2873         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
2874         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
2875                       E1000_READ_REG(&adapter->hw, TIDV),
2876                       E1000_READ_REG(&adapter->hw, TADV));
2877         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
2878                       E1000_READ_REG(&adapter->hw, RDTR),
2879                       E1000_READ_REG(&adapter->hw, RADV));
2880 #ifdef DBG_STATS
2881         device_printf(dev, "Packets not Avail = %ld\n", adapter->no_pkts_avail);
2882         device_printf(dev, "CleanTxInterrupts = %ld\n",
2883                       adapter->clean_tx_interrupts);
2884 #endif
2885         device_printf(dev, "fifo workaround = %lld, fifo_reset = %lld\n",
2886                       (long long)adapter->tx_fifo_wrk,
2887                       (long long)adapter->tx_fifo_reset);
2888         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2889                       E1000_READ_REG(&adapter->hw, TDH),
2890                       E1000_READ_REG(&adapter->hw, TDT));
2891         device_printf(dev, "Num Tx descriptors avail = %d\n",
2892                       adapter->num_tx_desc_avail);
2893         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
2894                       adapter->no_tx_desc_avail1);
2895         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
2896                       adapter->no_tx_desc_avail2);
2897         device_printf(dev, "Std mbuf failed = %ld\n",
2898                       adapter->mbuf_alloc_failed);
2899         device_printf(dev, "Std mbuf cluster failed = %ld\n",
2900                       adapter->mbuf_cluster_failed);
2901         device_printf(dev, "Driver dropped packets = %ld\n",
2902                       adapter->dropped_pkts);
2903 }
2904
2905 static void
2906 em_print_hw_stats(struct adapter *adapter)
2907 {
2908         device_t dev= adapter->dev;
2909
2910         device_printf(dev, "Excessive collisions = %lld\n",
2911                       (long long)adapter->stats.ecol);
2912         device_printf(dev, "Symbol errors = %lld\n",
2913                       (long long)adapter->stats.symerrs);
2914         device_printf(dev, "Sequence errors = %lld\n",
2915                       (long long)adapter->stats.sec);
2916         device_printf(dev, "Defer count = %lld\n",
2917                       (long long)adapter->stats.dc);
2918
2919         device_printf(dev, "Missed Packets = %lld\n",
2920                       (long long)adapter->stats.mpc);
2921         device_printf(dev, "Receive No Buffers = %lld\n",
2922                       (long long)adapter->stats.rnbc);
2923         device_printf(dev, "Receive length errors = %lld\n",
2924                       (long long)adapter->stats.rlec);
2925         device_printf(dev, "Receive errors = %lld\n",
2926                       (long long)adapter->stats.rxerrc);
2927         device_printf(dev, "Crc errors = %lld\n",
2928                       (long long)adapter->stats.crcerrs);
2929         device_printf(dev, "Alignment errors = %lld\n",
2930                       (long long)adapter->stats.algnerrc);
2931         device_printf(dev, "Carrier extension errors = %lld\n",
2932                       (long long)adapter->stats.cexterr);
2933
2934         device_printf(dev, "XON Rcvd = %lld\n",
2935                       (long long)adapter->stats.xonrxc);
2936         device_printf(dev, "XON Xmtd = %lld\n",
2937                       (long long)adapter->stats.xontxc);
2938         device_printf(dev, "XOFF Rcvd = %lld\n",
2939                       (long long)adapter->stats.xoffrxc);
2940         device_printf(dev, "XOFF Xmtd = %lld\n",
2941                       (long long)adapter->stats.xofftxc);
2942
2943         device_printf(dev, "Good Packets Rcvd = %lld\n",
2944                       (long long)adapter->stats.gprc);
2945         device_printf(dev, "Good Packets Xmtd = %lld\n",
2946                       (long long)adapter->stats.gptc);
2947 }
2948
2949 static int
2950 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
2951 {
2952         int error;
2953         int result;
2954         struct adapter *adapter;
2955
2956         result = -1;
2957         error = sysctl_handle_int(oidp, &result, 0, req);
2958
2959         if (error || !req->newptr)
2960                 return(error);
2961
2962         if (result == 1) {
2963                 adapter = (struct adapter *)arg1;
2964                 em_print_debug_info(adapter);
2965         }
2966
2967         return(error);
2968 }
2969
2970 static int
2971 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
2972 {
2973         int error;
2974         int result;
2975         struct adapter *adapter;
2976
2977         result = -1;
2978         error = sysctl_handle_int(oidp, &result, 0, req);
2979
2980         if (error || !req->newptr)
2981                 return(error);
2982
2983         if (result == 1) {
2984                 adapter = (struct adapter *)arg1;
2985                 em_print_hw_stats(adapter);
2986         }
2987
2988         return(error);
2989 }
2990
2991 static int
2992 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
2993 {
2994         struct em_int_delay_info *info;
2995         struct adapter *adapter;
2996         uint32_t regval;
2997         int error;
2998         int usecs;
2999         int ticks;
3000         int s;
3001
3002         info = (struct em_int_delay_info *)arg1;
3003         adapter = info->adapter;
3004         usecs = info->value;
3005         error = sysctl_handle_int(oidp, &usecs, 0, req);
3006         if (error != 0 || req->newptr == NULL)
3007                 return(error);
3008         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3009                 return(EINVAL);
3010         info->value = usecs;
3011         ticks = E1000_USECS_TO_TICKS(usecs);
3012
3013         s = splimp();
3014         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3015         regval = (regval & ~0xffff) | (ticks & 0xffff);
3016         /* Handle a few special cases. */
3017         switch (info->offset) {
3018         case E1000_RDTR:
3019         case E1000_82542_RDTR:
3020                 regval |= E1000_RDT_FPDB;
3021                 break;
3022         case E1000_TIDV:
3023         case E1000_82542_TIDV:
3024                 if (ticks == 0) {
3025                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3026                         /* Don't write 0 into the TIDV register. */
3027                         regval++;
3028                 } else
3029                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3030                 break;
3031         }
3032         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3033         splx(s);
3034         return(0);
3035 }
3036
3037 static void
3038 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3039                         const char *description, struct em_int_delay_info *info,
3040                         int offset, int value)
3041 {
3042         info->adapter = adapter;
3043         info->offset = offset;
3044         info->value = value;
3045         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3046                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3047                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3048                         info, 0, em_sysctl_int_delay, "I", description);
3049 }