sys/dev/netif/em/if_em.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4
   5 Copyright (c) 2001-2003, Intel Corporation
   6 All rights reserved.
   7
   8 Redistribution and use in source and binary forms, with or without
   9 modification, are permitted provided that the following conditions are met:
  10
  11  1. Redistributions of source code must retain the above copyright notice,
  12     this list of conditions and the following disclaimer.
  13
  14  2. Redistributions in binary form must reproduce the above copyright
  15     notice, this list of conditions and the following disclaimer in the
  16     documentation and/or other materials provided with the distribution.
  17
  18  3. Neither the name of the Intel Corporation nor the names of its
  19     contributors may be used to endorse or promote products derived from
  20     this software without specific prior written permission.
  21
  22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32 POSSIBILITY OF SUCH DAMAGE.
  33
  34 ***************************************************************************/
  35
  36 /*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.15 2003/06/09 22:10:15 pdeuskar Exp $*/
  37 /*$DragonFly: src/sys/dev/netif/em/if_em.c,v 1.18 2004/06/05 13:52:29 joerg Exp $*/
  38
  39 #include <dev/netif/em/if_em.h>
  40
  41 /*********************************************************************
  42  *  Set this to one to display debug statistics
  43  *********************************************************************/
  44 int             em_display_debug_stats = 0;
  45
  46 /*********************************************************************
  47  *  Driver version
  48  *********************************************************************/
  49
  50 char em_driver_version[] = "1.7.25";
  51
  52
  53 /*********************************************************************
  54  *  PCI Device ID Table
  55  *
  56  *  Used by probe to select devices to load on
  57  *  Last field stores an index into em_strings
  58  *  Last entry must be all 0s
  59  *
  60  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  61  *********************************************************************/
  62
  63 static em_vendor_info_t em_vendor_info_array[] =
  64 {
  65         /* Intel(R) PRO/1000 Network Connection */
  66         { 0x8086, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0},
  67         { 0x8086, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0},
  68         { 0x8086, 0x1004, PCI_ANY_ID, PCI_ANY_ID, 0},
  69         { 0x8086, 0x1008, PCI_ANY_ID, PCI_ANY_ID, 0},
  70         { 0x8086, 0x1009, PCI_ANY_ID, PCI_ANY_ID, 0},
  71         { 0x8086, 0x100C, PCI_ANY_ID, PCI_ANY_ID, 0},
  72         { 0x8086, 0x100D, PCI_ANY_ID, PCI_ANY_ID, 0},
  73         { 0x8086, 0x100E, PCI_ANY_ID, PCI_ANY_ID, 0},
  74         { 0x8086, 0x100F, PCI_ANY_ID, PCI_ANY_ID, 0},
  75         { 0x8086, 0x1010, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         { 0x8086, 0x1011, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         { 0x8086, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0},
  78         { 0x8086, 0x1013, PCI_ANY_ID, PCI_ANY_ID, 0},
  79         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
  80         { 0x8086, 0x1015, PCI_ANY_ID, PCI_ANY_ID, 0},
  81         { 0x8086, 0x1016, PCI_ANY_ID, PCI_ANY_ID, 0},
  82         { 0x8086, 0x1017, PCI_ANY_ID, PCI_ANY_ID, 0},
  83         { 0x8086, 0x1018, PCI_ANY_ID, PCI_ANY_ID, 0},
  84         { 0x8086, 0x1019, PCI_ANY_ID, PCI_ANY_ID, 0},
  85         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
  86         { 0x8086, 0x101D, PCI_ANY_ID, PCI_ANY_ID, 0},
  87         { 0x8086, 0x101E, PCI_ANY_ID, PCI_ANY_ID, 0},
  88         { 0x8086, 0x1026, PCI_ANY_ID, PCI_ANY_ID, 0},
  89         { 0x8086, 0x1027, PCI_ANY_ID, PCI_ANY_ID, 0},
  90         { 0x8086, 0x1028, PCI_ANY_ID, PCI_ANY_ID, 0},
  91         { 0x8086, 0x1075, PCI_ANY_ID, PCI_ANY_ID, 0},
  92         { 0x8086, 0x1076, PCI_ANY_ID, PCI_ANY_ID, 0},
  93         { 0x8086, 0x1077, PCI_ANY_ID, PCI_ANY_ID, 0},
  94         { 0x8086, 0x1078, PCI_ANY_ID, PCI_ANY_ID, 0},
  95         { 0x8086, 0x1079, PCI_ANY_ID, PCI_ANY_ID, 0},
  96         { 0x8086, 0x107A, PCI_ANY_ID, PCI_ANY_ID, 0},
  97         { 0x8086, 0x107B, PCI_ANY_ID, PCI_ANY_ID, 0},
  98         /* required last entry */
  99         { 0, 0, 0, 0, 0}
 100 };
 101
 102 /*********************************************************************
 103  *  Table of branding strings for all supported NICs.
 104  *********************************************************************/
 105
 106 static const char *em_strings[] = {
 107         "Intel(R) PRO/1000 Network Connection"
 108 };
 109
 110 /*********************************************************************
 111  *  Function prototypes
 112  *********************************************************************/
 113 static int      em_probe(device_t);
 114 static int      em_attach(device_t);
 115 static int      em_detach(device_t);
 116 static int      em_shutdown(device_t);
 117 static void     em_intr(void *);
 118 static void     em_start(struct ifnet *);
 119 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 120 static void     em_watchdog(struct ifnet *);
 121 static void     em_init(void *);
 122 static void     em_stop(void *);
 123 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 124 static int      em_media_change(struct ifnet *);
 125 static void     em_identify_hardware(struct adapter *);
 126 static void     em_local_timer(void *);
 127 static int      em_hardware_init(struct adapter *);
 128 static void     em_setup_interface(device_t, struct adapter *);
 129 static int      em_setup_transmit_structures(struct adapter *);
 130 static void     em_initialize_transmit_unit(struct adapter *);
 131 static int      em_setup_receive_structures(struct adapter *);
 132 static void     em_initialize_receive_unit(struct adapter *);
 133 static void     em_enable_intr(struct adapter *);
 134 static void     em_disable_intr(struct adapter *);
 135 static void     em_free_transmit_structures(struct adapter *);
 136 static void     em_free_receive_structures(struct adapter *);
 137 static void     em_update_stats_counters(struct adapter *);
 138 static void     em_clean_transmit_interrupts(struct adapter *);
 139 static int      em_allocate_receive_structures(struct adapter *);
 140 static int      em_allocate_transmit_structures(struct adapter *);
 141 static void     em_process_receive_interrupts(struct adapter *, int);
 142 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 143                                     struct mbuf *);
 144 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 145                                            uint32_t *, uint32_t *);
 146 static void     em_set_promisc(struct adapter *);
 147 static void     em_disable_promisc(struct adapter *);
 148 static void     em_set_multi(struct adapter *);
 149 static void     em_print_hw_stats(struct adapter *);
 150 static void     em_print_link_status(struct adapter *);
 151 static int      em_get_buf(int i, struct adapter *, struct mbuf *);
 152 static void     em_enable_vlans(struct adapter *);
 153 static int      em_encap(struct adapter *, struct mbuf *);
 154 static void     em_smartspeed(struct adapter *);
 155 static int      em_82547_fifo_workaround(struct adapter *, int);
 156 static void     em_82547_update_fifo_head(struct adapter *, int);
 157 static int      em_82547_tx_fifo_reset(struct adapter *);
 158 static void     em_82547_move_tail(void *arg);
 159 static int      em_dma_malloc(struct adapter *, bus_size_t,
 160                               struct em_dma_alloc *, int);
 161 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 162 static void     em_print_debug_info(struct adapter *);
 163 static int      em_is_valid_ether_addr(uint8_t *);
 164 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 165 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 166 static uint32_t em_fill_descriptors(uint64_t address, uint32_t length,
 167                                    PDESC_ARRAY desc_array);
 168 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 169 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 170                                         const char *,
 171                                         struct em_int_delay_info *, int, int);
 172
 173 /*********************************************************************
 174  *  FreeBSD Device Interface Entry Points
 175  *********************************************************************/
 176
 177 static device_method_t em_methods[] = {
 178         /* Device interface */
 179         DEVMETHOD(device_probe, em_probe),
 180         DEVMETHOD(device_attach, em_attach),
 181         DEVMETHOD(device_detach, em_detach),
 182         DEVMETHOD(device_shutdown, em_shutdown),
 183         {0, 0}
 184 };
 185
 186 static driver_t em_driver = {
 187         "em", em_methods, sizeof(struct adapter),
 188 };
 189
 190 static devclass_t em_devclass;
 191
 192 DECLARE_DUMMY_MODULE(if_em);
 193 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 194
 195 /*********************************************************************
 196  *  Tunable default values.
 197  *********************************************************************/
 198
 199 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 200 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 201
 202 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 203 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 204 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 205 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 206
 207 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 208 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 209 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 210 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 211
 212 /*********************************************************************
 213  *  Device identification routine
 214  *
 215  *  em_probe determines if the driver should be loaded on
 216  *  adapter based on PCI vendor/device id of the adapter.
 217  *
 218  *  return 0 on success, positive on failure
 219  *********************************************************************/
 220
 221 static int
 222 em_probe(device_t dev)
 223 {
 224         em_vendor_info_t *ent;
 225
 226         uint16_t pci_vendor_id = 0;
 227         uint16_t pci_device_id = 0;
 228         uint16_t pci_subvendor_id = 0;
 229         uint16_t pci_subdevice_id = 0;
 230         char adapter_name[60];
 231
 232         INIT_DEBUGOUT("em_probe: begin");
 233
 234         pci_vendor_id = pci_get_vendor(dev);
 235         if (pci_vendor_id != EM_VENDOR_ID)
 236                 return(ENXIO);
 237
 238         pci_device_id = pci_get_device(dev);
 239         pci_subvendor_id = pci_get_subvendor(dev);
 240         pci_subdevice_id = pci_get_subdevice(dev);
 241
 242         ent = em_vendor_info_array;
 243         while (ent->vendor_id != 0) {
 244                 if ((pci_vendor_id == ent->vendor_id) &&
 245                     (pci_device_id == ent->device_id) &&
 246
 247                     ((pci_subvendor_id == ent->subvendor_id) ||
 248                      (ent->subvendor_id == PCI_ANY_ID)) &&
 249
 250                     ((pci_subdevice_id == ent->subdevice_id) ||
 251                      (ent->subdevice_id == PCI_ANY_ID))) {
 252                         snprintf(adapter_name, sizeof(adapter_name),
 253                                  "%s, Version - %s",  em_strings[ent->index],
 254                                  em_driver_version);
 255                         device_set_desc_copy(dev, adapter_name);
 256                         return(0);
 257                 }
 258                 ent++;
 259         }
 260
 261         return(ENXIO);
 262 }
 263
 264 /*********************************************************************
 265  *  Device initialization routine
 266  *
 267  *  The attach entry point is called when the driver is being loaded.
 268  *  This routine identifies the type of hardware, allocates all resources
 269  *  and initializes the hardware.
 270  *
 271  *  return 0 on success, positive on failure
 272  *********************************************************************/
 273
 274 static int
 275 em_attach(device_t dev)
 276 {
 277         struct adapter *adapter;
 278         int tsize, rsize;
 279         int i, val, rid;
 280         int error = 0;
 281
 282         INIT_DEBUGOUT("em_attach: begin");
 283
 284         adapter = device_get_softc(dev);
 285
 286         bzero(adapter, sizeof(struct adapter));
 287
 288         callout_init(&adapter->timer);
 289         callout_init(&adapter->tx_fifo_timer);
 290
 291         adapter->dev = dev;
 292         adapter->osdep.dev = dev;
 293
 294         /* SYSCTL stuff */
 295         sysctl_ctx_init(&adapter->sysctl_ctx);
 296         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 297                                                SYSCTL_STATIC_CHILDREN(_hw),
 298                                                OID_AUTO,
 299                                                device_get_nameunit(dev),
 300                                                CTLFLAG_RD,
 301                                                0, "");
 302
 303         if (adapter->sysctl_tree == NULL) {
 304                 error = EIO;
 305                 goto fail;
 306         }
 307
 308         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 309                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 310                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 311                         (void *)adapter, 0,
 312                         em_sysctl_debug_info, "I", "Debug Information");
 313
 314         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 315                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 316                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 317                         (void *)adapter, 0,
 318                         em_sysctl_stats, "I", "Statistics");
 319
 320         /* Determine hardware revision */
 321         em_identify_hardware(adapter);
 322
 323         /* Set up some sysctls for the tunable interrupt delays */
 324         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 325                                 "receive interrupt delay in usecs",
 326                                 &adapter->rx_int_delay,
 327                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 328                                 em_rx_int_delay_dflt);
 329         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 330                                 "transmit interrupt delay in usecs",
 331                                 &adapter->tx_int_delay,
 332                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 333                                 em_tx_int_delay_dflt);
 334         if (adapter->hw.mac_type >= em_82540) {
 335                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 336                                         "receive interrupt delay limit in usecs",
 337                                         &adapter->rx_abs_int_delay,
 338                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 339                                         em_rx_abs_int_delay_dflt);
 340                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 341                                         "transmit interrupt delay limit in usecs",
 342                                         &adapter->tx_abs_int_delay,
 343                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 344                                         em_tx_abs_int_delay_dflt);
 345         }
 346
 347         /* Parameters (to be read from user) */
 348         adapter->num_tx_desc = EM_MAX_TXD;
 349         adapter->num_rx_desc = EM_MAX_RXD;
 350         adapter->hw.autoneg = DO_AUTO_NEG;
 351         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 352         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 353         adapter->hw.tbi_compatibility_en = TRUE;
 354         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 355
 356         /*
 357          * These parameters control the automatic generation(Tx) and
 358          * response(Rx) to Ethernet PAUSE frames.
 359          */
 360         adapter->hw.fc_high_water = FC_DEFAULT_HI_THRESH;
 361         adapter->hw.fc_low_water  = FC_DEFAULT_LO_THRESH;
 362         adapter->hw.fc_pause_time = FC_DEFAULT_TX_TIMER;
 363         adapter->hw.fc_send_xon   = TRUE;
 364         adapter->hw.fc = em_fc_full;
 365
 366         adapter->hw.phy_init_script = 1;
 367         adapter->hw.phy_reset_disable = FALSE;
 368
 369 #ifndef EM_MASTER_SLAVE
 370         adapter->hw.master_slave = em_ms_hw_default;
 371 #else
 372         adapter->hw.master_slave = EM_MASTER_SLAVE;
 373 #endif
 374
 375         /*
 376          * Set the max frame size assuming standard ethernet
 377          * sized frames
 378          */
 379         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 380
 381         adapter->hw.min_frame_size =
 382             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 383
 384         /*
 385          * This controls when hardware reports transmit completion
 386          * status.
 387          */
 388         adapter->hw.report_tx_early = 1;
 389
 390         rid = EM_MMBA;
 391         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 392                                                      &rid, RF_ACTIVE);
 393         if (!(adapter->res_memory)) {
 394                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 395                 error = ENXIO;
 396                 goto fail;
 397         }
 398         adapter->osdep.mem_bus_space_tag =
 399             rman_get_bustag(adapter->res_memory);
 400         adapter->osdep.mem_bus_space_handle =
 401             rman_get_bushandle(adapter->res_memory);
 402         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
 403
 404         if (adapter->hw.mac_type > em_82543) {
 405                 /* Figure our where our IO BAR is ? */
 406                 rid = EM_MMBA;
 407                 for (i = 0; i < 5; i++) {
 408                         val = pci_read_config(dev, rid, 4);
 409                         if (val & 0x00000001) {
 410                                 adapter->io_rid = rid;
 411                                 break;
 412                         }
 413                         rid += 4;
 414                 }
 415
 416                 adapter->res_ioport = bus_alloc_resource_any(dev,
 417                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
 418                 if (!(adapter->res_ioport)) {
 419                         device_printf(dev, "Unable to allocate bus resource: ioport\n");
 420                         error = ENXIO;
 421                         goto fail;
 422                 }
 423
 424                 adapter->hw.reg_io_tag = rman_get_bustag(adapter->res_ioport);
 425                 adapter->hw.reg_io_handle = rman_get_bushandle(adapter->res_ioport);
 426         }
 427
 428         rid = 0x0;
 429         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 430             &rid, RF_SHAREABLE | RF_ACTIVE);
 431         if (!(adapter->res_interrupt)) {
 432                 device_printf(dev, "Unable to allocate bus resource: interrupt\n");
 433                 error = ENXIO;
 434                 goto fail;
 435         }
 436
 437         adapter->hw.back = &adapter->osdep;
 438
 439         /* Initialize eeprom parameters */
 440         em_init_eeprom_params(&adapter->hw);
 441
 442         tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc);
 443
 444         /* Allocate Transmit Descriptor ring */
 445         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_WAITOK)) {
 446                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 447                 error = ENOMEM;
 448                 goto fail;
 449         }
 450         adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr;
 451
 452         rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc);
 453
 454         /* Allocate Receive Descriptor ring */
 455         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 456                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 457                 error = ENOMEM;
 458                 goto fail;
 459         }
 460         adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr;
 461
 462         /* Initialize the hardware */
 463         if (em_hardware_init(adapter)) {
 464                 device_printf(dev, "Unable to initialize the hardware\n");
 465                 error = EIO;
 466                 goto fail;
 467         }
 468
 469         /* Copy the permanent MAC address out of the EEPROM */
 470         if (em_read_mac_addr(&adapter->hw) < 0) {
 471                 device_printf(dev, "EEPROM read error while reading mac address\n");
 472                 error = EIO;
 473                 goto fail;
 474         }
 475
 476         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 477                 device_printf(dev, "Invalid mac address\n");
 478                 error = EIO;
 479                 goto fail;
 480         }
 481
 482         /* Setup OS specific network interface */
 483         em_setup_interface(dev, adapter);
 484
 485         /* Initialize statistics */
 486         em_clear_hw_cntrs(&adapter->hw);
 487         em_update_stats_counters(adapter);
 488         adapter->hw.get_link_status = 1;
 489         em_check_for_link(&adapter->hw);
 490
 491         /* Print the link status */
 492         if (adapter->link_active == 1) {
 493                 em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
 494                                         &adapter->link_duplex);
 495                 device_printf(dev, "Speed: %d Mbps, Duplex: %s\n",
 496                     adapter->link_speed,
 497                     adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
 498         } else
 499                 device_printf(dev, "Speed: N/A, Duplex:N/A\n");
 500
 501         /* Identify 82544 on PCIX */
 502         em_get_bus_info(&adapter->hw);
 503         if (adapter->hw.bus_type == em_bus_type_pcix &&
 504             adapter->hw.mac_type == em_82544)
 505                 adapter->pcix_82544 = TRUE;
 506         else
 507                 adapter->pcix_82544 = FALSE;
 508
 509         error = bus_setup_intr(dev, adapter->res_interrupt, INTR_TYPE_NET,
 510                            (void (*)(void *)) em_intr, adapter,
 511                            &adapter->int_handler_tag);
 512         if (error) {
 513                 device_printf(dev, "Error registering interrupt handler!\n");
 514                 ether_ifdetach(&adapter->interface_data.ac_if);
 515                 goto fail;
 516         }
 517
 518         INIT_DEBUGOUT("em_attach: end");
 519         return(0);
 520
 521 fail:
 522         em_detach(dev);
 523         return(error);
 524 }
 525
 526 /*********************************************************************
 527  *  Device removal routine
 528  *
 529  *  The detach entry point is called when the driver is being removed.
 530  *  This routine stops the adapter and deallocates all the resources
 531  *  that were allocated for driver operation.
 532  *
 533  *  return 0 on success, positive on failure
 534  *********************************************************************/
 535
 536 static int
 537 em_detach(device_t dev)
 538 {
 539         struct adapter * adapter = device_get_softc(dev);
 540         int s;
 541
 542         INIT_DEBUGOUT("em_detach: begin");
 543         s = splimp();
 544
 545         adapter->in_detach = 1;
 546
 547         if (device_is_attached(dev)) {
 548                 em_stop(adapter);
 549                 em_phy_hw_reset(&adapter->hw);
 550                 ether_ifdetach(&adapter->interface_data.ac_if);
 551         }
 552         bus_generic_detach(dev);
 553
 554         if (adapter->res_interrupt != NULL) {
 555                 bus_teardown_intr(dev, adapter->res_interrupt,
 556                                   adapter->int_handler_tag);
 557                 bus_release_resource(dev, SYS_RES_IRQ, 0,
 558                                      adapter->res_interrupt);
 559         }
 560         if (adapter->res_memory != NULL) {
 561                 bus_release_resource(dev, SYS_RES_MEMORY, EM_MMBA,
 562                                      adapter->res_memory);
 563         }
 564
 565         if (adapter->res_ioport != NULL) {
 566                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
 567                                      adapter->res_ioport);
 568         }
 569
 570         /* Free Transmit Descriptor ring */
 571         if (adapter->tx_desc_base != NULL) {
 572                 em_dma_free(adapter, &adapter->txdma);
 573                 adapter->tx_desc_base = NULL;
 574         }
 575
 576         /* Free Receive Descriptor ring */
 577         if (adapter->rx_desc_base != NULL) {
 578                 em_dma_free(adapter, &adapter->rxdma);
 579                 adapter->rx_desc_base = NULL;
 580         }
 581
 582         adapter->sysctl_tree = NULL;
 583         sysctl_ctx_free(&adapter->sysctl_ctx);
 584
 585         splx(s);
 586         return(0);
 587 }
 588
 589 /*********************************************************************
 590  *
 591  *  Shutdown entry point
 592  *
 593  **********************************************************************/
 594
 595 static int
 596 em_shutdown(device_t dev)
 597 {
 598         struct adapter *adapter = device_get_softc(dev);
 599         em_stop(adapter);
 600         return(0);
 601 }
 602
 603 /*********************************************************************
 604  *  Transmit entry point
 605  *
 606  *  em_start is called by the stack to initiate a transmit.
 607  *  The driver will remain in this routine as long as there are
 608  *  packets to transmit and transmit resources are available.
 609  *  In case resources are not available stack is notified and
 610  *  the packet is requeued.
 611  **********************************************************************/
 612
 613 static void
 614 em_start(struct ifnet *ifp)
 615 {
 616         int s;
 617         struct mbuf *m_head;
 618         struct adapter *adapter = ifp->if_softc;
 619
 620         if (!adapter->link_active)
 621                 return;
 622
 623         s = splimp();
 624         while (ifp->if_snd.ifq_head != NULL) {
 625                 IF_DEQUEUE(&ifp->if_snd, m_head);
 626
 627                 if (m_head == NULL)
 628                         break;
 629
 630                 if (em_encap(adapter, m_head)) {
 631                         ifp->if_flags |= IFF_OACTIVE;
 632                         IF_PREPEND(&ifp->if_snd, m_head);
 633                         break;
 634                 }
 635
 636                 /* Send a copy of the frame to the BPF listener */
 637                 BPF_MTAP(ifp, m_head);
 638
 639                 /* Set timeout in case hardware has problems transmitting */
 640                 ifp->if_timer = EM_TX_TIMEOUT;
 641         }
 642         splx(s);
 643 }
 644
 645 /*********************************************************************
 646  *  Ioctl entry point
 647  *
 648  *  em_ioctl is called when the user wants to configure the
 649  *  interface.
 650  *
 651  *  return 0 on success, positive on failure
 652  **********************************************************************/
 653
 654 static int
 655 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 656 {
 657         int s, mask, error = 0;
 658         struct ifreq *ifr = (struct ifreq *) data;
 659         struct adapter *adapter = ifp->if_softc;
 660
 661         s = splimp();
 662
 663         if (adapter->in_detach)
 664                 goto out;
 665
 666         switch (command) {
 667         case SIOCSIFADDR:
 668         case SIOCGIFADDR:
 669                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 670                 ether_ioctl(ifp, command, data);
 671                 break;
 672         case SIOCSIFMTU:
 673                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 674                 if (ifr->ifr_mtu > MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 675                         error = EINVAL;
 676                 } else {
 677                         ifp->if_mtu = ifr->ifr_mtu;
 678                         adapter->hw.max_frame_size =
 679                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 680                         em_init(adapter);
 681                 }
 682                 break;
 683         case SIOCSIFFLAGS:
 684                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 685                 if (ifp->if_flags & IFF_UP) {
 686                         if (!(ifp->if_flags & IFF_RUNNING))
 687                                 em_init(adapter);
 688                         em_disable_promisc(adapter);
 689                         em_set_promisc(adapter);
 690                 } else {
 691                         if (ifp->if_flags & IFF_RUNNING)
 692                                 em_stop(adapter);
 693                 }
 694                 break;
 695         case SIOCADDMULTI:
 696         case SIOCDELMULTI:
 697                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 698                 if (ifp->if_flags & IFF_RUNNING) {
 699                         em_disable_intr(adapter);
 700                         em_set_multi(adapter);
 701                         if (adapter->hw.mac_type == em_82542_rev2_0)
 702                                 em_initialize_receive_unit(adapter);
 703 #ifdef DEVICE_POLLING
 704                         if (!(ifp->if_flags & IFF_POLLING))
 705 #endif
 706                                 em_enable_intr(adapter);
 707                 }
 708                 break;
 709         case SIOCSIFMEDIA:
 710         case SIOCGIFMEDIA:
 711                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 712                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 713                 break;
 714         case SIOCSIFCAP:
 715                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 716                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 717                 if (mask & IFCAP_HWCSUM) {
 718                         if (IFCAP_HWCSUM & ifp->if_capenable)
 719                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 720                         else
 721                                 ifp->if_capenable |= IFCAP_HWCSUM;
 722                         if (ifp->if_flags & IFF_RUNNING)
 723                                 em_init(adapter);
 724                 }
 725                 break;
 726         default:
 727                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%x)\n", (int)command);
 728                 error = EINVAL;
 729         }
 730
 731 out:
 732         splx(s);
 733         return(error);
 734 }
 735
 736 /*********************************************************************
 737  *  Watchdog entry point
 738  *
 739  *  This routine is called whenever hardware quits transmitting.
 740  *
 741  **********************************************************************/
 742
 743 static void
 744 em_watchdog(struct ifnet *ifp)
 745 {
 746         struct adapter * adapter;
 747         adapter = ifp->if_softc;
 748
 749         /* If we are in this routine because of pause frames, then
 750          * don't reset the hardware.
 751          */
 752         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 753                 ifp->if_timer = EM_TX_TIMEOUT;
 754                 return;
 755         }
 756
 757         if (em_check_for_link(&adapter->hw))
 758                 if_printf(ifp, "watchdog timeout -- resetting\n");
 759
 760         ifp->if_flags &= ~IFF_RUNNING;
 761
 762         em_init(adapter);
 763
 764         ifp->if_oerrors++;
 765 }
 766
 767 /*********************************************************************
 768  *  Init entry point
 769  *
 770  *  This routine is used in two ways. It is used by the stack as
 771  *  init entry point in network interface structure. It is also used
 772  *  by the driver as a hw/sw initialization routine to get to a
 773  *  consistent state.
 774  *
 775  *  return 0 on success, positive on failure
 776  **********************************************************************/
 777
 778 static void
 779 em_init(void *arg)
 780 {
 781         int s;
 782         struct adapter *adapter = arg;
 783         struct ifnet *ifp = &adapter->interface_data.ac_if;
 784
 785         INIT_DEBUGOUT("em_init: begin");
 786
 787         s = splimp();
 788
 789         em_stop(adapter);
 790
 791         /* Get the latest mac address, User can use a LAA */
 792         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
 793               ETHER_ADDR_LEN);
 794
 795         /* Initialize the hardware */
 796         if (em_hardware_init(adapter)) {
 797                 if_printf(ifp, "Unable to initialize the hardware\n");
 798                 splx(s);
 799                 return;
 800         }
 801
 802         em_enable_vlans(adapter);
 803
 804         /* Prepare transmit descriptors and buffers */
 805         if (em_setup_transmit_structures(adapter)) {
 806                 if_printf(ifp, "Could not setup transmit structures\n");
 807                 em_stop(adapter);
 808                 splx(s);
 809                 return;
 810         }
 811         em_initialize_transmit_unit(adapter);
 812
 813         /* Setup Multicast table */
 814         em_set_multi(adapter);
 815
 816         /* Prepare receive descriptors and buffers */
 817         if (em_setup_receive_structures(adapter)) {
 818                 if_printf(ifp, "Could not setup receive structures\n");
 819                 em_stop(adapter);
 820                 splx(s);
 821                 return;
 822         }
 823         em_initialize_receive_unit(adapter);
 824
 825         /* Don't loose promiscuous settings */
 826         em_set_promisc(adapter);
 827
 828         ifp->if_flags |= IFF_RUNNING;
 829         ifp->if_flags &= ~IFF_OACTIVE;
 830
 831         if (adapter->hw.mac_type >= em_82543) {
 832                 if (ifp->if_capenable & IFCAP_TXCSUM)
 833                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
 834                 else
 835                         ifp->if_hwassist = 0;
 836         }
 837
 838         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 839         em_clear_hw_cntrs(&adapter->hw);
 840 #ifdef DEVICE_POLLING
 841         /*
 842          * Only enable interrupts if we are not polling, make sure
 843          * they are off otherwise.
 844          */
 845         if (ifp->if_flags & IFF_POLLING)
 846                 em_disable_intr(adapter);
 847         else
 848 #endif /* DEVICE_POLLING */
 849                 em_enable_intr(adapter);
 850
 851         /* Don't reset the phy next time init gets called */
 852         adapter->hw.phy_reset_disable = TRUE;
 853
 854         splx(s);
 855 }
 856
 857 #ifdef DEVICE_POLLING
 858 static poll_handler_t em_poll;
 859
 860 static void
 861 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 862 {
 863         struct adapter *adapter = ifp->if_softc;
 864         uint32_t reg_icr;
 865
 866         if (cmd == POLL_DEREGISTER) {       /* final call, enable interrupts */
 867                 em_enable_intr(adapter);
 868                 return;
 869         }
 870         if (cmd == POLL_AND_CHECK_STATUS) {
 871                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 872                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 873                         callout_stop(&adapter->timer);
 874                         adapter->hw.get_link_status = 1;
 875                         em_check_for_link(&adapter->hw);
 876                         em_print_link_status(adapter);
 877                         callout_reset(&adapter->timer, 2*hz, em_local_timer,
 878                                       adapter);
 879                 }
 880         }
 881         if (ifp->if_flags & IFF_RUNNING) {
 882                 em_process_receive_interrupts(adapter, count);
 883                 em_clean_transmit_interrupts(adapter);
 884         }
 885
 886         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 887                 em_start(ifp);
 888 }
 889 #endif /* DEVICE_POLLING */
 890
 891 /*********************************************************************
 892  *
 893  *  Interrupt Service routine
 894  *
 895  **********************************************************************/
 896 static void
 897 em_intr(void *arg)
 898 {
 899         uint32_t loop_cnt = EM_MAX_INTR;
 900         uint32_t reg_icr;
 901         struct ifnet *ifp;
 902         struct adapter *adapter = arg;
 903
 904         ifp = &adapter->interface_data.ac_if;
 905
 906 #ifdef DEVICE_POLLING
 907         if (ifp->if_flags & IFF_POLLING)
 908                 return;
 909
 910         if (ether_poll_register(em_poll, ifp)) {
 911                 em_disable_intr(adapter);
 912                 em_poll(ifp, 0, 1);
 913                 return;
 914         }
 915 #endif /* DEVICE_POLLING */
 916
 917         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 918         if (!reg_icr)
 919                 return;
 920
 921         /* Link status change */
 922         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 923                 callout_stop(&adapter->timer);
 924                 adapter->hw.get_link_status = 1;
 925                 em_check_for_link(&adapter->hw);
 926                 em_print_link_status(adapter);
 927                 callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 928         }
 929
 930         while (loop_cnt > 0) {
 931                 if (ifp->if_flags & IFF_RUNNING) {
 932                         em_process_receive_interrupts(adapter, -1);
 933                         em_clean_transmit_interrupts(adapter);
 934                 }
 935                 loop_cnt--;
 936         }
 937
 938         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 939                 em_start(ifp);
 940 }
 941
 942 /*********************************************************************
 943  *
 944  *  Media Ioctl callback
 945  *
 946  *  This routine is called whenever the user queries the status of
 947  *  the interface using ifconfig.
 948  *
 949  **********************************************************************/
 950 static void
 951 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 952 {
 953         struct adapter * adapter = ifp->if_softc;
 954
 955         INIT_DEBUGOUT("em_media_status: begin");
 956
 957         em_check_for_link(&adapter->hw);
 958         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
 959                 if (adapter->link_active == 0) {
 960                         em_get_speed_and_duplex(&adapter->hw,
 961                                                 &adapter->link_speed,
 962                                                 &adapter->link_duplex);
 963                         adapter->link_active = 1;
 964                 }
 965         } else {
 966                 if (adapter->link_active == 1) {
 967                         adapter->link_speed = 0;
 968                         adapter->link_duplex = 0;
 969                         adapter->link_active = 0;
 970                 }
 971         }
 972
 973         ifmr->ifm_status = IFM_AVALID;
 974         ifmr->ifm_active = IFM_ETHER;
 975
 976         if (!adapter->link_active)
 977                 return;
 978
 979         ifmr->ifm_status |= IFM_ACTIVE;
 980
 981         if (adapter->hw.media_type == em_media_type_fiber) {
 982                 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 983         } else {
 984                 switch (adapter->link_speed) {
 985                 case 10:
 986                         ifmr->ifm_active |= IFM_10_T;
 987                         break;
 988                 case 100:
 989                         ifmr->ifm_active |= IFM_100_TX;
 990                         break;
 991                 case 1000:
 992                         ifmr->ifm_active |= IFM_1000_TX;
 993                         break;
 994                 }
 995                 if (adapter->link_duplex == FULL_DUPLEX)
 996                         ifmr->ifm_active |= IFM_FDX;
 997                 else
 998                         ifmr->ifm_active |= IFM_HDX;
 999         }
1000 }
1001
1002 /*********************************************************************
1003  *
1004  *  Media Ioctl callback
1005  *
1006  *  This routine is called when the user changes speed/duplex using
1007  *  media/mediopt option with ifconfig.
1008  *
1009  **********************************************************************/
1010 static int
1011 em_media_change(struct ifnet *ifp)
1012 {
1013         struct adapter * adapter = ifp->if_softc;
1014         struct ifmedia  *ifm = &adapter->media;
1015
1016         INIT_DEBUGOUT("em_media_change: begin");
1017
1018         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1019                 return(EINVAL);
1020
1021         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1022         case IFM_AUTO:
1023                 adapter->hw.autoneg = DO_AUTO_NEG;
1024                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1025                 break;
1026         case IFM_1000_SX:
1027         case IFM_1000_TX:
1028                 adapter->hw.autoneg = DO_AUTO_NEG;
1029                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1030                 break;
1031         case IFM_100_TX:
1032                 adapter->hw.autoneg = FALSE;
1033                 adapter->hw.autoneg_advertised = 0;
1034                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1035                         adapter->hw.forced_speed_duplex = em_100_full;
1036                 else
1037                         adapter->hw.forced_speed_duplex = em_100_half;
1038                 break;
1039         case IFM_10_T:
1040                 adapter->hw.autoneg = FALSE;
1041                 adapter->hw.autoneg_advertised = 0;
1042                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1043                         adapter->hw.forced_speed_duplex = em_10_full;
1044                 else
1045                         adapter->hw.forced_speed_duplex = em_10_half;
1046                 break;
1047         default:
1048                 if_printf(ifp, "Unsupported media type\n");
1049         }
1050         /*
1051          * As the speed/duplex settings may have changed we need to
1052          * reset the PHY.
1053          */
1054         adapter->hw.phy_reset_disable = FALSE;
1055
1056         em_init(adapter);
1057
1058         return(0);
1059 }
1060
1061 static void
1062 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1063          int error)
1064 {
1065         struct em_q *q = arg;
1066
1067         if (error)
1068                 return;
1069         KASSERT(nsegs <= EM_MAX_SCATTER,
1070                 ("Too many DMA segments returned when mapping tx packet"));
1071         q->nsegs = nsegs;
1072         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1073 }
1074
1075 #define EM_FIFO_HDR              0x10
1076 #define EM_82547_PKT_THRESH      0x3e0
1077 #define EM_82547_TX_FIFO_SIZE    0x2800
1078 #define EM_82547_TX_FIFO_BEGIN   0xf00
1079 /*********************************************************************
1080  *
1081  *  This routine maps the mbufs to tx descriptors.
1082  *
1083  *  return 0 on success, positive on failure
1084  **********************************************************************/
1085 static int
1086 em_encap(struct adapter *adapter, struct mbuf *m_head)
1087 {
1088         uint32_t txd_upper;
1089         uint32_t txd_lower, txd_used = 0, txd_saved = 0;
1090         int i, j, error;
1091         uint64_t address;
1092
1093         /* For 82544 Workaround */
1094         DESC_ARRAY desc_array;
1095         uint32_t array_elements;
1096         uint32_t counter;
1097
1098 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1099         struct ifvlan *ifv = NULL;
1100 #else
1101         struct m_tag *mtag;
1102 #endif
1103         struct em_q q;
1104         struct em_buffer *tx_buffer = NULL;
1105         struct em_tx_desc *current_tx_desc = NULL;
1106         struct ifnet *ifp = &adapter->interface_data.ac_if;
1107
1108         /*
1109          * Force a cleanup if number of TX descriptors
1110          * available hits the threshold
1111          */
1112         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1113                 em_clean_transmit_interrupts(adapter);
1114                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1115                         adapter->no_tx_desc_avail1++;
1116                         return(ENOBUFS);
1117                 }
1118         }
1119         /*
1120          * Map the packet for DMA.
1121          */
1122         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q.map)) {
1123                 adapter->no_tx_map_avail++;
1124                 return(ENOMEM);
1125         }
1126         error = bus_dmamap_load_mbuf(adapter->txtag, q.map, m_head, em_tx_cb,
1127                                      &q, BUS_DMA_NOWAIT);
1128         if (error != 0) {
1129                 adapter->no_tx_dma_setup++;
1130                 bus_dmamap_destroy(adapter->txtag, q.map);
1131                 return(error);
1132         }
1133         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1134
1135         if (q.nsegs > adapter->num_tx_desc_avail) {
1136                 adapter->no_tx_desc_avail2++;
1137                 bus_dmamap_unload(adapter->txtag, q.map);
1138                 bus_dmamap_destroy(adapter->txtag, q.map);
1139                 return(ENOBUFS);
1140         }
1141
1142         if (ifp->if_hwassist > 0) {
1143                 em_transmit_checksum_setup(adapter,  m_head,
1144                                            &txd_upper, &txd_lower);
1145         }
1146         else
1147                 txd_upper = txd_lower = 0;
1148
1149         /* Find out if we are in vlan mode */
1150 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1151         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1152             m_head->m_pkthdr.rcvif != NULL &&
1153             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1154                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1155 #else
1156         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1157 #endif
1158
1159         i = adapter->next_avail_tx_desc;
1160         if (adapter->pcix_82544) {
1161                 txd_saved = i;
1162                 txd_used = 0;
1163         }
1164         for (j = 0; j < q.nsegs; j++) {
1165                 /* If adapter is 82544 and on PCIX bus */
1166                 if(adapter->pcix_82544) {
1167                         array_elements = 0;
1168                         address = htole64(q.segs[j].ds_addr);
1169                         /*
1170                          * Check the Address and Length combination and
1171                          * split the data accordingly
1172                          */
1173                         array_elements = em_fill_descriptors(address,
1174                                                              htole32(q.segs[j].ds_len),
1175                                                              &desc_array);
1176                         for (counter = 0; counter < array_elements; counter++) {
1177                                 if (txd_used == adapter->num_tx_desc_avail) {
1178                                         adapter->next_avail_tx_desc = txd_saved;
1179                                         adapter->no_tx_desc_avail2++;
1180                                         bus_dmamap_unload(adapter->txtag, q.map);
1181                                         bus_dmamap_destroy(adapter->txtag, q.map);
1182                                         return(ENOBUFS);
1183                                 }
1184                                 tx_buffer = &adapter->tx_buffer_area[i];
1185                                 current_tx_desc = &adapter->tx_desc_base[i];
1186                                 current_tx_desc->buffer_addr = htole64(
1187                                 desc_array.descriptor[counter].address);
1188                                 current_tx_desc->lower.data = htole32(
1189                                 (adapter->txd_cmd | txd_lower |
1190                                 (uint16_t)desc_array.descriptor[counter].length));
1191                                 current_tx_desc->upper.data = htole32((txd_upper));
1192                                 if (++i == adapter->num_tx_desc)
1193                                         i = 0;
1194
1195                                 tx_buffer->m_head = NULL;
1196                                 txd_used++;
1197                         }
1198                 } else {
1199                         tx_buffer = &adapter->tx_buffer_area[i];
1200                         current_tx_desc = &adapter->tx_desc_base[i];
1201
1202                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1203                         current_tx_desc->lower.data = htole32(
1204                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1205                         current_tx_desc->upper.data = htole32(txd_upper);
1206
1207                         if (++i == adapter->num_tx_desc)
1208                                 i = 0;
1209
1210                         tx_buffer->m_head = NULL;
1211                 }
1212         }
1213
1214         adapter->next_avail_tx_desc = i;
1215         if (adapter->pcix_82544)
1216                 adapter->num_tx_desc_avail -= txd_used;
1217         else
1218                 adapter->num_tx_desc_avail -= q.nsegs;
1219
1220 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1221         if (ifv != NULL) {
1222                 /* Set the vlan id */
1223                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1224 #else
1225         if (mtag != NULL) {
1226                 /* Set the vlan id */
1227                 current_tx_desc->upper.fields.special = htole16(VLAN_TAG_VALUE(mtag));
1228 #endif
1229
1230                 /* Tell hardware to add tag */
1231                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1232         }
1233
1234         tx_buffer->m_head = m_head;
1235         tx_buffer->map = q.map;
1236         bus_dmamap_sync(adapter->txtag, q.map, BUS_DMASYNC_PREWRITE);
1237
1238         /*
1239          * Last Descriptor of Packet needs End Of Packet (EOP)
1240          */
1241         current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1242
1243         /*
1244          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1245          * that this frame is available to transmit.
1246          */
1247         if (adapter->hw.mac_type == em_82547 &&
1248             adapter->link_duplex == HALF_DUPLEX) {
1249                 em_82547_move_tail(adapter);
1250         } else {
1251                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1252                 if (adapter->hw.mac_type == em_82547) {
1253                         em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len);
1254                 }
1255         }
1256
1257         return(0);
1258 }
1259
1260 /*********************************************************************
1261  *
1262  * 82547 workaround to avoid controller hang in half-duplex environment.
1263  * The workaround is to avoid queuing a large packet that would span
1264  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1265  * in this case. We do that only when FIFO is quiescent.
1266  *
1267  **********************************************************************/
1268 static void
1269 em_82547_move_tail(void *arg)
1270 {
1271         int s;
1272         struct adapter *adapter = arg;
1273         uint16_t hw_tdt;
1274         uint16_t sw_tdt;
1275         struct em_tx_desc *tx_desc;
1276         uint16_t length = 0;
1277         boolean_t eop = 0;
1278
1279         s = splimp();
1280         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1281         sw_tdt = adapter->next_avail_tx_desc;
1282
1283         while (hw_tdt != sw_tdt) {
1284                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1285                 length += tx_desc->lower.flags.length;
1286                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1287                 if(++hw_tdt == adapter->num_tx_desc)
1288                         hw_tdt = 0;
1289
1290                 if(eop) {
1291                         if (em_82547_fifo_workaround(adapter, length)) {
1292                                 adapter->tx_fifo_wrk++;
1293                                 callout_reset(&adapter->tx_fifo_timer, 1,
1294                                         em_82547_move_tail, adapter);
1295                                 break;
1296                         }
1297                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1298                         em_82547_update_fifo_head(adapter, length);
1299                         length = 0;
1300                 }
1301         }
1302         splx(s);
1303 }
1304
1305 static int
1306 em_82547_fifo_workaround(struct adapter *adapter, int len)
1307 {
1308         int fifo_space, fifo_pkt_len;
1309
1310         fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1311
1312         if (adapter->link_duplex == HALF_DUPLEX) {
1313                 fifo_space = EM_82547_TX_FIFO_SIZE - adapter->tx_fifo_head;
1314
1315                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1316                         if (em_82547_tx_fifo_reset(adapter))
1317                                 return(0);
1318                         else
1319                                 return(1);
1320                 }
1321         }
1322
1323         return(0);
1324 }
1325
1326 static void
1327 em_82547_update_fifo_head(struct adapter *adapter, int len)
1328 {
1329         int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1330
1331         /* tx_fifo_head is always 16 byte aligned */
1332         adapter->tx_fifo_head += fifo_pkt_len;
1333         if (adapter->tx_fifo_head >= EM_82547_TX_FIFO_SIZE)
1334                 adapter->tx_fifo_head -= EM_82547_TX_FIFO_SIZE;
1335 }
1336
1337 static int
1338 em_82547_tx_fifo_reset(struct adapter *adapter)
1339 {
1340         uint32_t tctl;
1341
1342         if ( (E1000_READ_REG(&adapter->hw, TDT) ==
1343               E1000_READ_REG(&adapter->hw, TDH)) &&
1344              (E1000_READ_REG(&adapter->hw, TDFT) ==
1345               E1000_READ_REG(&adapter->hw, TDFH)) &&
1346              (E1000_READ_REG(&adapter->hw, TDFTS) ==
1347               E1000_READ_REG(&adapter->hw, TDFHS)) &&
1348              (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1349
1350                 /* Disable TX unit */
1351                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1352                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1353
1354                 /* Reset FIFO pointers */
1355                 E1000_WRITE_REG(&adapter->hw, TDFT, EM_82547_TX_FIFO_BEGIN);
1356                 E1000_WRITE_REG(&adapter->hw, TDFH, EM_82547_TX_FIFO_BEGIN);
1357                 E1000_WRITE_REG(&adapter->hw, TDFTS, EM_82547_TX_FIFO_BEGIN);
1358                 E1000_WRITE_REG(&adapter->hw, TDFHS, EM_82547_TX_FIFO_BEGIN);
1359
1360                 /* Re-enable TX unit */
1361                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1362                 E1000_WRITE_FLUSH(&adapter->hw);
1363
1364                 adapter->tx_fifo_head = 0;
1365                 adapter->tx_fifo_reset++;
1366
1367                 return(TRUE);
1368         }
1369         else {
1370                 return(FALSE);
1371         }
1372 }
1373
1374 static void
1375 em_set_promisc(struct adapter *adapter)
1376 {
1377         uint32_t reg_rctl;
1378         struct ifnet *ifp = &adapter->interface_data.ac_if;
1379
1380         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1381
1382         if (ifp->if_flags & IFF_PROMISC) {
1383                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1384                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1385         } else if (ifp->if_flags & IFF_ALLMULTI) {
1386                 reg_rctl |= E1000_RCTL_MPE;
1387                 reg_rctl &= ~E1000_RCTL_UPE;
1388                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1389         }
1390 }
1391
1392 static void
1393 em_disable_promisc(struct adapter *adapter)
1394 {
1395         uint32_t reg_rctl;
1396
1397         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1398
1399         reg_rctl &=  (~E1000_RCTL_UPE);
1400         reg_rctl &=  (~E1000_RCTL_MPE);
1401         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1402 }
1403
1404 /*********************************************************************
1405  *  Multicast Update
1406  *
1407  *  This routine is called whenever multicast address list is updated.
1408  *
1409  **********************************************************************/
1410
1411 static void
1412 em_set_multi(struct adapter *adapter)
1413 {
1414         uint32_t reg_rctl = 0;
1415         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1416         struct ifmultiaddr *ifma;
1417         int mcnt = 0;
1418         struct ifnet *ifp = &adapter->interface_data.ac_if;
1419
1420         IOCTL_DEBUGOUT("em_set_multi: begin");
1421
1422         if (adapter->hw.mac_type == em_82542_rev2_0) {
1423                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1424                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1425                         em_pci_clear_mwi(&adapter->hw);
1426                 reg_rctl |= E1000_RCTL_RST;
1427                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1428                 msec_delay(5);
1429         }
1430
1431         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1432                 if (ifma->ifma_addr->sa_family != AF_LINK)
1433                         continue;
1434
1435                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1436                         break;
1437
1438                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1439                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1440                 mcnt++;
1441         }
1442
1443         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1444                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1445                 reg_rctl |= E1000_RCTL_MPE;
1446                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1447         } else
1448                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1449
1450         if (adapter->hw.mac_type == em_82542_rev2_0) {
1451                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1452                 reg_rctl &= ~E1000_RCTL_RST;
1453                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1454                 msec_delay(5);
1455                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1456                         em_pci_set_mwi(&adapter->hw);
1457         }
1458 }
1459
1460 /*********************************************************************
1461  *  Timer routine
1462  *
1463  *  This routine checks for link status and updates statistics.
1464  *
1465  **********************************************************************/
1466
1467 static void
1468 em_local_timer(void *arg)
1469 {
1470         int s;
1471         struct ifnet *ifp;
1472         struct adapter *adapter = arg;
1473         ifp = &adapter->interface_data.ac_if;
1474
1475         s = splimp();
1476
1477         em_check_for_link(&adapter->hw);
1478         em_print_link_status(adapter);
1479         em_update_stats_counters(adapter);
1480         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1481                 em_print_hw_stats(adapter);
1482         em_smartspeed(adapter);
1483
1484         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
1485
1486         splx(s);
1487 }
1488
1489 static void
1490 em_print_link_status(struct adapter *adapter)
1491 {
1492         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1493                 if (adapter->link_active == 0) {
1494                         em_get_speed_and_duplex(&adapter->hw,
1495                                                 &adapter->link_speed,
1496                                                 &adapter->link_duplex);
1497                         device_printf(adapter->dev, "Link is up %d Mbps %s\n",
1498                                adapter->link_speed,
1499                                ((adapter->link_duplex == FULL_DUPLEX) ?
1500                                 "Full Duplex" : "Half Duplex"));
1501                         adapter->link_active = 1;
1502                         adapter->smartspeed = 0;
1503                 }
1504         } else {
1505                 if (adapter->link_active == 1) {
1506                         adapter->link_speed = 0;
1507                         adapter->link_duplex = 0;
1508                         device_printf(adapter->dev, "Link is Down\n");
1509                         adapter->link_active = 0;
1510                 }
1511         }
1512 }
1513
1514 /*********************************************************************
1515  *
1516  *  This routine disables all traffic on the adapter by issuing a
1517  *  global reset on the MAC and deallocates TX/RX buffers.
1518  *
1519  **********************************************************************/
1520
1521 static void
1522 em_stop(void *arg)
1523 {
1524         struct ifnet   *ifp;
1525         struct adapter * adapter = arg;
1526         ifp = &adapter->interface_data.ac_if;
1527
1528         INIT_DEBUGOUT("em_stop: begin");
1529         em_disable_intr(adapter);
1530         em_reset_hw(&adapter->hw);
1531         callout_stop(&adapter->timer);
1532         callout_stop(&adapter->tx_fifo_timer);
1533         em_free_transmit_structures(adapter);
1534         em_free_receive_structures(adapter);
1535
1536         /* Tell the stack that the interface is no longer active */
1537         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1538         ifp->if_timer = 0;
1539 }
1540
1541 /*********************************************************************
1542  *
1543  *  Determine hardware revision.
1544  *
1545  **********************************************************************/
1546 static void
1547 em_identify_hardware(struct adapter * adapter)
1548 {
1549         device_t dev = adapter->dev;
1550
1551         /* Make sure our PCI config space has the necessary stuff set */
1552         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1553         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1554               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1555                 device_printf(dev, "Memory Access and/or Bus Master bits were not set!\n");
1556                 adapter->hw.pci_cmd_word |=
1557                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1558                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1559         }
1560
1561         /* Save off the information about this board */
1562         adapter->hw.vendor_id = pci_get_vendor(dev);
1563         adapter->hw.device_id = pci_get_device(dev);
1564         adapter->hw.revision_id = pci_get_revid(dev);
1565         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1566         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1567
1568         /* Identify the MAC */
1569         if (em_set_mac_type(&adapter->hw))
1570                 device_printf(dev, "Unknown MAC Type\n");
1571
1572         if (adapter->hw.mac_type == em_82541 ||
1573             adapter->hw.mac_type == em_82541_rev_2 ||
1574             adapter->hw.mac_type == em_82547 ||
1575             adapter->hw.mac_type == em_82547_rev_2)
1576                 adapter->hw.phy_init_script = TRUE;
1577 }
1578
1579 /*********************************************************************
1580  *
1581  *  Initialize the hardware to a configuration as specified by the
1582  *  adapter structure. The controller is reset, the EEPROM is
1583  *  verified, the MAC address is set, then the shared initialization
1584  *  routines are called.
1585  *
1586  **********************************************************************/
1587 static int
1588 em_hardware_init(struct adapter *adapter)
1589 {
1590         INIT_DEBUGOUT("em_hardware_init: begin");
1591         /* Issue a global reset */
1592         em_reset_hw(&adapter->hw);
1593
1594         /* When hardware is reset, fifo_head is also reset */
1595         adapter->tx_fifo_head = 0;
1596
1597         /* Make sure we have a good EEPROM before we read from it */
1598         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
1599                 device_printf(adapter->dev, "The EEPROM Checksum Is Not Valid\n");
1600                 return(EIO);
1601         }
1602
1603         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
1604                 device_printf(adapter->dev, "EEPROM read error while reading part number\n");
1605                 return(EIO);
1606         }
1607
1608         if (em_init_hw(&adapter->hw) < 0) {
1609                 device_printf(adapter->dev, "Hardware Initialization Failed");
1610                 return(EIO);
1611         }
1612
1613         em_check_for_link(&adapter->hw);
1614         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)
1615                 adapter->link_active = 1;
1616         else
1617                 adapter->link_active = 0;
1618
1619         if (adapter->link_active) {
1620                 em_get_speed_and_duplex(&adapter->hw,
1621                                         &adapter->link_speed,
1622                                         &adapter->link_duplex);
1623         } else {
1624                 adapter->link_speed = 0;
1625                 adapter->link_duplex = 0;
1626         }
1627
1628         return(0);
1629 }
1630
1631 /*********************************************************************
1632  *
1633  *  Setup networking device structure and register an interface.
1634  *
1635  **********************************************************************/
1636 static void
1637 em_setup_interface(device_t dev, struct adapter *adapter)
1638 {
1639         struct ifnet   *ifp;
1640         INIT_DEBUGOUT("em_setup_interface: begin");
1641
1642         ifp = &adapter->interface_data.ac_if;
1643         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1644         ifp->if_mtu = ETHERMTU;
1645         ifp->if_output = ether_output;
1646         ifp->if_baudrate = 1000000000;
1647         ifp->if_init =  em_init;
1648         ifp->if_softc = adapter;
1649         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1650         ifp->if_ioctl = em_ioctl;
1651         ifp->if_start = em_start;
1652         ifp->if_watchdog = em_watchdog;
1653         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1;
1654
1655         ether_ifattach(ifp, adapter->hw.mac_addr);
1656
1657         if (adapter->hw.mac_type >= em_82543) {
1658                 ifp->if_capabilities = IFCAP_HWCSUM;
1659                 ifp->if_capenable = ifp->if_capabilities;
1660         }
1661
1662         /*
1663          * Tell the upper layer(s) we support long frames.
1664          */
1665         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1666 #if defined(__FreeBSD__) && __FreeBSD_version >= 500000
1667         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1668 #endif
1669
1670         /*
1671          * Specify the media types supported by this adapter and register
1672          * callbacks to update media and link information
1673          */
1674         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
1675                      em_media_status);
1676         if (adapter->hw.media_type == em_media_type_fiber) {
1677                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1678                             0, NULL);
1679                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1680                             0, NULL);
1681         } else {
1682                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1683                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1684                             0, NULL);
1685                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
1686                             0, NULL);
1687                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1688                             0, NULL);
1689                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX | IFM_FDX,
1690                             0, NULL);
1691                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX, 0, NULL);
1692         }
1693         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1694         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1695 }
1696
1697 /*********************************************************************
1698  *
1699  *  Workaround for SmartSpeed on 82541 and 82547 controllers
1700  *
1701  **********************************************************************/
1702 static void
1703 em_smartspeed(struct adapter *adapter)
1704 {
1705         uint16_t phy_tmp;
1706
1707         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
1708             !adapter->hw.autoneg ||
1709             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
1710                 return;
1711
1712         if (adapter->smartspeed == 0) {
1713                 /*
1714                  * If Master/Slave config fault is asserted twice,
1715                  * we assume back-to-back.
1716                  */
1717                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1718                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
1719                         return;
1720                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1721                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
1722                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL,
1723                                         &phy_tmp);
1724                         if (phy_tmp & CR_1000T_MS_ENABLE) {
1725                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
1726                                 em_write_phy_reg(&adapter->hw,
1727                                                  PHY_1000T_CTRL, phy_tmp);
1728                                 adapter->smartspeed++;
1729                                 if (adapter->hw.autoneg &&
1730                                     !em_phy_setup_autoneg(&adapter->hw) &&
1731                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
1732                                                      &phy_tmp)) {
1733                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1734                                                     MII_CR_RESTART_AUTO_NEG);
1735                                         em_write_phy_reg(&adapter->hw,
1736                                                          PHY_CTRL, phy_tmp);
1737                                 }
1738                         }
1739                 }
1740                 return;
1741         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
1742                 /* If still no link, perhaps using 2/3 pair cable */
1743                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
1744                 phy_tmp |= CR_1000T_MS_ENABLE;
1745                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
1746                 if (adapter->hw.autoneg &&
1747                     !em_phy_setup_autoneg(&adapter->hw) &&
1748                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
1749                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1750                                     MII_CR_RESTART_AUTO_NEG);
1751                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
1752                 }
1753         }
1754         /* Restart process after EM_SMARTSPEED_MAX iterations */
1755         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
1756                 adapter->smartspeed = 0;
1757 }
1758
1759 /*
1760  * Manage DMA'able memory.
1761  */
1762 static void
1763 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1764 {
1765         if (error)
1766                 return;
1767         *(bus_addr_t*) arg = segs->ds_addr;
1768 }
1769
1770 static int
1771 em_dma_malloc(struct adapter *adapter, bus_size_t size,
1772               struct em_dma_alloc *dma, int mapflags)
1773 {
1774         int r;
1775         device_t dev = adapter->dev;
1776
1777         r = bus_dma_tag_create(NULL,                    /* parent */
1778                                PAGE_SIZE, 0,            /* alignment, bounds */
1779                                BUS_SPACE_MAXADDR,       /* lowaddr */
1780                                BUS_SPACE_MAXADDR,       /* highaddr */
1781                                NULL, NULL,              /* filter, filterarg */
1782                                size,                    /* maxsize */
1783                                1,                       /* nsegments */
1784                                size,                    /* maxsegsize */
1785                                BUS_DMA_ALLOCNOW,        /* flags */
1786                                &dma->dma_tag);
1787         if (r != 0) {
1788                 device_printf(dev, "em_dma_malloc: bus_dma_tag_create failed; "
1789                               "error %u\n", r);
1790                 goto fail_0;
1791         }
1792
1793         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1794         if (r != 0) {
1795                 device_printf(dev, "em_dma_malloc: bus_dmamap_create failed; "
1796                               "error %u\n", r);
1797                 goto fail_1;
1798         }
1799
1800         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1801                              BUS_DMA_NOWAIT, &dma->dma_map);
1802         if (r != 0) {
1803                 device_printf(dev, "em_dma_malloc: bus_dmammem_alloc failed; "
1804                               "size %llu, error %d\n", (uintmax_t)size, r);
1805                 goto fail_2;
1806         }
1807
1808         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1809                             size,
1810                             em_dmamap_cb,
1811                             &dma->dma_paddr,
1812                             mapflags | BUS_DMA_NOWAIT);
1813         if (r != 0) {
1814                 device_printf(dev, "em_dma_malloc: bus_dmamap_load failed; "
1815                               "error %u\n", r);
1816                 goto fail_3;
1817         }
1818
1819         dma->dma_size = size;
1820         return(0);
1821
1822 fail_3:
1823         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1824 fail_2:
1825         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1826 fail_1:
1827         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1828         bus_dma_tag_destroy(dma->dma_tag);
1829 fail_0:
1830         dma->dma_map = NULL;
1831         dma->dma_tag = NULL;
1832         return(r);
1833 }
1834
1835 static void
1836 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
1837 {
1838         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1839         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1840         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1841         bus_dma_tag_destroy(dma->dma_tag);
1842 }
1843
1844 /*********************************************************************
1845  *
1846  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1847  *  the information needed to transmit a packet on the wire.
1848  *
1849  **********************************************************************/
1850 static int
1851 em_allocate_transmit_structures(struct adapter * adapter)
1852 {
1853         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
1854             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
1855         if (adapter->tx_buffer_area == NULL) {
1856                 device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
1857                 return(ENOMEM);
1858         }
1859
1860         return(0);
1861 }
1862
1863 /*********************************************************************
1864  *
1865  *  Allocate and initialize transmit structures.
1866  *
1867  **********************************************************************/
1868 static int
1869 em_setup_transmit_structures(struct adapter * adapter)
1870 {
1871         /*
1872          * Setup DMA descriptor areas.
1873          */
1874         if (bus_dma_tag_create(NULL,                    /* parent */
1875                                PAGE_SIZE, 0,            /* alignment, bounds */
1876                                BUS_SPACE_MAXADDR,       /* lowaddr */
1877                                BUS_SPACE_MAXADDR,       /* highaddr */
1878                                NULL, NULL,              /* filter, filterarg */
1879                                MCLBYTES * 8,            /* maxsize */
1880                                EM_MAX_SCATTER,          /* nsegments */
1881                                MCLBYTES * 8,            /* maxsegsize */
1882                                BUS_DMA_ALLOCNOW,        /* flags */
1883                                &adapter->txtag)) {
1884                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
1885                 return(ENOMEM);
1886         }
1887
1888         if (em_allocate_transmit_structures(adapter))
1889                 return(ENOMEM);
1890
1891         bzero((void *) adapter->tx_desc_base,
1892               (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
1893
1894         adapter->next_avail_tx_desc = 0;
1895         adapter->oldest_used_tx_desc = 0;
1896
1897         /* Set number of descriptors available */
1898         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1899
1900         /* Set checksum context */
1901         adapter->active_checksum_context = OFFLOAD_NONE;
1902
1903         return(0);
1904 }
1905
1906 /*********************************************************************
1907  *
1908  *  Enable transmit unit.
1909  *
1910  **********************************************************************/
1911 static void
1912 em_initialize_transmit_unit(struct adapter * adapter)
1913 {
1914         uint32_t reg_tctl;
1915         uint32_t reg_tipg = 0;
1916         uint64_t bus_addr;
1917
1918         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
1919
1920         /* Setup the Base and Length of the Tx Descriptor Ring */
1921         bus_addr = adapter->txdma.dma_paddr;
1922         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
1923         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
1924         E1000_WRITE_REG(&adapter->hw, TDLEN,
1925                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
1926
1927         /* Setup the HW Tx Head and Tail descriptor pointers */
1928         E1000_WRITE_REG(&adapter->hw, TDH, 0);
1929         E1000_WRITE_REG(&adapter->hw, TDT, 0);
1930
1931         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1932                      E1000_READ_REG(&adapter->hw, TDBAL),
1933                      E1000_READ_REG(&adapter->hw, TDLEN));
1934
1935         /* Set the default values for the Tx Inter Packet Gap timer */
1936         switch (adapter->hw.mac_type) {
1937         case em_82542_rev2_0:
1938         case em_82542_rev2_1:
1939                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
1940                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1941                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1942                 break;
1943         default:
1944                 if (adapter->hw.media_type == em_media_type_fiber)
1945                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1946                 else
1947                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1948                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1949                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1950         }
1951
1952         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
1953         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
1954         if (adapter->hw.mac_type >= em_82540)
1955                 E1000_WRITE_REG(&adapter->hw, TADV,
1956                                 adapter->tx_abs_int_delay.value);
1957
1958         /* Program the Transmit Control Register */
1959         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
1960                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1961         if (adapter->link_duplex == 1)
1962                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1963         else
1964                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1965         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
1966
1967         /* Setup Transmit Descriptor Settings for this adapter */
1968         adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
1969
1970         if (adapter->tx_int_delay.value > 0)
1971                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1972 }
1973
1974 /*********************************************************************
1975  *
1976  *  Free all transmit related data structures.
1977  *
1978  **********************************************************************/
1979 static void
1980 em_free_transmit_structures(struct adapter * adapter)
1981 {
1982         struct em_buffer *tx_buffer;
1983         int i;
1984
1985         INIT_DEBUGOUT("free_transmit_structures: begin");
1986
1987         if (adapter->tx_buffer_area != NULL) {
1988                 tx_buffer = adapter->tx_buffer_area;
1989                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
1990                         if (tx_buffer->m_head != NULL) {
1991                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1992                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
1993                                 m_freem(tx_buffer->m_head);
1994                         }
1995                         tx_buffer->m_head = NULL;
1996                 }
1997         }
1998         if (adapter->tx_buffer_area != NULL) {
1999                 free(adapter->tx_buffer_area, M_DEVBUF);
2000                 adapter->tx_buffer_area = NULL;
2001         }
2002         if (adapter->txtag != NULL) {
2003                 bus_dma_tag_destroy(adapter->txtag);
2004                 adapter->txtag = NULL;
2005         }
2006 }
2007
2008 /*********************************************************************
2009  *
2010  *  The offload context needs to be set when we transfer the first
2011  *  packet of a particular protocol (TCP/UDP). We change the
2012  *  context only if the protocol type changes.
2013  *
2014  **********************************************************************/
2015 static void
2016 em_transmit_checksum_setup(struct adapter * adapter,
2017                            struct mbuf *mp,
2018                            uint32_t *txd_upper,
2019                            uint32_t *txd_lower)
2020 {
2021         struct em_context_desc *TXD;
2022         struct em_buffer *tx_buffer;
2023         int curr_txd;
2024
2025         if (mp->m_pkthdr.csum_flags) {
2026                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2027                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2028                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2029                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2030                                 return;
2031                         else
2032                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2033                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2034                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2035                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2036                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2037                                 return;
2038                         else
2039                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2040                 } else {
2041                         *txd_upper = 0;
2042                         *txd_lower = 0;
2043                         return;
2044                 }
2045         } else {
2046                 *txd_upper = 0;
2047                 *txd_lower = 0;
2048                 return;
2049         }
2050
2051         /* If we reach this point, the checksum offload context
2052          * needs to be reset.
2053          */
2054         curr_txd = adapter->next_avail_tx_desc;
2055         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2056         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2057
2058         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2059         TXD->lower_setup.ip_fields.ipcso =
2060             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2061         TXD->lower_setup.ip_fields.ipcse =
2062             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2063
2064         TXD->upper_setup.tcp_fields.tucss =
2065             ETHER_HDR_LEN + sizeof(struct ip);
2066         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2067
2068         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2069                 TXD->upper_setup.tcp_fields.tucso =
2070                     ETHER_HDR_LEN + sizeof(struct ip) +
2071                     offsetof(struct tcphdr, th_sum);
2072         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2073                 TXD->upper_setup.tcp_fields.tucso =
2074                         ETHER_HDR_LEN + sizeof(struct ip) +
2075                         offsetof(struct udphdr, uh_sum);
2076         }
2077
2078         TXD->tcp_seg_setup.data = htole32(0);
2079         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2080
2081         tx_buffer->m_head = NULL;
2082
2083         if (++curr_txd == adapter->num_tx_desc)
2084                 curr_txd = 0;
2085
2086         adapter->num_tx_desc_avail--;
2087         adapter->next_avail_tx_desc = curr_txd;
2088 }
2089
2090 /**********************************************************************
2091  *
2092  *  Examine each tx_buffer in the used queue. If the hardware is done
2093  *  processing the packet then free associated resources. The
2094  *  tx_buffer is put back on the free queue.
2095  *
2096  **********************************************************************/
2097 static void
2098 em_clean_transmit_interrupts(struct adapter *adapter)
2099 {
2100         int s;
2101         int i, num_avail;
2102         struct em_buffer *tx_buffer;
2103         struct em_tx_desc *tx_desc;
2104         struct ifnet *ifp = &adapter->interface_data.ac_if;
2105
2106         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2107                 return;
2108
2109         s = splimp();
2110 #ifdef DBG_STATS
2111         adapter->clean_tx_interrupts++;
2112 #endif
2113         num_avail = adapter->num_tx_desc_avail;
2114         i = adapter->oldest_used_tx_desc;
2115
2116         tx_buffer = &adapter->tx_buffer_area[i];
2117         tx_desc = &adapter->tx_desc_base[i];
2118
2119         while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2120                 tx_desc->upper.data = 0;
2121                 num_avail++;
2122
2123                 if (tx_buffer->m_head) {
2124                         ifp->if_opackets++;
2125                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2126                                         BUS_DMASYNC_POSTWRITE);
2127                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2128                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2129
2130                         m_freem(tx_buffer->m_head);
2131                         tx_buffer->m_head = NULL;
2132                 }
2133
2134                 if (++i == adapter->num_tx_desc)
2135                         i = 0;
2136
2137                 tx_buffer = &adapter->tx_buffer_area[i];
2138                 tx_desc = &adapter->tx_desc_base[i];
2139         }
2140
2141         adapter->oldest_used_tx_desc = i;
2142
2143         /*
2144          * If we have enough room, clear IFF_OACTIVE to tell the stack
2145          * that it is OK to send packets.
2146          * If there are no pending descriptors, clear the timeout. Otherwise,
2147          * if some descriptors have been freed, restart the timeout.
2148          */
2149         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2150                 ifp->if_flags &= ~IFF_OACTIVE;
2151                 if (num_avail == adapter->num_tx_desc)
2152                         ifp->if_timer = 0;
2153                 else if (num_avail == adapter->num_tx_desc_avail)
2154                         ifp->if_timer = EM_TX_TIMEOUT;
2155         }
2156         adapter->num_tx_desc_avail = num_avail;
2157         splx(s);
2158 }
2159
2160 /*********************************************************************
2161  *
2162  *  Get a buffer from system mbuf buffer pool.
2163  *
2164  **********************************************************************/
2165 static int
2166 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp)
2167 {
2168         struct mbuf *mp = nmp;
2169         struct em_buffer *rx_buffer;
2170         struct ifnet *ifp;
2171         bus_addr_t paddr;
2172         int error;
2173
2174         ifp = &adapter->interface_data.ac_if;
2175
2176         if (mp == NULL) {
2177                 mp = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR);
2178                 if (mp == NULL) {
2179                         adapter->mbuf_cluster_failed++;
2180                         return(ENOBUFS);
2181                 }
2182                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2183         } else {
2184                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2185                 mp->m_data = mp->m_ext.ext_buf;
2186                 mp->m_next = NULL;
2187         }
2188         if (ifp->if_mtu <= ETHERMTU)
2189                 m_adj(mp, ETHER_ALIGN);
2190
2191         rx_buffer = &adapter->rx_buffer_area[i];
2192
2193         /*
2194          * Using memory from the mbuf cluster pool, invoke the
2195          * bus_dma machinery to arrange the memory mapping.
2196          */
2197         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2198                                 mtod(mp, void *), mp->m_len,
2199                                 em_dmamap_cb, &paddr, 0);
2200         if (error) {
2201                 m_free(mp);
2202                 return(error);
2203         }
2204         rx_buffer->m_head = mp;
2205         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2206         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2207
2208         return(0);
2209 }
2210
2211 /*********************************************************************
2212  *
2213  *  Allocate memory for rx_buffer structures. Since we use one
2214  *  rx_buffer per received packet, the maximum number of rx_buffer's
2215  *  that we'll need is equal to the number of receive descriptors
2216  *  that we've allocated.
2217  *
2218  **********************************************************************/
2219 static int
2220 em_allocate_receive_structures(struct adapter *adapter)
2221 {
2222         int i, error, size;
2223         struct em_buffer *rx_buffer;
2224
2225         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2226         adapter->rx_buffer_area = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2227
2228         error = bus_dma_tag_create(NULL,                /* parent */
2229                                    PAGE_SIZE, 0,        /* alignment, bounds */
2230                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2231                                    BUS_SPACE_MAXADDR,   /* highaddr */
2232                                    NULL, NULL,          /* filter, filterarg */
2233                                    MCLBYTES,            /* maxsize */
2234                                    1,                   /* nsegments */
2235                                    MCLBYTES,            /* maxsegsize */
2236                                    BUS_DMA_ALLOCNOW,    /* flags */
2237                                    &adapter->rxtag);
2238         if (error != 0) {
2239                 device_printf(adapter->dev, "em_allocate_receive_structures: "
2240                               "bus_dma_tag_create failed; error %u\n", error);
2241                 goto fail_0;
2242         }
2243
2244         rx_buffer = adapter->rx_buffer_area;
2245         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2246                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2247                                           &rx_buffer->map);
2248                 if (error != 0) {
2249                         device_printf(adapter->dev,
2250                                       "em_allocate_receive_structures: "
2251                                       "bus_dmamap_create failed; error %u\n",
2252                                       error);
2253                         goto fail_1;
2254                 }
2255         }
2256
2257         for (i = 0; i < adapter->num_rx_desc; i++) {
2258                 error = em_get_buf(i, adapter, NULL);
2259                 if (error != 0) {
2260                         adapter->rx_buffer_area[i].m_head = NULL;
2261                         adapter->rx_desc_base[i].buffer_addr = 0;
2262                         return(error);
2263                 }
2264         }
2265
2266         return(0);
2267
2268 fail_1:
2269         bus_dma_tag_destroy(adapter->rxtag);
2270 fail_0:
2271         adapter->rxtag = NULL;
2272         free(adapter->rx_buffer_area, M_DEVBUF);
2273         adapter->rx_buffer_area = NULL;
2274         return(error);
2275 }
2276
2277 /*********************************************************************
2278  *
2279  *  Allocate and initialize receive structures.
2280  *
2281  **********************************************************************/
2282 static int
2283 em_setup_receive_structures(struct adapter *adapter)
2284 {
2285         bzero((void *) adapter->rx_desc_base,
2286               (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2287
2288         if (em_allocate_receive_structures(adapter))
2289                 return(ENOMEM);
2290
2291         /* Setup our descriptor pointers */
2292         adapter->next_rx_desc_to_check = 0;
2293         return(0);
2294 }
2295
2296 /*********************************************************************
2297  *
2298  *  Enable receive unit.
2299  *
2300  **********************************************************************/
2301 static void
2302 em_initialize_receive_unit(struct adapter *adapter)
2303 {
2304         uint32_t reg_rctl;
2305         uint32_t reg_rxcsum;
2306         struct ifnet *ifp;
2307         uint64_t bus_addr;
2308
2309         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2310
2311         ifp = &adapter->interface_data.ac_if;
2312
2313         /* Make sure receives are disabled while setting up the descriptor ring */
2314         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2315
2316         /* Set the Receive Delay Timer Register */
2317         E1000_WRITE_REG(&adapter->hw, RDTR,
2318                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2319
2320         if(adapter->hw.mac_type >= em_82540) {
2321                 E1000_WRITE_REG(&adapter->hw, RADV,
2322                                 adapter->rx_abs_int_delay.value);
2323
2324                 /* Set the interrupt throttling rate.  Value is calculated
2325                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
2326                  */
2327 #define MAX_INTS_PER_SEC        8000
2328 #define DEFAULT_ITR             1000000000/(MAX_INTS_PER_SEC * 256)
2329                 E1000_WRITE_REG(&adapter->hw, ITR, DEFAULT_ITR);
2330         }
2331
2332         /* Setup the Base and Length of the Rx Descriptor Ring */
2333         bus_addr = adapter->rxdma.dma_paddr;
2334         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2335         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2336         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2337                         sizeof(struct em_rx_desc));
2338
2339         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2340         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2341         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2342
2343         /* Setup the Receive Control Register */
2344         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2345                    E1000_RCTL_RDMTS_HALF |
2346                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2347
2348         if (adapter->hw.tbi_compatibility_on == TRUE)
2349                 reg_rctl |= E1000_RCTL_SBP;
2350
2351         switch (adapter->rx_buffer_len) {
2352         default:
2353         case EM_RXBUFFER_2048:
2354                 reg_rctl |= E1000_RCTL_SZ_2048;
2355                 break;
2356         case EM_RXBUFFER_4096:
2357                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2358                 break;
2359         case EM_RXBUFFER_8192:
2360                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2361                 break;
2362         case EM_RXBUFFER_16384:
2363                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2364                 break;
2365         }
2366
2367         if (ifp->if_mtu > ETHERMTU)
2368                 reg_rctl |= E1000_RCTL_LPE;
2369
2370         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2371         if ((adapter->hw.mac_type >= em_82543) &&
2372             (ifp->if_capenable & IFCAP_RXCSUM)) {
2373                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2374                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2375                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2376         }
2377
2378         /* Enable Receives */
2379         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2380 }
2381
2382 /*********************************************************************
2383  *
2384  *  Free receive related data structures.
2385  *
2386  **********************************************************************/
2387 static void
2388 em_free_receive_structures(struct adapter *adapter)
2389 {
2390         struct em_buffer *rx_buffer;
2391         int i;
2392
2393         INIT_DEBUGOUT("free_receive_structures: begin");
2394
2395         if (adapter->rx_buffer_area != NULL) {
2396                 rx_buffer = adapter->rx_buffer_area;
2397                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2398                         if (rx_buffer->map != NULL) {
2399                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2400                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2401                         }
2402                         if (rx_buffer->m_head != NULL)
2403                                 m_freem(rx_buffer->m_head);
2404                         rx_buffer->m_head = NULL;
2405                 }
2406         }
2407         if (adapter->rx_buffer_area != NULL) {
2408                 free(adapter->rx_buffer_area, M_DEVBUF);
2409                 adapter->rx_buffer_area = NULL;
2410         }
2411         if (adapter->rxtag != NULL) {
2412                 bus_dma_tag_destroy(adapter->rxtag);
2413                 adapter->rxtag = NULL;
2414         }
2415 }
2416
2417 /*********************************************************************
2418  *
2419  *  This routine executes in interrupt context. It replenishes
2420  *  the mbufs in the descriptor and sends data which has been
2421  *  dma'ed into host memory to upper layer.
2422  *
2423  *  We loop at most count times if count is > 0, or until done if
2424  *  count < 0.
2425  *
2426  *********************************************************************/
2427 static void
2428 em_process_receive_interrupts(struct adapter *adapter, int count)
2429 {
2430         struct ifnet *ifp;
2431         struct mbuf *mp;
2432 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2433         struct ether_header *eh;
2434 #endif
2435         uint8_t accept_frame = 0;
2436         uint8_t eop = 0;
2437         uint16_t len, desc_len, prev_len_adj;
2438         int i;
2439
2440         /* Pointer to the receive descriptor being examined. */
2441         struct em_rx_desc *current_desc;
2442
2443         ifp = &adapter->interface_data.ac_if;
2444         i = adapter->next_rx_desc_to_check;
2445         current_desc = &adapter->rx_desc_base[i];
2446
2447         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
2448 #ifdef DBG_STATS
2449                 adapter->no_pkts_avail++;
2450 #endif
2451                 return;
2452         }
2453         while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0)) {
2454                 mp = adapter->rx_buffer_area[i].m_head;
2455                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2456                                 BUS_DMASYNC_POSTREAD);
2457
2458                 accept_frame = 1;
2459                 prev_len_adj = 0;
2460                 desc_len = le16toh(current_desc->length);
2461                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2462                         count--;
2463                         eop = 1;
2464                         if (desc_len < ETHER_CRC_LEN) {
2465                                 len = 0;
2466                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2467                         }
2468                         else {
2469                                 len = desc_len - ETHER_CRC_LEN;
2470                         }
2471                 } else {
2472                         eop = 0;
2473                         len = desc_len;
2474                 }
2475
2476                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2477                         uint8_t last_byte;
2478                         uint32_t pkt_len = desc_len;
2479
2480                         if (adapter->fmp != NULL)
2481                                 pkt_len += adapter->fmp->m_pkthdr.len;
2482
2483                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
2484
2485                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
2486                                        current_desc->errors,
2487                                        pkt_len, last_byte)) {
2488                                 em_tbi_adjust_stats(&adapter->hw,
2489                                                     &adapter->stats,
2490                                                     pkt_len,
2491                                                     adapter->hw.mac_addr);
2492                                 if (len > 0)
2493                                         len--;
2494                         }
2495                         else {
2496                                 accept_frame = 0;
2497                         }
2498                 }
2499
2500                 if (accept_frame) {
2501                         if (em_get_buf(i, adapter, NULL) == ENOBUFS) {
2502                                 adapter->dropped_pkts++;
2503                                 em_get_buf(i, adapter, mp);
2504                                 if (adapter->fmp != NULL)
2505                                         m_freem(adapter->fmp);
2506                                 adapter->fmp = NULL;
2507                                 adapter->lmp = NULL;
2508                                 break;
2509                         }
2510
2511                         /* Assign correct length to the current fragment */
2512                         mp->m_len = len;
2513
2514                         if (adapter->fmp == NULL) {
2515                                 mp->m_pkthdr.len = len;
2516                                 adapter->fmp = mp;       /* Store the first mbuf */
2517                                 adapter->lmp = mp;
2518                         } else {
2519                                 /* Chain mbuf's together */
2520                                 mp->m_flags &= ~M_PKTHDR;
2521                                 /*
2522                                  * Adjust length of previous mbuf in chain if we
2523                                  * received less than 4 bytes in the last descriptor.
2524                                  */
2525                                 if (prev_len_adj > 0) {
2526                                         adapter->lmp->m_len -= prev_len_adj;
2527                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
2528                                 }
2529                                 adapter->lmp->m_next = mp;
2530                                 adapter->lmp = adapter->lmp->m_next;
2531                                 adapter->fmp->m_pkthdr.len += len;
2532                         }
2533
2534                         if (eop) {
2535                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2536                                 ifp->if_ipackets++;
2537
2538 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2539                                 eh = mtod(adapter->fmp, struct ether_header *);
2540                                 /* Remove ethernet header from mbuf */
2541                                 m_adj(adapter->fmp, sizeof(struct ether_header));
2542                                 em_receive_checksum(adapter, current_desc,
2543                                                     adapter->fmp);
2544                                 if (current_desc->status & E1000_RXD_STAT_VP)
2545                                         VLAN_INPUT_TAG(eh, adapter->fmp,
2546                                                        (current_desc->special &
2547                                                         E1000_RXD_SPC_VLAN_MASK));
2548                                 else
2549                                         ether_input(ifp, eh, adapter->fmp);
2550 #else
2551                                 em_receive_checksum(adapter, current_desc,
2552                                                     adapter->fmp);
2553                                 if (current_desc->status & E1000_RXD_STAT_VP)
2554                                         VLAN_INPUT_TAG(ifp, adapter->fmp,
2555                                                        (current_desc->special &
2556                                                         E1000_RXD_SPC_VLAN_MASK),
2557                                                        adapter->fmp = NULL);
2558
2559                                 if (adapter->fmp != NULL)
2560                                         (*ifp->if_input)(ifp, adapter->fmp);
2561 #endif
2562                                 adapter->fmp = NULL;
2563                                 adapter->lmp = NULL;
2564                         }
2565                 } else {
2566                         adapter->dropped_pkts++;
2567                         em_get_buf(i, adapter, mp);
2568                         if (adapter->fmp != NULL)
2569                                 m_freem(adapter->fmp);
2570                         adapter->fmp = NULL;
2571                         adapter->lmp = NULL;
2572                 }
2573
2574                 /* Zero out the receive descriptors status  */
2575                 current_desc->status = 0;
2576
2577                 /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
2578                 E1000_WRITE_REG(&adapter->hw, RDT, i);
2579
2580                 /* Advance our pointers to the next descriptor */
2581                 if (++i == adapter->num_rx_desc) {
2582                         i = 0;
2583                         current_desc = adapter->rx_desc_base;
2584                 } else
2585                         current_desc++;
2586         }
2587         adapter->next_rx_desc_to_check = i;
2588 }
2589
2590 /*********************************************************************
2591  *
2592  *  Verify that the hardware indicated that the checksum is valid.
2593  *  Inform the stack about the status of checksum so that stack
2594  *  doesn't spend time verifying the checksum.
2595  *
2596  *********************************************************************/
2597 static void
2598 em_receive_checksum(struct adapter *adapter,
2599                     struct em_rx_desc *rx_desc,
2600                     struct mbuf *mp)
2601 {
2602         /* 82543 or newer only */
2603         if ((adapter->hw.mac_type < em_82543) ||
2604             /* Ignore Checksum bit is set */
2605             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
2606                 mp->m_pkthdr.csum_flags = 0;
2607                 return;
2608         }
2609
2610         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
2611                 /* Did it pass? */
2612                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
2613                         /* IP Checksum Good */
2614                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2615                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2616                 } else {
2617                         mp->m_pkthdr.csum_flags = 0;
2618                 }
2619         }
2620
2621         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
2622                 /* Did it pass? */
2623                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
2624                         mp->m_pkthdr.csum_flags |=
2625                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2626                         mp->m_pkthdr.csum_data = htons(0xffff);
2627                 }
2628         }
2629 }
2630
2631
2632 static void
2633 em_enable_vlans(struct adapter *adapter)
2634 {
2635         uint32_t ctrl;
2636
2637         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
2638
2639         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
2640         ctrl |= E1000_CTRL_VME;
2641         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
2642 }
2643
2644 static void
2645 em_enable_intr(struct adapter *adapter)
2646 {
2647         E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
2648 }
2649
2650 static void
2651 em_disable_intr(struct adapter *adapter)
2652 {
2653         E1000_WRITE_REG(&adapter->hw, IMC,
2654                         (0xffffffff & ~E1000_IMC_RXSEQ));
2655 }
2656
2657 static int
2658 em_is_valid_ether_addr(uint8_t *addr)
2659 {
2660         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2661
2662         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN)))
2663                 return(FALSE);
2664         else
2665                 return(TRUE);
2666 }
2667
2668 void
2669 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2670 {
2671         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
2672 }
2673
2674 void
2675 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2676 {
2677         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
2678 }
2679
2680 void
2681 em_pci_set_mwi(struct em_hw *hw)
2682 {
2683         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2684                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
2685 }
2686
2687 void
2688 em_pci_clear_mwi(struct em_hw *hw)
2689 {
2690         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2691                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
2692 }
2693
2694 uint32_t
2695 em_read_reg_io(struct em_hw *hw, uint32_t offset)
2696 {
2697         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2698         return(bus_space_read_4(hw->reg_io_tag, hw->reg_io_handle, 4));
2699 }
2700
2701 void
2702 em_write_reg_io(struct em_hw *hw, uint32_t offset, uint32_t value)
2703 {
2704         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2705         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 4, value);
2706 }
2707
2708 /*********************************************************************
2709  * 82544 Coexistence issue workaround.
2710  *    There are 2 issues.
2711  *      1. Transmit Hang issue.
2712  *    To detect this issue, following equation can be used...
2713  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2714  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
2715  *
2716  *      2. DAC issue.
2717  *    To detect this issue, following equation can be used...
2718  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2719  *          If SUM[3:0] is in between 9 to c, we will have this issue.
2720  *
2721  *
2722  *    WORKAROUND:
2723  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
2724  *          9,a,b,c (DAC)
2725  *
2726 *************************************************************************/
2727 static uint32_t
2728 em_fill_descriptors(uint64_t address, uint32_t length, PDESC_ARRAY desc_array)
2729 {
2730         /* Since issue is sensitive to length and address.*/
2731         /* Let us first check the address...*/
2732         uint32_t safe_terminator;
2733         if (length <= 4) {
2734                 desc_array->descriptor[0].address = address;
2735                 desc_array->descriptor[0].length = length;
2736                 desc_array->elements = 1;
2737                 return(desc_array->elements);
2738         }
2739         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
2740         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
2741         if (safe_terminator == 0 ||
2742             (safe_terminator > 4 && safe_terminator < 9) ||
2743             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
2744                 desc_array->descriptor[0].address = address;
2745                 desc_array->descriptor[0].length = length;
2746                 desc_array->elements = 1;
2747                 return(desc_array->elements);
2748         }
2749
2750         desc_array->descriptor[0].address = address;
2751         desc_array->descriptor[0].length = length - 4;
2752         desc_array->descriptor[1].address = address + (length - 4);
2753         desc_array->descriptor[1].length = 4;
2754         desc_array->elements = 2;
2755         return(desc_array->elements);
2756 }
2757
2758 /**********************************************************************
2759  *
2760  *  Update the board statistics counters.
2761  *
2762  **********************************************************************/
2763 static void
2764 em_update_stats_counters(struct adapter *adapter)
2765 {
2766         struct ifnet   *ifp;
2767
2768         if (adapter->hw.media_type == em_media_type_copper ||
2769             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
2770                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
2771                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
2772         }
2773         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
2774         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
2775         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
2776         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
2777
2778         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
2779         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
2780         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
2781         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
2782         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
2783         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
2784         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
2785         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
2786         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
2787         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
2788         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
2789         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
2790         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
2791         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
2792         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
2793         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
2794         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
2795         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
2796         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
2797         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
2798
2799         /* For the 64-bit byte counters the low dword must be read first. */
2800         /* Both registers clear on the read of the high dword */
2801
2802         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
2803         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
2804         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
2805         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
2806
2807         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
2808         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
2809         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
2810         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
2811         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
2812
2813         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
2814         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
2815         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
2816         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
2817
2818         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
2819         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
2820         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
2821         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
2822         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
2823         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
2824         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
2825         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
2826         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
2827         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
2828
2829         if (adapter->hw.mac_type >= em_82543) {
2830                 adapter->stats.algnerrc +=
2831                     E1000_READ_REG(&adapter->hw, ALGNERRC);
2832                 adapter->stats.rxerrc +=
2833                     E1000_READ_REG(&adapter->hw, RXERRC);
2834                 adapter->stats.tncrs +=
2835                     E1000_READ_REG(&adapter->hw, TNCRS);
2836                 adapter->stats.cexterr +=
2837                     E1000_READ_REG(&adapter->hw, CEXTERR);
2838                 adapter->stats.tsctc +=
2839                     E1000_READ_REG(&adapter->hw, TSCTC);
2840                 adapter->stats.tsctfc +=
2841                     E1000_READ_REG(&adapter->hw, TSCTFC);
2842         }
2843         ifp = &adapter->interface_data.ac_if;
2844
2845         /* Fill out the OS statistics structure */
2846         ifp->if_ibytes = adapter->stats.gorcl;
2847         ifp->if_obytes = adapter->stats.gotcl;
2848         ifp->if_imcasts = adapter->stats.mprc;
2849         ifp->if_collisions = adapter->stats.colc;
2850
2851         /* Rx Errors */
2852         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
2853             adapter->stats.crcerrs + adapter->stats.algnerrc +
2854             adapter->stats.rlec + adapter->stats.rnbc +
2855             adapter->stats.mpc + adapter->stats.cexterr;
2856
2857         /* Tx Errors */
2858         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol;
2859 }
2860
2861
2862 /**********************************************************************
2863  *
2864  *  This routine is called only when em_display_debug_stats is enabled.
2865  *  This routine provides a way to take a look at important statistics
2866  *  maintained by the driver and hardware.
2867  *
2868  **********************************************************************/
2869 static void
2870 em_print_debug_info(struct adapter *adapter)
2871 {
2872         device_t dev= adapter->dev;
2873         uint8_t *hw_addr = adapter->hw.hw_addr;
2874
2875         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
2876         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
2877                       E1000_READ_REG(&adapter->hw, TIDV),
2878                       E1000_READ_REG(&adapter->hw, TADV));
2879         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
2880                       E1000_READ_REG(&adapter->hw, RDTR),
2881                       E1000_READ_REG(&adapter->hw, RADV));
2882 #ifdef DBG_STATS
2883         device_printf(dev, "Packets not Avail = %ld\n", adapter->no_pkts_avail);
2884         device_printf(dev, "CleanTxInterrupts = %ld\n",
2885                       adapter->clean_tx_interrupts);
2886 #endif
2887         device_printf(dev, "fifo workaround = %lld, fifo_reset = %lld\n",
2888                       (long long)adapter->tx_fifo_wrk,
2889                       (long long)adapter->tx_fifo_reset);
2890         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2891                       E1000_READ_REG(&adapter->hw, TDH),
2892                       E1000_READ_REG(&adapter->hw, TDT));
2893         device_printf(dev, "Num Tx descriptors avail = %d\n",
2894                       adapter->num_tx_desc_avail);
2895         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
2896                       adapter->no_tx_desc_avail1);
2897         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
2898                       adapter->no_tx_desc_avail2);
2899         device_printf(dev, "Std mbuf failed = %ld\n",
2900                       adapter->mbuf_alloc_failed);
2901         device_printf(dev, "Std mbuf cluster failed = %ld\n",
2902                       adapter->mbuf_cluster_failed);
2903         device_printf(dev, "Driver dropped packets = %ld\n",
2904                       adapter->dropped_pkts);
2905 }
2906
2907 static void
2908 em_print_hw_stats(struct adapter *adapter)
2909 {
2910         device_t dev= adapter->dev;
2911
2912         device_printf(dev, "Excessive collisions = %lld\n",
2913                       (long long)adapter->stats.ecol);
2914         device_printf(dev, "Symbol errors = %lld\n",
2915                       (long long)adapter->stats.symerrs);
2916         device_printf(dev, "Sequence errors = %lld\n",
2917                       (long long)adapter->stats.sec);
2918         device_printf(dev, "Defer count = %lld\n",
2919                       (long long)adapter->stats.dc);
2920
2921         device_printf(dev, "Missed Packets = %lld\n",
2922                       (long long)adapter->stats.mpc);
2923         device_printf(dev, "Receive No Buffers = %lld\n",
2924                       (long long)adapter->stats.rnbc);
2925         device_printf(dev, "Receive length errors = %lld\n",
2926                       (long long)adapter->stats.rlec);
2927         device_printf(dev, "Receive errors = %lld\n",
2928                       (long long)adapter->stats.rxerrc);
2929         device_printf(dev, "Crc errors = %lld\n",
2930                       (long long)adapter->stats.crcerrs);
2931         device_printf(dev, "Alignment errors = %lld\n",
2932                       (long long)adapter->stats.algnerrc);
2933         device_printf(dev, "Carrier extension errors = %lld\n",
2934                       (long long)adapter->stats.cexterr);
2935
2936         device_printf(dev, "XON Rcvd = %lld\n",
2937                       (long long)adapter->stats.xonrxc);
2938         device_printf(dev, "XON Xmtd = %lld\n",
2939                       (long long)adapter->stats.xontxc);
2940         device_printf(dev, "XOFF Rcvd = %lld\n",
2941                       (long long)adapter->stats.xoffrxc);
2942         device_printf(dev, "XOFF Xmtd = %lld\n",
2943                       (long long)adapter->stats.xofftxc);
2944
2945         device_printf(dev, "Good Packets Rcvd = %lld\n",
2946                       (long long)adapter->stats.gprc);
2947         device_printf(dev, "Good Packets Xmtd = %lld\n",
2948                       (long long)adapter->stats.gptc);
2949 }
2950
2951 static int
2952 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
2953 {
2954         int error;
2955         int result;
2956         struct adapter *adapter;
2957
2958         result = -1;
2959         error = sysctl_handle_int(oidp, &result, 0, req);
2960
2961         if (error || !req->newptr)
2962                 return(error);
2963
2964         if (result == 1) {
2965                 adapter = (struct adapter *)arg1;
2966                 em_print_debug_info(adapter);
2967         }
2968
2969         return(error);
2970 }
2971
2972 static int
2973 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
2974 {
2975         int error;
2976         int result;
2977         struct adapter *adapter;
2978
2979         result = -1;
2980         error = sysctl_handle_int(oidp, &result, 0, req);
2981
2982         if (error || !req->newptr)
2983                 return(error);
2984
2985         if (result == 1) {
2986                 adapter = (struct adapter *)arg1;
2987                 em_print_hw_stats(adapter);
2988         }
2989
2990         return(error);
2991 }
2992
2993 static int
2994 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
2995 {
2996         struct em_int_delay_info *info;
2997         struct adapter *adapter;
2998         uint32_t regval;
2999         int error;
3000         int usecs;
3001         int ticks;
3002         int s;
3003
3004         info = (struct em_int_delay_info *)arg1;
3005         adapter = info->adapter;
3006         usecs = info->value;
3007         error = sysctl_handle_int(oidp, &usecs, 0, req);
3008         if (error != 0 || req->newptr == NULL)
3009                 return(error);
3010         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3011                 return(EINVAL);
3012         info->value = usecs;
3013         ticks = E1000_USECS_TO_TICKS(usecs);
3014
3015         s = splimp();
3016         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3017         regval = (regval & ~0xffff) | (ticks & 0xffff);
3018         /* Handle a few special cases. */
3019         switch (info->offset) {
3020         case E1000_RDTR:
3021         case E1000_82542_RDTR:
3022                 regval |= E1000_RDT_FPDB;
3023                 break;
3024         case E1000_TIDV:
3025         case E1000_82542_TIDV:
3026                 if (ticks == 0) {
3027                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3028                         /* Don't write 0 into the TIDV register. */
3029                         regval++;
3030                 } else
3031                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3032                 break;
3033         }
3034         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3035         splx(s);
3036         return(0);
3037 }
3038
3039 static void
3040 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3041                         const char *description, struct em_int_delay_info *info,
3042                         int offset, int value)
3043 {
3044         info->adapter = adapter;
3045         info->offset = offset;
3046         info->value = value;
3047         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3048                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3049                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3050                         info, 0, em_sysctl_int_delay, "I", description);
3051 }