sys/dev/netif/em/if_em.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4
   5 Copyright (c) 2001-2003, Intel Corporation
   6 All rights reserved.
   7
   8 Redistribution and use in source and binary forms, with or without
   9 modification, are permitted provided that the following conditions are met:
  10
  11  1. Redistributions of source code must retain the above copyright notice,
  12     this list of conditions and the following disclaimer.
  13
  14  2. Redistributions in binary form must reproduce the above copyright
  15     notice, this list of conditions and the following disclaimer in the
  16     documentation and/or other materials provided with the distribution.
  17
  18  3. Neither the name of the Intel Corporation nor the names of its
  19     contributors may be used to endorse or promote products derived from
  20     this software without specific prior written permission.
  21
  22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32 POSSIBILITY OF SUCH DAMAGE.
  33
  34 ***************************************************************************/
  35
  36 /*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.15 2003/06/09 22:10:15 pdeuskar Exp $*/
  37 /*$DragonFly: src/sys/dev/netif/em/if_em.c,v 1.35 2005/06/09 02:03:38 hsu Exp $*/
  38
  39 #include "if_em.h"
  40 #include <net/ifq_var.h>
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             em_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50
  51 char em_driver_version[] = "1.7.25";
  52
  53
  54 /*********************************************************************
  55  *  PCI Device ID Table
  56  *
  57  *  Used by probe to select devices to load on
  58  *  Last field stores an index into em_strings
  59  *  Last entry must be all 0s
  60  *
  61  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  62  *********************************************************************/
  63
  64 static em_vendor_info_t em_vendor_info_array[] =
  65 {
  66         /* Intel(R) PRO/1000 Network Connection */
  67         { 0x8086, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0},
  68         { 0x8086, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0},
  69         { 0x8086, 0x1004, PCI_ANY_ID, PCI_ANY_ID, 0},
  70         { 0x8086, 0x1008, PCI_ANY_ID, PCI_ANY_ID, 0},
  71         { 0x8086, 0x1009, PCI_ANY_ID, PCI_ANY_ID, 0},
  72         { 0x8086, 0x100C, PCI_ANY_ID, PCI_ANY_ID, 0},
  73         { 0x8086, 0x100D, PCI_ANY_ID, PCI_ANY_ID, 0},
  74         { 0x8086, 0x100E, PCI_ANY_ID, PCI_ANY_ID, 0},
  75         { 0x8086, 0x100F, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         { 0x8086, 0x1010, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         { 0x8086, 0x1011, PCI_ANY_ID, PCI_ANY_ID, 0},
  78         { 0x8086, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0},
  79         { 0x8086, 0x1013, PCI_ANY_ID, PCI_ANY_ID, 0},
  80         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
  81         { 0x8086, 0x1015, PCI_ANY_ID, PCI_ANY_ID, 0},
  82         { 0x8086, 0x1016, PCI_ANY_ID, PCI_ANY_ID, 0},
  83         { 0x8086, 0x1017, PCI_ANY_ID, PCI_ANY_ID, 0},
  84         { 0x8086, 0x1018, PCI_ANY_ID, PCI_ANY_ID, 0},
  85         { 0x8086, 0x1019, PCI_ANY_ID, PCI_ANY_ID, 0},
  86         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
  87         { 0x8086, 0x101D, PCI_ANY_ID, PCI_ANY_ID, 0},
  88         { 0x8086, 0x101E, PCI_ANY_ID, PCI_ANY_ID, 0},
  89         { 0x8086, 0x1026, PCI_ANY_ID, PCI_ANY_ID, 0},
  90         { 0x8086, 0x1027, PCI_ANY_ID, PCI_ANY_ID, 0},
  91         { 0x8086, 0x1028, PCI_ANY_ID, PCI_ANY_ID, 0},
  92         { 0x8086, 0x1075, PCI_ANY_ID, PCI_ANY_ID, 0},
  93         { 0x8086, 0x1076, PCI_ANY_ID, PCI_ANY_ID, 0},
  94         { 0x8086, 0x1077, PCI_ANY_ID, PCI_ANY_ID, 0},
  95         { 0x8086, 0x1078, PCI_ANY_ID, PCI_ANY_ID, 0},
  96         { 0x8086, 0x1079, PCI_ANY_ID, PCI_ANY_ID, 0},
  97         { 0x8086, 0x107A, PCI_ANY_ID, PCI_ANY_ID, 0},
  98         { 0x8086, 0x107B, PCI_ANY_ID, PCI_ANY_ID, 0},
  99         /* required last entry */
 100         { 0, 0, 0, 0, 0}
 101 };
 102
 103 /*********************************************************************
 104  *  Table of branding strings for all supported NICs.
 105  *********************************************************************/
 106
 107 static const char *em_strings[] = {
 108         "Intel(R) PRO/1000 Network Connection"
 109 };
 110
 111 /*********************************************************************
 112  *  Function prototypes
 113  *********************************************************************/
 114 static int      em_probe(device_t);
 115 static int      em_attach(device_t);
 116 static int      em_detach(device_t);
 117 static int      em_shutdown(device_t);
 118 static void     em_intr(void *);
 119 static void     em_start(struct ifnet *);
 120 static void     em_start_serialized(struct ifnet *);
 121 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 122 static void     em_watchdog(struct ifnet *);
 123 static void     em_init(void *);
 124 static void     em_init_serialized(void *);
 125 static void     em_stop(void *);
 126 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 127 static int      em_media_change(struct ifnet *);
 128 static void     em_identify_hardware(struct adapter *);
 129 static void     em_local_timer(void *);
 130 static int      em_hardware_init(struct adapter *);
 131 static void     em_setup_interface(device_t, struct adapter *);
 132 static int      em_setup_transmit_structures(struct adapter *);
 133 static void     em_initialize_transmit_unit(struct adapter *);
 134 static int      em_setup_receive_structures(struct adapter *);
 135 static void     em_initialize_receive_unit(struct adapter *);
 136 static void     em_enable_intr(struct adapter *);
 137 static void     em_disable_intr(struct adapter *);
 138 static void     em_free_transmit_structures(struct adapter *);
 139 static void     em_free_receive_structures(struct adapter *);
 140 static void     em_update_stats_counters(struct adapter *);
 141 static void     em_clean_transmit_interrupts(struct adapter *);
 142 static int      em_allocate_receive_structures(struct adapter *);
 143 static int      em_allocate_transmit_structures(struct adapter *);
 144 static void     em_process_receive_interrupts(struct adapter *, int);
 145 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 146                                     struct mbuf *);
 147 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 148                                            uint32_t *, uint32_t *);
 149 static void     em_set_promisc(struct adapter *);
 150 static void     em_disable_promisc(struct adapter *);
 151 static void     em_set_multi(struct adapter *);
 152 static void     em_print_hw_stats(struct adapter *);
 153 static void     em_print_link_status(struct adapter *);
 154 static int      em_get_buf(int i, struct adapter *, struct mbuf *, int how);
 155 static void     em_enable_vlans(struct adapter *);
 156 static int      em_encap(struct adapter *, struct mbuf *);
 157 static void     em_smartspeed(struct adapter *);
 158 static int      em_82547_fifo_workaround(struct adapter *, int);
 159 static void     em_82547_update_fifo_head(struct adapter *, int);
 160 static int      em_82547_tx_fifo_reset(struct adapter *);
 161 static void     em_82547_move_tail(void *arg);
 162 static void     em_82547_move_tail_serialized(void *arg);
 163 static int      em_dma_malloc(struct adapter *, bus_size_t,
 164                               struct em_dma_alloc *, int);
 165 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 166 static void     em_print_debug_info(struct adapter *);
 167 static int      em_is_valid_ether_addr(uint8_t *);
 168 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 169 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 170 static uint32_t em_fill_descriptors(uint64_t address, uint32_t length,
 171                                    PDESC_ARRAY desc_array);
 172 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 173 static int      em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 174 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 175                                         const char *,
 176                                         struct em_int_delay_info *, int, int);
 177
 178 /*********************************************************************
 179  *  FreeBSD Device Interface Entry Points
 180  *********************************************************************/
 181
 182 static device_method_t em_methods[] = {
 183         /* Device interface */
 184         DEVMETHOD(device_probe, em_probe),
 185         DEVMETHOD(device_attach, em_attach),
 186         DEVMETHOD(device_detach, em_detach),
 187         DEVMETHOD(device_shutdown, em_shutdown),
 188         {0, 0}
 189 };
 190
 191 static driver_t em_driver = {
 192         "em", em_methods, sizeof(struct adapter),
 193 };
 194
 195 static devclass_t em_devclass;
 196
 197 DECLARE_DUMMY_MODULE(if_em);
 198 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 199
 200 /*********************************************************************
 201  *  Tunable default values.
 202  *********************************************************************/
 203
 204 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 205 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 206
 207 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 208 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 209 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 210 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 211 static int em_int_throttle_ceil = 10000;
 212
 213 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 214 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 215 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 216 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 217 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
 218
 219 /*********************************************************************
 220  *  Device identification routine
 221  *
 222  *  em_probe determines if the driver should be loaded on
 223  *  adapter based on PCI vendor/device id of the adapter.
 224  *
 225  *  return 0 on success, positive on failure
 226  *********************************************************************/
 227
 228 static int
 229 em_probe(device_t dev)
 230 {
 231         em_vendor_info_t *ent;
 232
 233         uint16_t pci_vendor_id = 0;
 234         uint16_t pci_device_id = 0;
 235         uint16_t pci_subvendor_id = 0;
 236         uint16_t pci_subdevice_id = 0;
 237         char adapter_name[60];
 238
 239         INIT_DEBUGOUT("em_probe: begin");
 240
 241         pci_vendor_id = pci_get_vendor(dev);
 242         if (pci_vendor_id != EM_VENDOR_ID)
 243                 return(ENXIO);
 244
 245         pci_device_id = pci_get_device(dev);
 246         pci_subvendor_id = pci_get_subvendor(dev);
 247         pci_subdevice_id = pci_get_subdevice(dev);
 248
 249         ent = em_vendor_info_array;
 250         while (ent->vendor_id != 0) {
 251                 if ((pci_vendor_id == ent->vendor_id) &&
 252                     (pci_device_id == ent->device_id) &&
 253
 254                     ((pci_subvendor_id == ent->subvendor_id) ||
 255                      (ent->subvendor_id == PCI_ANY_ID)) &&
 256
 257                     ((pci_subdevice_id == ent->subdevice_id) ||
 258                      (ent->subdevice_id == PCI_ANY_ID))) {
 259                         snprintf(adapter_name, sizeof(adapter_name),
 260                                  "%s, Version - %s",  em_strings[ent->index],
 261                                  em_driver_version);
 262                         device_set_desc_copy(dev, adapter_name);
 263                         return(0);
 264                 }
 265                 ent++;
 266         }
 267
 268         return(ENXIO);
 269 }
 270
 271 /*********************************************************************
 272  *  Device initialization routine
 273  *
 274  *  The attach entry point is called when the driver is being loaded.
 275  *  This routine identifies the type of hardware, allocates all resources
 276  *  and initializes the hardware.
 277  *
 278  *  return 0 on success, positive on failure
 279  *********************************************************************/
 280
 281 static int
 282 em_attach(device_t dev)
 283 {
 284         struct adapter *adapter;
 285         int tsize, rsize;
 286         int i, val, rid;
 287         int error = 0;
 288
 289         INIT_DEBUGOUT("em_attach: begin");
 290
 291         adapter = device_get_softc(dev);
 292
 293         bzero(adapter, sizeof(struct adapter));
 294
 295         lwkt_serialize_init(&adapter->serializer);
 296
 297         callout_init(&adapter->timer);
 298         callout_init(&adapter->tx_fifo_timer);
 299
 300         adapter->dev = dev;
 301         adapter->osdep.dev = dev;
 302
 303         /* SYSCTL stuff */
 304         sysctl_ctx_init(&adapter->sysctl_ctx);
 305         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 306                                                SYSCTL_STATIC_CHILDREN(_hw),
 307                                                OID_AUTO,
 308                                                device_get_nameunit(dev),
 309                                                CTLFLAG_RD,
 310                                                0, "");
 311
 312         if (adapter->sysctl_tree == NULL) {
 313                 error = EIO;
 314                 goto fail;
 315         }
 316
 317         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 318                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 319                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 320                         (void *)adapter, 0,
 321                         em_sysctl_debug_info, "I", "Debug Information");
 322
 323         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 324                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 325                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 326                         (void *)adapter, 0,
 327                         em_sysctl_stats, "I", "Statistics");
 328
 329         /* Determine hardware revision */
 330         em_identify_hardware(adapter);
 331
 332         /* Set up some sysctls for the tunable interrupt delays */
 333         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 334                                 "receive interrupt delay in usecs",
 335                                 &adapter->rx_int_delay,
 336                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 337                                 em_rx_int_delay_dflt);
 338         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 339                                 "transmit interrupt delay in usecs",
 340                                 &adapter->tx_int_delay,
 341                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 342                                 em_tx_int_delay_dflt);
 343         if (adapter->hw.mac_type >= em_82540) {
 344                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 345                                         "receive interrupt delay limit in usecs",
 346                                         &adapter->rx_abs_int_delay,
 347                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 348                                         em_rx_abs_int_delay_dflt);
 349                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 350                                         "transmit interrupt delay limit in usecs",
 351                                         &adapter->tx_abs_int_delay,
 352                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 353                                         em_tx_abs_int_delay_dflt);
 354                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 355                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 356                         OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW,
 357                         adapter, 0, em_sysctl_int_throttle, "I", NULL);
 358         }
 359
 360         /* Parameters (to be read from user) */
 361         adapter->num_tx_desc = EM_MAX_TXD;
 362         adapter->num_rx_desc = EM_MAX_RXD;
 363         adapter->hw.autoneg = DO_AUTO_NEG;
 364         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 365         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 366         adapter->hw.tbi_compatibility_en = TRUE;
 367         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 368
 369         /*
 370          * These parameters control the automatic generation(Tx) and
 371          * response(Rx) to Ethernet PAUSE frames.
 372          */
 373         adapter->hw.fc_high_water = FC_DEFAULT_HI_THRESH;
 374         adapter->hw.fc_low_water  = FC_DEFAULT_LO_THRESH;
 375         adapter->hw.fc_pause_time = FC_DEFAULT_TX_TIMER;
 376         adapter->hw.fc_send_xon   = TRUE;
 377         adapter->hw.fc = em_fc_full;
 378
 379         adapter->hw.phy_init_script = 1;
 380         adapter->hw.phy_reset_disable = FALSE;
 381
 382 #ifndef EM_MASTER_SLAVE
 383         adapter->hw.master_slave = em_ms_hw_default;
 384 #else
 385         adapter->hw.master_slave = EM_MASTER_SLAVE;
 386 #endif
 387
 388         /*
 389          * Set the max frame size assuming standard ethernet
 390          * sized frames
 391          */
 392         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 393
 394         adapter->hw.min_frame_size =
 395             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 396
 397         /*
 398          * This controls when hardware reports transmit completion
 399          * status.
 400          */
 401         adapter->hw.report_tx_early = 1;
 402
 403         rid = EM_MMBA;
 404         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 405                                                      &rid, RF_ACTIVE);
 406         if (!(adapter->res_memory)) {
 407                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 408                 error = ENXIO;
 409                 goto fail;
 410         }
 411         adapter->osdep.mem_bus_space_tag =
 412             rman_get_bustag(adapter->res_memory);
 413         adapter->osdep.mem_bus_space_handle =
 414             rman_get_bushandle(adapter->res_memory);
 415         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
 416
 417         if (adapter->hw.mac_type > em_82543) {
 418                 /* Figure our where our IO BAR is ? */
 419                 rid = EM_MMBA;
 420                 for (i = 0; i < 5; i++) {
 421                         val = pci_read_config(dev, rid, 4);
 422                         if (val & 0x00000001) {
 423                                 adapter->io_rid = rid;
 424                                 break;
 425                         }
 426                         rid += 4;
 427                 }
 428
 429                 adapter->res_ioport = bus_alloc_resource_any(dev,
 430                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
 431                 if (!(adapter->res_ioport)) {
 432                         device_printf(dev, "Unable to allocate bus resource: ioport\n");
 433                         error = ENXIO;
 434                         goto fail;
 435                 }
 436
 437                 adapter->hw.reg_io_tag = rman_get_bustag(adapter->res_ioport);
 438                 adapter->hw.reg_io_handle = rman_get_bushandle(adapter->res_ioport);
 439         }
 440
 441         rid = 0x0;
 442         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 443             &rid, RF_SHAREABLE | RF_ACTIVE);
 444         if (!(adapter->res_interrupt)) {
 445                 device_printf(dev, "Unable to allocate bus resource: interrupt\n");
 446                 error = ENXIO;
 447                 goto fail;
 448         }
 449
 450         adapter->hw.back = &adapter->osdep;
 451
 452         /* Initialize eeprom parameters */
 453         em_init_eeprom_params(&adapter->hw);
 454
 455         tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc);
 456
 457         /* Allocate Transmit Descriptor ring */
 458         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_WAITOK)) {
 459                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 460                 error = ENOMEM;
 461                 goto fail;
 462         }
 463         adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr;
 464
 465         rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc);
 466
 467         /* Allocate Receive Descriptor ring */
 468         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_WAITOK)) {
 469                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 470                 error = ENOMEM;
 471                 goto fail;
 472         }
 473         adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr;
 474
 475         /* Initialize the hardware */
 476         if (em_hardware_init(adapter)) {
 477                 device_printf(dev, "Unable to initialize the hardware\n");
 478                 error = EIO;
 479                 goto fail;
 480         }
 481
 482         /* Copy the permanent MAC address out of the EEPROM */
 483         if (em_read_mac_addr(&adapter->hw) < 0) {
 484                 device_printf(dev, "EEPROM read error while reading mac address\n");
 485                 error = EIO;
 486                 goto fail;
 487         }
 488
 489         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 490                 device_printf(dev, "Invalid mac address\n");
 491                 error = EIO;
 492                 goto fail;
 493         }
 494
 495         /* Setup OS specific network interface */
 496         em_setup_interface(dev, adapter);
 497
 498         /* Initialize statistics */
 499         em_clear_hw_cntrs(&adapter->hw);
 500         em_update_stats_counters(adapter);
 501         adapter->hw.get_link_status = 1;
 502         em_check_for_link(&adapter->hw);
 503
 504         /* Print the link status */
 505         if (adapter->link_active == 1) {
 506                 em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
 507                                         &adapter->link_duplex);
 508                 device_printf(dev, "Speed: %d Mbps, Duplex: %s\n",
 509                     adapter->link_speed,
 510                     adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
 511         } else
 512                 device_printf(dev, "Speed: N/A, Duplex:N/A\n");
 513
 514         /* Identify 82544 on PCIX */
 515         em_get_bus_info(&adapter->hw);
 516         if (adapter->hw.bus_type == em_bus_type_pcix &&
 517             adapter->hw.mac_type == em_82544)
 518                 adapter->pcix_82544 = TRUE;
 519         else
 520                 adapter->pcix_82544 = FALSE;
 521
 522         error = bus_setup_intr(dev, adapter->res_interrupt, INTR_TYPE_MISC,
 523                            (void (*)(void *)) em_intr, adapter,
 524                            &adapter->int_handler_tag, &adapter->serializer);
 525         if (error) {
 526                 device_printf(dev, "Error registering interrupt handler!\n");
 527                 ether_ifdetach(&adapter->interface_data.ac_if);
 528                 goto fail;
 529         }
 530
 531         INIT_DEBUGOUT("em_attach: end");
 532         return(0);
 533
 534 fail:
 535         em_detach(dev);
 536         return(error);
 537 }
 538
 539 /*********************************************************************
 540  *  Device removal routine
 541  *
 542  *  The detach entry point is called when the driver is being removed.
 543  *  This routine stops the adapter and deallocates all the resources
 544  *  that were allocated for driver operation.
 545  *
 546  *  return 0 on success, positive on failure
 547  *********************************************************************/
 548
 549 static int
 550 em_detach(device_t dev)
 551 {
 552         struct adapter * adapter = device_get_softc(dev);
 553
 554         INIT_DEBUGOUT("em_detach: begin");
 555
 556         lwkt_serialize_enter(&adapter->serializer);
 557         adapter->in_detach = 1;
 558
 559         if (device_is_attached(dev)) {
 560                 em_stop(adapter);
 561                 em_phy_hw_reset(&adapter->hw);
 562                 ether_ifdetach(&adapter->interface_data.ac_if);
 563         }
 564         bus_generic_detach(dev);
 565
 566         if (adapter->res_interrupt != NULL) {
 567                 bus_teardown_intr(dev, adapter->res_interrupt,
 568                                   adapter->int_handler_tag);
 569                 bus_release_resource(dev, SYS_RES_IRQ, 0,
 570                                      adapter->res_interrupt);
 571         }
 572         if (adapter->res_memory != NULL) {
 573                 bus_release_resource(dev, SYS_RES_MEMORY, EM_MMBA,
 574                                      adapter->res_memory);
 575         }
 576
 577         if (adapter->res_ioport != NULL) {
 578                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
 579                                      adapter->res_ioport);
 580         }
 581
 582         /* Free Transmit Descriptor ring */
 583         if (adapter->tx_desc_base != NULL) {
 584                 em_dma_free(adapter, &adapter->txdma);
 585                 adapter->tx_desc_base = NULL;
 586         }
 587
 588         /* Free Receive Descriptor ring */
 589         if (adapter->rx_desc_base != NULL) {
 590                 em_dma_free(adapter, &adapter->rxdma);
 591                 adapter->rx_desc_base = NULL;
 592         }
 593
 594         adapter->sysctl_tree = NULL;
 595         sysctl_ctx_free(&adapter->sysctl_ctx);
 596
 597         lwkt_serialize_exit(&adapter->serializer);
 598         return(0);
 599 }
 600
 601 /*********************************************************************
 602  *
 603  *  Shutdown entry point
 604  *
 605  **********************************************************************/
 606
 607 static int
 608 em_shutdown(device_t dev)
 609 {
 610         struct adapter *adapter = device_get_softc(dev);
 611         em_stop(adapter);
 612         return(0);
 613 }
 614
 615 /*********************************************************************
 616  *  Transmit entry point
 617  *
 618  *  em_start is called by the stack to initiate a transmit.
 619  *  The driver will remain in this routine as long as there are
 620  *  packets to transmit and transmit resources are available.
 621  *  In case resources are not available stack is notified and
 622  *  the packet is requeued.
 623  **********************************************************************/
 624
 625 static void
 626 em_start(struct ifnet *ifp)
 627 {
 628         struct adapter *adapter = ifp->if_softc;
 629
 630         lwkt_serialize_enter(&adapter->serializer);
 631         em_start_serialized(ifp);
 632         lwkt_serialize_exit(&adapter->serializer);
 633 }
 634
 635 static void
 636 em_start_serialized(struct ifnet *ifp)
 637 {
 638         struct mbuf *m_head;
 639         struct adapter *adapter = ifp->if_softc;
 640
 641         if (!adapter->link_active)
 642                 return;
 643         while (!ifq_is_empty(&ifp->if_snd)) {
 644                 m_head = ifq_poll(&ifp->if_snd);
 645
 646                 if (m_head == NULL)
 647                         break;
 648
 649                 if (em_encap(adapter, m_head)) {
 650                         ifp->if_flags |= IFF_OACTIVE;
 651                         break;
 652                 }
 653                 m_head = ifq_dequeue(&ifp->if_snd);
 654
 655                 /* Send a copy of the frame to the BPF listener */
 656                 BPF_MTAP(ifp, m_head);
 657
 658                 /* Set timeout in case hardware has problems transmitting */
 659                 ifp->if_timer = EM_TX_TIMEOUT;
 660         }
 661 }
 662
 663 /*********************************************************************
 664  *  Ioctl entry point
 665  *
 666  *  em_ioctl is called when the user wants to configure the
 667  *  interface.
 668  *
 669  *  return 0 on success, positive on failure
 670  **********************************************************************/
 671
 672 static int
 673 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 674 {
 675         int mask, error = 0;
 676         struct ifreq *ifr = (struct ifreq *) data;
 677         struct adapter *adapter = ifp->if_softc;
 678
 679         lwkt_serialize_enter(&adapter->serializer);
 680
 681         if (adapter->in_detach)
 682                 goto out;
 683
 684         switch (command) {
 685         case SIOCSIFADDR:
 686         case SIOCGIFADDR:
 687                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 688                 lwkt_serialize_exit(&adapter->serializer);
 689                 ether_ioctl(ifp, command, data);
 690                 lwkt_serialize_enter(&adapter->serializer);
 691                 break;
 692         case SIOCSIFMTU:
 693                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 694                 if (ifr->ifr_mtu > MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 695                         error = EINVAL;
 696                 } else {
 697                         ifp->if_mtu = ifr->ifr_mtu;
 698                         adapter->hw.max_frame_size =
 699                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 700                         em_init_serialized(adapter);
 701                 }
 702                 break;
 703         case SIOCSIFFLAGS:
 704                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 705                 if (ifp->if_flags & IFF_UP) {
 706                         if (!(ifp->if_flags & IFF_RUNNING))
 707                                 em_init_serialized(adapter);
 708                         em_disable_promisc(adapter);
 709                         em_set_promisc(adapter);
 710                 } else {
 711                         if (ifp->if_flags & IFF_RUNNING)
 712                                 em_stop(adapter);
 713                 }
 714                 break;
 715         case SIOCADDMULTI:
 716         case SIOCDELMULTI:
 717                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 718                 if (ifp->if_flags & IFF_RUNNING) {
 719                         em_disable_intr(adapter);
 720                         em_set_multi(adapter);
 721                         if (adapter->hw.mac_type == em_82542_rev2_0)
 722                                 em_initialize_receive_unit(adapter);
 723                         em_enable_intr(adapter);
 724                 }
 725                 break;
 726         case SIOCSIFMEDIA:
 727         case SIOCGIFMEDIA:
 728                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 729                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 730                 break;
 731         case SIOCSIFCAP:
 732                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 733                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 734                 if (mask & IFCAP_HWCSUM) {
 735                         if (IFCAP_HWCSUM & ifp->if_capenable)
 736                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 737                         else
 738                                 ifp->if_capenable |= IFCAP_HWCSUM;
 739                         if (ifp->if_flags & IFF_RUNNING)
 740                                 em_init_serialized(adapter);
 741                 }
 742                 break;
 743         default:
 744                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%x)\n", (int)command);
 745                 error = EINVAL;
 746         }
 747
 748 out:
 749         lwkt_serialize_exit(&adapter->serializer);
 750         return(error);
 751 }
 752
 753 /*********************************************************************
 754  *  Watchdog entry point
 755  *
 756  *  This routine is called whenever hardware quits transmitting.
 757  *
 758  **********************************************************************/
 759
 760 static void
 761 em_watchdog(struct ifnet *ifp)
 762 {
 763         struct adapter * adapter;
 764         adapter = ifp->if_softc;
 765
 766         /* If we are in this routine because of pause frames, then
 767          * don't reset the hardware.
 768          */
 769         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 770                 ifp->if_timer = EM_TX_TIMEOUT;
 771                 return;
 772         }
 773
 774         if (em_check_for_link(&adapter->hw))
 775                 if_printf(ifp, "watchdog timeout -- resetting\n");
 776
 777         ifp->if_flags &= ~IFF_RUNNING;
 778
 779         em_init(adapter);
 780
 781         ifp->if_oerrors++;
 782 }
 783
 784 /*********************************************************************
 785  *  Init entry point
 786  *
 787  *  This routine is used in two ways. It is used by the stack as
 788  *  init entry point in network interface structure. It is also used
 789  *  by the driver as a hw/sw initialization routine to get to a
 790  *  consistent state.
 791  *
 792  *  return 0 on success, positive on failure
 793  **********************************************************************/
 794
 795 static void
 796 em_init(void *arg)
 797 {
 798         struct adapter *adapter = arg;
 799
 800         lwkt_serialize_enter(&adapter->serializer);
 801         em_init_serialized(arg);
 802         lwkt_serialize_exit(&adapter->serializer);
 803 }
 804
 805 static void
 806 em_init_serialized(void *arg)
 807 {
 808         struct adapter *adapter = arg;
 809         struct ifnet *ifp = &adapter->interface_data.ac_if;
 810
 811         INIT_DEBUGOUT("em_init: begin");
 812
 813         em_stop(adapter);
 814
 815         /* Get the latest mac address, User can use a LAA */
 816         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
 817               ETHER_ADDR_LEN);
 818
 819         /* Initialize the hardware */
 820         if (em_hardware_init(adapter)) {
 821                 if_printf(ifp, "Unable to initialize the hardware\n");
 822                 return;
 823         }
 824
 825         em_enable_vlans(adapter);
 826
 827         /* Prepare transmit descriptors and buffers */
 828         if (em_setup_transmit_structures(adapter)) {
 829                 if_printf(ifp, "Could not setup transmit structures\n");
 830                 em_stop(adapter);
 831                 return;
 832         }
 833         em_initialize_transmit_unit(adapter);
 834
 835         /* Setup Multicast table */
 836         em_set_multi(adapter);
 837
 838         /* Prepare receive descriptors and buffers */
 839         if (em_setup_receive_structures(adapter)) {
 840                 if_printf(ifp, "Could not setup receive structures\n");
 841                 em_stop(adapter);
 842                 return;
 843         }
 844         em_initialize_receive_unit(adapter);
 845
 846         /* Don't loose promiscuous settings */
 847         em_set_promisc(adapter);
 848
 849         ifp->if_flags |= IFF_RUNNING;
 850         ifp->if_flags &= ~IFF_OACTIVE;
 851
 852         if (adapter->hw.mac_type >= em_82543) {
 853                 if (ifp->if_capenable & IFCAP_TXCSUM)
 854                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
 855                 else
 856                         ifp->if_hwassist = 0;
 857         }
 858
 859         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 860         em_clear_hw_cntrs(&adapter->hw);
 861         em_enable_intr(adapter);
 862
 863         /* Don't reset the phy next time init gets called */
 864         adapter->hw.phy_reset_disable = TRUE;
 865 }
 866
 867 #ifdef DEVICE_POLLING
 868
 869 static void
 870 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 871 {
 872         struct adapter *adapter = ifp->if_softc;
 873         uint32_t reg_icr;
 874
 875         lwkt_serialize_enter(&adapter->serializer);
 876         switch(cmd) {
 877         case POLL_REGISTER:
 878                 em_disable_intr(adapter);
 879                 break;
 880         case POLL_DEREGISTER:
 881                 em_enable_intr(adapter);
 882                 break;
 883         case POLL_AND_CHECK_STATUS:
 884                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 885                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 886                         callout_stop(&adapter->timer);
 887                         adapter->hw.get_link_status = 1;
 888                         em_check_for_link(&adapter->hw);
 889                         em_print_link_status(adapter);
 890                         callout_reset(&adapter->timer, 2*hz, em_local_timer,
 891                                       adapter);
 892                 }
 893                 /* fall through */
 894         case POLL_ONLY:
 895                 if (ifp->if_flags & IFF_RUNNING) {
 896                         em_process_receive_interrupts(adapter, count);
 897                         em_clean_transmit_interrupts(adapter);
 898                 }
 899                 if (ifp->if_flags & IFF_RUNNING) {
 900                         if (!ifq_is_empty(&ifp->if_snd))
 901                                 em_start_serialized(ifp);
 902                 }
 903                 break;
 904         }
 905         lwkt_serialize_exit(&adapter->serializer);
 906 }
 907
 908 #endif /* DEVICE_POLLING */
 909
 910 /*********************************************************************
 911  *
 912  *  Interrupt Service routine
 913  *
 914  **********************************************************************/
 915 static void
 916 em_intr(void *arg)
 917 {
 918         uint32_t reg_icr;
 919         struct ifnet *ifp;
 920         struct adapter *adapter = arg;
 921
 922         ifp = &adapter->interface_data.ac_if;
 923
 924         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 925         if (!reg_icr)
 926                 return;
 927
 928         /* Link status change */
 929         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 930                 callout_stop(&adapter->timer);
 931                 adapter->hw.get_link_status = 1;
 932                 em_check_for_link(&adapter->hw);
 933                 em_print_link_status(adapter);
 934                 callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 935         }
 936
 937         /*
 938          * note: do not attempt to improve efficiency by looping.  This
 939          * only results in unnecessary piecemeal collection of received
 940          * packets and unnecessary piecemeal cleanups of the transmit ring.
 941          */
 942         if (ifp->if_flags & IFF_RUNNING) {
 943                 em_process_receive_interrupts(adapter, -1);
 944                 em_clean_transmit_interrupts(adapter);
 945         }
 946
 947         if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd))
 948                 em_start_serialized(ifp);
 949 }
 950
 951 /*********************************************************************
 952  *
 953  *  Media Ioctl callback
 954  *
 955  *  This routine is called whenever the user queries the status of
 956  *  the interface using ifconfig.
 957  *
 958  **********************************************************************/
 959 static void
 960 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 961 {
 962         struct adapter * adapter = ifp->if_softc;
 963
 964         INIT_DEBUGOUT("em_media_status: begin");
 965
 966         em_check_for_link(&adapter->hw);
 967         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
 968                 if (adapter->link_active == 0) {
 969                         em_get_speed_and_duplex(&adapter->hw,
 970                                                 &adapter->link_speed,
 971                                                 &adapter->link_duplex);
 972                         adapter->link_active = 1;
 973                 }
 974         } else {
 975                 if (adapter->link_active == 1) {
 976                         adapter->link_speed = 0;
 977                         adapter->link_duplex = 0;
 978                         adapter->link_active = 0;
 979                 }
 980         }
 981
 982         ifmr->ifm_status = IFM_AVALID;
 983         ifmr->ifm_active = IFM_ETHER;
 984
 985         if (!adapter->link_active)
 986                 return;
 987
 988         ifmr->ifm_status |= IFM_ACTIVE;
 989
 990         if (adapter->hw.media_type == em_media_type_fiber) {
 991                 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 992         } else {
 993                 switch (adapter->link_speed) {
 994                 case 10:
 995                         ifmr->ifm_active |= IFM_10_T;
 996                         break;
 997                 case 100:
 998                         ifmr->ifm_active |= IFM_100_TX;
 999                         break;
1000                 case 1000:
1001                         ifmr->ifm_active |= IFM_1000_T;
1002                         break;
1003                 }
1004                 if (adapter->link_duplex == FULL_DUPLEX)
1005                         ifmr->ifm_active |= IFM_FDX;
1006                 else
1007                         ifmr->ifm_active |= IFM_HDX;
1008         }
1009 }
1010
1011 /*********************************************************************
1012  *
1013  *  Media Ioctl callback
1014  *
1015  *  This routine is called when the user changes speed/duplex using
1016  *  media/mediopt option with ifconfig.
1017  *
1018  **********************************************************************/
1019 static int
1020 em_media_change(struct ifnet *ifp)
1021 {
1022         struct adapter * adapter = ifp->if_softc;
1023         struct ifmedia  *ifm = &adapter->media;
1024
1025         INIT_DEBUGOUT("em_media_change: begin");
1026
1027         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1028                 return(EINVAL);
1029
1030         lwkt_serialize_enter(&adapter->serializer);
1031
1032         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1033         case IFM_AUTO:
1034                 adapter->hw.autoneg = DO_AUTO_NEG;
1035                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1036                 break;
1037         case IFM_1000_SX:
1038         case IFM_1000_T:
1039                 adapter->hw.autoneg = DO_AUTO_NEG;
1040                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1041                 break;
1042         case IFM_100_TX:
1043                 adapter->hw.autoneg = FALSE;
1044                 adapter->hw.autoneg_advertised = 0;
1045                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1046                         adapter->hw.forced_speed_duplex = em_100_full;
1047                 else
1048                         adapter->hw.forced_speed_duplex = em_100_half;
1049                 break;
1050         case IFM_10_T:
1051                 adapter->hw.autoneg = FALSE;
1052                 adapter->hw.autoneg_advertised = 0;
1053                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1054                         adapter->hw.forced_speed_duplex = em_10_full;
1055                 else
1056                         adapter->hw.forced_speed_duplex = em_10_half;
1057                 break;
1058         default:
1059                 if_printf(ifp, "Unsupported media type\n");
1060         }
1061         /*
1062          * As the speed/duplex settings may have changed we need to
1063          * reset the PHY.
1064          */
1065         adapter->hw.phy_reset_disable = FALSE;
1066
1067         em_init_serialized(adapter);
1068
1069         lwkt_serialize_exit(&adapter->serializer);
1070         return(0);
1071 }
1072
1073 static void
1074 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1075          int error)
1076 {
1077         struct em_q *q = arg;
1078
1079         if (error)
1080                 return;
1081         KASSERT(nsegs <= EM_MAX_SCATTER,
1082                 ("Too many DMA segments returned when mapping tx packet"));
1083         q->nsegs = nsegs;
1084         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1085 }
1086
1087 #define EM_FIFO_HDR              0x10
1088 #define EM_82547_PKT_THRESH      0x3e0
1089 #define EM_82547_TX_FIFO_SIZE    0x2800
1090 #define EM_82547_TX_FIFO_BEGIN   0xf00
1091 /*********************************************************************
1092  *
1093  *  This routine maps the mbufs to tx descriptors.
1094  *
1095  *  return 0 on success, positive on failure
1096  **********************************************************************/
1097 static int
1098 em_encap(struct adapter *adapter, struct mbuf *m_head)
1099 {
1100         uint32_t txd_upper;
1101         uint32_t txd_lower, txd_used = 0, txd_saved = 0;
1102         int i, j, error;
1103         uint64_t address;
1104
1105         /* For 82544 Workaround */
1106         DESC_ARRAY desc_array;
1107         uint32_t array_elements;
1108         uint32_t counter;
1109
1110         struct ifvlan *ifv = NULL;
1111         struct em_q q;
1112         struct em_buffer *tx_buffer = NULL;
1113         struct em_tx_desc *current_tx_desc = NULL;
1114         struct ifnet *ifp = &adapter->interface_data.ac_if;
1115
1116         /*
1117          * Force a cleanup if number of TX descriptors
1118          * available hits the threshold
1119          */
1120         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1121                 em_clean_transmit_interrupts(adapter);
1122                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1123                         adapter->no_tx_desc_avail1++;
1124                         return(ENOBUFS);
1125                 }
1126         }
1127         /*
1128          * Map the packet for DMA.
1129          */
1130         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q.map)) {
1131                 adapter->no_tx_map_avail++;
1132                 return(ENOMEM);
1133         }
1134         error = bus_dmamap_load_mbuf(adapter->txtag, q.map, m_head, em_tx_cb,
1135                                      &q, BUS_DMA_NOWAIT);
1136         if (error != 0) {
1137                 adapter->no_tx_dma_setup++;
1138                 bus_dmamap_destroy(adapter->txtag, q.map);
1139                 return(error);
1140         }
1141         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1142
1143         if (q.nsegs > adapter->num_tx_desc_avail) {
1144                 adapter->no_tx_desc_avail2++;
1145                 bus_dmamap_unload(adapter->txtag, q.map);
1146                 bus_dmamap_destroy(adapter->txtag, q.map);
1147                 return(ENOBUFS);
1148         }
1149
1150         if (ifp->if_hwassist > 0) {
1151                 em_transmit_checksum_setup(adapter,  m_head,
1152                                            &txd_upper, &txd_lower);
1153         }
1154         else
1155                 txd_upper = txd_lower = 0;
1156
1157         /* Find out if we are in vlan mode */
1158         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1159             m_head->m_pkthdr.rcvif != NULL &&
1160             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1161                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1162
1163         i = adapter->next_avail_tx_desc;
1164         if (adapter->pcix_82544) {
1165                 txd_saved = i;
1166                 txd_used = 0;
1167         }
1168         for (j = 0; j < q.nsegs; j++) {
1169                 /* If adapter is 82544 and on PCIX bus */
1170                 if(adapter->pcix_82544) {
1171                         array_elements = 0;
1172                         address = htole64(q.segs[j].ds_addr);
1173                         /*
1174                          * Check the Address and Length combination and
1175                          * split the data accordingly
1176                          */
1177                         array_elements = em_fill_descriptors(address,
1178                                                              htole32(q.segs[j].ds_len),
1179                                                              &desc_array);
1180                         for (counter = 0; counter < array_elements; counter++) {
1181                                 if (txd_used == adapter->num_tx_desc_avail) {
1182                                         adapter->next_avail_tx_desc = txd_saved;
1183                                         adapter->no_tx_desc_avail2++;
1184                                         bus_dmamap_unload(adapter->txtag, q.map);
1185                                         bus_dmamap_destroy(adapter->txtag, q.map);
1186                                         return(ENOBUFS);
1187                                 }
1188                                 tx_buffer = &adapter->tx_buffer_area[i];
1189                                 current_tx_desc = &adapter->tx_desc_base[i];
1190                                 current_tx_desc->buffer_addr = htole64(
1191                                 desc_array.descriptor[counter].address);
1192                                 current_tx_desc->lower.data = htole32(
1193                                 (adapter->txd_cmd | txd_lower |
1194                                 (uint16_t)desc_array.descriptor[counter].length));
1195                                 current_tx_desc->upper.data = htole32((txd_upper));
1196                                 if (++i == adapter->num_tx_desc)
1197                                         i = 0;
1198
1199                                 tx_buffer->m_head = NULL;
1200                                 txd_used++;
1201                         }
1202                 } else {
1203                         tx_buffer = &adapter->tx_buffer_area[i];
1204                         current_tx_desc = &adapter->tx_desc_base[i];
1205
1206                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1207                         current_tx_desc->lower.data = htole32(
1208                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1209                         current_tx_desc->upper.data = htole32(txd_upper);
1210
1211                         if (++i == adapter->num_tx_desc)
1212                                 i = 0;
1213
1214                         tx_buffer->m_head = NULL;
1215                 }
1216         }
1217
1218         adapter->next_avail_tx_desc = i;
1219         if (adapter->pcix_82544)
1220                 adapter->num_tx_desc_avail -= txd_used;
1221         else
1222                 adapter->num_tx_desc_avail -= q.nsegs;
1223
1224         if (ifv != NULL) {
1225                 /* Set the vlan id */
1226                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1227
1228                 /* Tell hardware to add tag */
1229                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1230         }
1231
1232         tx_buffer->m_head = m_head;
1233         tx_buffer->map = q.map;
1234         bus_dmamap_sync(adapter->txtag, q.map, BUS_DMASYNC_PREWRITE);
1235
1236         /*
1237          * Last Descriptor of Packet needs End Of Packet (EOP)
1238          */
1239         current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1240
1241         /*
1242          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1243          * that this frame is available to transmit.
1244          */
1245         if (adapter->hw.mac_type == em_82547 &&
1246             adapter->link_duplex == HALF_DUPLEX) {
1247                 em_82547_move_tail(adapter);
1248         } else {
1249                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1250                 if (adapter->hw.mac_type == em_82547) {
1251                         em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len);
1252                 }
1253         }
1254
1255         return(0);
1256 }
1257
1258 /*********************************************************************
1259  *
1260  * 82547 workaround to avoid controller hang in half-duplex environment.
1261  * The workaround is to avoid queuing a large packet that would span
1262  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1263  * in this case. We do that only when FIFO is quiescent.
1264  *
1265  **********************************************************************/
1266 static void
1267 em_82547_move_tail(void *arg)
1268 {
1269         struct adapter *adapter = arg;
1270
1271         lwkt_serialize_enter(&adapter->serializer);
1272         em_82547_move_tail_serialized(arg);
1273         lwkt_serialize_exit(&adapter->serializer);
1274 }
1275
1276 static void
1277 em_82547_move_tail_serialized(void *arg)
1278 {
1279         struct adapter *adapter = arg;
1280         uint16_t hw_tdt;
1281         uint16_t sw_tdt;
1282         struct em_tx_desc *tx_desc;
1283         uint16_t length = 0;
1284         boolean_t eop = 0;
1285
1286         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1287         sw_tdt = adapter->next_avail_tx_desc;
1288
1289         while (hw_tdt != sw_tdt) {
1290                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1291                 length += tx_desc->lower.flags.length;
1292                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1293                 if(++hw_tdt == adapter->num_tx_desc)
1294                         hw_tdt = 0;
1295
1296                 if(eop) {
1297                         if (em_82547_fifo_workaround(adapter, length)) {
1298                                 adapter->tx_fifo_wrk++;
1299                                 callout_reset(&adapter->tx_fifo_timer, 1,
1300                                         em_82547_move_tail, adapter);
1301                                 break;
1302                         }
1303                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1304                         em_82547_update_fifo_head(adapter, length);
1305                         length = 0;
1306                 }
1307         }
1308 }
1309
1310 static int
1311 em_82547_fifo_workaround(struct adapter *adapter, int len)
1312 {
1313         int fifo_space, fifo_pkt_len;
1314
1315         fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1316
1317         if (adapter->link_duplex == HALF_DUPLEX) {
1318                 fifo_space = EM_82547_TX_FIFO_SIZE - adapter->tx_fifo_head;
1319
1320                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1321                         if (em_82547_tx_fifo_reset(adapter))
1322                                 return(0);
1323                         else
1324                                 return(1);
1325                 }
1326         }
1327
1328         return(0);
1329 }
1330
1331 static void
1332 em_82547_update_fifo_head(struct adapter *adapter, int len)
1333 {
1334         int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1335
1336         /* tx_fifo_head is always 16 byte aligned */
1337         adapter->tx_fifo_head += fifo_pkt_len;
1338         if (adapter->tx_fifo_head >= EM_82547_TX_FIFO_SIZE)
1339                 adapter->tx_fifo_head -= EM_82547_TX_FIFO_SIZE;
1340 }
1341
1342 static int
1343 em_82547_tx_fifo_reset(struct adapter *adapter)
1344 {
1345         uint32_t tctl;
1346
1347         if ( (E1000_READ_REG(&adapter->hw, TDT) ==
1348               E1000_READ_REG(&adapter->hw, TDH)) &&
1349              (E1000_READ_REG(&adapter->hw, TDFT) ==
1350               E1000_READ_REG(&adapter->hw, TDFH)) &&
1351              (E1000_READ_REG(&adapter->hw, TDFTS) ==
1352               E1000_READ_REG(&adapter->hw, TDFHS)) &&
1353              (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1354
1355                 /* Disable TX unit */
1356                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1357                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1358
1359                 /* Reset FIFO pointers */
1360                 E1000_WRITE_REG(&adapter->hw, TDFT, EM_82547_TX_FIFO_BEGIN);
1361                 E1000_WRITE_REG(&adapter->hw, TDFH, EM_82547_TX_FIFO_BEGIN);
1362                 E1000_WRITE_REG(&adapter->hw, TDFTS, EM_82547_TX_FIFO_BEGIN);
1363                 E1000_WRITE_REG(&adapter->hw, TDFHS, EM_82547_TX_FIFO_BEGIN);
1364
1365                 /* Re-enable TX unit */
1366                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1367                 E1000_WRITE_FLUSH(&adapter->hw);
1368
1369                 adapter->tx_fifo_head = 0;
1370                 adapter->tx_fifo_reset++;
1371
1372                 return(TRUE);
1373         }
1374         else {
1375                 return(FALSE);
1376         }
1377 }
1378
1379 static void
1380 em_set_promisc(struct adapter *adapter)
1381 {
1382         uint32_t reg_rctl;
1383         struct ifnet *ifp = &adapter->interface_data.ac_if;
1384
1385         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1386
1387         if (ifp->if_flags & IFF_PROMISC) {
1388                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1389                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1390         } else if (ifp->if_flags & IFF_ALLMULTI) {
1391                 reg_rctl |= E1000_RCTL_MPE;
1392                 reg_rctl &= ~E1000_RCTL_UPE;
1393                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1394         }
1395 }
1396
1397 static void
1398 em_disable_promisc(struct adapter *adapter)
1399 {
1400         uint32_t reg_rctl;
1401
1402         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1403
1404         reg_rctl &=  (~E1000_RCTL_UPE);
1405         reg_rctl &=  (~E1000_RCTL_MPE);
1406         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1407 }
1408
1409 /*********************************************************************
1410  *  Multicast Update
1411  *
1412  *  This routine is called whenever multicast address list is updated.
1413  *
1414  **********************************************************************/
1415
1416 static void
1417 em_set_multi(struct adapter *adapter)
1418 {
1419         uint32_t reg_rctl = 0;
1420         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1421         struct ifmultiaddr *ifma;
1422         int mcnt = 0;
1423         struct ifnet *ifp = &adapter->interface_data.ac_if;
1424
1425         IOCTL_DEBUGOUT("em_set_multi: begin");
1426
1427         if (adapter->hw.mac_type == em_82542_rev2_0) {
1428                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1429                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1430                         em_pci_clear_mwi(&adapter->hw);
1431                 reg_rctl |= E1000_RCTL_RST;
1432                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1433                 msec_delay(5);
1434         }
1435
1436         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1437                 if (ifma->ifma_addr->sa_family != AF_LINK)
1438                         continue;
1439
1440                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1441                         break;
1442
1443                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1444                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1445                 mcnt++;
1446         }
1447
1448         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1449                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1450                 reg_rctl |= E1000_RCTL_MPE;
1451                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1452         } else
1453                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1454
1455         if (adapter->hw.mac_type == em_82542_rev2_0) {
1456                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1457                 reg_rctl &= ~E1000_RCTL_RST;
1458                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1459                 msec_delay(5);
1460                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1461                         em_pci_set_mwi(&adapter->hw);
1462         }
1463 }
1464
1465 /*********************************************************************
1466  *  Timer routine
1467  *
1468  *  This routine checks for link status and updates statistics.
1469  *
1470  **********************************************************************/
1471
1472 static void
1473 em_local_timer(void *arg)
1474 {
1475         struct ifnet *ifp;
1476         struct adapter *adapter = arg;
1477         ifp = &adapter->interface_data.ac_if;
1478
1479         lwkt_serialize_enter(&adapter->serializer);
1480
1481         em_check_for_link(&adapter->hw);
1482         em_print_link_status(adapter);
1483         em_update_stats_counters(adapter);
1484         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1485                 em_print_hw_stats(adapter);
1486         em_smartspeed(adapter);
1487
1488         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
1489
1490         lwkt_serialize_exit(&adapter->serializer);
1491 }
1492
1493 static void
1494 em_print_link_status(struct adapter *adapter)
1495 {
1496         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1497                 if (adapter->link_active == 0) {
1498                         em_get_speed_and_duplex(&adapter->hw,
1499                                                 &adapter->link_speed,
1500                                                 &adapter->link_duplex);
1501                         device_printf(adapter->dev, "Link is up %d Mbps %s\n",
1502                                adapter->link_speed,
1503                                ((adapter->link_duplex == FULL_DUPLEX) ?
1504                                 "Full Duplex" : "Half Duplex"));
1505                         adapter->link_active = 1;
1506                         adapter->smartspeed = 0;
1507                 }
1508         } else {
1509                 if (adapter->link_active == 1) {
1510                         adapter->link_speed = 0;
1511                         adapter->link_duplex = 0;
1512                         device_printf(adapter->dev, "Link is Down\n");
1513                         adapter->link_active = 0;
1514                 }
1515         }
1516 }
1517
1518 /*********************************************************************
1519  *
1520  *  This routine disables all traffic on the adapter by issuing a
1521  *  global reset on the MAC and deallocates TX/RX buffers.
1522  *
1523  **********************************************************************/
1524
1525 static void
1526 em_stop(void *arg)
1527 {
1528         struct ifnet   *ifp;
1529         struct adapter * adapter = arg;
1530         ifp = &adapter->interface_data.ac_if;
1531
1532         INIT_DEBUGOUT("em_stop: begin");
1533         em_disable_intr(adapter);
1534         em_reset_hw(&adapter->hw);
1535         callout_stop(&adapter->timer);
1536         callout_stop(&adapter->tx_fifo_timer);
1537         em_free_transmit_structures(adapter);
1538         em_free_receive_structures(adapter);
1539
1540         /* Tell the stack that the interface is no longer active */
1541         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1542         ifp->if_timer = 0;
1543 }
1544
1545 /*********************************************************************
1546  *
1547  *  Determine hardware revision.
1548  *
1549  **********************************************************************/
1550 static void
1551 em_identify_hardware(struct adapter * adapter)
1552 {
1553         device_t dev = adapter->dev;
1554
1555         /* Make sure our PCI config space has the necessary stuff set */
1556         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1557         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1558               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1559                 device_printf(dev, "Memory Access and/or Bus Master bits were not set!\n");
1560                 adapter->hw.pci_cmd_word |=
1561                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1562                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1563         }
1564
1565         /* Save off the information about this board */
1566         adapter->hw.vendor_id = pci_get_vendor(dev);
1567         adapter->hw.device_id = pci_get_device(dev);
1568         adapter->hw.revision_id = pci_get_revid(dev);
1569         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1570         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1571
1572         /* Identify the MAC */
1573         if (em_set_mac_type(&adapter->hw))
1574                 device_printf(dev, "Unknown MAC Type\n");
1575
1576         if (adapter->hw.mac_type == em_82541 ||
1577             adapter->hw.mac_type == em_82541_rev_2 ||
1578             adapter->hw.mac_type == em_82547 ||
1579             adapter->hw.mac_type == em_82547_rev_2)
1580                 adapter->hw.phy_init_script = TRUE;
1581 }
1582
1583 /*********************************************************************
1584  *
1585  *  Initialize the hardware to a configuration as specified by the
1586  *  adapter structure. The controller is reset, the EEPROM is
1587  *  verified, the MAC address is set, then the shared initialization
1588  *  routines are called.
1589  *
1590  **********************************************************************/
1591 static int
1592 em_hardware_init(struct adapter *adapter)
1593 {
1594         INIT_DEBUGOUT("em_hardware_init: begin");
1595         /* Issue a global reset */
1596         em_reset_hw(&adapter->hw);
1597
1598         /* When hardware is reset, fifo_head is also reset */
1599         adapter->tx_fifo_head = 0;
1600
1601         /* Make sure we have a good EEPROM before we read from it */
1602         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
1603                 device_printf(adapter->dev, "The EEPROM Checksum Is Not Valid\n");
1604                 return(EIO);
1605         }
1606
1607         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
1608                 device_printf(adapter->dev, "EEPROM read error while reading part number\n");
1609                 return(EIO);
1610         }
1611
1612         if (em_init_hw(&adapter->hw) < 0) {
1613                 device_printf(adapter->dev, "Hardware Initialization Failed");
1614                 return(EIO);
1615         }
1616
1617         em_check_for_link(&adapter->hw);
1618         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)
1619                 adapter->link_active = 1;
1620         else
1621                 adapter->link_active = 0;
1622
1623         if (adapter->link_active) {
1624                 em_get_speed_and_duplex(&adapter->hw,
1625                                         &adapter->link_speed,
1626                                         &adapter->link_duplex);
1627         } else {
1628                 adapter->link_speed = 0;
1629                 adapter->link_duplex = 0;
1630         }
1631
1632         return(0);
1633 }
1634
1635 /*********************************************************************
1636  *
1637  *  Setup networking device structure and register an interface.
1638  *
1639  **********************************************************************/
1640 static void
1641 em_setup_interface(device_t dev, struct adapter *adapter)
1642 {
1643         struct ifnet   *ifp;
1644         INIT_DEBUGOUT("em_setup_interface: begin");
1645
1646         ifp = &adapter->interface_data.ac_if;
1647         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1648         ifp->if_mtu = ETHERMTU;
1649         ifp->if_baudrate = 1000000000;
1650         ifp->if_init =  em_init;
1651         ifp->if_softc = adapter;
1652         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1653         ifp->if_ioctl = em_ioctl;
1654         ifp->if_start = em_start;
1655 #ifdef DEVICE_POLLING
1656         ifp->if_poll = em_poll;
1657 #endif
1658         ifp->if_watchdog = em_watchdog;
1659         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
1660         ifq_set_ready(&ifp->if_snd);
1661
1662         if (adapter->hw.mac_type >= em_82543)
1663                 ifp->if_capabilities |= IFCAP_HWCSUM;
1664
1665         ifp->if_capenable = ifp->if_capabilities;
1666
1667         ether_ifattach(ifp, adapter->hw.mac_addr);
1668
1669         /*
1670          * Tell the upper layer(s) we support long frames.
1671          */
1672         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1673         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1674
1675         /*
1676          * Specify the media types supported by this adapter and register
1677          * callbacks to update media and link information
1678          */
1679         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
1680                      em_media_status);
1681         if (adapter->hw.media_type == em_media_type_fiber) {
1682                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1683                             0, NULL);
1684                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1685                             0, NULL);
1686         } else {
1687                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1688                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1689                             0, NULL);
1690                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
1691                             0, NULL);
1692                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1693                             0, NULL);
1694                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1695                             0, NULL);
1696                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1697         }
1698         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1699         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1700 }
1701
1702 /*********************************************************************
1703  *
1704  *  Workaround for SmartSpeed on 82541 and 82547 controllers
1705  *
1706  **********************************************************************/
1707 static void
1708 em_smartspeed(struct adapter *adapter)
1709 {
1710         uint16_t phy_tmp;
1711
1712         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
1713             !adapter->hw.autoneg ||
1714             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
1715                 return;
1716
1717         if (adapter->smartspeed == 0) {
1718                 /*
1719                  * If Master/Slave config fault is asserted twice,
1720                  * we assume back-to-back.
1721                  */
1722                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1723                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
1724                         return;
1725                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1726                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
1727                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL,
1728                                         &phy_tmp);
1729                         if (phy_tmp & CR_1000T_MS_ENABLE) {
1730                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
1731                                 em_write_phy_reg(&adapter->hw,
1732                                                  PHY_1000T_CTRL, phy_tmp);
1733                                 adapter->smartspeed++;
1734                                 if (adapter->hw.autoneg &&
1735                                     !em_phy_setup_autoneg(&adapter->hw) &&
1736                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
1737                                                      &phy_tmp)) {
1738                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1739                                                     MII_CR_RESTART_AUTO_NEG);
1740                                         em_write_phy_reg(&adapter->hw,
1741                                                          PHY_CTRL, phy_tmp);
1742                                 }
1743                         }
1744                 }
1745                 return;
1746         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
1747                 /* If still no link, perhaps using 2/3 pair cable */
1748                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
1749                 phy_tmp |= CR_1000T_MS_ENABLE;
1750                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
1751                 if (adapter->hw.autoneg &&
1752                     !em_phy_setup_autoneg(&adapter->hw) &&
1753                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
1754                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1755                                     MII_CR_RESTART_AUTO_NEG);
1756                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
1757                 }
1758         }
1759         /* Restart process after EM_SMARTSPEED_MAX iterations */
1760         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
1761                 adapter->smartspeed = 0;
1762 }
1763
1764 /*
1765  * Manage DMA'able memory.
1766  */
1767 static void
1768 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1769 {
1770         if (error)
1771                 return;
1772         *(bus_addr_t*) arg = segs->ds_addr;
1773 }
1774
1775 static int
1776 em_dma_malloc(struct adapter *adapter, bus_size_t size,
1777               struct em_dma_alloc *dma, int mapflags)
1778 {
1779         int r;
1780         device_t dev = adapter->dev;
1781
1782         r = bus_dma_tag_create(NULL,                    /* parent */
1783                                PAGE_SIZE, 0,            /* alignment, bounds */
1784                                BUS_SPACE_MAXADDR,       /* lowaddr */
1785                                BUS_SPACE_MAXADDR,       /* highaddr */
1786                                NULL, NULL,              /* filter, filterarg */
1787                                size,                    /* maxsize */
1788                                1,                       /* nsegments */
1789                                size,                    /* maxsegsize */
1790                                BUS_DMA_ALLOCNOW,        /* flags */
1791                                &dma->dma_tag);
1792         if (r != 0) {
1793                 device_printf(dev, "em_dma_malloc: bus_dma_tag_create failed; "
1794                               "error %u\n", r);
1795                 goto fail_0;
1796         }
1797
1798         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1799                              BUS_DMA_NOWAIT, &dma->dma_map);
1800         if (r != 0) {
1801                 device_printf(dev, "em_dma_malloc: bus_dmammem_alloc failed; "
1802                               "size %llu, error %d\n", (uintmax_t)size, r);
1803                 goto fail_2;
1804         }
1805
1806         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1807                             size,
1808                             em_dmamap_cb,
1809                             &dma->dma_paddr,
1810                             mapflags | BUS_DMA_NOWAIT);
1811         if (r != 0) {
1812                 device_printf(dev, "em_dma_malloc: bus_dmamap_load failed; "
1813                               "error %u\n", r);
1814                 goto fail_3;
1815         }
1816
1817         dma->dma_size = size;
1818         return(0);
1819
1820 fail_3:
1821         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1822 fail_2:
1823         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1824         bus_dma_tag_destroy(dma->dma_tag);
1825 fail_0:
1826         dma->dma_map = NULL;
1827         dma->dma_tag = NULL;
1828         return(r);
1829 }
1830
1831 static void
1832 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
1833 {
1834         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1835         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1836         bus_dma_tag_destroy(dma->dma_tag);
1837 }
1838
1839 /*********************************************************************
1840  *
1841  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1842  *  the information needed to transmit a packet on the wire.
1843  *
1844  **********************************************************************/
1845 static int
1846 em_allocate_transmit_structures(struct adapter * adapter)
1847 {
1848         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
1849             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
1850         if (adapter->tx_buffer_area == NULL) {
1851                 device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
1852                 return(ENOMEM);
1853         }
1854
1855         return(0);
1856 }
1857
1858 /*********************************************************************
1859  *
1860  *  Allocate and initialize transmit structures.
1861  *
1862  **********************************************************************/
1863 static int
1864 em_setup_transmit_structures(struct adapter * adapter)
1865 {
1866         /*
1867          * Setup DMA descriptor areas.
1868          */
1869         if (bus_dma_tag_create(NULL,                    /* parent */
1870                                1, 0,                    /* alignment, bounds */
1871                                BUS_SPACE_MAXADDR,       /* lowaddr */
1872                                BUS_SPACE_MAXADDR,       /* highaddr */
1873                                NULL, NULL,              /* filter, filterarg */
1874                                MCLBYTES * 8,            /* maxsize */
1875                                EM_MAX_SCATTER,          /* nsegments */
1876                                MCLBYTES * 8,            /* maxsegsize */
1877                                BUS_DMA_ALLOCNOW,        /* flags */
1878                                &adapter->txtag)) {
1879                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
1880                 return(ENOMEM);
1881         }
1882
1883         if (em_allocate_transmit_structures(adapter))
1884                 return(ENOMEM);
1885
1886         bzero((void *) adapter->tx_desc_base,
1887               (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
1888
1889         adapter->next_avail_tx_desc = 0;
1890         adapter->oldest_used_tx_desc = 0;
1891
1892         /* Set number of descriptors available */
1893         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1894
1895         /* Set checksum context */
1896         adapter->active_checksum_context = OFFLOAD_NONE;
1897
1898         return(0);
1899 }
1900
1901 /*********************************************************************
1902  *
1903  *  Enable transmit unit.
1904  *
1905  **********************************************************************/
1906 static void
1907 em_initialize_transmit_unit(struct adapter * adapter)
1908 {
1909         uint32_t reg_tctl;
1910         uint32_t reg_tipg = 0;
1911         uint64_t bus_addr;
1912
1913         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
1914
1915         /* Setup the Base and Length of the Tx Descriptor Ring */
1916         bus_addr = adapter->txdma.dma_paddr;
1917         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
1918         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
1919         E1000_WRITE_REG(&adapter->hw, TDLEN,
1920                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
1921
1922         /* Setup the HW Tx Head and Tail descriptor pointers */
1923         E1000_WRITE_REG(&adapter->hw, TDH, 0);
1924         E1000_WRITE_REG(&adapter->hw, TDT, 0);
1925
1926         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1927                      E1000_READ_REG(&adapter->hw, TDBAL),
1928                      E1000_READ_REG(&adapter->hw, TDLEN));
1929
1930         /* Set the default values for the Tx Inter Packet Gap timer */
1931         switch (adapter->hw.mac_type) {
1932         case em_82542_rev2_0:
1933         case em_82542_rev2_1:
1934                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
1935                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1936                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1937                 break;
1938         default:
1939                 if (adapter->hw.media_type == em_media_type_fiber)
1940                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1941                 else
1942                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1943                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1944                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1945         }
1946
1947         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
1948         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
1949         if (adapter->hw.mac_type >= em_82540)
1950                 E1000_WRITE_REG(&adapter->hw, TADV,
1951                                 adapter->tx_abs_int_delay.value);
1952
1953         /* Program the Transmit Control Register */
1954         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
1955                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1956         if (adapter->link_duplex == 1)
1957                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1958         else
1959                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1960         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
1961
1962         /* Setup Transmit Descriptor Settings for this adapter */
1963         adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
1964
1965         if (adapter->tx_int_delay.value > 0)
1966                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
1967 }
1968
1969 /*********************************************************************
1970  *
1971  *  Free all transmit related data structures.
1972  *
1973  **********************************************************************/
1974 static void
1975 em_free_transmit_structures(struct adapter * adapter)
1976 {
1977         struct em_buffer *tx_buffer;
1978         int i;
1979
1980         INIT_DEBUGOUT("free_transmit_structures: begin");
1981
1982         if (adapter->tx_buffer_area != NULL) {
1983                 tx_buffer = adapter->tx_buffer_area;
1984                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
1985                         if (tx_buffer->m_head != NULL) {
1986                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1987                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
1988                                 m_freem(tx_buffer->m_head);
1989                         }
1990                         tx_buffer->m_head = NULL;
1991                 }
1992         }
1993         if (adapter->tx_buffer_area != NULL) {
1994                 free(adapter->tx_buffer_area, M_DEVBUF);
1995                 adapter->tx_buffer_area = NULL;
1996         }
1997         if (adapter->txtag != NULL) {
1998                 bus_dma_tag_destroy(adapter->txtag);
1999                 adapter->txtag = NULL;
2000         }
2001 }
2002
2003 /*********************************************************************
2004  *
2005  *  The offload context needs to be set when we transfer the first
2006  *  packet of a particular protocol (TCP/UDP). We change the
2007  *  context only if the protocol type changes.
2008  *
2009  **********************************************************************/
2010 static void
2011 em_transmit_checksum_setup(struct adapter * adapter,
2012                            struct mbuf *mp,
2013                            uint32_t *txd_upper,
2014                            uint32_t *txd_lower)
2015 {
2016         struct em_context_desc *TXD;
2017         struct em_buffer *tx_buffer;
2018         int curr_txd;
2019
2020         if (mp->m_pkthdr.csum_flags) {
2021                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2022                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2023                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2024                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2025                                 return;
2026                         else
2027                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2028                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2029                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2030                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2031                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2032                                 return;
2033                         else
2034                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2035                 } else {
2036                         *txd_upper = 0;
2037                         *txd_lower = 0;
2038                         return;
2039                 }
2040         } else {
2041                 *txd_upper = 0;
2042                 *txd_lower = 0;
2043                 return;
2044         }
2045
2046         /* If we reach this point, the checksum offload context
2047          * needs to be reset.
2048          */
2049         curr_txd = adapter->next_avail_tx_desc;
2050         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2051         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2052
2053         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2054         TXD->lower_setup.ip_fields.ipcso =
2055             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2056         TXD->lower_setup.ip_fields.ipcse =
2057             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2058
2059         TXD->upper_setup.tcp_fields.tucss =
2060             ETHER_HDR_LEN + sizeof(struct ip);
2061         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2062
2063         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2064                 TXD->upper_setup.tcp_fields.tucso =
2065                     ETHER_HDR_LEN + sizeof(struct ip) +
2066                     offsetof(struct tcphdr, th_sum);
2067         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2068                 TXD->upper_setup.tcp_fields.tucso =
2069                         ETHER_HDR_LEN + sizeof(struct ip) +
2070                         offsetof(struct udphdr, uh_sum);
2071         }
2072
2073         TXD->tcp_seg_setup.data = htole32(0);
2074         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2075
2076         tx_buffer->m_head = NULL;
2077
2078         if (++curr_txd == adapter->num_tx_desc)
2079                 curr_txd = 0;
2080
2081         adapter->num_tx_desc_avail--;
2082         adapter->next_avail_tx_desc = curr_txd;
2083 }
2084
2085 /**********************************************************************
2086  *
2087  *  Examine each tx_buffer in the used queue. If the hardware is done
2088  *  processing the packet then free associated resources. The
2089  *  tx_buffer is put back on the free queue.
2090  *
2091  **********************************************************************/
2092
2093 static void
2094 em_clean_transmit_interrupts(struct adapter *adapter)
2095 {
2096         int i, num_avail;
2097         struct em_buffer *tx_buffer;
2098         struct em_tx_desc *tx_desc;
2099         struct ifnet *ifp = &adapter->interface_data.ac_if;
2100
2101         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2102                 return;
2103
2104 #ifdef DBG_STATS
2105         adapter->clean_tx_interrupts++;
2106 #endif
2107         num_avail = adapter->num_tx_desc_avail;
2108         i = adapter->oldest_used_tx_desc;
2109
2110         tx_buffer = &adapter->tx_buffer_area[i];
2111         tx_desc = &adapter->tx_desc_base[i];
2112
2113         while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2114                 tx_desc->upper.data = 0;
2115                 num_avail++;
2116
2117                 if (tx_buffer->m_head) {
2118                         ifp->if_opackets++;
2119                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2120                                         BUS_DMASYNC_POSTWRITE);
2121                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2122                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2123
2124                         m_freem(tx_buffer->m_head);
2125                         tx_buffer->m_head = NULL;
2126                 }
2127
2128                 if (++i == adapter->num_tx_desc)
2129                         i = 0;
2130
2131                 tx_buffer = &adapter->tx_buffer_area[i];
2132                 tx_desc = &adapter->tx_desc_base[i];
2133         }
2134
2135         adapter->oldest_used_tx_desc = i;
2136
2137         /*
2138          * If we have enough room, clear IFF_OACTIVE to tell the stack
2139          * that it is OK to send packets.
2140          * If there are no pending descriptors, clear the timeout. Otherwise,
2141          * if some descriptors have been freed, restart the timeout.
2142          */
2143         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2144                 ifp->if_flags &= ~IFF_OACTIVE;
2145                 if (num_avail == adapter->num_tx_desc)
2146                         ifp->if_timer = 0;
2147                 else if (num_avail == adapter->num_tx_desc_avail)
2148                         ifp->if_timer = EM_TX_TIMEOUT;
2149         }
2150         adapter->num_tx_desc_avail = num_avail;
2151 }
2152
2153 /*********************************************************************
2154  *
2155  *  Get a buffer from system mbuf buffer pool.
2156  *
2157  **********************************************************************/
2158 static int
2159 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp, int how)
2160 {
2161         struct mbuf *mp = nmp;
2162         struct em_buffer *rx_buffer;
2163         struct ifnet *ifp;
2164         bus_addr_t paddr;
2165         int error;
2166
2167         ifp = &adapter->interface_data.ac_if;
2168
2169         if (mp == NULL) {
2170                 mp = m_getcl(how, MT_DATA, M_PKTHDR);
2171                 if (mp == NULL) {
2172                         adapter->mbuf_cluster_failed++;
2173                         return(ENOBUFS);
2174                 }
2175                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2176         } else {
2177                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2178                 mp->m_data = mp->m_ext.ext_buf;
2179                 mp->m_next = NULL;
2180         }
2181         if (ifp->if_mtu <= ETHERMTU)
2182                 m_adj(mp, ETHER_ALIGN);
2183
2184         rx_buffer = &adapter->rx_buffer_area[i];
2185
2186         /*
2187          * Using memory from the mbuf cluster pool, invoke the
2188          * bus_dma machinery to arrange the memory mapping.
2189          */
2190         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2191                                 mtod(mp, void *), mp->m_len,
2192                                 em_dmamap_cb, &paddr, 0);
2193         if (error) {
2194                 m_free(mp);
2195                 return(error);
2196         }
2197         rx_buffer->m_head = mp;
2198         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2199         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2200
2201         return(0);
2202 }
2203
2204 /*********************************************************************
2205  *
2206  *  Allocate memory for rx_buffer structures. Since we use one
2207  *  rx_buffer per received packet, the maximum number of rx_buffer's
2208  *  that we'll need is equal to the number of receive descriptors
2209  *  that we've allocated.
2210  *
2211  **********************************************************************/
2212 static int
2213 em_allocate_receive_structures(struct adapter *adapter)
2214 {
2215         int i, error, size;
2216         struct em_buffer *rx_buffer;
2217
2218         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2219         adapter->rx_buffer_area = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2220
2221         error = bus_dma_tag_create(NULL,                /* parent */
2222                                    1, 0,                /* alignment, bounds */
2223                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2224                                    BUS_SPACE_MAXADDR,   /* highaddr */
2225                                    NULL, NULL,          /* filter, filterarg */
2226                                    MCLBYTES,            /* maxsize */
2227                                    1,                   /* nsegments */
2228                                    MCLBYTES,            /* maxsegsize */
2229                                    BUS_DMA_ALLOCNOW,    /* flags */
2230                                    &adapter->rxtag);
2231         if (error != 0) {
2232                 device_printf(adapter->dev, "em_allocate_receive_structures: "
2233                               "bus_dma_tag_create failed; error %u\n", error);
2234                 goto fail_0;
2235         }
2236
2237         rx_buffer = adapter->rx_buffer_area;
2238         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2239                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2240                                           &rx_buffer->map);
2241                 if (error != 0) {
2242                         device_printf(adapter->dev,
2243                                       "em_allocate_receive_structures: "
2244                                       "bus_dmamap_create failed; error %u\n",
2245                                       error);
2246                         goto fail_1;
2247                 }
2248         }
2249
2250         for (i = 0; i < adapter->num_rx_desc; i++) {
2251                 error = em_get_buf(i, adapter, NULL, MB_WAIT);
2252                 if (error != 0) {
2253                         adapter->rx_buffer_area[i].m_head = NULL;
2254                         adapter->rx_desc_base[i].buffer_addr = 0;
2255                         return(error);
2256                 }
2257         }
2258
2259         return(0);
2260
2261 fail_1:
2262         bus_dma_tag_destroy(adapter->rxtag);
2263 fail_0:
2264         adapter->rxtag = NULL;
2265         free(adapter->rx_buffer_area, M_DEVBUF);
2266         adapter->rx_buffer_area = NULL;
2267         return(error);
2268 }
2269
2270 /*********************************************************************
2271  *
2272  *  Allocate and initialize receive structures.
2273  *
2274  **********************************************************************/
2275 static int
2276 em_setup_receive_structures(struct adapter *adapter)
2277 {
2278         bzero((void *) adapter->rx_desc_base,
2279               (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2280
2281         if (em_allocate_receive_structures(adapter))
2282                 return(ENOMEM);
2283
2284         /* Setup our descriptor pointers */
2285         adapter->next_rx_desc_to_check = 0;
2286         return(0);
2287 }
2288
2289 /*********************************************************************
2290  *
2291  *  Enable receive unit.
2292  *
2293  **********************************************************************/
2294 static void
2295 em_initialize_receive_unit(struct adapter *adapter)
2296 {
2297         uint32_t reg_rctl;
2298         uint32_t reg_rxcsum;
2299         struct ifnet *ifp;
2300         uint64_t bus_addr;
2301
2302         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2303
2304         ifp = &adapter->interface_data.ac_if;
2305
2306         /* Make sure receives are disabled while setting up the descriptor ring */
2307         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2308
2309         /* Set the Receive Delay Timer Register */
2310         E1000_WRITE_REG(&adapter->hw, RDTR,
2311                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2312
2313         if(adapter->hw.mac_type >= em_82540) {
2314                 E1000_WRITE_REG(&adapter->hw, RADV,
2315                                 adapter->rx_abs_int_delay.value);
2316
2317                 /* Set the interrupt throttling rate in 256ns increments */
2318                 if (em_int_throttle_ceil) {
2319                         E1000_WRITE_REG(&adapter->hw, ITR,
2320                                 1000000000 / 256 / em_int_throttle_ceil);
2321                 } else {
2322                         E1000_WRITE_REG(&adapter->hw, ITR, 0);
2323                 }
2324         }
2325
2326         /* Setup the Base and Length of the Rx Descriptor Ring */
2327         bus_addr = adapter->rxdma.dma_paddr;
2328         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2329         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2330         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2331                         sizeof(struct em_rx_desc));
2332
2333         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2334         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2335         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2336
2337         /* Setup the Receive Control Register */
2338         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2339                    E1000_RCTL_RDMTS_HALF |
2340                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2341
2342         if (adapter->hw.tbi_compatibility_on == TRUE)
2343                 reg_rctl |= E1000_RCTL_SBP;
2344
2345         switch (adapter->rx_buffer_len) {
2346         default:
2347         case EM_RXBUFFER_2048:
2348                 reg_rctl |= E1000_RCTL_SZ_2048;
2349                 break;
2350         case EM_RXBUFFER_4096:
2351                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2352                 break;
2353         case EM_RXBUFFER_8192:
2354                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2355                 break;
2356         case EM_RXBUFFER_16384:
2357                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2358                 break;
2359         }
2360
2361         if (ifp->if_mtu > ETHERMTU)
2362                 reg_rctl |= E1000_RCTL_LPE;
2363
2364         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2365         if ((adapter->hw.mac_type >= em_82543) &&
2366             (ifp->if_capenable & IFCAP_RXCSUM)) {
2367                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2368                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2369                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2370         }
2371
2372         /* Enable Receives */
2373         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2374 }
2375
2376 /*********************************************************************
2377  *
2378  *  Free receive related data structures.
2379  *
2380  **********************************************************************/
2381 static void
2382 em_free_receive_structures(struct adapter *adapter)
2383 {
2384         struct em_buffer *rx_buffer;
2385         int i;
2386
2387         INIT_DEBUGOUT("free_receive_structures: begin");
2388
2389         if (adapter->rx_buffer_area != NULL) {
2390                 rx_buffer = adapter->rx_buffer_area;
2391                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2392                         if (rx_buffer->map != NULL) {
2393                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2394                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2395                         }
2396                         if (rx_buffer->m_head != NULL)
2397                                 m_freem(rx_buffer->m_head);
2398                         rx_buffer->m_head = NULL;
2399                 }
2400         }
2401         if (adapter->rx_buffer_area != NULL) {
2402                 free(adapter->rx_buffer_area, M_DEVBUF);
2403                 adapter->rx_buffer_area = NULL;
2404         }
2405         if (adapter->rxtag != NULL) {
2406                 bus_dma_tag_destroy(adapter->rxtag);
2407                 adapter->rxtag = NULL;
2408         }
2409 }
2410
2411 /*********************************************************************
2412  *
2413  *  This routine executes in interrupt context. It replenishes
2414  *  the mbufs in the descriptor and sends data which has been
2415  *  dma'ed into host memory to upper layer.
2416  *
2417  *  We loop at most count times if count is > 0, or until done if
2418  *  count < 0.
2419  *
2420  *********************************************************************/
2421 static void
2422 em_process_receive_interrupts(struct adapter *adapter, int count)
2423 {
2424         struct ifnet *ifp;
2425         struct mbuf *mp;
2426         uint8_t accept_frame = 0;
2427         uint8_t eop = 0;
2428         uint16_t len, desc_len, prev_len_adj;
2429         int i;
2430
2431         /* Pointer to the receive descriptor being examined. */
2432         struct em_rx_desc *current_desc;
2433
2434         ifp = &adapter->interface_data.ac_if;
2435         i = adapter->next_rx_desc_to_check;
2436         current_desc = &adapter->rx_desc_base[i];
2437
2438         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
2439 #ifdef DBG_STATS
2440                 adapter->no_pkts_avail++;
2441 #endif
2442                 return;
2443         }
2444         while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0)) {
2445                 mp = adapter->rx_buffer_area[i].m_head;
2446                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2447                                 BUS_DMASYNC_POSTREAD);
2448
2449                 accept_frame = 1;
2450                 prev_len_adj = 0;
2451                 desc_len = le16toh(current_desc->length);
2452                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2453                         count--;
2454                         eop = 1;
2455                         if (desc_len < ETHER_CRC_LEN) {
2456                                 len = 0;
2457                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2458                         }
2459                         else {
2460                                 len = desc_len - ETHER_CRC_LEN;
2461                         }
2462                 } else {
2463                         eop = 0;
2464                         len = desc_len;
2465                 }
2466
2467                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2468                         uint8_t last_byte;
2469                         uint32_t pkt_len = desc_len;
2470
2471                         if (adapter->fmp != NULL)
2472                                 pkt_len += adapter->fmp->m_pkthdr.len;
2473
2474                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
2475
2476                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
2477                                        current_desc->errors,
2478                                        pkt_len, last_byte)) {
2479                                 em_tbi_adjust_stats(&adapter->hw,
2480                                                     &adapter->stats,
2481                                                     pkt_len,
2482                                                     adapter->hw.mac_addr);
2483                                 if (len > 0)
2484                                         len--;
2485                         }
2486                         else {
2487                                 accept_frame = 0;
2488                         }
2489                 }
2490
2491                 if (accept_frame) {
2492                         if (em_get_buf(i, adapter, NULL, MB_DONTWAIT) == ENOBUFS) {
2493                                 adapter->dropped_pkts++;
2494                                 em_get_buf(i, adapter, mp, MB_DONTWAIT);
2495                                 if (adapter->fmp != NULL)
2496                                         m_freem(adapter->fmp);
2497                                 adapter->fmp = NULL;
2498                                 adapter->lmp = NULL;
2499                                 break;
2500                         }
2501
2502                         /* Assign correct length to the current fragment */
2503                         mp->m_len = len;
2504
2505                         if (adapter->fmp == NULL) {
2506                                 mp->m_pkthdr.len = len;
2507                                 adapter->fmp = mp;       /* Store the first mbuf */
2508                                 adapter->lmp = mp;
2509                         } else {
2510                                 /* Chain mbuf's together */
2511                                 /*
2512                                  * Adjust length of previous mbuf in chain if we
2513                                  * received less than 4 bytes in the last descriptor.
2514                                  */
2515                                 if (prev_len_adj > 0) {
2516                                         adapter->lmp->m_len -= prev_len_adj;
2517                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
2518                                 }
2519                                 adapter->lmp->m_next = mp;
2520                                 adapter->lmp = adapter->lmp->m_next;
2521                                 adapter->fmp->m_pkthdr.len += len;
2522                         }
2523
2524                         if (eop) {
2525                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2526                                 ifp->if_ipackets++;
2527
2528                                 em_receive_checksum(adapter, current_desc,
2529                                                     adapter->fmp);
2530                                 if (current_desc->status & E1000_RXD_STAT_VP)
2531                                         VLAN_INPUT_TAG(adapter->fmp,
2532                                                        (current_desc->special &
2533                                                         E1000_RXD_SPC_VLAN_MASK));
2534                                 else
2535                                         (*ifp->if_input)(ifp, adapter->fmp);
2536                                 adapter->fmp = NULL;
2537                                 adapter->lmp = NULL;
2538                         }
2539                 } else {
2540                         adapter->dropped_pkts++;
2541                         em_get_buf(i, adapter, mp, MB_DONTWAIT);
2542                         if (adapter->fmp != NULL)
2543                                 m_freem(adapter->fmp);
2544                         adapter->fmp = NULL;
2545                         adapter->lmp = NULL;
2546                 }
2547
2548                 /* Zero out the receive descriptors status  */
2549                 current_desc->status = 0;
2550
2551                 /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
2552                 E1000_WRITE_REG(&adapter->hw, RDT, i);
2553
2554                 /* Advance our pointers to the next descriptor */
2555                 if (++i == adapter->num_rx_desc) {
2556                         i = 0;
2557                         current_desc = adapter->rx_desc_base;
2558                 } else
2559                         current_desc++;
2560         }
2561         adapter->next_rx_desc_to_check = i;
2562 }
2563
2564 /*********************************************************************
2565  *
2566  *  Verify that the hardware indicated that the checksum is valid.
2567  *  Inform the stack about the status of checksum so that stack
2568  *  doesn't spend time verifying the checksum.
2569  *
2570  *********************************************************************/
2571 static void
2572 em_receive_checksum(struct adapter *adapter,
2573                     struct em_rx_desc *rx_desc,
2574                     struct mbuf *mp)
2575 {
2576         /* 82543 or newer only */
2577         if ((adapter->hw.mac_type < em_82543) ||
2578             /* Ignore Checksum bit is set */
2579             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
2580                 mp->m_pkthdr.csum_flags = 0;
2581                 return;
2582         }
2583
2584         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
2585                 /* Did it pass? */
2586                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
2587                         /* IP Checksum Good */
2588                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2589                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2590                 } else {
2591                         mp->m_pkthdr.csum_flags = 0;
2592                 }
2593         }
2594
2595         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
2596                 /* Did it pass? */
2597                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
2598                         mp->m_pkthdr.csum_flags |=
2599                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2600                         mp->m_pkthdr.csum_data = htons(0xffff);
2601                 }
2602         }
2603 }
2604
2605
2606 static void
2607 em_enable_vlans(struct adapter *adapter)
2608 {
2609         uint32_t ctrl;
2610
2611         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
2612
2613         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
2614         ctrl |= E1000_CTRL_VME;
2615         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
2616 }
2617
2618 /*
2619  * note: we must call bus_enable_intr() prior to enabling the hardware
2620  * interrupt and bus_disable_intr() after disabling the hardware interrupt
2621  * in order to avoid handler execution races from scheduled interrupt
2622  * threads.
2623  */
2624 static void
2625 em_enable_intr(struct adapter *adapter)
2626 {
2627         struct ifnet *ifp = &adapter->interface_data.ac_if;
2628
2629         if ((ifp->if_flags & IFF_POLLING) == 0) {
2630                 lwkt_serialize_handler_enable(&adapter->serializer);
2631                 E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
2632         }
2633 }
2634
2635 static void
2636 em_disable_intr(struct adapter *adapter)
2637 {
2638         E1000_WRITE_REG(&adapter->hw, IMC,
2639                         (0xffffffff & ~E1000_IMC_RXSEQ));
2640         lwkt_serialize_handler_disable(&adapter->serializer);
2641 }
2642
2643 static int
2644 em_is_valid_ether_addr(uint8_t *addr)
2645 {
2646         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2647
2648         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN)))
2649                 return(FALSE);
2650         else
2651                 return(TRUE);
2652 }
2653
2654 void
2655 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2656 {
2657         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
2658 }
2659
2660 void
2661 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2662 {
2663         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
2664 }
2665
2666 void
2667 em_pci_set_mwi(struct em_hw *hw)
2668 {
2669         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2670                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
2671 }
2672
2673 void
2674 em_pci_clear_mwi(struct em_hw *hw)
2675 {
2676         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2677                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
2678 }
2679
2680 uint32_t
2681 em_read_reg_io(struct em_hw *hw, uint32_t offset)
2682 {
2683         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2684         return(bus_space_read_4(hw->reg_io_tag, hw->reg_io_handle, 4));
2685 }
2686
2687 void
2688 em_write_reg_io(struct em_hw *hw, uint32_t offset, uint32_t value)
2689 {
2690         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2691         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 4, value);
2692 }
2693
2694 /*********************************************************************
2695  * 82544 Coexistence issue workaround.
2696  *    There are 2 issues.
2697  *      1. Transmit Hang issue.
2698  *    To detect this issue, following equation can be used...
2699  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2700  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
2701  *
2702  *      2. DAC issue.
2703  *    To detect this issue, following equation can be used...
2704  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2705  *          If SUM[3:0] is in between 9 to c, we will have this issue.
2706  *
2707  *
2708  *    WORKAROUND:
2709  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
2710  *          9,a,b,c (DAC)
2711  *
2712 *************************************************************************/
2713 static uint32_t
2714 em_fill_descriptors(uint64_t address, uint32_t length, PDESC_ARRAY desc_array)
2715 {
2716         /* Since issue is sensitive to length and address.*/
2717         /* Let us first check the address...*/
2718         uint32_t safe_terminator;
2719         if (length <= 4) {
2720                 desc_array->descriptor[0].address = address;
2721                 desc_array->descriptor[0].length = length;
2722                 desc_array->elements = 1;
2723                 return(desc_array->elements);
2724         }
2725         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
2726         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
2727         if (safe_terminator == 0 ||
2728             (safe_terminator > 4 && safe_terminator < 9) ||
2729             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
2730                 desc_array->descriptor[0].address = address;
2731                 desc_array->descriptor[0].length = length;
2732                 desc_array->elements = 1;
2733                 return(desc_array->elements);
2734         }
2735
2736         desc_array->descriptor[0].address = address;
2737         desc_array->descriptor[0].length = length - 4;
2738         desc_array->descriptor[1].address = address + (length - 4);
2739         desc_array->descriptor[1].length = 4;
2740         desc_array->elements = 2;
2741         return(desc_array->elements);
2742 }
2743
2744 /**********************************************************************
2745  *
2746  *  Update the board statistics counters.
2747  *
2748  **********************************************************************/
2749 static void
2750 em_update_stats_counters(struct adapter *adapter)
2751 {
2752         struct ifnet   *ifp;
2753
2754         if (adapter->hw.media_type == em_media_type_copper ||
2755             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
2756                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
2757                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
2758         }
2759         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
2760         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
2761         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
2762         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
2763
2764         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
2765         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
2766         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
2767         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
2768         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
2769         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
2770         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
2771         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
2772         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
2773         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
2774         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
2775         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
2776         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
2777         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
2778         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
2779         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
2780         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
2781         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
2782         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
2783         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
2784
2785         /* For the 64-bit byte counters the low dword must be read first. */
2786         /* Both registers clear on the read of the high dword */
2787
2788         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
2789         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
2790         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
2791         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
2792
2793         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
2794         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
2795         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
2796         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
2797         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
2798
2799         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
2800         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
2801         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
2802         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
2803
2804         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
2805         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
2806         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
2807         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
2808         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
2809         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
2810         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
2811         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
2812         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
2813         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
2814
2815         if (adapter->hw.mac_type >= em_82543) {
2816                 adapter->stats.algnerrc +=
2817                     E1000_READ_REG(&adapter->hw, ALGNERRC);
2818                 adapter->stats.rxerrc +=
2819                     E1000_READ_REG(&adapter->hw, RXERRC);
2820                 adapter->stats.tncrs +=
2821                     E1000_READ_REG(&adapter->hw, TNCRS);
2822                 adapter->stats.cexterr +=
2823                     E1000_READ_REG(&adapter->hw, CEXTERR);
2824                 adapter->stats.tsctc +=
2825                     E1000_READ_REG(&adapter->hw, TSCTC);
2826                 adapter->stats.tsctfc +=
2827                     E1000_READ_REG(&adapter->hw, TSCTFC);
2828         }
2829         ifp = &adapter->interface_data.ac_if;
2830
2831         /* Fill out the OS statistics structure */
2832         ifp->if_ibytes = adapter->stats.gorcl;
2833         ifp->if_obytes = adapter->stats.gotcl;
2834         ifp->if_imcasts = adapter->stats.mprc;
2835         ifp->if_collisions = adapter->stats.colc;
2836
2837         /* Rx Errors */
2838         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
2839             adapter->stats.crcerrs + adapter->stats.algnerrc +
2840             adapter->stats.rlec + adapter->stats.rnbc +
2841             adapter->stats.mpc + adapter->stats.cexterr;
2842
2843         /* Tx Errors */
2844         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol;
2845 }
2846
2847
2848 /**********************************************************************
2849  *
2850  *  This routine is called only when em_display_debug_stats is enabled.
2851  *  This routine provides a way to take a look at important statistics
2852  *  maintained by the driver and hardware.
2853  *
2854  **********************************************************************/
2855 static void
2856 em_print_debug_info(struct adapter *adapter)
2857 {
2858         device_t dev= adapter->dev;
2859         uint8_t *hw_addr = adapter->hw.hw_addr;
2860
2861         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
2862         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
2863                       E1000_READ_REG(&adapter->hw, TIDV),
2864                       E1000_READ_REG(&adapter->hw, TADV));
2865         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
2866                       E1000_READ_REG(&adapter->hw, RDTR),
2867                       E1000_READ_REG(&adapter->hw, RADV));
2868 #ifdef DBG_STATS
2869         device_printf(dev, "Packets not Avail = %ld\n", adapter->no_pkts_avail);
2870         device_printf(dev, "CleanTxInterrupts = %ld\n",
2871                       adapter->clean_tx_interrupts);
2872 #endif
2873         device_printf(dev, "fifo workaround = %lld, fifo_reset = %lld\n",
2874                       (long long)adapter->tx_fifo_wrk,
2875                       (long long)adapter->tx_fifo_reset);
2876         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2877                       E1000_READ_REG(&adapter->hw, TDH),
2878                       E1000_READ_REG(&adapter->hw, TDT));
2879         device_printf(dev, "Num Tx descriptors avail = %d\n",
2880                       adapter->num_tx_desc_avail);
2881         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
2882                       adapter->no_tx_desc_avail1);
2883         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
2884                       adapter->no_tx_desc_avail2);
2885         device_printf(dev, "Std mbuf failed = %ld\n",
2886                       adapter->mbuf_alloc_failed);
2887         device_printf(dev, "Std mbuf cluster failed = %ld\n",
2888                       adapter->mbuf_cluster_failed);
2889         device_printf(dev, "Driver dropped packets = %ld\n",
2890                       adapter->dropped_pkts);
2891 }
2892
2893 static void
2894 em_print_hw_stats(struct adapter *adapter)
2895 {
2896         device_t dev= adapter->dev;
2897
2898         device_printf(dev, "Adapter: %p\n", adapter);
2899
2900         device_printf(dev, "Excessive collisions = %lld\n",
2901                       (long long)adapter->stats.ecol);
2902         device_printf(dev, "Symbol errors = %lld\n",
2903                       (long long)adapter->stats.symerrs);
2904         device_printf(dev, "Sequence errors = %lld\n",
2905                       (long long)adapter->stats.sec);
2906         device_printf(dev, "Defer count = %lld\n",
2907                       (long long)adapter->stats.dc);
2908
2909         device_printf(dev, "Missed Packets = %lld\n",
2910                       (long long)adapter->stats.mpc);
2911         device_printf(dev, "Receive No Buffers = %lld\n",
2912                       (long long)adapter->stats.rnbc);
2913         device_printf(dev, "Receive length errors = %lld\n",
2914                       (long long)adapter->stats.rlec);
2915         device_printf(dev, "Receive errors = %lld\n",
2916                       (long long)adapter->stats.rxerrc);
2917         device_printf(dev, "Crc errors = %lld\n",
2918                       (long long)adapter->stats.crcerrs);
2919         device_printf(dev, "Alignment errors = %lld\n",
2920                       (long long)adapter->stats.algnerrc);
2921         device_printf(dev, "Carrier extension errors = %lld\n",
2922                       (long long)adapter->stats.cexterr);
2923
2924         device_printf(dev, "XON Rcvd = %lld\n",
2925                       (long long)adapter->stats.xonrxc);
2926         device_printf(dev, "XON Xmtd = %lld\n",
2927                       (long long)adapter->stats.xontxc);
2928         device_printf(dev, "XOFF Rcvd = %lld\n",
2929                       (long long)adapter->stats.xoffrxc);
2930         device_printf(dev, "XOFF Xmtd = %lld\n",
2931                       (long long)adapter->stats.xofftxc);
2932
2933         device_printf(dev, "Good Packets Rcvd = %lld\n",
2934                       (long long)adapter->stats.gprc);
2935         device_printf(dev, "Good Packets Xmtd = %lld\n",
2936                       (long long)adapter->stats.gptc);
2937 }
2938
2939 static int
2940 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
2941 {
2942         int error;
2943         int result;
2944         struct adapter *adapter;
2945
2946         result = -1;
2947         error = sysctl_handle_int(oidp, &result, 0, req);
2948
2949         if (error || !req->newptr)
2950                 return(error);
2951
2952         if (result == 1) {
2953                 adapter = (struct adapter *)arg1;
2954                 em_print_debug_info(adapter);
2955         }
2956
2957         return(error);
2958 }
2959
2960 static int
2961 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
2962 {
2963         int error;
2964         int result;
2965         struct adapter *adapter;
2966
2967         result = -1;
2968         error = sysctl_handle_int(oidp, &result, 0, req);
2969
2970         if (error || !req->newptr)
2971                 return(error);
2972
2973         if (result == 1) {
2974                 adapter = (struct adapter *)arg1;
2975                 em_print_hw_stats(adapter);
2976         }
2977
2978         return(error);
2979 }
2980
2981 static int
2982 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
2983 {
2984         struct em_int_delay_info *info;
2985         struct adapter *adapter;
2986         uint32_t regval;
2987         int error;
2988         int usecs;
2989         int ticks;
2990
2991         info = (struct em_int_delay_info *)arg1;
2992         adapter = info->adapter;
2993         usecs = info->value;
2994         error = sysctl_handle_int(oidp, &usecs, 0, req);
2995         if (error != 0 || req->newptr == NULL)
2996                 return(error);
2997         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
2998                 return(EINVAL);
2999         info->value = usecs;
3000         ticks = E1000_USECS_TO_TICKS(usecs);
3001
3002         lwkt_serialize_enter(&adapter->serializer);
3003         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3004         regval = (regval & ~0xffff) | (ticks & 0xffff);
3005         /* Handle a few special cases. */
3006         switch (info->offset) {
3007         case E1000_RDTR:
3008         case E1000_82542_RDTR:
3009                 regval |= E1000_RDT_FPDB;
3010                 break;
3011         case E1000_TIDV:
3012         case E1000_82542_TIDV:
3013                 if (ticks == 0) {
3014                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3015                         /* Don't write 0 into the TIDV register. */
3016                         regval++;
3017                 } else
3018                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3019                 break;
3020         }
3021         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3022         lwkt_serialize_exit(&adapter->serializer);
3023         return(0);
3024 }
3025
3026 static void
3027 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3028                         const char *description, struct em_int_delay_info *info,
3029                         int offset, int value)
3030 {
3031         info->adapter = adapter;
3032         info->offset = offset;
3033         info->value = value;
3034         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3035                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3036                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3037                         info, 0, em_sysctl_int_delay, "I", description);
3038 }
3039
3040 static int
3041 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
3042 {
3043         struct adapter *adapter = (void *)arg1;
3044         int error;
3045         int throttle;
3046
3047         throttle = em_int_throttle_ceil;
3048         error = sysctl_handle_int(oidp, &throttle, 0, req);
3049         if (error || req->newptr == NULL)
3050                 return error;
3051         if (throttle < 0 || throttle > 1000000000 / 256)
3052                 return EINVAL;
3053         if (throttle) {
3054                 /*
3055                  * Set the interrupt throttling rate in 256ns increments,
3056                  * recalculate sysctl value assignment to get exact frequency.
3057                  */
3058                 throttle = 1000000000 / 256 / throttle;
3059                 lwkt_serialize_enter(&adapter->serializer);
3060                 em_int_throttle_ceil = 1000000000 / 256 / throttle;
3061                 E1000_WRITE_REG(&adapter->hw, ITR, throttle);
3062                 lwkt_serialize_exit(&adapter->serializer);
3063         } else {
3064                 lwkt_serialize_enter(&adapter->serializer);
3065                 em_int_throttle_ceil = 0;
3066                 E1000_WRITE_REG(&adapter->hw, ITR, 0);
3067                 lwkt_serialize_exit(&adapter->serializer);
3068         }
3069         device_printf(adapter->dev, "Interrupt moderation set to %d/sec\n",
3070                         em_int_throttle_ceil);
3071         return 0;
3072 }
3073