sys/dev/netif/em/if_em.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2001-2003, Intel Corporation
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12  2. Redistributions in binary form must reproduce the above copyright
  13     notice, this list of conditions and the following disclaimer in the
  14     documentation and/or other materials provided with the distribution.
  15
  16  3. Neither the name of the Intel Corporation nor the names of its
  17     contributors may be used to endorse or promote products derived from
  18     this software without specific prior written permission.
  19
  20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30 POSSIBILITY OF SUCH DAMAGE.
  31
  32 ***************************************************************************/
  33
  34 /*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.15 2003/06/09 22:10:15 pdeuskar Exp $*/
  35 /*$DragonFly: src/sys/dev/netif/em/if_em.c,v 1.14 2004/05/11 22:55:15 joerg Exp $*/
  36
  37 #include <dev/netif/em/if_em.h>
  38
  39 /*********************************************************************
  40  *  Set this to one to display debug statistics
  41  *********************************************************************/
  42 int             em_display_debug_stats = 0;
  43
  44 /*********************************************************************
  45  *  Driver version
  46  *********************************************************************/
  47
  48 char em_driver_version[] = "1.7.25";
  49
  50
  51 /*********************************************************************
  52  *  PCI Device ID Table
  53  *
  54  *  Used by probe to select devices to load on
  55  *  Last field stores an index into em_strings
  56  *  Last entry must be all 0s
  57  *
  58  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  59  *********************************************************************/
  60
  61 static em_vendor_info_t em_vendor_info_array[] =
  62 {
  63         /* Intel(R) PRO/1000 Network Connection */
  64         { 0x8086, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0},
  65         { 0x8086, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0},
  66         { 0x8086, 0x1004, PCI_ANY_ID, PCI_ANY_ID, 0},
  67         { 0x8086, 0x1008, PCI_ANY_ID, PCI_ANY_ID, 0},
  68         { 0x8086, 0x1009, PCI_ANY_ID, PCI_ANY_ID, 0},
  69         { 0x8086, 0x100C, PCI_ANY_ID, PCI_ANY_ID, 0},
  70         { 0x8086, 0x100D, PCI_ANY_ID, PCI_ANY_ID, 0},
  71         { 0x8086, 0x100E, PCI_ANY_ID, PCI_ANY_ID, 0},
  72         { 0x8086, 0x100F, PCI_ANY_ID, PCI_ANY_ID, 0},
  73         { 0x8086, 0x1010, PCI_ANY_ID, PCI_ANY_ID, 0},
  74         { 0x8086, 0x1011, PCI_ANY_ID, PCI_ANY_ID, 0},
  75         { 0x8086, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         { 0x8086, 0x1013, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
  78         { 0x8086, 0x1015, PCI_ANY_ID, PCI_ANY_ID, 0},
  79         { 0x8086, 0x1016, PCI_ANY_ID, PCI_ANY_ID, 0},
  80         { 0x8086, 0x1017, PCI_ANY_ID, PCI_ANY_ID, 0},
  81         { 0x8086, 0x1018, PCI_ANY_ID, PCI_ANY_ID, 0},
  82         { 0x8086, 0x1019, PCI_ANY_ID, PCI_ANY_ID, 0},
  83         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
  84         { 0x8086, 0x101D, PCI_ANY_ID, PCI_ANY_ID, 0},
  85         { 0x8086, 0x101E, PCI_ANY_ID, PCI_ANY_ID, 0},
  86         { 0x8086, 0x1026, PCI_ANY_ID, PCI_ANY_ID, 0},
  87         { 0x8086, 0x1027, PCI_ANY_ID, PCI_ANY_ID, 0},
  88         { 0x8086, 0x1028, PCI_ANY_ID, PCI_ANY_ID, 0},
  89         { 0x8086, 0x1075, PCI_ANY_ID, PCI_ANY_ID, 0},
  90         { 0x8086, 0x1076, PCI_ANY_ID, PCI_ANY_ID, 0},
  91         { 0x8086, 0x1077, PCI_ANY_ID, PCI_ANY_ID, 0},
  92         { 0x8086, 0x1078, PCI_ANY_ID, PCI_ANY_ID, 0},
  93         { 0x8086, 0x1079, PCI_ANY_ID, PCI_ANY_ID, 0},
  94         { 0x8086, 0x107A, PCI_ANY_ID, PCI_ANY_ID, 0},
  95         { 0x8086, 0x107B, PCI_ANY_ID, PCI_ANY_ID, 0},
  96         /* required last entry */
  97         { 0, 0, 0, 0, 0}
  98 };
  99
 100 /*********************************************************************
 101  *  Table of branding strings for all supported NICs.
 102  *********************************************************************/
 103
 104 static const char *em_strings[] = {
 105         "Intel(R) PRO/1000 Network Connection"
 106 };
 107
 108 /*********************************************************************
 109  *  Function prototypes
 110  *********************************************************************/
 111 static int      em_probe(device_t);
 112 static int      em_attach(device_t);
 113 static int      em_detach(device_t);
 114 static int      em_shutdown(device_t);
 115 static void     em_intr(void *);
 116 static void     em_start(struct ifnet *);
 117 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 118 static void     em_watchdog(struct ifnet *);
 119 static void     em_init(void *);
 120 static void     em_stop(void *);
 121 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 122 static int      em_media_change(struct ifnet *);
 123 static void     em_identify_hardware(struct adapter *);
 124 static int      em_allocate_pci_resources(struct adapter *);
 125 static void     em_free_pci_resources(struct adapter *);
 126 static void     em_local_timer(void *);
 127 static int      em_hardware_init(struct adapter *);
 128 static void     em_setup_interface(device_t, struct adapter *);
 129 static int      em_setup_transmit_structures(struct adapter *);
 130 static void     em_initialize_transmit_unit(struct adapter *);
 131 static int      em_setup_receive_structures(struct adapter *);
 132 static void     em_initialize_receive_unit(struct adapter *);
 133 static void     em_enable_intr(struct adapter *);
 134 static void     em_disable_intr(struct adapter *);
 135 static void     em_free_transmit_structures(struct adapter *);
 136 static void     em_free_receive_structures(struct adapter *);
 137 static void     em_update_stats_counters(struct adapter *);
 138 static void     em_clean_transmit_interrupts(struct adapter *);
 139 static int      em_allocate_receive_structures(struct adapter *);
 140 static int      em_allocate_transmit_structures(struct adapter *);
 141 static void     em_process_receive_interrupts(struct adapter *, int);
 142 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 143                                     struct mbuf *);
 144 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 145                                            uint32_t *, uint32_t *);
 146 static void     em_set_promisc(struct adapter *);
 147 static void     em_disable_promisc(struct adapter *);
 148 static void     em_set_multi(struct adapter *);
 149 static void     em_print_hw_stats(struct adapter *);
 150 static void     em_print_link_status(struct adapter *);
 151 static int      em_get_buf(int i, struct adapter *, struct mbuf *);
 152 static void     em_enable_vlans(struct adapter *);
 153 static int      em_encap(struct adapter *, struct mbuf *);
 154 static void     em_smartspeed(struct adapter *);
 155 static int      em_82547_fifo_workaround(struct adapter *, int);
 156 static void     em_82547_update_fifo_head(struct adapter *, int);
 157 static int      em_82547_tx_fifo_reset(struct adapter *);
 158 static void     em_82547_move_tail(void *arg);
 159 static int      em_dma_malloc(struct adapter *, bus_size_t,
 160                               struct em_dma_alloc *, int);
 161 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 162 static void     em_print_debug_info(struct adapter *);
 163 static int      em_is_valid_ether_addr(uint8_t *);
 164 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 165 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 166 static uint32_t em_fill_descriptors(uint64_t address, uint32_t length,
 167                                    PDESC_ARRAY desc_array);
 168 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 169 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 170                                         const char *,
 171                                         struct em_int_delay_info *, int, int);
 172
 173 /*********************************************************************
 174  *  FreeBSD Device Interface Entry Points
 175  *********************************************************************/
 176
 177 static device_method_t em_methods[] = {
 178         /* Device interface */
 179         DEVMETHOD(device_probe, em_probe),
 180         DEVMETHOD(device_attach, em_attach),
 181         DEVMETHOD(device_detach, em_detach),
 182         DEVMETHOD(device_shutdown, em_shutdown),
 183         {0, 0}
 184 };
 185
 186 static driver_t em_driver = {
 187         "em", em_methods, sizeof(struct adapter),
 188 };
 189
 190 static devclass_t em_devclass;
 191
 192 DECLARE_DUMMY_MODULE(if_em);
 193 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 194
 195 /*********************************************************************
 196  *  Tunable default values.
 197  *********************************************************************/
 198
 199 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 200 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 201
 202 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 203 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 204 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 205 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 206
 207 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 208 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 209 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 210 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 211
 212 /*********************************************************************
 213  *  Device identification routine
 214  *
 215  *  em_probe determines if the driver should be loaded on
 216  *  adapter based on PCI vendor/device id of the adapter.
 217  *
 218  *  return 0 on success, positive on failure
 219  *********************************************************************/
 220
 221 static int
 222 em_probe(device_t dev)
 223 {
 224         em_vendor_info_t *ent;
 225
 226         uint16_t pci_vendor_id = 0;
 227         uint16_t pci_device_id = 0;
 228         uint16_t pci_subvendor_id = 0;
 229         uint16_t pci_subdevice_id = 0;
 230         char adapter_name[60];
 231
 232         INIT_DEBUGOUT("em_probe: begin");
 233
 234         pci_vendor_id = pci_get_vendor(dev);
 235         if (pci_vendor_id != EM_VENDOR_ID)
 236                 return(ENXIO);
 237
 238         pci_device_id = pci_get_device(dev);
 239         pci_subvendor_id = pci_get_subvendor(dev);
 240         pci_subdevice_id = pci_get_subdevice(dev);
 241
 242         ent = em_vendor_info_array;
 243         while (ent->vendor_id != 0) {
 244                 if ((pci_vendor_id == ent->vendor_id) &&
 245                     (pci_device_id == ent->device_id) &&
 246
 247                     ((pci_subvendor_id == ent->subvendor_id) ||
 248                      (ent->subvendor_id == PCI_ANY_ID)) &&
 249
 250                     ((pci_subdevice_id == ent->subdevice_id) ||
 251                      (ent->subdevice_id == PCI_ANY_ID))) {
 252                         snprintf(adapter_name, sizeof(adapter_name),
 253                                  "%s, Version - %s",  em_strings[ent->index],
 254                                  em_driver_version);
 255                         device_set_desc_copy(dev, adapter_name);
 256                         return(0);
 257                 }
 258                 ent++;
 259         }
 260
 261         return(ENXIO);
 262 }
 263
 264 /*********************************************************************
 265  *  Device initialization routine
 266  *
 267  *  The attach entry point is called when the driver is being loaded.
 268  *  This routine identifies the type of hardware, allocates all resources
 269  *  and initializes the hardware.
 270  *
 271  *  return 0 on success, positive on failure
 272  *********************************************************************/
 273
 274 static int
 275 em_attach(device_t dev)
 276 {
 277         struct adapter *adapter;
 278         int s;
 279         int tsize, rsize;
 280         int error = 0;
 281
 282         INIT_DEBUGOUT("em_attach: begin");
 283         s = splimp();
 284
 285         adapter = device_get_softc(dev);
 286
 287         bzero(adapter, sizeof(struct adapter));
 288         adapter->dev = dev;
 289         adapter->osdep.dev = dev;
 290         adapter->unit = device_get_unit(dev);
 291
 292         /* SYSCTL stuff */
 293         sysctl_ctx_init(&adapter->sysctl_ctx);
 294         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 295                                                SYSCTL_STATIC_CHILDREN(_hw),
 296                                                OID_AUTO,
 297                                                device_get_nameunit(dev),
 298                                                CTLFLAG_RD,
 299                                                0, "");
 300
 301         if (adapter->sysctl_tree == NULL) {
 302                 error = EIO;
 303                 goto err_sysctl;
 304         }
 305
 306         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 307                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 308                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 309                         (void *)adapter, 0,
 310                         em_sysctl_debug_info, "I", "Debug Information");
 311
 312         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 313                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 314                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 315                         (void *)adapter, 0,
 316                         em_sysctl_stats, "I", "Statistics");
 317
 318         callout_init(&adapter->timer);
 319         callout_init(&adapter->tx_fifo_timer);
 320
 321         /* Determine hardware revision */
 322         em_identify_hardware(adapter);
 323
 324         /* Set up some sysctls for the tunable interrupt delays */
 325         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 326                                 "receive interrupt delay in usecs",
 327                                 &adapter->rx_int_delay,
 328                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 329                                 em_rx_int_delay_dflt);
 330         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 331                                 "transmit interrupt delay in usecs",
 332                                 &adapter->tx_int_delay,
 333                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 334                                 em_tx_int_delay_dflt);
 335         if (adapter->hw.mac_type >= em_82540) {
 336                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 337                                         "receive interrupt delay limit in usecs",
 338                                         &adapter->rx_abs_int_delay,
 339                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 340                                         em_rx_abs_int_delay_dflt);
 341                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 342                                         "transmit interrupt delay limit in usecs",
 343                                         &adapter->tx_abs_int_delay,
 344                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 345                                         em_tx_abs_int_delay_dflt);
 346         }
 347
 348         /* Parameters (to be read from user) */
 349         adapter->num_tx_desc = EM_MAX_TXD;
 350         adapter->num_rx_desc = EM_MAX_RXD;
 351         adapter->hw.autoneg = DO_AUTO_NEG;
 352         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 353         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 354         adapter->hw.tbi_compatibility_en = TRUE;
 355         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 356
 357         /*
 358          * These parameters control the automatic generation(Tx) and
 359          * response(Rx) to Ethernet PAUSE frames.
 360          */
 361         adapter->hw.fc_high_water = FC_DEFAULT_HI_THRESH;
 362         adapter->hw.fc_low_water  = FC_DEFAULT_LO_THRESH;
 363         adapter->hw.fc_pause_time = FC_DEFAULT_TX_TIMER;
 364         adapter->hw.fc_send_xon   = TRUE;
 365         adapter->hw.fc = em_fc_full;
 366
 367         adapter->hw.phy_init_script = 1;
 368         adapter->hw.phy_reset_disable = FALSE;
 369
 370 #ifndef EM_MASTER_SLAVE
 371         adapter->hw.master_slave = em_ms_hw_default;
 372 #else
 373         adapter->hw.master_slave = EM_MASTER_SLAVE;
 374 #endif
 375
 376         /*
 377          * Set the max frame size assuming standard ethernet
 378          * sized frames
 379          */
 380         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 381
 382         adapter->hw.min_frame_size =
 383             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 384
 385         /*
 386          * This controls when hardware reports transmit completion
 387          * status.
 388          */
 389         adapter->hw.report_tx_early = 1;
 390
 391         if (em_allocate_pci_resources(adapter)) {
 392                 device_printf(dev, "Allocation of PCI resources failed\n");
 393                 error = ENXIO;
 394                 goto err_pci;
 395         }
 396
 397         /* Initialize eeprom parameters */
 398         em_init_eeprom_params(&adapter->hw);
 399
 400         tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc);
 401
 402         /* Allocate Transmit Descriptor ring */
 403         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_WAITOK)) {
 404                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 405                 error = ENOMEM;
 406                 goto err_tx_desc;
 407         }
 408         adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr;
 409
 410         rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc);
 411
 412         /* Allocate Receive Descriptor ring */
 413         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 414                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 415                 error = ENOMEM;
 416                 goto err_rx_desc;
 417         }
 418         adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr;
 419
 420         /* Initialize the hardware */
 421         if (em_hardware_init(adapter)) {
 422                 device_printf(dev, "Unable to initialize the hardware\n");
 423                 error = EIO;
 424                 goto err_hw_init;
 425         }
 426
 427         /* Copy the permanent MAC address out of the EEPROM */
 428         if (em_read_mac_addr(&adapter->hw) < 0) {
 429                 device_printf(dev, "EEPROM read error while reading mac address\n");
 430                 error = EIO;
 431                 goto err_mac_addr;
 432         }
 433
 434         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 435                 device_printf(dev, "Invalid mac address\n");
 436                 error = EIO;
 437                 goto err_mac_addr;
 438         }
 439
 440         bcopy(adapter->hw.mac_addr, adapter->interface_data.ac_enaddr,
 441               ETHER_ADDR_LEN);
 442
 443         /* Setup OS specific network interface */
 444         em_setup_interface(dev, adapter);
 445
 446         /* Initialize statistics */
 447         em_clear_hw_cntrs(&adapter->hw);
 448         em_update_stats_counters(adapter);
 449         adapter->hw.get_link_status = 1;
 450         em_check_for_link(&adapter->hw);
 451
 452         /* Print the link status */
 453         if (adapter->link_active == 1) {
 454                 em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
 455                                         &adapter->link_duplex);
 456                 device_printf(dev, "Speed: %d Mbps, Duplex: %s\n",
 457                     adapter->link_speed,
 458                     adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
 459         } else
 460                 device_printf(dev, "Speed: N/A, Duplex:N/A\n");
 461
 462         /* Identify 82544 on PCIX */
 463         em_get_bus_info(&adapter->hw);
 464         if (adapter->hw.bus_type == em_bus_type_pcix &&
 465             adapter->hw.mac_type == em_82544)
 466                 adapter->pcix_82544 = TRUE;
 467         else
 468                 adapter->pcix_82544 = FALSE;
 469         INIT_DEBUGOUT("em_attach: end");
 470         splx(s);
 471         return(0);
 472
 473 err_mac_addr:
 474 err_hw_init:
 475         em_dma_free(adapter, &adapter->rxdma);
 476 err_rx_desc:
 477         em_dma_free(adapter, &adapter->txdma);
 478 err_tx_desc:
 479 err_pci:
 480         em_free_pci_resources(adapter);
 481         sysctl_ctx_free(&adapter->sysctl_ctx);
 482 err_sysctl:
 483         splx(s);
 484         return(error);
 485 }
 486
 487 /*********************************************************************
 488  *  Device removal routine
 489  *
 490  *  The detach entry point is called when the driver is being removed.
 491  *  This routine stops the adapter and deallocates all the resources
 492  *  that were allocated for driver operation.
 493  *
 494  *  return 0 on success, positive on failure
 495  *********************************************************************/
 496
 497 static int
 498 em_detach(device_t dev)
 499 {
 500         struct adapter * adapter = device_get_softc(dev);
 501         struct ifnet *ifp = &adapter->interface_data.ac_if;
 502         int s;
 503
 504         INIT_DEBUGOUT("em_detach: begin");
 505         s = splimp();
 506
 507         adapter->in_detach = 1;
 508
 509         em_stop(adapter);
 510         em_phy_hw_reset(&adapter->hw);
 511         ether_ifdetach(&adapter->interface_data.ac_if);
 512         em_free_pci_resources(adapter);
 513         bus_generic_detach(dev);
 514
 515         /* Free Transmit Descriptor ring */
 516         if (adapter->tx_desc_base != NULL) {
 517                 em_dma_free(adapter, &adapter->txdma);
 518                 adapter->tx_desc_base = NULL;
 519         }
 520
 521         /* Free Receive Descriptor ring */
 522         if (adapter->rx_desc_base != NULL) {
 523                 em_dma_free(adapter, &adapter->rxdma);
 524                 adapter->rx_desc_base = NULL;
 525         }
 526
 527         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
 528         ifp->if_timer = 0;
 529
 530         adapter->sysctl_tree = NULL;
 531         sysctl_ctx_free(&adapter->sysctl_ctx);
 532
 533         splx(s);
 534         return(0);
 535 }
 536
 537 /*********************************************************************
 538  *
 539  *  Shutdown entry point
 540  *
 541  **********************************************************************/
 542
 543 static int
 544 em_shutdown(device_t dev)
 545 {
 546         struct adapter *adapter = device_get_softc(dev);
 547         em_stop(adapter);
 548         return(0);
 549 }
 550
 551 /*********************************************************************
 552  *  Transmit entry point
 553  *
 554  *  em_start is called by the stack to initiate a transmit.
 555  *  The driver will remain in this routine as long as there are
 556  *  packets to transmit and transmit resources are available.
 557  *  In case resources are not available stack is notified and
 558  *  the packet is requeued.
 559  **********************************************************************/
 560
 561 static void
 562 em_start(struct ifnet *ifp)
 563 {
 564         int s;
 565         struct mbuf *m_head;
 566         struct adapter *adapter = ifp->if_softc;
 567
 568         if (!adapter->link_active)
 569                 return;
 570
 571         s = splimp();
 572         while (ifp->if_snd.ifq_head != NULL) {
 573                 IF_DEQUEUE(&ifp->if_snd, m_head);
 574
 575                 if (m_head == NULL)
 576                         break;
 577
 578                 if (em_encap(adapter, m_head)) {
 579                         ifp->if_flags |= IFF_OACTIVE;
 580                         IF_PREPEND(&ifp->if_snd, m_head);
 581                         break;
 582                 }
 583
 584                 /* Send a copy of the frame to the BPF listener */
 585                 BPF_MTAP(ifp, m_head);
 586
 587                 /* Set timeout in case hardware has problems transmitting */
 588                 ifp->if_timer = EM_TX_TIMEOUT;
 589         }
 590         splx(s);
 591 }
 592
 593 /*********************************************************************
 594  *  Ioctl entry point
 595  *
 596  *  em_ioctl is called when the user wants to configure the
 597  *  interface.
 598  *
 599  *  return 0 on success, positive on failure
 600  **********************************************************************/
 601
 602 static int
 603 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 604 {
 605         int s, mask, error = 0;
 606         struct ifreq *ifr = (struct ifreq *) data;
 607         struct adapter *adapter = ifp->if_softc;
 608
 609         s = splimp();
 610
 611         if (adapter->in_detach)
 612                 goto out;
 613
 614         switch (command) {
 615         case SIOCSIFADDR:
 616         case SIOCGIFADDR:
 617                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 618                 ether_ioctl(ifp, command, data);
 619                 break;
 620         case SIOCSIFMTU:
 621                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 622                 if (ifr->ifr_mtu > MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 623                         error = EINVAL;
 624                 } else {
 625                         ifp->if_mtu = ifr->ifr_mtu;
 626                         adapter->hw.max_frame_size =
 627                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 628                         em_init(adapter);
 629                 }
 630                 break;
 631         case SIOCSIFFLAGS:
 632                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 633                 if (ifp->if_flags & IFF_UP) {
 634                         if (!(ifp->if_flags & IFF_RUNNING))
 635                                 em_init(adapter);
 636                         em_disable_promisc(adapter);
 637                         em_set_promisc(adapter);
 638                 } else {
 639                         if (ifp->if_flags & IFF_RUNNING)
 640                                 em_stop(adapter);
 641                 }
 642                 break;
 643         case SIOCADDMULTI:
 644         case SIOCDELMULTI:
 645                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 646                 if (ifp->if_flags & IFF_RUNNING) {
 647                         em_disable_intr(adapter);
 648                         em_set_multi(adapter);
 649                         if (adapter->hw.mac_type == em_82542_rev2_0)
 650                                 em_initialize_receive_unit(adapter);
 651 #ifdef DEVICE_POLLING
 652                         if (!(ifp->if_flags & IFF_POLLING))
 653 #endif
 654                                 em_enable_intr(adapter);
 655                 }
 656                 break;
 657         case SIOCSIFMEDIA:
 658         case SIOCGIFMEDIA:
 659                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 660                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 661                 break;
 662         case SIOCSIFCAP:
 663                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 664                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 665                 if (mask & IFCAP_HWCSUM) {
 666                         if (IFCAP_HWCSUM & ifp->if_capenable)
 667                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 668                         else
 669                                 ifp->if_capenable |= IFCAP_HWCSUM;
 670                         if (ifp->if_flags & IFF_RUNNING)
 671                                 em_init(adapter);
 672                 }
 673                 break;
 674         default:
 675                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%x)\n", (int)command);
 676                 error = EINVAL;
 677         }
 678
 679 out:
 680         splx(s);
 681         return(error);
 682 }
 683
 684 /*********************************************************************
 685  *  Watchdog entry point
 686  *
 687  *  This routine is called whenever hardware quits transmitting.
 688  *
 689  **********************************************************************/
 690
 691 static void
 692 em_watchdog(struct ifnet *ifp)
 693 {
 694         struct adapter * adapter;
 695         adapter = ifp->if_softc;
 696
 697         /* If we are in this routine because of pause frames, then
 698          * don't reset the hardware.
 699          */
 700         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 701                 ifp->if_timer = EM_TX_TIMEOUT;
 702                 return;
 703         }
 704
 705         if (em_check_for_link(&adapter->hw))
 706                 if_printf(ifp, "watchdog timeout -- resetting\n");
 707
 708         ifp->if_flags &= ~IFF_RUNNING;
 709
 710         em_init(adapter);
 711
 712         ifp->if_oerrors++;
 713 }
 714
 715 /*********************************************************************
 716  *  Init entry point
 717  *
 718  *  This routine is used in two ways. It is used by the stack as
 719  *  init entry point in network interface structure. It is also used
 720  *  by the driver as a hw/sw initialization routine to get to a
 721  *  consistent state.
 722  *
 723  *  return 0 on success, positive on failure
 724  **********************************************************************/
 725
 726 static void
 727 em_init(void *arg)
 728 {
 729         int s;
 730         struct adapter *adapter = arg;
 731         struct ifnet *ifp = &adapter->interface_data.ac_if;
 732
 733         INIT_DEBUGOUT("em_init: begin");
 734
 735         s = splimp();
 736
 737         em_stop(adapter);
 738
 739         /* Get the latest mac address, User can use a LAA */
 740         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
 741               ETHER_ADDR_LEN);
 742
 743         /* Initialize the hardware */
 744         if (em_hardware_init(adapter)) {
 745                 if_printf(ifp, "Unable to initialize the hardware\n");
 746                 splx(s);
 747                 return;
 748         }
 749
 750         em_enable_vlans(adapter);
 751
 752         /* Prepare transmit descriptors and buffers */
 753         if (em_setup_transmit_structures(adapter)) {
 754                 if_printf(ifp, "Could not setup transmit structures\n");
 755                 em_stop(adapter);
 756                 splx(s);
 757                 return;
 758         }
 759         em_initialize_transmit_unit(adapter);
 760
 761         /* Setup Multicast table */
 762         em_set_multi(adapter);
 763
 764         /* Prepare receive descriptors and buffers */
 765         if (em_setup_receive_structures(adapter)) {
 766                 if_printf(ifp, "Could not setup receive structures\n");
 767                 em_stop(adapter);
 768                 splx(s);
 769                 return;
 770         }
 771         em_initialize_receive_unit(adapter);
 772
 773         /* Don't loose promiscuous settings */
 774         em_set_promisc(adapter);
 775
 776         ifp->if_flags |= IFF_RUNNING;
 777         ifp->if_flags &= ~IFF_OACTIVE;
 778
 779         if (adapter->hw.mac_type >= em_82543) {
 780                 if (ifp->if_capenable & IFCAP_TXCSUM)
 781                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
 782                 else
 783                         ifp->if_hwassist = 0;
 784         }
 785
 786         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 787         em_clear_hw_cntrs(&adapter->hw);
 788 #ifdef DEVICE_POLLING
 789         /*
 790          * Only enable interrupts if we are not polling, make sure
 791          * they are off otherwise.
 792          */
 793         if (ifp->if_flags & IFF_POLLING)
 794                 em_disable_intr(adapter);
 795         else
 796 #endif /* DEVICE_POLLING */
 797                 em_enable_intr(adapter);
 798
 799         /* Don't reset the phy next time init gets called */
 800         adapter->hw.phy_reset_disable = TRUE;
 801
 802         splx(s);
 803 }
 804
 805 #ifdef DEVICE_POLLING
 806 static poll_handler_t em_poll;
 807
 808 static void
 809 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 810 {
 811         struct adapter *adapter = ifp->if_softc;
 812         uint32_t reg_icr;
 813
 814         if (cmd == POLL_DEREGISTER) {       /* final call, enable interrupts */
 815                 em_enable_intr(adapter);
 816                 return;
 817         }
 818         if (cmd == POLL_AND_CHECK_STATUS) {
 819                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 820                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 821                         callout_stop(&adapter->timer);
 822                         adapter->hw.get_link_status = 1;
 823                         em_check_for_link(&adapter->hw);
 824                         em_print_link_status(adapter);
 825                         callout_reset(&adapter->timer, 2*hz, em_local_timer,
 826                                       adapter);
 827                 }
 828         }
 829         if (ifp->if_flags & IFF_RUNNING) {
 830                 em_process_receive_interrupts(adapter, count);
 831                 em_clean_transmit_interrupts(adapter);
 832         }
 833
 834         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 835                 em_start(ifp);
 836 }
 837 #endif /* DEVICE_POLLING */
 838
 839 /*********************************************************************
 840  *
 841  *  Interrupt Service routine
 842  *
 843  **********************************************************************/
 844 static void
 845 em_intr(void *arg)
 846 {
 847         uint32_t loop_cnt = EM_MAX_INTR;
 848         uint32_t reg_icr;
 849         struct ifnet *ifp;
 850         struct adapter *adapter = arg;
 851
 852         ifp = &adapter->interface_data.ac_if;
 853
 854 #ifdef DEVICE_POLLING
 855         if (ifp->if_flags & IFF_POLLING)
 856                 return;
 857
 858         if (ether_poll_register(em_poll, ifp)) {
 859                 em_disable_intr(adapter);
 860                 em_poll(ifp, 0, 1);
 861                 return;
 862         }
 863 #endif /* DEVICE_POLLING */
 864
 865         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 866         if (!reg_icr)
 867                 return;
 868
 869         /* Link status change */
 870         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 871                 callout_stop(&adapter->timer);
 872                 adapter->hw.get_link_status = 1;
 873                 em_check_for_link(&adapter->hw);
 874                 em_print_link_status(adapter);
 875                 callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
 876         }
 877
 878         while (loop_cnt > 0) {
 879                 if (ifp->if_flags & IFF_RUNNING) {
 880                         em_process_receive_interrupts(adapter, -1);
 881                         em_clean_transmit_interrupts(adapter);
 882                 }
 883                 loop_cnt--;
 884         }
 885
 886         if (ifp->if_flags & IFF_RUNNING && ifp->if_snd.ifq_head != NULL)
 887                 em_start(ifp);
 888 }
 889
 890 /*********************************************************************
 891  *
 892  *  Media Ioctl callback
 893  *
 894  *  This routine is called whenever the user queries the status of
 895  *  the interface using ifconfig.
 896  *
 897  **********************************************************************/
 898 static void
 899 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 900 {
 901         struct adapter * adapter = ifp->if_softc;
 902
 903         INIT_DEBUGOUT("em_media_status: begin");
 904
 905         em_check_for_link(&adapter->hw);
 906         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
 907                 if (adapter->link_active == 0) {
 908                         em_get_speed_and_duplex(&adapter->hw,
 909                                                 &adapter->link_speed,
 910                                                 &adapter->link_duplex);
 911                         adapter->link_active = 1;
 912                 }
 913         } else {
 914                 if (adapter->link_active == 1) {
 915                         adapter->link_speed = 0;
 916                         adapter->link_duplex = 0;
 917                         adapter->link_active = 0;
 918                 }
 919         }
 920
 921         ifmr->ifm_status = IFM_AVALID;
 922         ifmr->ifm_active = IFM_ETHER;
 923
 924         if (!adapter->link_active)
 925                 return;
 926
 927         ifmr->ifm_status |= IFM_ACTIVE;
 928
 929         if (adapter->hw.media_type == em_media_type_fiber) {
 930                 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
 931         } else {
 932                 switch (adapter->link_speed) {
 933                 case 10:
 934                         ifmr->ifm_active |= IFM_10_T;
 935                         break;
 936                 case 100:
 937                         ifmr->ifm_active |= IFM_100_TX;
 938                         break;
 939                 case 1000:
 940                         ifmr->ifm_active |= IFM_1000_TX;
 941                         break;
 942                 }
 943                 if (adapter->link_duplex == FULL_DUPLEX)
 944                         ifmr->ifm_active |= IFM_FDX;
 945                 else
 946                         ifmr->ifm_active |= IFM_HDX;
 947         }
 948 }
 949
 950 /*********************************************************************
 951  *
 952  *  Media Ioctl callback
 953  *
 954  *  This routine is called when the user changes speed/duplex using
 955  *  media/mediopt option with ifconfig.
 956  *
 957  **********************************************************************/
 958 static int
 959 em_media_change(struct ifnet *ifp)
 960 {
 961         struct adapter * adapter = ifp->if_softc;
 962         struct ifmedia  *ifm = &adapter->media;
 963
 964         INIT_DEBUGOUT("em_media_change: begin");
 965
 966         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
 967                 return(EINVAL);
 968
 969         switch (IFM_SUBTYPE(ifm->ifm_media)) {
 970         case IFM_AUTO:
 971                 adapter->hw.autoneg = DO_AUTO_NEG;
 972                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 973                 break;
 974         case IFM_1000_SX:
 975         case IFM_1000_TX:
 976                 adapter->hw.autoneg = DO_AUTO_NEG;
 977                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
 978                 break;
 979         case IFM_100_TX:
 980                 adapter->hw.autoneg = FALSE;
 981                 adapter->hw.autoneg_advertised = 0;
 982                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 983                         adapter->hw.forced_speed_duplex = em_100_full;
 984                 else
 985                         adapter->hw.forced_speed_duplex = em_100_half;
 986                 break;
 987         case IFM_10_T:
 988                 adapter->hw.autoneg = FALSE;
 989                 adapter->hw.autoneg_advertised = 0;
 990                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
 991                         adapter->hw.forced_speed_duplex = em_10_full;
 992                 else
 993                         adapter->hw.forced_speed_duplex = em_10_half;
 994                 break;
 995         default:
 996                 if_printf(ifp, "Unsupported media type\n");
 997         }
 998         /*
 999          * As the speed/duplex settings may have changed we need to
1000          * reset the PHY.
1001          */
1002         adapter->hw.phy_reset_disable = FALSE;
1003
1004         em_init(adapter);
1005
1006         return(0);
1007 }
1008
1009 static void
1010 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1011          int error)
1012 {
1013         struct em_q *q = arg;
1014
1015         if (error)
1016                 return;
1017         KASSERT(nsegs <= EM_MAX_SCATTER,
1018                 ("Too many DMA segments returned when mapping tx packet"));
1019         q->nsegs = nsegs;
1020         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1021 }
1022
1023 #define EM_FIFO_HDR              0x10
1024 #define EM_82547_PKT_THRESH      0x3e0
1025 #define EM_82547_TX_FIFO_SIZE    0x2800
1026 #define EM_82547_TX_FIFO_BEGIN   0xf00
1027 /*********************************************************************
1028  *
1029  *  This routine maps the mbufs to tx descriptors.
1030  *
1031  *  return 0 on success, positive on failure
1032  **********************************************************************/
1033 static int
1034 em_encap(struct adapter *adapter, struct mbuf *m_head)
1035 {
1036         uint32_t txd_upper;
1037         uint32_t txd_lower, txd_used = 0, txd_saved = 0;
1038         int i, j, error;
1039         uint64_t address;
1040
1041         /* For 82544 Workaround */
1042         DESC_ARRAY desc_array;
1043         uint32_t array_elements;
1044         uint32_t counter;
1045
1046 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1047         struct ifvlan *ifv = NULL;
1048 #else
1049         struct m_tag *mtag;
1050 #endif
1051         struct em_q q;
1052         struct em_buffer *tx_buffer = NULL;
1053         struct em_tx_desc *current_tx_desc = NULL;
1054         struct ifnet *ifp = &adapter->interface_data.ac_if;
1055
1056         /*
1057          * Force a cleanup if number of TX descriptors
1058          * available hits the threshold
1059          */
1060         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1061                 em_clean_transmit_interrupts(adapter);
1062                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1063                         adapter->no_tx_desc_avail1++;
1064                         return(ENOBUFS);
1065                 }
1066         }
1067         /*
1068          * Map the packet for DMA.
1069          */
1070         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q.map)) {
1071                 adapter->no_tx_map_avail++;
1072                 return(ENOMEM);
1073         }
1074         error = bus_dmamap_load_mbuf(adapter->txtag, q.map, m_head, em_tx_cb,
1075                                      &q, BUS_DMA_NOWAIT);
1076         if (error != 0) {
1077                 adapter->no_tx_dma_setup++;
1078                 bus_dmamap_destroy(adapter->txtag, q.map);
1079                 return(error);
1080         }
1081         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1082
1083         if (q.nsegs > adapter->num_tx_desc_avail) {
1084                 adapter->no_tx_desc_avail2++;
1085                 bus_dmamap_destroy(adapter->txtag, q.map);
1086                 return(ENOBUFS);
1087         }
1088
1089         if (ifp->if_hwassist > 0) {
1090                 em_transmit_checksum_setup(adapter,  m_head,
1091                                            &txd_upper, &txd_lower);
1092         }
1093         else
1094                 txd_upper = txd_lower = 0;
1095
1096         /* Find out if we are in vlan mode */
1097 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1098         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1099             m_head->m_pkthdr.rcvif != NULL &&
1100             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1101                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1102 #else
1103         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1104 #endif
1105
1106         i = adapter->next_avail_tx_desc;
1107         if (adapter->pcix_82544) {
1108                 txd_saved = i;
1109                 txd_used = 0;
1110         }
1111         for (j = 0; j < q.nsegs; j++) {
1112                 /* If adapter is 82544 and on PCIX bus */
1113                 if(adapter->pcix_82544) {
1114                         array_elements = 0;
1115                         address = htole64(q.segs[j].ds_addr);
1116                         /*
1117                          * Check the Address and Length combination and
1118                          * split the data accordingly
1119                          */
1120                         array_elements = em_fill_descriptors(address,
1121                                                              htole32(q.segs[j].ds_len),
1122                                                              &desc_array);
1123                         for (counter = 0; counter < array_elements; counter++) {
1124                                 if (txd_used == adapter->num_tx_desc_avail) {
1125                                         adapter->next_avail_tx_desc = txd_saved;
1126                                         adapter->no_tx_desc_avail2++;
1127                                         bus_dmamap_destroy(adapter->txtag, q.map);
1128                                         return(ENOBUFS);
1129                                 }
1130                                 tx_buffer = &adapter->tx_buffer_area[i];
1131                                 current_tx_desc = &adapter->tx_desc_base[i];
1132                                 current_tx_desc->buffer_addr = htole64(
1133                                 desc_array.descriptor[counter].address);
1134                                 current_tx_desc->lower.data = htole32(
1135                                 (adapter->txd_cmd | txd_lower |
1136                                 (uint16_t)desc_array.descriptor[counter].length));
1137                                 current_tx_desc->upper.data = htole32((txd_upper));
1138                                 if (++i == adapter->num_tx_desc)
1139                                         i = 0;
1140
1141                                 tx_buffer->m_head = NULL;
1142                                 txd_used++;
1143                         }
1144                 } else {
1145                         tx_buffer = &adapter->tx_buffer_area[i];
1146                         current_tx_desc = &adapter->tx_desc_base[i];
1147
1148                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1149                         current_tx_desc->lower.data = htole32(
1150                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1151                         current_tx_desc->upper.data = htole32(txd_upper);
1152
1153                         if (++i == adapter->num_tx_desc)
1154                                 i = 0;
1155
1156                         tx_buffer->m_head = NULL;
1157                 }
1158         }
1159
1160         adapter->next_avail_tx_desc = i;
1161         if (adapter->pcix_82544)
1162                 adapter->num_tx_desc_avail -= txd_used;
1163         else
1164                 adapter->num_tx_desc_avail -= q.nsegs;
1165
1166 #if defined(__DragonFly__) || __FreeBSD_version < 500000
1167         if (ifv != NULL) {
1168                 /* Set the vlan id */
1169                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1170 #else
1171         if (mtag != NULL) {
1172                 /* Set the vlan id */
1173                 current_tx_desc->upper.fields.special = htole16(VLAN_TAG_VALUE(mtag));
1174 #endif
1175
1176                 /* Tell hardware to add tag */
1177                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1178         }
1179
1180         tx_buffer->m_head = m_head;
1181         tx_buffer->map = q.map;
1182         bus_dmamap_sync(adapter->txtag, q.map, BUS_DMASYNC_PREWRITE);
1183
1184         /*
1185          * Last Descriptor of Packet needs End Of Packet (EOP)
1186          */
1187         current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1188
1189         /*
1190          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1191          * that this frame is available to transmit.
1192          */
1193         if (adapter->hw.mac_type == em_82547 &&
1194             adapter->link_duplex == HALF_DUPLEX) {
1195                 em_82547_move_tail(adapter);
1196         } else {
1197                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1198                 if (adapter->hw.mac_type == em_82547) {
1199                         em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len);
1200                 }
1201         }
1202
1203         return(0);
1204 }
1205
1206 /*********************************************************************
1207  *
1208  * 82547 workaround to avoid controller hang in half-duplex environment.
1209  * The workaround is to avoid queuing a large packet that would span
1210  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1211  * in this case. We do that only when FIFO is quiescent.
1212  *
1213  **********************************************************************/
1214 static void
1215 em_82547_move_tail(void *arg)
1216 {
1217         int s;
1218         struct adapter *adapter = arg;
1219         uint16_t hw_tdt;
1220         uint16_t sw_tdt;
1221         struct em_tx_desc *tx_desc;
1222         uint16_t length = 0;
1223         boolean_t eop = 0;
1224
1225         s = splimp();
1226         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1227         sw_tdt = adapter->next_avail_tx_desc;
1228
1229         while (hw_tdt != sw_tdt) {
1230                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1231                 length += tx_desc->lower.flags.length;
1232                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1233                 if(++hw_tdt == adapter->num_tx_desc)
1234                         hw_tdt = 0;
1235
1236                 if(eop) {
1237                         if (em_82547_fifo_workaround(adapter, length)) {
1238                                 adapter->tx_fifo_wrk++;
1239                                 callout_reset(&adapter->tx_fifo_timer, 1,
1240                                         em_82547_move_tail, adapter);
1241                                 break;
1242                         }
1243                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1244                         em_82547_update_fifo_head(adapter, length);
1245                         length = 0;
1246                 }
1247         }
1248         splx(s);
1249 }
1250
1251 static int
1252 em_82547_fifo_workaround(struct adapter *adapter, int len)
1253 {
1254         int fifo_space, fifo_pkt_len;
1255
1256         fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1257
1258         if (adapter->link_duplex == HALF_DUPLEX) {
1259                 fifo_space = EM_82547_TX_FIFO_SIZE - adapter->tx_fifo_head;
1260
1261                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1262                         if (em_82547_tx_fifo_reset(adapter))
1263                                 return(0);
1264                         else
1265                                 return(1);
1266                 }
1267         }
1268
1269         return(0);
1270 }
1271
1272 static void
1273 em_82547_update_fifo_head(struct adapter *adapter, int len)
1274 {
1275         int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1276
1277         /* tx_fifo_head is always 16 byte aligned */
1278         adapter->tx_fifo_head += fifo_pkt_len;
1279         if (adapter->tx_fifo_head >= EM_82547_TX_FIFO_SIZE)
1280                 adapter->tx_fifo_head -= EM_82547_TX_FIFO_SIZE;
1281 }
1282
1283 static int
1284 em_82547_tx_fifo_reset(struct adapter *adapter)
1285 {
1286         uint32_t tctl;
1287
1288         if ( (E1000_READ_REG(&adapter->hw, TDT) ==
1289               E1000_READ_REG(&adapter->hw, TDH)) &&
1290              (E1000_READ_REG(&adapter->hw, TDFT) ==
1291               E1000_READ_REG(&adapter->hw, TDFH)) &&
1292              (E1000_READ_REG(&adapter->hw, TDFTS) ==
1293               E1000_READ_REG(&adapter->hw, TDFHS)) &&
1294              (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1295
1296                 /* Disable TX unit */
1297                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1298                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1299
1300                 /* Reset FIFO pointers */
1301                 E1000_WRITE_REG(&adapter->hw, TDFT, EM_82547_TX_FIFO_BEGIN);
1302                 E1000_WRITE_REG(&adapter->hw, TDFH, EM_82547_TX_FIFO_BEGIN);
1303                 E1000_WRITE_REG(&adapter->hw, TDFTS, EM_82547_TX_FIFO_BEGIN);
1304                 E1000_WRITE_REG(&adapter->hw, TDFHS, EM_82547_TX_FIFO_BEGIN);
1305
1306                 /* Re-enable TX unit */
1307                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1308                 E1000_WRITE_FLUSH(&adapter->hw);
1309
1310                 adapter->tx_fifo_head = 0;
1311                 adapter->tx_fifo_reset++;
1312
1313                 return(TRUE);
1314         }
1315         else {
1316                 return(FALSE);
1317         }
1318 }
1319
1320 static void
1321 em_set_promisc(struct adapter *adapter)
1322 {
1323         uint32_t reg_rctl;
1324         struct ifnet *ifp = &adapter->interface_data.ac_if;
1325
1326         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1327
1328         if (ifp->if_flags & IFF_PROMISC) {
1329                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1330                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1331         } else if (ifp->if_flags & IFF_ALLMULTI) {
1332                 reg_rctl |= E1000_RCTL_MPE;
1333                 reg_rctl &= ~E1000_RCTL_UPE;
1334                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1335         }
1336 }
1337
1338 static void
1339 em_disable_promisc(struct adapter *adapter)
1340 {
1341         uint32_t reg_rctl;
1342
1343         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1344
1345         reg_rctl &=  (~E1000_RCTL_UPE);
1346         reg_rctl &=  (~E1000_RCTL_MPE);
1347         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1348 }
1349
1350 /*********************************************************************
1351  *  Multicast Update
1352  *
1353  *  This routine is called whenever multicast address list is updated.
1354  *
1355  **********************************************************************/
1356
1357 static void
1358 em_set_multi(struct adapter *adapter)
1359 {
1360         uint32_t reg_rctl = 0;
1361         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1362         struct ifmultiaddr *ifma;
1363         int mcnt = 0;
1364         struct ifnet *ifp = &adapter->interface_data.ac_if;
1365
1366         IOCTL_DEBUGOUT("em_set_multi: begin");
1367
1368         if (adapter->hw.mac_type == em_82542_rev2_0) {
1369                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1370                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1371                         em_pci_clear_mwi(&adapter->hw);
1372                 reg_rctl |= E1000_RCTL_RST;
1373                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1374                 msec_delay(5);
1375         }
1376
1377         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1378                 if (ifma->ifma_addr->sa_family != AF_LINK)
1379                         continue;
1380
1381                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1382                         break;
1383
1384                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1385                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1386                 mcnt++;
1387         }
1388
1389         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1390                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1391                 reg_rctl |= E1000_RCTL_MPE;
1392                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1393         } else
1394                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1395
1396         if (adapter->hw.mac_type == em_82542_rev2_0) {
1397                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1398                 reg_rctl &= ~E1000_RCTL_RST;
1399                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1400                 msec_delay(5);
1401                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1402                         em_pci_set_mwi(&adapter->hw);
1403         }
1404 }
1405
1406 /*********************************************************************
1407  *  Timer routine
1408  *
1409  *  This routine checks for link status and updates statistics.
1410  *
1411  **********************************************************************/
1412
1413 static void
1414 em_local_timer(void *arg)
1415 {
1416         int s;
1417         struct ifnet *ifp;
1418         struct adapter *adapter = arg;
1419         ifp = &adapter->interface_data.ac_if;
1420
1421         s = splimp();
1422
1423         em_check_for_link(&adapter->hw);
1424         em_print_link_status(adapter);
1425         em_update_stats_counters(adapter);
1426         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1427                 em_print_hw_stats(adapter);
1428         em_smartspeed(adapter);
1429
1430         callout_reset(&adapter->timer, 2*hz, em_local_timer, adapter);
1431
1432         splx(s);
1433 }
1434
1435 static void
1436 em_print_link_status(struct adapter *adapter)
1437 {
1438         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1439                 if (adapter->link_active == 0) {
1440                         em_get_speed_and_duplex(&adapter->hw,
1441                                                 &adapter->link_speed,
1442                                                 &adapter->link_duplex);
1443                         device_printf(adapter->dev, "Link is up %d Mbps %s\n",
1444                                adapter->link_speed,
1445                                ((adapter->link_duplex == FULL_DUPLEX) ?
1446                                 "Full Duplex" : "Half Duplex"));
1447                         adapter->link_active = 1;
1448                         adapter->smartspeed = 0;
1449                 }
1450         } else {
1451                 if (adapter->link_active == 1) {
1452                         adapter->link_speed = 0;
1453                         adapter->link_duplex = 0;
1454                         device_printf(adapter->dev, "Link is Down\n");
1455                         adapter->link_active = 0;
1456                 }
1457         }
1458 }
1459
1460 /*********************************************************************
1461  *
1462  *  This routine disables all traffic on the adapter by issuing a
1463  *  global reset on the MAC and deallocates TX/RX buffers.
1464  *
1465  **********************************************************************/
1466
1467 static void
1468 em_stop(void *arg)
1469 {
1470         struct ifnet   *ifp;
1471         struct adapter * adapter = arg;
1472         ifp = &adapter->interface_data.ac_if;
1473
1474         INIT_DEBUGOUT("em_stop: begin");
1475         em_disable_intr(adapter);
1476         em_reset_hw(&adapter->hw);
1477         callout_stop(&adapter->timer);
1478         callout_stop(&adapter->tx_fifo_timer);
1479         em_free_transmit_structures(adapter);
1480         em_free_receive_structures(adapter);
1481
1482         /* Tell the stack that the interface is no longer active */
1483         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1484 }
1485
1486 /*********************************************************************
1487  *
1488  *  Determine hardware revision.
1489  *
1490  **********************************************************************/
1491 static void
1492 em_identify_hardware(struct adapter * adapter)
1493 {
1494         device_t dev = adapter->dev;
1495
1496         /* Make sure our PCI config space has the necessary stuff set */
1497         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1498         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1499               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1500                 device_printf(dev, "Memory Access and/or Bus Master bits were not set!\n");
1501                 adapter->hw.pci_cmd_word |=
1502                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1503                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1504         }
1505
1506         /* Save off the information about this board */
1507         adapter->hw.vendor_id = pci_get_vendor(dev);
1508         adapter->hw.device_id = pci_get_device(dev);
1509         adapter->hw.revision_id = pci_get_revid(dev);
1510         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1511         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1512
1513         /* Identify the MAC */
1514         if (em_set_mac_type(&adapter->hw))
1515                 device_printf(dev, "Unknown MAC Type\n");
1516
1517         if (adapter->hw.mac_type == em_82541 ||
1518             adapter->hw.mac_type == em_82541_rev_2 ||
1519             adapter->hw.mac_type == em_82547 ||
1520             adapter->hw.mac_type == em_82547_rev_2)
1521                 adapter->hw.phy_init_script = TRUE;
1522 }
1523
1524 static int
1525 em_allocate_pci_resources(struct adapter *adapter)
1526 {
1527         int i, val, rid;
1528         device_t dev = adapter->dev;
1529
1530         rid = EM_MMBA;
1531         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1532                                                      &rid, RF_ACTIVE);
1533         if (!(adapter->res_memory)) {
1534                 device_printf(dev, "Unable to allocate bus resource: memory\n");
1535                 return(ENXIO);
1536         }
1537         adapter->osdep.mem_bus_space_tag =
1538         rman_get_bustag(adapter->res_memory);
1539         adapter->osdep.mem_bus_space_handle =
1540         rman_get_bushandle(adapter->res_memory);
1541         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
1542
1543         if (adapter->hw.mac_type > em_82543) {
1544                 /* Figure our where our IO BAR is ? */
1545                 rid = EM_MMBA;
1546                 for (i = 0; i < 5; i++) {
1547                         val = pci_read_config(dev, rid, 4);
1548                         if (val & 0x00000001) {
1549                                 adapter->io_rid = rid;
1550                                 break;
1551                         }
1552                         rid += 4;
1553                 }
1554
1555                 adapter->res_ioport = bus_alloc_resource_any(dev,
1556                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
1557                 if (!(adapter->res_ioport)) {
1558                         device_printf(dev, "Unable to allocate bus resource: ioport\n");
1559                         return(ENXIO);
1560                 }
1561
1562                 adapter->hw.reg_io_tag = rman_get_bustag(adapter->res_ioport);
1563                 adapter->hw.reg_io_handle = rman_get_bushandle(adapter->res_ioport);
1564         }
1565
1566         rid = 0x0;
1567         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1568             &rid, RF_SHAREABLE | RF_ACTIVE);
1569         if (!(adapter->res_interrupt)) {
1570                 device_printf(dev, "Unable to allocate bus resource: interrupt\n");
1571                 return(ENXIO);
1572         }
1573         if (bus_setup_intr(dev, adapter->res_interrupt, INTR_TYPE_NET,
1574                            (void (*)(void *)) em_intr, adapter,
1575                            &adapter->int_handler_tag)) {
1576                 device_printf(dev, "Error registering interrupt handler!\n");
1577                 return(ENXIO);
1578         }
1579
1580         adapter->hw.back = &adapter->osdep;
1581
1582         return(0);
1583 }
1584
1585 static void
1586 em_free_pci_resources(struct adapter *adapter)
1587 {
1588         device_t dev = adapter->dev;
1589
1590         if (adapter->res_interrupt != NULL) {
1591                 bus_teardown_intr(dev, adapter->res_interrupt,
1592                                   adapter->int_handler_tag);
1593                 bus_release_resource(dev, SYS_RES_IRQ, 0,
1594                                      adapter->res_interrupt);
1595         }
1596         if (adapter->res_memory != NULL) {
1597                 bus_release_resource(dev, SYS_RES_MEMORY, EM_MMBA,
1598                                      adapter->res_memory);
1599         }
1600
1601         if (adapter->res_ioport != NULL) {
1602                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
1603                                      adapter->res_ioport);
1604         }
1605 }
1606
1607 /*********************************************************************
1608  *
1609  *  Initialize the hardware to a configuration as specified by the
1610  *  adapter structure. The controller is reset, the EEPROM is
1611  *  verified, the MAC address is set, then the shared initialization
1612  *  routines are called.
1613  *
1614  **********************************************************************/
1615 static int
1616 em_hardware_init(struct adapter *adapter)
1617 {
1618         INIT_DEBUGOUT("em_hardware_init: begin");
1619         /* Issue a global reset */
1620         em_reset_hw(&adapter->hw);
1621
1622         /* When hardware is reset, fifo_head is also reset */
1623         adapter->tx_fifo_head = 0;
1624
1625         /* Make sure we have a good EEPROM before we read from it */
1626         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
1627                 device_printf(adapter->dev, "The EEPROM Checksum Is Not Valid\n");
1628                 return(EIO);
1629         }
1630
1631         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
1632                 device_printf(adapter->dev, "EEPROM read error while reading part number\n");
1633                 return(EIO);
1634         }
1635
1636         if (em_init_hw(&adapter->hw) < 0) {
1637                 device_printf(adapter->dev, "Hardware Initialization Failed");
1638                 return(EIO);
1639         }
1640
1641         em_check_for_link(&adapter->hw);
1642         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)
1643                 adapter->link_active = 1;
1644         else
1645                 adapter->link_active = 0;
1646
1647         if (adapter->link_active) {
1648                 em_get_speed_and_duplex(&adapter->hw,
1649                                         &adapter->link_speed,
1650                                         &adapter->link_duplex);
1651         } else {
1652                 adapter->link_speed = 0;
1653                 adapter->link_duplex = 0;
1654         }
1655
1656         return(0);
1657 }
1658
1659 /*********************************************************************
1660  *
1661  *  Setup networking device structure and register an interface.
1662  *
1663  **********************************************************************/
1664 static void
1665 em_setup_interface(device_t dev, struct adapter *adapter)
1666 {
1667         struct ifnet   *ifp;
1668         INIT_DEBUGOUT("em_setup_interface: begin");
1669
1670         ifp = &adapter->interface_data.ac_if;
1671         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1672         ifp->if_mtu = ETHERMTU;
1673         ifp->if_output = ether_output;
1674         ifp->if_baudrate = 1000000000;
1675         ifp->if_init =  em_init;
1676         ifp->if_softc = adapter;
1677         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1678         ifp->if_ioctl = em_ioctl;
1679         ifp->if_start = em_start;
1680         ifp->if_watchdog = em_watchdog;
1681         ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 1;
1682
1683         ether_ifattach(ifp, adapter->interface_data.ac_enaddr);
1684
1685         if (adapter->hw.mac_type >= em_82543) {
1686                 ifp->if_capabilities = IFCAP_HWCSUM;
1687                 ifp->if_capenable = ifp->if_capabilities;
1688         }
1689
1690         /*
1691          * Tell the upper layer(s) we support long frames.
1692          */
1693         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1694 #if defined(__FreeBSD__) && __FreeBSD_version >= 500000
1695         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1696 #endif
1697
1698         /*
1699          * Specify the media types supported by this adapter and register
1700          * callbacks to update media and link information
1701          */
1702         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
1703                      em_media_status);
1704         if (adapter->hw.media_type == em_media_type_fiber) {
1705                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1706                             0, NULL);
1707                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1708                             0, NULL);
1709         } else {
1710                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1711                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1712                             0, NULL);
1713                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
1714                             0, NULL);
1715                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1716                             0, NULL);
1717                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX | IFM_FDX,
1718                             0, NULL);
1719                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_TX, 0, NULL);
1720         }
1721         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1722         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1723 }
1724
1725 /*********************************************************************
1726  *
1727  *  Workaround for SmartSpeed on 82541 and 82547 controllers
1728  *
1729  **********************************************************************/
1730 static void
1731 em_smartspeed(struct adapter *adapter)
1732 {
1733         uint16_t phy_tmp;
1734
1735         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
1736             !adapter->hw.autoneg ||
1737             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
1738                 return;
1739
1740         if (adapter->smartspeed == 0) {
1741                 /*
1742                  * If Master/Slave config fault is asserted twice,
1743                  * we assume back-to-back.
1744                  */
1745                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1746                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
1747                         return;
1748                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1749                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
1750                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL,
1751                                         &phy_tmp);
1752                         if (phy_tmp & CR_1000T_MS_ENABLE) {
1753                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
1754                                 em_write_phy_reg(&adapter->hw,
1755                                                  PHY_1000T_CTRL, phy_tmp);
1756                                 adapter->smartspeed++;
1757                                 if (adapter->hw.autoneg &&
1758                                     !em_phy_setup_autoneg(&adapter->hw) &&
1759                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
1760                                                      &phy_tmp)) {
1761                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1762                                                     MII_CR_RESTART_AUTO_NEG);
1763                                         em_write_phy_reg(&adapter->hw,
1764                                                          PHY_CTRL, phy_tmp);
1765                                 }
1766                         }
1767                 }
1768                 return;
1769         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
1770                 /* If still no link, perhaps using 2/3 pair cable */
1771                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
1772                 phy_tmp |= CR_1000T_MS_ENABLE;
1773                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
1774                 if (adapter->hw.autoneg &&
1775                     !em_phy_setup_autoneg(&adapter->hw) &&
1776                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
1777                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1778                                     MII_CR_RESTART_AUTO_NEG);
1779                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
1780                 }
1781         }
1782         /* Restart process after EM_SMARTSPEED_MAX iterations */
1783         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
1784                 adapter->smartspeed = 0;
1785 }
1786
1787 /*
1788  * Manage DMA'able memory.
1789  */
1790 static void
1791 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1792 {
1793         if (error)
1794                 return;
1795         *(bus_addr_t*) arg = segs->ds_addr;
1796 }
1797
1798 static int
1799 em_dma_malloc(struct adapter *adapter, bus_size_t size,
1800               struct em_dma_alloc *dma, int mapflags)
1801 {
1802         int r;
1803         device_t dev = adapter->dev;
1804
1805         r = bus_dma_tag_create(NULL,                    /* parent */
1806                                PAGE_SIZE, 0,            /* alignment, bounds */
1807                                BUS_SPACE_MAXADDR,       /* lowaddr */
1808                                BUS_SPACE_MAXADDR,       /* highaddr */
1809                                NULL, NULL,              /* filter, filterarg */
1810                                size,                    /* maxsize */
1811                                1,                       /* nsegments */
1812                                size,                    /* maxsegsize */
1813                                BUS_DMA_ALLOCNOW,        /* flags */
1814                                &dma->dma_tag);
1815         if (r != 0) {
1816                 device_printf(dev, "em_dma_malloc: bus_dma_tag_create failed; "
1817                               "error %u\n", r);
1818                 goto fail_0;
1819         }
1820
1821         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1822         if (r != 0) {
1823                 device_printf(dev, "em_dma_malloc: bus_dmamap_create failed; "
1824                               "error %u\n", r);
1825                 goto fail_1;
1826         }
1827
1828         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1829                              BUS_DMA_NOWAIT, &dma->dma_map);
1830         if (r != 0) {
1831                 device_printf(dev, "em_dma_malloc: bus_dmammem_alloc failed; "
1832                               "size %llu, error %d\n", (uintmax_t)size, r);
1833                 goto fail_2;
1834         }
1835
1836         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1837                             size,
1838                             em_dmamap_cb,
1839                             &dma->dma_paddr,
1840                             mapflags | BUS_DMA_NOWAIT);
1841         if (r != 0) {
1842                 device_printf(dev, "em_dma_malloc: bus_dmamap_load failed; "
1843                               "error %u\n", r);
1844                 goto fail_3;
1845         }
1846
1847         dma->dma_size = size;
1848         return(0);
1849
1850 fail_3:
1851         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1852 fail_2:
1853         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1854 fail_1:
1855         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1856         bus_dma_tag_destroy(dma->dma_tag);
1857 fail_0:
1858         dma->dma_map = NULL;
1859         dma->dma_tag = NULL;
1860         return(r);
1861 }
1862
1863 static void
1864 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
1865 {
1866         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1867         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1868         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1869         bus_dma_tag_destroy(dma->dma_tag);
1870 }
1871
1872 /*********************************************************************
1873  *
1874  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1875  *  the information needed to transmit a packet on the wire.
1876  *
1877  **********************************************************************/
1878 static int
1879 em_allocate_transmit_structures(struct adapter * adapter)
1880 {
1881         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
1882             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
1883         if (adapter->tx_buffer_area == NULL) {
1884                 device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
1885                 return(ENOMEM);
1886         }
1887
1888         return(0);
1889 }
1890
1891 /*********************************************************************
1892  *
1893  *  Allocate and initialize transmit structures.
1894  *
1895  **********************************************************************/
1896 static int
1897 em_setup_transmit_structures(struct adapter * adapter)
1898 {
1899         /*
1900          * Setup DMA descriptor areas.
1901          */
1902         if (bus_dma_tag_create(NULL,                    /* parent */
1903                                PAGE_SIZE, 0,            /* alignment, bounds */
1904                                BUS_SPACE_MAXADDR,       /* lowaddr */
1905                                BUS_SPACE_MAXADDR,       /* highaddr */
1906                                NULL, NULL,              /* filter, filterarg */
1907                                MCLBYTES * 8,            /* maxsize */
1908                                EM_MAX_SCATTER,          /* nsegments */
1909                                MCLBYTES * 8,            /* maxsegsize */
1910                                BUS_DMA_ALLOCNOW,        /* flags */
1911                                &adapter->txtag)) {
1912                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
1913                 return(ENOMEM);
1914         }
1915
1916         if (em_allocate_transmit_structures(adapter))
1917                 return(ENOMEM);
1918
1919         bzero((void *) adapter->tx_desc_base,
1920               (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
1921
1922         adapter->next_avail_tx_desc = 0;
1923         adapter->oldest_used_tx_desc = 0;
1924
1925         /* Set number of descriptors available */
1926         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1927
1928         /* Set checksum context */
1929         adapter->active_checksum_context = OFFLOAD_NONE;
1930
1931         return(0);
1932 }
1933
1934 /*********************************************************************
1935  *
1936  *  Enable transmit unit.
1937  *
1938  **********************************************************************/
1939 static void
1940 em_initialize_transmit_unit(struct adapter * adapter)
1941 {
1942         uint32_t reg_tctl;
1943         uint32_t reg_tipg = 0;
1944         uint64_t bus_addr;
1945
1946         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
1947
1948         /* Setup the Base and Length of the Tx Descriptor Ring */
1949         bus_addr = adapter->txdma.dma_paddr;
1950         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
1951         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
1952         E1000_WRITE_REG(&adapter->hw, TDLEN,
1953                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
1954
1955         /* Setup the HW Tx Head and Tail descriptor pointers */
1956         E1000_WRITE_REG(&adapter->hw, TDH, 0);
1957         E1000_WRITE_REG(&adapter->hw, TDT, 0);
1958
1959         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1960                      E1000_READ_REG(&adapter->hw, TDBAL),
1961                      E1000_READ_REG(&adapter->hw, TDLEN));
1962
1963         /* Set the default values for the Tx Inter Packet Gap timer */
1964         switch (adapter->hw.mac_type) {
1965         case em_82542_rev2_0:
1966         case em_82542_rev2_1:
1967                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
1968                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1969                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1970                 break;
1971         default:
1972                 if (adapter->hw.media_type == em_media_type_fiber)
1973                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1974                 else
1975                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1976                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1977                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1978         }
1979
1980         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
1981         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
1982         if (adapter->hw.mac_type >= em_82540)
1983                 E1000_WRITE_REG(&adapter->hw, TADV,
1984                                 adapter->tx_abs_int_delay.value);
1985
1986         /* Program the Transmit Control Register */
1987         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
1988                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1989         if (adapter->link_duplex == 1)
1990                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1991         else
1992                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
1993         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
1994
1995         /* Setup Transmit Descriptor Settings for this adapter */
1996         adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
1997
1998         if (adapter->tx_int_delay.value > 0)
1999                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2000 }
2001
2002 /*********************************************************************
2003  *
2004  *  Free all transmit related data structures.
2005  *
2006  **********************************************************************/
2007 static void
2008 em_free_transmit_structures(struct adapter * adapter)
2009 {
2010         struct em_buffer *tx_buffer;
2011         int i;
2012
2013         INIT_DEBUGOUT("free_transmit_structures: begin");
2014
2015         if (adapter->tx_buffer_area != NULL) {
2016                 tx_buffer = adapter->tx_buffer_area;
2017                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2018                         if (tx_buffer->m_head != NULL) {
2019                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2020                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2021                                 m_freem(tx_buffer->m_head);
2022                         }
2023                         tx_buffer->m_head = NULL;
2024                 }
2025         }
2026         if (adapter->tx_buffer_area != NULL) {
2027                 free(adapter->tx_buffer_area, M_DEVBUF);
2028                 adapter->tx_buffer_area = NULL;
2029         }
2030         if (adapter->txtag != NULL) {
2031                 bus_dma_tag_destroy(adapter->txtag);
2032                 adapter->txtag = NULL;
2033         }
2034 }
2035
2036 /*********************************************************************
2037  *
2038  *  The offload context needs to be set when we transfer the first
2039  *  packet of a particular protocol (TCP/UDP). We change the
2040  *  context only if the protocol type changes.
2041  *
2042  **********************************************************************/
2043 static void
2044 em_transmit_checksum_setup(struct adapter * adapter,
2045                            struct mbuf *mp,
2046                            uint32_t *txd_upper,
2047                            uint32_t *txd_lower)
2048 {
2049         struct em_context_desc *TXD;
2050         struct em_buffer *tx_buffer;
2051         int curr_txd;
2052
2053         if (mp->m_pkthdr.csum_flags) {
2054                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2055                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2056                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2057                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2058                                 return;
2059                         else
2060                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2061                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2062                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2063                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2064                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2065                                 return;
2066                         else
2067                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2068                 } else {
2069                         *txd_upper = 0;
2070                         *txd_lower = 0;
2071                         return;
2072                 }
2073         } else {
2074                 *txd_upper = 0;
2075                 *txd_lower = 0;
2076                 return;
2077         }
2078
2079         /* If we reach this point, the checksum offload context
2080          * needs to be reset.
2081          */
2082         curr_txd = adapter->next_avail_tx_desc;
2083         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2084         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2085
2086         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2087         TXD->lower_setup.ip_fields.ipcso =
2088             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2089         TXD->lower_setup.ip_fields.ipcse =
2090             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2091
2092         TXD->upper_setup.tcp_fields.tucss =
2093             ETHER_HDR_LEN + sizeof(struct ip);
2094         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2095
2096         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2097                 TXD->upper_setup.tcp_fields.tucso =
2098                     ETHER_HDR_LEN + sizeof(struct ip) +
2099                     offsetof(struct tcphdr, th_sum);
2100         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2101                 TXD->upper_setup.tcp_fields.tucso =
2102                         ETHER_HDR_LEN + sizeof(struct ip) +
2103                         offsetof(struct udphdr, uh_sum);
2104         }
2105
2106         TXD->tcp_seg_setup.data = htole32(0);
2107         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2108
2109         tx_buffer->m_head = NULL;
2110
2111         if (++curr_txd == adapter->num_tx_desc)
2112                 curr_txd = 0;
2113
2114         adapter->num_tx_desc_avail--;
2115         adapter->next_avail_tx_desc = curr_txd;
2116 }
2117
2118 /**********************************************************************
2119  *
2120  *  Examine each tx_buffer in the used queue. If the hardware is done
2121  *  processing the packet then free associated resources. The
2122  *  tx_buffer is put back on the free queue.
2123  *
2124  **********************************************************************/
2125 static void
2126 em_clean_transmit_interrupts(struct adapter *adapter)
2127 {
2128         int s;
2129         int i, num_avail;
2130         struct em_buffer *tx_buffer;
2131         struct em_tx_desc *tx_desc;
2132         struct ifnet *ifp = &adapter->interface_data.ac_if;
2133
2134         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2135                 return;
2136
2137         s = splimp();
2138 #ifdef DBG_STATS
2139         adapter->clean_tx_interrupts++;
2140 #endif
2141         num_avail = adapter->num_tx_desc_avail;
2142         i = adapter->oldest_used_tx_desc;
2143
2144         tx_buffer = &adapter->tx_buffer_area[i];
2145         tx_desc = &adapter->tx_desc_base[i];
2146
2147         while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2148                 tx_desc->upper.data = 0;
2149                 num_avail++;
2150
2151                 if (tx_buffer->m_head) {
2152                         ifp->if_opackets++;
2153                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2154                                         BUS_DMASYNC_POSTWRITE);
2155                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2156                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2157
2158                         m_freem(tx_buffer->m_head);
2159                         tx_buffer->m_head = NULL;
2160                 }
2161
2162                 if (++i == adapter->num_tx_desc)
2163                         i = 0;
2164
2165                 tx_buffer = &adapter->tx_buffer_area[i];
2166                 tx_desc = &adapter->tx_desc_base[i];
2167         }
2168
2169         adapter->oldest_used_tx_desc = i;
2170
2171         /*
2172          * If we have enough room, clear IFF_OACTIVE to tell the stack
2173          * that it is OK to send packets.
2174          * If there are no pending descriptors, clear the timeout. Otherwise,
2175          * if some descriptors have been freed, restart the timeout.
2176          */
2177         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2178                 ifp->if_flags &= ~IFF_OACTIVE;
2179                 if (num_avail == adapter->num_tx_desc)
2180                         ifp->if_timer = 0;
2181                 else if (num_avail == adapter->num_tx_desc_avail)
2182                         ifp->if_timer = EM_TX_TIMEOUT;
2183         }
2184         adapter->num_tx_desc_avail = num_avail;
2185         splx(s);
2186 }
2187
2188 /*********************************************************************
2189  *
2190  *  Get a buffer from system mbuf buffer pool.
2191  *
2192  **********************************************************************/
2193 static int
2194 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp)
2195 {
2196         struct mbuf *mp = nmp;
2197         struct em_buffer *rx_buffer;
2198         struct ifnet *ifp;
2199         bus_addr_t paddr;
2200         int error;
2201
2202         ifp = &adapter->interface_data.ac_if;
2203
2204         if (mp == NULL) {
2205                 mp = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2206                 if (mp == NULL) {
2207                         adapter->mbuf_cluster_failed++;
2208                         return(ENOBUFS);
2209                 }
2210                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2211         } else {
2212                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2213                 mp->m_data = mp->m_ext.ext_buf;
2214                 mp->m_next = NULL;
2215         }
2216         if (ifp->if_mtu <= ETHERMTU)
2217                 m_adj(mp, ETHER_ALIGN);
2218
2219         rx_buffer = &adapter->rx_buffer_area[i];
2220
2221         /*
2222          * Using memory from the mbuf cluster pool, invoke the
2223          * bus_dma machinery to arrange the memory mapping.
2224          */
2225         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2226                                 mtod(mp, void *), mp->m_len,
2227                                 em_dmamap_cb, &paddr, 0);
2228         if (error) {
2229                 m_free(mp);
2230                 return(error);
2231         }
2232         rx_buffer->m_head = mp;
2233         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2234         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2235
2236         return(0);
2237 }
2238
2239 /*********************************************************************
2240  *
2241  *  Allocate memory for rx_buffer structures. Since we use one
2242  *  rx_buffer per received packet, the maximum number of rx_buffer's
2243  *  that we'll need is equal to the number of receive descriptors
2244  *  that we've allocated.
2245  *
2246  **********************************************************************/
2247 static int
2248 em_allocate_receive_structures(struct adapter *adapter)
2249 {
2250         int i, error, size;
2251         struct em_buffer *rx_buffer;
2252
2253         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2254         adapter->rx_buffer_area = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2255
2256         error = bus_dma_tag_create(NULL,                /* parent */
2257                                    PAGE_SIZE, 0,        /* alignment, bounds */
2258                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2259                                    BUS_SPACE_MAXADDR,   /* highaddr */
2260                                    NULL, NULL,          /* filter, filterarg */
2261                                    MCLBYTES,            /* maxsize */
2262                                    1,                   /* nsegments */
2263                                    MCLBYTES,            /* maxsegsize */
2264                                    BUS_DMA_ALLOCNOW,    /* flags */
2265                                    &adapter->rxtag);
2266         if (error != 0) {
2267                 device_printf(adapter->dev, "em_allocate_receive_structures: "
2268                               "bus_dma_tag_create failed; error %u\n", error);
2269                 goto fail_0;
2270         }
2271
2272         rx_buffer = adapter->rx_buffer_area;
2273         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2274                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2275                                           &rx_buffer->map);
2276                 if (error != 0) {
2277                         device_printf(adapter->dev,
2278                                       "em_allocate_receive_structures: "
2279                                       "bus_dmamap_create failed; error %u\n",
2280                                       error);
2281                         goto fail_1;
2282                 }
2283         }
2284
2285         for (i = 0; i < adapter->num_rx_desc; i++) {
2286                 error = em_get_buf(i, adapter, NULL);
2287                 if (error != 0) {
2288                         adapter->rx_buffer_area[i].m_head = NULL;
2289                         adapter->rx_desc_base[i].buffer_addr = 0;
2290                         return(error);
2291                 }
2292         }
2293
2294         return(0);
2295
2296 fail_1:
2297         bus_dma_tag_destroy(adapter->rxtag);
2298 fail_0:
2299         adapter->rxtag = NULL;
2300         free(adapter->rx_buffer_area, M_DEVBUF);
2301         adapter->rx_buffer_area = NULL;
2302         return(error);
2303 }
2304
2305 /*********************************************************************
2306  *
2307  *  Allocate and initialize receive structures.
2308  *
2309  **********************************************************************/
2310 static int
2311 em_setup_receive_structures(struct adapter *adapter)
2312 {
2313         bzero((void *) adapter->rx_desc_base,
2314               (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2315
2316         if (em_allocate_receive_structures(adapter))
2317                 return(ENOMEM);
2318
2319         /* Setup our descriptor pointers */
2320         adapter->next_rx_desc_to_check = 0;
2321         return(0);
2322 }
2323
2324 /*********************************************************************
2325  *
2326  *  Enable receive unit.
2327  *
2328  **********************************************************************/
2329 static void
2330 em_initialize_receive_unit(struct adapter *adapter)
2331 {
2332         uint32_t reg_rctl;
2333         uint32_t reg_rxcsum;
2334         struct ifnet *ifp;
2335         uint64_t bus_addr;
2336
2337         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2338
2339         ifp = &adapter->interface_data.ac_if;
2340
2341         /* Make sure receives are disabled while setting up the descriptor ring */
2342         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2343
2344         /* Set the Receive Delay Timer Register */
2345         E1000_WRITE_REG(&adapter->hw, RDTR,
2346                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2347
2348         if(adapter->hw.mac_type >= em_82540) {
2349                 E1000_WRITE_REG(&adapter->hw, RADV,
2350                                 adapter->rx_abs_int_delay.value);
2351
2352                 /* Set the interrupt throttling rate.  Value is calculated
2353                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
2354                  */
2355 #define MAX_INTS_PER_SEC        8000
2356 #define DEFAULT_ITR             1000000000/(MAX_INTS_PER_SEC * 256)
2357                 E1000_WRITE_REG(&adapter->hw, ITR, DEFAULT_ITR);
2358         }
2359
2360         /* Setup the Base and Length of the Rx Descriptor Ring */
2361         bus_addr = adapter->rxdma.dma_paddr;
2362         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2363         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2364         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2365                         sizeof(struct em_rx_desc));
2366
2367         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2368         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2369         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2370
2371         /* Setup the Receive Control Register */
2372         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2373                    E1000_RCTL_RDMTS_HALF |
2374                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2375
2376         if (adapter->hw.tbi_compatibility_on == TRUE)
2377                 reg_rctl |= E1000_RCTL_SBP;
2378
2379         switch (adapter->rx_buffer_len) {
2380         default:
2381         case EM_RXBUFFER_2048:
2382                 reg_rctl |= E1000_RCTL_SZ_2048;
2383                 break;
2384         case EM_RXBUFFER_4096:
2385                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2386                 break;
2387         case EM_RXBUFFER_8192:
2388                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2389                 break;
2390         case EM_RXBUFFER_16384:
2391                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2392                 break;
2393         }
2394
2395         if (ifp->if_mtu > ETHERMTU)
2396                 reg_rctl |= E1000_RCTL_LPE;
2397
2398         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2399         if ((adapter->hw.mac_type >= em_82543) &&
2400             (ifp->if_capenable & IFCAP_RXCSUM)) {
2401                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2402                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2403                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2404         }
2405
2406         /* Enable Receives */
2407         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2408 }
2409
2410 /*********************************************************************
2411  *
2412  *  Free receive related data structures.
2413  *
2414  **********************************************************************/
2415 static void
2416 em_free_receive_structures(struct adapter *adapter)
2417 {
2418         struct em_buffer *rx_buffer;
2419         int i;
2420
2421         INIT_DEBUGOUT("free_receive_structures: begin");
2422
2423         if (adapter->rx_buffer_area != NULL) {
2424                 rx_buffer = adapter->rx_buffer_area;
2425                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2426                         if (rx_buffer->map != NULL) {
2427                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2428                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2429                         }
2430                         if (rx_buffer->m_head != NULL)
2431                                 m_freem(rx_buffer->m_head);
2432                         rx_buffer->m_head = NULL;
2433                 }
2434         }
2435         if (adapter->rx_buffer_area != NULL) {
2436                 free(adapter->rx_buffer_area, M_DEVBUF);
2437                 adapter->rx_buffer_area = NULL;
2438         }
2439         if (adapter->rxtag != NULL) {
2440                 bus_dma_tag_destroy(adapter->rxtag);
2441                 adapter->rxtag = NULL;
2442         }
2443 }
2444
2445 /*********************************************************************
2446  *
2447  *  This routine executes in interrupt context. It replenishes
2448  *  the mbufs in the descriptor and sends data which has been
2449  *  dma'ed into host memory to upper layer.
2450  *
2451  *  We loop at most count times if count is > 0, or until done if
2452  *  count < 0.
2453  *
2454  *********************************************************************/
2455 static void
2456 em_process_receive_interrupts(struct adapter *adapter, int count)
2457 {
2458         struct ifnet *ifp;
2459         struct mbuf *mp;
2460 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2461         struct ether_header *eh;
2462 #endif
2463         uint8_t accept_frame = 0;
2464         uint8_t eop = 0;
2465         uint16_t len, desc_len, prev_len_adj;
2466         int i;
2467
2468         /* Pointer to the receive descriptor being examined. */
2469         struct em_rx_desc *current_desc;
2470
2471         ifp = &adapter->interface_data.ac_if;
2472         i = adapter->next_rx_desc_to_check;
2473         current_desc = &adapter->rx_desc_base[i];
2474
2475         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
2476 #ifdef DBG_STATS
2477                 adapter->no_pkts_avail++;
2478 #endif
2479                 return;
2480         }
2481         while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0)) {
2482                 mp = adapter->rx_buffer_area[i].m_head;
2483                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2484                                 BUS_DMASYNC_POSTREAD);
2485
2486                 accept_frame = 1;
2487                 prev_len_adj = 0;
2488                 desc_len = le16toh(current_desc->length);
2489                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2490                         count--;
2491                         eop = 1;
2492                         if (desc_len < ETHER_CRC_LEN) {
2493                                 len = 0;
2494                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2495                         }
2496                         else {
2497                                 len = desc_len - ETHER_CRC_LEN;
2498                         }
2499                 } else {
2500                         eop = 0;
2501                         len = desc_len;
2502                 }
2503
2504                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2505                         uint8_t last_byte;
2506                         uint32_t pkt_len = desc_len;
2507
2508                         if (adapter->fmp != NULL)
2509                                 pkt_len += adapter->fmp->m_pkthdr.len;
2510
2511                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
2512
2513                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
2514                                        current_desc->errors,
2515                                        pkt_len, last_byte)) {
2516                                 em_tbi_adjust_stats(&adapter->hw,
2517                                                     &adapter->stats,
2518                                                     pkt_len,
2519                                                     adapter->hw.mac_addr);
2520                                 if (len > 0)
2521                                         len--;
2522                         }
2523                         else {
2524                                 accept_frame = 0;
2525                         }
2526                 }
2527
2528                 if (accept_frame) {
2529                         if (em_get_buf(i, adapter, NULL) == ENOBUFS) {
2530                                 adapter->dropped_pkts++;
2531                                 em_get_buf(i, adapter, mp);
2532                                 if (adapter->fmp != NULL)
2533                                         m_freem(adapter->fmp);
2534                                 adapter->fmp = NULL;
2535                                 adapter->lmp = NULL;
2536                                 break;
2537                         }
2538
2539                         /* Assign correct length to the current fragment */
2540                         mp->m_len = len;
2541
2542                         if (adapter->fmp == NULL) {
2543                                 mp->m_pkthdr.len = len;
2544                                 adapter->fmp = mp;       /* Store the first mbuf */
2545                                 adapter->lmp = mp;
2546                         } else {
2547                                 /* Chain mbuf's together */
2548                                 mp->m_flags &= ~M_PKTHDR;
2549                                 /*
2550                                  * Adjust length of previous mbuf in chain if we
2551                                  * received less than 4 bytes in the last descriptor.
2552                                  */
2553                                 if (prev_len_adj > 0) {
2554                                         adapter->lmp->m_len -= prev_len_adj;
2555                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
2556                                 }
2557                                 adapter->lmp->m_next = mp;
2558                                 adapter->lmp = adapter->lmp->m_next;
2559                                 adapter->fmp->m_pkthdr.len += len;
2560                         }
2561
2562                         if (eop) {
2563                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2564                                 ifp->if_ipackets++;
2565
2566 #if defined(__DragonFly__) || __FreeBSD_version < 500000
2567                                 eh = mtod(adapter->fmp, struct ether_header *);
2568                                 /* Remove ethernet header from mbuf */
2569                                 m_adj(adapter->fmp, sizeof(struct ether_header));
2570                                 em_receive_checksum(adapter, current_desc,
2571                                                     adapter->fmp);
2572                                 if (current_desc->status & E1000_RXD_STAT_VP)
2573                                         VLAN_INPUT_TAG(eh, adapter->fmp,
2574                                                        (current_desc->special &
2575                                                         E1000_RXD_SPC_VLAN_MASK));
2576                                 else
2577                                         ether_input(ifp, eh, adapter->fmp);
2578 #else
2579                                 em_receive_checksum(adapter, current_desc,
2580                                                     adapter->fmp);
2581                                 if (current_desc->status & E1000_RXD_STAT_VP)
2582                                         VLAN_INPUT_TAG(ifp, adapter->fmp,
2583                                                        (current_desc->special &
2584                                                         E1000_RXD_SPC_VLAN_MASK),
2585                                                        adapter->fmp = NULL);
2586
2587                                 if (adapter->fmp != NULL)
2588                                         (*ifp->if_input)(ifp, adapter->fmp);
2589 #endif
2590                                 adapter->fmp = NULL;
2591                                 adapter->lmp = NULL;
2592                         }
2593                 } else {
2594                         adapter->dropped_pkts++;
2595                         em_get_buf(i, adapter, mp);
2596                         if (adapter->fmp != NULL)
2597                                 m_freem(adapter->fmp);
2598                         adapter->fmp = NULL;
2599                         adapter->lmp = NULL;
2600                 }
2601
2602                 /* Zero out the receive descriptors status  */
2603                 current_desc->status = 0;
2604
2605                 /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
2606                 E1000_WRITE_REG(&adapter->hw, RDT, i);
2607
2608                 /* Advance our pointers to the next descriptor */
2609                 if (++i == adapter->num_rx_desc) {
2610                         i = 0;
2611                         current_desc = adapter->rx_desc_base;
2612                 } else
2613                         current_desc++;
2614         }
2615         adapter->next_rx_desc_to_check = i;
2616 }
2617
2618 /*********************************************************************
2619  *
2620  *  Verify that the hardware indicated that the checksum is valid.
2621  *  Inform the stack about the status of checksum so that stack
2622  *  doesn't spend time verifying the checksum.
2623  *
2624  *********************************************************************/
2625 static void
2626 em_receive_checksum(struct adapter *adapter,
2627                     struct em_rx_desc *rx_desc,
2628                     struct mbuf *mp)
2629 {
2630         /* 82543 or newer only */
2631         if ((adapter->hw.mac_type < em_82543) ||
2632             /* Ignore Checksum bit is set */
2633             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
2634                 mp->m_pkthdr.csum_flags = 0;
2635                 return;
2636         }
2637
2638         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
2639                 /* Did it pass? */
2640                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
2641                         /* IP Checksum Good */
2642                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2643                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2644                 } else {
2645                         mp->m_pkthdr.csum_flags = 0;
2646                 }
2647         }
2648
2649         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
2650                 /* Did it pass? */
2651                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
2652                         mp->m_pkthdr.csum_flags |=
2653                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2654                         mp->m_pkthdr.csum_data = htons(0xffff);
2655                 }
2656         }
2657 }
2658
2659
2660 static void
2661 em_enable_vlans(struct adapter *adapter)
2662 {
2663         uint32_t ctrl;
2664
2665         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
2666
2667         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
2668         ctrl |= E1000_CTRL_VME;
2669         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
2670 }
2671
2672 static void
2673 em_enable_intr(struct adapter *adapter)
2674 {
2675         E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
2676 }
2677
2678 static void
2679 em_disable_intr(struct adapter *adapter)
2680 {
2681         E1000_WRITE_REG(&adapter->hw, IMC,
2682                         (0xffffffff & ~E1000_IMC_RXSEQ));
2683 }
2684
2685 static int
2686 em_is_valid_ether_addr(uint8_t *addr)
2687 {
2688         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2689
2690         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN)))
2691                 return(FALSE);
2692         else
2693                 return(TRUE);
2694 }
2695
2696 void
2697 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2698 {
2699         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
2700 }
2701
2702 void
2703 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2704 {
2705         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
2706 }
2707
2708 void
2709 em_pci_set_mwi(struct em_hw *hw)
2710 {
2711         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2712                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
2713 }
2714
2715 void
2716 em_pci_clear_mwi(struct em_hw *hw)
2717 {
2718         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2719                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
2720 }
2721
2722 uint32_t
2723 em_read_reg_io(struct em_hw *hw, uint32_t offset)
2724 {
2725         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2726         return(bus_space_read_4(hw->reg_io_tag, hw->reg_io_handle, 4));
2727 }
2728
2729 void
2730 em_write_reg_io(struct em_hw *hw, uint32_t offset, uint32_t value)
2731 {
2732         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2733         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 4, value);
2734 }
2735
2736 /*********************************************************************
2737  * 82544 Coexistence issue workaround.
2738  *    There are 2 issues.
2739  *      1. Transmit Hang issue.
2740  *    To detect this issue, following equation can be used...
2741  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2742  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
2743  *
2744  *      2. DAC issue.
2745  *    To detect this issue, following equation can be used...
2746  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2747  *          If SUM[3:0] is in between 9 to c, we will have this issue.
2748  *
2749  *
2750  *    WORKAROUND:
2751  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
2752  *          9,a,b,c (DAC)
2753  *
2754 *************************************************************************/
2755 static uint32_t
2756 em_fill_descriptors(uint64_t address, uint32_t length, PDESC_ARRAY desc_array)
2757 {
2758         /* Since issue is sensitive to length and address.*/
2759         /* Let us first check the address...*/
2760         uint32_t safe_terminator;
2761         if (length <= 4) {
2762                 desc_array->descriptor[0].address = address;
2763                 desc_array->descriptor[0].length = length;
2764                 desc_array->elements = 1;
2765                 return(desc_array->elements);
2766         }
2767         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
2768         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
2769         if (safe_terminator == 0 ||
2770             (safe_terminator > 4 && safe_terminator < 9) ||
2771             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
2772                 desc_array->descriptor[0].address = address;
2773                 desc_array->descriptor[0].length = length;
2774                 desc_array->elements = 1;
2775                 return(desc_array->elements);
2776         }
2777
2778         desc_array->descriptor[0].address = address;
2779         desc_array->descriptor[0].length = length - 4;
2780         desc_array->descriptor[1].address = address + (length - 4);
2781         desc_array->descriptor[1].length = 4;
2782         desc_array->elements = 2;
2783         return(desc_array->elements);
2784 }
2785
2786 /**********************************************************************
2787  *
2788  *  Update the board statistics counters.
2789  *
2790  **********************************************************************/
2791 static void
2792 em_update_stats_counters(struct adapter *adapter)
2793 {
2794         struct ifnet   *ifp;
2795
2796         if (adapter->hw.media_type == em_media_type_copper ||
2797             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
2798                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
2799                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
2800         }
2801         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
2802         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
2803         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
2804         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
2805
2806         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
2807         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
2808         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
2809         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
2810         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
2811         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
2812         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
2813         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
2814         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
2815         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
2816         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
2817         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
2818         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
2819         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
2820         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
2821         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
2822         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
2823         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
2824         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
2825         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
2826
2827         /* For the 64-bit byte counters the low dword must be read first. */
2828         /* Both registers clear on the read of the high dword */
2829
2830         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
2831         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
2832         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
2833         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
2834
2835         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
2836         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
2837         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
2838         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
2839         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
2840
2841         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
2842         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
2843         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
2844         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
2845
2846         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
2847         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
2848         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
2849         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
2850         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
2851         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
2852         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
2853         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
2854         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
2855         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
2856
2857         if (adapter->hw.mac_type >= em_82543) {
2858                 adapter->stats.algnerrc +=
2859                     E1000_READ_REG(&adapter->hw, ALGNERRC);
2860                 adapter->stats.rxerrc +=
2861                     E1000_READ_REG(&adapter->hw, RXERRC);
2862                 adapter->stats.tncrs +=
2863                     E1000_READ_REG(&adapter->hw, TNCRS);
2864                 adapter->stats.cexterr +=
2865                     E1000_READ_REG(&adapter->hw, CEXTERR);
2866                 adapter->stats.tsctc +=
2867                     E1000_READ_REG(&adapter->hw, TSCTC);
2868                 adapter->stats.tsctfc +=
2869                     E1000_READ_REG(&adapter->hw, TSCTFC);
2870         }
2871         ifp = &adapter->interface_data.ac_if;
2872
2873         /* Fill out the OS statistics structure */
2874         ifp->if_ibytes = adapter->stats.gorcl;
2875         ifp->if_obytes = adapter->stats.gotcl;
2876         ifp->if_imcasts = adapter->stats.mprc;
2877         ifp->if_collisions = adapter->stats.colc;
2878
2879         /* Rx Errors */
2880         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
2881             adapter->stats.crcerrs + adapter->stats.algnerrc +
2882             adapter->stats.rlec + adapter->stats.rnbc +
2883             adapter->stats.mpc + adapter->stats.cexterr;
2884
2885         /* Tx Errors */
2886         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol;
2887 }
2888
2889
2890 /**********************************************************************
2891  *
2892  *  This routine is called only when em_display_debug_stats is enabled.
2893  *  This routine provides a way to take a look at important statistics
2894  *  maintained by the driver and hardware.
2895  *
2896  **********************************************************************/
2897 static void
2898 em_print_debug_info(struct adapter *adapter)
2899 {
2900         device_t dev= adapter->dev;
2901         uint8_t *hw_addr = adapter->hw.hw_addr;
2902
2903         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
2904         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
2905                       E1000_READ_REG(&adapter->hw, TIDV),
2906                       E1000_READ_REG(&adapter->hw, TADV));
2907         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
2908                       E1000_READ_REG(&adapter->hw, RDTR),
2909                       E1000_READ_REG(&adapter->hw, RADV));
2910 #ifdef DBG_STATS
2911         device_printf(dev, "Packets not Avail = %ld\n", adapter->no_pkts_avail);
2912         device_printf(dev, "CleanTxInterrupts = %ld\n",
2913                       adapter->clean_tx_interrupts);
2914 #endif
2915         device_printf(dev, "fifo workaround = %lld, fifo_reset = %lld\n",
2916                       (long long)adapter->tx_fifo_wrk,
2917                       (long long)adapter->tx_fifo_reset);
2918         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2919                       E1000_READ_REG(&adapter->hw, TDH),
2920                       E1000_READ_REG(&adapter->hw, TDT));
2921         device_printf(dev, "Num Tx descriptors avail = %d\n",
2922                       adapter->num_tx_desc_avail);
2923         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
2924                       adapter->no_tx_desc_avail1);
2925         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
2926                       adapter->no_tx_desc_avail2);
2927         device_printf(dev, "Std mbuf failed = %ld\n",
2928                       adapter->mbuf_alloc_failed);
2929         device_printf(dev, "Std mbuf cluster failed = %ld\n",
2930                       adapter->mbuf_cluster_failed);
2931         device_printf(dev, "Driver dropped packets = %ld\n",
2932                       adapter->dropped_pkts);
2933 }
2934
2935 static void
2936 em_print_hw_stats(struct adapter *adapter)
2937 {
2938         device_t dev= adapter->dev;
2939
2940         device_printf(dev, "Excessive collisions = %lld\n",
2941                       (long long)adapter->stats.ecol);
2942         device_printf(dev, "Symbol errors = %lld\n",
2943                       (long long)adapter->stats.symerrs);
2944         device_printf(dev, "Sequence errors = %lld\n",
2945                       (long long)adapter->stats.sec);
2946         device_printf(dev, "Defer count = %lld\n",
2947                       (long long)adapter->stats.dc);
2948
2949         device_printf(dev, "Missed Packets = %lld\n",
2950                       (long long)adapter->stats.mpc);
2951         device_printf(dev, "Receive No Buffers = %lld\n",
2952                       (long long)adapter->stats.rnbc);
2953         device_printf(dev, "Receive length errors = %lld\n",
2954                       (long long)adapter->stats.rlec);
2955         device_printf(dev, "Receive errors = %lld\n",
2956                       (long long)adapter->stats.rxerrc);
2957         device_printf(dev, "Crc errors = %lld\n",
2958                       (long long)adapter->stats.crcerrs);
2959         device_printf(dev, "Alignment errors = %lld\n",
2960                       (long long)adapter->stats.algnerrc);
2961         device_printf(dev, "Carrier extension errors = %lld\n",
2962                       (long long)adapter->stats.cexterr);
2963
2964         device_printf(dev, "XON Rcvd = %lld\n",
2965                       (long long)adapter->stats.xonrxc);
2966         device_printf(dev, "XON Xmtd = %lld\n",
2967                       (long long)adapter->stats.xontxc);
2968         device_printf(dev, "XOFF Rcvd = %lld\n",
2969                       (long long)adapter->stats.xoffrxc);
2970         device_printf(dev, "XOFF Xmtd = %lld\n",
2971                       (long long)adapter->stats.xofftxc);
2972
2973         device_printf(dev, "Good Packets Rcvd = %lld\n",
2974                       (long long)adapter->stats.gprc);
2975         device_printf(dev, "Good Packets Xmtd = %lld\n",
2976                       (long long)adapter->stats.gptc);
2977 }
2978
2979 static int
2980 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
2981 {
2982         int error;
2983         int result;
2984         struct adapter *adapter;
2985
2986         result = -1;
2987         error = sysctl_handle_int(oidp, &result, 0, req);
2988
2989         if (error || !req->newptr)
2990                 return(error);
2991
2992         if (result == 1) {
2993                 adapter = (struct adapter *)arg1;
2994                 em_print_debug_info(adapter);
2995         }
2996
2997         return(error);
2998 }
2999
3000 static int
3001 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
3002 {
3003         int error;
3004         int result;
3005         struct adapter *adapter;
3006
3007         result = -1;
3008         error = sysctl_handle_int(oidp, &result, 0, req);
3009
3010         if (error || !req->newptr)
3011                 return(error);
3012
3013         if (result == 1) {
3014                 adapter = (struct adapter *)arg1;
3015                 em_print_hw_stats(adapter);
3016         }
3017
3018         return(error);
3019 }
3020
3021 static int
3022 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3023 {
3024         struct em_int_delay_info *info;
3025         struct adapter *adapter;
3026         uint32_t regval;
3027         int error;
3028         int usecs;
3029         int ticks;
3030         int s;
3031
3032         info = (struct em_int_delay_info *)arg1;
3033         adapter = info->adapter;
3034         usecs = info->value;
3035         error = sysctl_handle_int(oidp, &usecs, 0, req);
3036         if (error != 0 || req->newptr == NULL)
3037                 return(error);
3038         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3039                 return(EINVAL);
3040         info->value = usecs;
3041         ticks = E1000_USECS_TO_TICKS(usecs);
3042
3043         s = splimp();
3044         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3045         regval = (regval & ~0xffff) | (ticks & 0xffff);
3046         /* Handle a few special cases. */
3047         switch (info->offset) {
3048         case E1000_RDTR:
3049         case E1000_82542_RDTR:
3050                 regval |= E1000_RDT_FPDB;
3051                 break;
3052         case E1000_TIDV:
3053         case E1000_82542_TIDV:
3054                 if (ticks == 0) {
3055                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3056                         /* Don't write 0 into the TIDV register. */
3057                         regval++;
3058                 } else
3059                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3060                 break;
3061         }
3062         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3063         splx(s);
3064         return(0);
3065 }
3066
3067 static void
3068 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3069                         const char *description, struct em_int_delay_info *info,
3070                         int offset, int value)
3071 {
3072         info->adapter = adapter;
3073         info->offset = offset;
3074         info->value = value;
3075         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3076                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3077                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3078                         info, 0, em_sysctl_int_delay, "I", description);
3079 }