sys/dev/netif/em/if_em.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4
   5 Copyright (c) 2001-2003, Intel Corporation
   6 All rights reserved.
   7
   8 Redistribution and use in source and binary forms, with or without
   9 modification, are permitted provided that the following conditions are met:
  10
  11  1. Redistributions of source code must retain the above copyright notice,
  12     this list of conditions and the following disclaimer.
  13
  14  2. Redistributions in binary form must reproduce the above copyright
  15     notice, this list of conditions and the following disclaimer in the
  16     documentation and/or other materials provided with the distribution.
  17
  18  3. Neither the name of the Intel Corporation nor the names of its
  19     contributors may be used to endorse or promote products derived from
  20     this software without specific prior written permission.
  21
  22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32 POSSIBILITY OF SUCH DAMAGE.
  33
  34 ***************************************************************************/
  35
  36 /*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.15 2003/06/09 22:10:15 pdeuskar Exp $*/
  37 /*$DragonFly: src/sys/dev/netif/em/if_em.c,v 1.40 2005/10/17 06:18:36 sephe Exp $*/
  38
  39 #include <dev/netif/em/if_em.h>
  40 #include <net/ifq_var.h>
  41
  42 /*********************************************************************
  43  *  Set this to one to display debug statistics
  44  *********************************************************************/
  45 int             em_display_debug_stats = 0;
  46
  47 /*********************************************************************
  48  *  Driver version
  49  *********************************************************************/
  50
  51 char em_driver_version[] = "1.7.35";
  52
  53
  54 /*********************************************************************
  55  *  PCI Device ID Table
  56  *
  57  *  Used by probe to select devices to load on
  58  *  Last field stores an index into em_strings
  59  *  Last entry must be all 0s
  60  *
  61  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
  62  *********************************************************************/
  63
  64 static em_vendor_info_t em_vendor_info_array[] =
  65 {
  66         /* Intel(R) PRO/1000 Network Connection */
  67         { 0x8086, 0x1000, PCI_ANY_ID, PCI_ANY_ID, 0},
  68         { 0x8086, 0x1001, PCI_ANY_ID, PCI_ANY_ID, 0},
  69         { 0x8086, 0x1004, PCI_ANY_ID, PCI_ANY_ID, 0},
  70         { 0x8086, 0x1008, PCI_ANY_ID, PCI_ANY_ID, 0},
  71         { 0x8086, 0x1009, PCI_ANY_ID, PCI_ANY_ID, 0},
  72         { 0x8086, 0x100C, PCI_ANY_ID, PCI_ANY_ID, 0},
  73         { 0x8086, 0x100D, PCI_ANY_ID, PCI_ANY_ID, 0},
  74         { 0x8086, 0x100E, PCI_ANY_ID, PCI_ANY_ID, 0},
  75         { 0x8086, 0x100F, PCI_ANY_ID, PCI_ANY_ID, 0},
  76         { 0x8086, 0x1010, PCI_ANY_ID, PCI_ANY_ID, 0},
  77         { 0x8086, 0x1011, PCI_ANY_ID, PCI_ANY_ID, 0},
  78         { 0x8086, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0},
  79         { 0x8086, 0x1013, PCI_ANY_ID, PCI_ANY_ID, 0},
  80         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
  81         { 0x8086, 0x1015, PCI_ANY_ID, PCI_ANY_ID, 0},
  82         { 0x8086, 0x1016, PCI_ANY_ID, PCI_ANY_ID, 0},
  83         { 0x8086, 0x1017, PCI_ANY_ID, PCI_ANY_ID, 0},
  84         { 0x8086, 0x1018, PCI_ANY_ID, PCI_ANY_ID, 0},
  85         { 0x8086, 0x1019, PCI_ANY_ID, PCI_ANY_ID, 0},
  86         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
  87         { 0x8086, 0x101D, PCI_ANY_ID, PCI_ANY_ID, 0},
  88         { 0x8086, 0x101E, PCI_ANY_ID, PCI_ANY_ID, 0},
  89         { 0x8086, 0x1026, PCI_ANY_ID, PCI_ANY_ID, 0},
  90         { 0x8086, 0x1027, PCI_ANY_ID, PCI_ANY_ID, 0},
  91         { 0x8086, 0x1028, PCI_ANY_ID, PCI_ANY_ID, 0},
  92         { 0x8086, 0x1075, PCI_ANY_ID, PCI_ANY_ID, 0},
  93         { 0x8086, 0x1076, PCI_ANY_ID, PCI_ANY_ID, 0},
  94         { 0x8086, 0x1077, PCI_ANY_ID, PCI_ANY_ID, 0},
  95         { 0x8086, 0x1078, PCI_ANY_ID, PCI_ANY_ID, 0},
  96         { 0x8086, 0x1079, PCI_ANY_ID, PCI_ANY_ID, 0},
  97         { 0x8086, 0x107A, PCI_ANY_ID, PCI_ANY_ID, 0},
  98         { 0x8086, 0x107B, PCI_ANY_ID, PCI_ANY_ID, 0},
  99         { 0x8086, 0x107C, PCI_ANY_ID, PCI_ANY_ID, 0},
 100         { 0x8086, 0x108A, PCI_ANY_ID, PCI_ANY_ID, 0},
 101         /* required last entry */
 102         { 0, 0, 0, 0, 0}
 103 };
 104
 105 /*********************************************************************
 106  *  Table of branding strings for all supported NICs.
 107  *********************************************************************/
 108
 109 static const char *em_strings[] = {
 110         "Intel(R) PRO/1000 Network Connection"
 111 };
 112
 113 /*********************************************************************
 114  *  Function prototypes
 115  *********************************************************************/
 116 static int      em_probe(device_t);
 117 static int      em_attach(device_t);
 118 static int      em_detach(device_t);
 119 static int      em_shutdown(device_t);
 120 static void     em_intr(void *);
 121 static void     em_start(struct ifnet *);
 122 static void     em_start_serialized(struct ifnet *);
 123 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 124 static void     em_watchdog(struct ifnet *);
 125 static void     em_init(void *);
 126 static void     em_init_serialized(void *);
 127 static void     em_stop(void *);
 128 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 129 static int      em_media_change(struct ifnet *);
 130 static void     em_identify_hardware(struct adapter *);
 131 static void     em_local_timer(void *);
 132 static int      em_hardware_init(struct adapter *);
 133 static void     em_setup_interface(device_t, struct adapter *);
 134 static int      em_setup_transmit_structures(struct adapter *);
 135 static void     em_initialize_transmit_unit(struct adapter *);
 136 static int      em_setup_receive_structures(struct adapter *);
 137 static void     em_initialize_receive_unit(struct adapter *);
 138 static void     em_enable_intr(struct adapter *);
 139 static void     em_disable_intr(struct adapter *);
 140 static void     em_free_transmit_structures(struct adapter *);
 141 static void     em_free_receive_structures(struct adapter *);
 142 static void     em_update_stats_counters(struct adapter *);
 143 static void     em_clean_transmit_interrupts(struct adapter *);
 144 static int      em_allocate_receive_structures(struct adapter *);
 145 static int      em_allocate_transmit_structures(struct adapter *);
 146 static void     em_process_receive_interrupts(struct adapter *, int);
 147 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 148                                     struct mbuf *);
 149 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 150                                            uint32_t *, uint32_t *);
 151 static void     em_set_promisc(struct adapter *);
 152 static void     em_disable_promisc(struct adapter *);
 153 static void     em_set_multi(struct adapter *);
 154 static void     em_print_hw_stats(struct adapter *);
 155 static void     em_print_link_status(struct adapter *);
 156 static int      em_get_buf(int i, struct adapter *, struct mbuf *, int how);
 157 static void     em_enable_vlans(struct adapter *);
 158 static int      em_encap(struct adapter *, struct mbuf *);
 159 static void     em_smartspeed(struct adapter *);
 160 static int      em_82547_fifo_workaround(struct adapter *, int);
 161 static void     em_82547_update_fifo_head(struct adapter *, int);
 162 static int      em_82547_tx_fifo_reset(struct adapter *);
 163 static void     em_82547_move_tail(void *arg);
 164 static void     em_82547_move_tail_serialized(void *arg);
 165 static int      em_dma_malloc(struct adapter *, bus_size_t,
 166                               struct em_dma_alloc *, int);
 167 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 168 static void     em_print_debug_info(struct adapter *);
 169 static int      em_is_valid_ether_addr(uint8_t *);
 170 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 171 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 172 static uint32_t em_fill_descriptors(uint64_t address, uint32_t length,
 173                                    PDESC_ARRAY desc_array);
 174 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 175 static int      em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 176 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 177                                         const char *,
 178                                         struct em_int_delay_info *, int, int);
 179
 180 /*********************************************************************
 181  *  FreeBSD Device Interface Entry Points
 182  *********************************************************************/
 183
 184 static device_method_t em_methods[] = {
 185         /* Device interface */
 186         DEVMETHOD(device_probe, em_probe),
 187         DEVMETHOD(device_attach, em_attach),
 188         DEVMETHOD(device_detach, em_detach),
 189         DEVMETHOD(device_shutdown, em_shutdown),
 190         {0, 0}
 191 };
 192
 193 static driver_t em_driver = {
 194         "em", em_methods, sizeof(struct adapter),
 195 };
 196
 197 static devclass_t em_devclass;
 198
 199 DECLARE_DUMMY_MODULE(if_em);
 200 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 201
 202 /*********************************************************************
 203  *  Tunable default values.
 204  *********************************************************************/
 205
 206 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 207 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 208
 209 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 210 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 211 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 212 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 213 static int em_int_throttle_ceil = 10000;
 214
 215 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 216 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 217 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 218 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 219 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
 220
 221 /*********************************************************************
 222  *  Device identification routine
 223  *
 224  *  em_probe determines if the driver should be loaded on
 225  *  adapter based on PCI vendor/device id of the adapter.
 226  *
 227  *  return 0 on success, positive on failure
 228  *********************************************************************/
 229
 230 static int
 231 em_probe(device_t dev)
 232 {
 233         em_vendor_info_t *ent;
 234
 235         uint16_t pci_vendor_id = 0;
 236         uint16_t pci_device_id = 0;
 237         uint16_t pci_subvendor_id = 0;
 238         uint16_t pci_subdevice_id = 0;
 239         char adapter_name[60];
 240
 241         INIT_DEBUGOUT("em_probe: begin");
 242
 243         pci_vendor_id = pci_get_vendor(dev);
 244         if (pci_vendor_id != EM_VENDOR_ID)
 245                 return(ENXIO);
 246
 247         pci_device_id = pci_get_device(dev);
 248         pci_subvendor_id = pci_get_subvendor(dev);
 249         pci_subdevice_id = pci_get_subdevice(dev);
 250
 251         ent = em_vendor_info_array;
 252         while (ent->vendor_id != 0) {
 253                 if ((pci_vendor_id == ent->vendor_id) &&
 254                     (pci_device_id == ent->device_id) &&
 255
 256                     ((pci_subvendor_id == ent->subvendor_id) ||
 257                      (ent->subvendor_id == PCI_ANY_ID)) &&
 258
 259                     ((pci_subdevice_id == ent->subdevice_id) ||
 260                      (ent->subdevice_id == PCI_ANY_ID))) {
 261                         snprintf(adapter_name, sizeof(adapter_name),
 262                                  "%s, Version - %s",  em_strings[ent->index],
 263                                  em_driver_version);
 264                         device_set_desc_copy(dev, adapter_name);
 265                         return(0);
 266                 }
 267                 ent++;
 268         }
 269
 270         return(ENXIO);
 271 }
 272
 273 /*********************************************************************
 274  *  Device initialization routine
 275  *
 276  *  The attach entry point is called when the driver is being loaded.
 277  *  This routine identifies the type of hardware, allocates all resources
 278  *  and initializes the hardware.
 279  *
 280  *  return 0 on success, positive on failure
 281  *********************************************************************/
 282
 283 static int
 284 em_attach(device_t dev)
 285 {
 286         struct adapter *adapter;
 287         int tsize, rsize;
 288         int i, val, rid;
 289         int error = 0;
 290
 291         INIT_DEBUGOUT("em_attach: begin");
 292
 293         adapter = device_get_softc(dev);
 294
 295         lwkt_serialize_init(&adapter->serializer);
 296
 297         callout_init(&adapter->timer);
 298         callout_init(&adapter->tx_fifo_timer);
 299
 300         adapter->dev = dev;
 301         adapter->osdep.dev = dev;
 302
 303         /* SYSCTL stuff */
 304         sysctl_ctx_init(&adapter->sysctl_ctx);
 305         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 306                                                SYSCTL_STATIC_CHILDREN(_hw),
 307                                                OID_AUTO,
 308                                                device_get_nameunit(dev),
 309                                                CTLFLAG_RD,
 310                                                0, "");
 311
 312         if (adapter->sysctl_tree == NULL) {
 313                 error = EIO;
 314                 goto fail;
 315         }
 316
 317         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 318                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 319                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 320                         (void *)adapter, 0,
 321                         em_sysctl_debug_info, "I", "Debug Information");
 322
 323         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 324                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 325                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 326                         (void *)adapter, 0,
 327                         em_sysctl_stats, "I", "Statistics");
 328
 329         /* Determine hardware revision */
 330         em_identify_hardware(adapter);
 331
 332         /* Set up some sysctls for the tunable interrupt delays */
 333         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 334                                 "receive interrupt delay in usecs",
 335                                 &adapter->rx_int_delay,
 336                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 337                                 em_rx_int_delay_dflt);
 338         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 339                                 "transmit interrupt delay in usecs",
 340                                 &adapter->tx_int_delay,
 341                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 342                                 em_tx_int_delay_dflt);
 343         if (adapter->hw.mac_type >= em_82540) {
 344                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 345                                         "receive interrupt delay limit in usecs",
 346                                         &adapter->rx_abs_int_delay,
 347                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 348                                         em_rx_abs_int_delay_dflt);
 349                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 350                                         "transmit interrupt delay limit in usecs",
 351                                         &adapter->tx_abs_int_delay,
 352                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 353                                         em_tx_abs_int_delay_dflt);
 354                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 355                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 356                         OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW,
 357                         adapter, 0, em_sysctl_int_throttle, "I", NULL);
 358         }
 359
 360         /* Parameters (to be read from user) */
 361         adapter->num_tx_desc = EM_MAX_TXD;
 362         adapter->num_rx_desc = EM_MAX_RXD;
 363         adapter->hw.autoneg = DO_AUTO_NEG;
 364         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 365         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 366         adapter->hw.tbi_compatibility_en = TRUE;
 367         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 368
 369         /*
 370          * These parameters control the automatic generation(Tx) and
 371          * response(Rx) to Ethernet PAUSE frames.
 372          */
 373         adapter->hw.fc_high_water = FC_DEFAULT_HI_THRESH;
 374         adapter->hw.fc_low_water  = FC_DEFAULT_LO_THRESH;
 375         adapter->hw.fc_pause_time = FC_DEFAULT_TX_TIMER;
 376         adapter->hw.fc_send_xon   = TRUE;
 377         adapter->hw.fc = em_fc_full;
 378
 379         adapter->hw.phy_init_script = 1;
 380         adapter->hw.phy_reset_disable = FALSE;
 381
 382 #ifndef EM_MASTER_SLAVE
 383         adapter->hw.master_slave = em_ms_hw_default;
 384 #else
 385         adapter->hw.master_slave = EM_MASTER_SLAVE;
 386 #endif
 387
 388         /*
 389          * Set the max frame size assuming standard ethernet
 390          * sized frames
 391          */
 392         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 393
 394         adapter->hw.min_frame_size =
 395             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 396
 397         /*
 398          * This controls when hardware reports transmit completion
 399          * status.
 400          */
 401         adapter->hw.report_tx_early = 1;
 402
 403         rid = EM_MMBA;
 404         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 405                                                      &rid, RF_ACTIVE);
 406         if (!(adapter->res_memory)) {
 407                 device_printf(dev, "Unable to allocate bus resource: memory\n");
 408                 error = ENXIO;
 409                 goto fail;
 410         }
 411         adapter->osdep.mem_bus_space_tag =
 412             rman_get_bustag(adapter->res_memory);
 413         adapter->osdep.mem_bus_space_handle =
 414             rman_get_bushandle(adapter->res_memory);
 415         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
 416
 417         if (adapter->hw.mac_type > em_82543) {
 418                 /* Figure our where our IO BAR is ? */
 419                 rid = EM_MMBA;
 420                 for (i = 0; i < 5; i++) {
 421                         val = pci_read_config(dev, rid, 4);
 422                         if (val & 0x00000001) {
 423                                 adapter->io_rid = rid;
 424                                 break;
 425                         }
 426                         rid += 4;
 427                 }
 428
 429                 adapter->res_ioport = bus_alloc_resource_any(dev,
 430                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
 431                 if (!(adapter->res_ioport)) {
 432                         device_printf(dev, "Unable to allocate bus resource: ioport\n");
 433                         error = ENXIO;
 434                         goto fail;
 435                 }
 436
 437                 adapter->hw.reg_io_tag = rman_get_bustag(adapter->res_ioport);
 438                 adapter->hw.reg_io_handle = rman_get_bushandle(adapter->res_ioport);
 439         }
 440
 441         rid = 0x0;
 442         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 443             &rid, RF_SHAREABLE | RF_ACTIVE);
 444         if (!(adapter->res_interrupt)) {
 445                 device_printf(dev, "Unable to allocate bus resource: interrupt\n");
 446                 error = ENXIO;
 447                 goto fail;
 448         }
 449
 450         adapter->hw.back = &adapter->osdep;
 451
 452         /* Initialize eeprom parameters */
 453         em_init_eeprom_params(&adapter->hw);
 454
 455         tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc);
 456
 457         /* Allocate Transmit Descriptor ring */
 458         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_WAITOK)) {
 459                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 460                 error = ENOMEM;
 461                 goto fail;
 462         }
 463         adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr;
 464
 465         rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc);
 466
 467         /* Allocate Receive Descriptor ring */
 468         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_WAITOK)) {
 469                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 470                 error = ENOMEM;
 471                 goto fail;
 472         }
 473         adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr;
 474
 475         /* Initialize the hardware */
 476         if (em_hardware_init(adapter)) {
 477                 device_printf(dev, "Unable to initialize the hardware\n");
 478                 error = EIO;
 479                 goto fail;
 480         }
 481
 482         /* Copy the permanent MAC address out of the EEPROM */
 483         if (em_read_mac_addr(&adapter->hw) < 0) {
 484                 device_printf(dev, "EEPROM read error while reading mac address\n");
 485                 error = EIO;
 486                 goto fail;
 487         }
 488
 489         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 490                 device_printf(dev, "Invalid mac address\n");
 491                 error = EIO;
 492                 goto fail;
 493         }
 494
 495         /* Setup OS specific network interface */
 496         em_setup_interface(dev, adapter);
 497
 498         /* Initialize statistics */
 499         em_clear_hw_cntrs(&adapter->hw);
 500         em_update_stats_counters(adapter);
 501         adapter->hw.get_link_status = 1;
 502         em_check_for_link(&adapter->hw);
 503
 504         /* Print the link status */
 505         if (adapter->link_active == 1) {
 506                 em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
 507                                         &adapter->link_duplex);
 508                 device_printf(dev, "Speed: %d Mbps, Duplex: %s\n",
 509                     adapter->link_speed,
 510                     adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
 511         } else
 512                 device_printf(dev, "Speed: N/A, Duplex:N/A\n");
 513
 514         /* Identify 82544 on PCIX */
 515         em_get_bus_info(&adapter->hw);
 516         if (adapter->hw.bus_type == em_bus_type_pcix &&
 517             adapter->hw.mac_type == em_82544)
 518                 adapter->pcix_82544 = TRUE;
 519         else
 520                 adapter->pcix_82544 = FALSE;
 521
 522         error = bus_setup_intr(dev, adapter->res_interrupt, 0,
 523                            (void (*)(void *)) em_intr, adapter,
 524                            &adapter->int_handler_tag, &adapter->serializer);
 525         if (error) {
 526                 device_printf(dev, "Error registering interrupt handler!\n");
 527                 ether_ifdetach(&adapter->interface_data.ac_if);
 528                 goto fail;
 529         }
 530
 531         INIT_DEBUGOUT("em_attach: end");
 532         return(0);
 533
 534 fail:
 535         em_detach(dev);
 536         return(error);
 537 }
 538
 539 /*********************************************************************
 540  *  Device removal routine
 541  *
 542  *  The detach entry point is called when the driver is being removed.
 543  *  This routine stops the adapter and deallocates all the resources
 544  *  that were allocated for driver operation.
 545  *
 546  *  return 0 on success, positive on failure
 547  *********************************************************************/
 548
 549 static int
 550 em_detach(device_t dev)
 551 {
 552         struct adapter * adapter = device_get_softc(dev);
 553
 554         INIT_DEBUGOUT("em_detach: begin");
 555
 556         lwkt_serialize_enter(&adapter->serializer);
 557         adapter->in_detach = 1;
 558
 559         if (device_is_attached(dev)) {
 560                 em_stop(adapter);
 561                 em_phy_hw_reset(&adapter->hw);
 562                 ether_ifdetach(&adapter->interface_data.ac_if);
 563         }
 564         bus_generic_detach(dev);
 565
 566         if (adapter->res_interrupt != NULL) {
 567                 bus_teardown_intr(dev, adapter->res_interrupt,
 568                                   adapter->int_handler_tag);
 569                 bus_release_resource(dev, SYS_RES_IRQ, 0,
 570                                      adapter->res_interrupt);
 571         }
 572         if (adapter->res_memory != NULL) {
 573                 bus_release_resource(dev, SYS_RES_MEMORY, EM_MMBA,
 574                                      adapter->res_memory);
 575         }
 576
 577         if (adapter->res_ioport != NULL) {
 578                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
 579                                      adapter->res_ioport);
 580         }
 581
 582         /* Free Transmit Descriptor ring */
 583         if (adapter->tx_desc_base != NULL) {
 584                 em_dma_free(adapter, &adapter->txdma);
 585                 adapter->tx_desc_base = NULL;
 586         }
 587
 588         /* Free Receive Descriptor ring */
 589         if (adapter->rx_desc_base != NULL) {
 590                 em_dma_free(adapter, &adapter->rxdma);
 591                 adapter->rx_desc_base = NULL;
 592         }
 593
 594         adapter->sysctl_tree = NULL;
 595         sysctl_ctx_free(&adapter->sysctl_ctx);
 596
 597         lwkt_serialize_exit(&adapter->serializer);
 598         return(0);
 599 }
 600
 601 /*********************************************************************
 602  *
 603  *  Shutdown entry point
 604  *
 605  **********************************************************************/
 606
 607 static int
 608 em_shutdown(device_t dev)
 609 {
 610         struct adapter *adapter = device_get_softc(dev);
 611         em_stop(adapter);
 612         return(0);
 613 }
 614
 615 /*********************************************************************
 616  *  Transmit entry point
 617  *
 618  *  em_start is called by the stack to initiate a transmit.
 619  *  The driver will remain in this routine as long as there are
 620  *  packets to transmit and transmit resources are available.
 621  *  In case resources are not available stack is notified and
 622  *  the packet is requeued.
 623  **********************************************************************/
 624
 625 static void
 626 em_start(struct ifnet *ifp)
 627 {
 628         struct adapter *adapter = ifp->if_softc;
 629
 630         lwkt_serialize_enter(&adapter->serializer);
 631         em_start_serialized(ifp);
 632         lwkt_serialize_exit(&adapter->serializer);
 633 }
 634
 635 static void
 636 em_start_serialized(struct ifnet *ifp)
 637 {
 638         struct mbuf *m_head;
 639         struct adapter *adapter = ifp->if_softc;
 640
 641         if (!adapter->link_active)
 642                 return;
 643         while (!ifq_is_empty(&ifp->if_snd)) {
 644                 m_head = ifq_poll(&ifp->if_snd);
 645
 646                 if (m_head == NULL)
 647                         break;
 648
 649                 if (em_encap(adapter, m_head)) {
 650                         ifp->if_flags |= IFF_OACTIVE;
 651                         break;
 652                 }
 653                 m_head = ifq_dequeue(&ifp->if_snd);
 654
 655                 /* Send a copy of the frame to the BPF listener */
 656                 BPF_MTAP(ifp, m_head);
 657
 658                 /* Set timeout in case hardware has problems transmitting */
 659                 ifp->if_timer = EM_TX_TIMEOUT;
 660         }
 661 }
 662
 663 /*********************************************************************
 664  *  Ioctl entry point
 665  *
 666  *  em_ioctl is called when the user wants to configure the
 667  *  interface.
 668  *
 669  *  return 0 on success, positive on failure
 670  **********************************************************************/
 671
 672 static int
 673 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 674 {
 675         int mask, error = 0;
 676         struct ifreq *ifr = (struct ifreq *) data;
 677         struct adapter *adapter = ifp->if_softc;
 678
 679         lwkt_serialize_enter(&adapter->serializer);
 680
 681         if (adapter->in_detach)
 682                 goto out;
 683
 684         switch (command) {
 685         case SIOCSIFADDR:
 686         case SIOCGIFADDR:
 687                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFADDR (Get/Set Interface Addr)");
 688                 lwkt_serialize_exit(&adapter->serializer);
 689                 ether_ioctl(ifp, command, data);
 690                 lwkt_serialize_enter(&adapter->serializer);
 691                 break;
 692         case SIOCSIFMTU:
 693                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 694                 if (ifr->ifr_mtu > MAX_JUMBO_FRAME_SIZE - ETHER_HDR_LEN) {
 695                         error = EINVAL;
 696                 } else {
 697                         ifp->if_mtu = ifr->ifr_mtu;
 698                         adapter->hw.max_frame_size =
 699                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 700                         em_init_serialized(adapter);
 701                 }
 702                 break;
 703         case SIOCSIFFLAGS:
 704                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
 705                 if (ifp->if_flags & IFF_UP) {
 706                         if (!(ifp->if_flags & IFF_RUNNING))
 707                                 em_init_serialized(adapter);
 708                         em_disable_promisc(adapter);
 709                         em_set_promisc(adapter);
 710                 } else {
 711                         if (ifp->if_flags & IFF_RUNNING)
 712                                 em_stop(adapter);
 713                 }
 714                 break;
 715         case SIOCADDMULTI:
 716         case SIOCDELMULTI:
 717                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 718                 if (ifp->if_flags & IFF_RUNNING) {
 719                         em_disable_intr(adapter);
 720                         em_set_multi(adapter);
 721                         if (adapter->hw.mac_type == em_82542_rev2_0)
 722                                 em_initialize_receive_unit(adapter);
 723                         em_enable_intr(adapter);
 724                 }
 725                 break;
 726         case SIOCSIFMEDIA:
 727         case SIOCGIFMEDIA:
 728                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
 729                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 730                 break;
 731         case SIOCSIFCAP:
 732                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 733                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 734                 if (mask & IFCAP_HWCSUM) {
 735                         if (IFCAP_HWCSUM & ifp->if_capenable)
 736                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 737                         else
 738                                 ifp->if_capenable |= IFCAP_HWCSUM;
 739                         if (ifp->if_flags & IFF_RUNNING)
 740                                 em_init_serialized(adapter);
 741                 }
 742                 break;
 743         default:
 744                 IOCTL_DEBUGOUT1("ioctl received: UNKNOWN (0x%x)", (int)command);
 745                 error = EINVAL;
 746         }
 747
 748 out:
 749         lwkt_serialize_exit(&adapter->serializer);
 750         return(error);
 751 }
 752
 753 /*********************************************************************
 754  *  Watchdog entry point
 755  *
 756  *  This routine is called whenever hardware quits transmitting.
 757  *
 758  **********************************************************************/
 759
 760 static void
 761 em_watchdog(struct ifnet *ifp)
 762 {
 763         struct adapter * adapter;
 764         adapter = ifp->if_softc;
 765
 766         /* If we are in this routine because of pause frames, then
 767          * don't reset the hardware.
 768          */
 769         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 770                 ifp->if_timer = EM_TX_TIMEOUT;
 771                 return;
 772         }
 773
 774         if (em_check_for_link(&adapter->hw))
 775                 if_printf(ifp, "watchdog timeout -- resetting\n");
 776
 777         ifp->if_flags &= ~IFF_RUNNING;
 778
 779         em_init(adapter);
 780
 781         ifp->if_oerrors++;
 782 }
 783
 784 /*********************************************************************
 785  *  Init entry point
 786  *
 787  *  This routine is used in two ways. It is used by the stack as
 788  *  init entry point in network interface structure. It is also used
 789  *  by the driver as a hw/sw initialization routine to get to a
 790  *  consistent state.
 791  *
 792  *  return 0 on success, positive on failure
 793  **********************************************************************/
 794
 795 static void
 796 em_init(void *arg)
 797 {
 798         struct adapter *adapter = arg;
 799
 800         lwkt_serialize_enter(&adapter->serializer);
 801         em_init_serialized(arg);
 802         lwkt_serialize_exit(&adapter->serializer);
 803 }
 804
 805 static void
 806 em_init_serialized(void *arg)
 807 {
 808         struct adapter *adapter = arg;
 809         uint32_t pba;
 810         struct ifnet *ifp = &adapter->interface_data.ac_if;
 811
 812         INIT_DEBUGOUT("em_init: begin");
 813
 814         em_stop(adapter);
 815
 816         /*
 817          * Packet Buffer Allocation (PBA)
 818          * Writing PBA sets the receive portion of the buffer
 819          * the remainder is used for the transmit buffer.
 820          *
 821          * Devices before the 82547 had a Packet Buffer of 64K.
 822          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
 823          * After the 82547 the buffer was reduced to 40K.
 824          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
 825          *   Note: default does not leave enough room for Jumbo Frame >10k.
 826          */
 827         if(adapter->hw.mac_type < em_82547) {
 828                 /* Total FIFO is 64K */
 829                 if(adapter->rx_buffer_len > EM_RXBUFFER_8192)
 830                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
 831                 else
 832                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
 833         } else {
 834                 /* Total FIFO is 40K */
 835                 if(adapter->hw.max_frame_size > EM_RXBUFFER_8192) {
 836                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
 837                 } else {
 838                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
 839                 }
 840                 adapter->tx_fifo_head = 0;
 841                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
 842                 adapter->tx_fifo_size =
 843                         (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
 844         }
 845         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
 846         E1000_WRITE_REG(&adapter->hw, PBA, pba);
 847
 848         /* Get the latest mac address, User can use a LAA */
 849         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
 850               ETHER_ADDR_LEN);
 851
 852         /* Initialize the hardware */
 853         if (em_hardware_init(adapter)) {
 854                 if_printf(ifp, "Unable to initialize the hardware\n");
 855                 return;
 856         }
 857
 858         em_enable_vlans(adapter);
 859
 860         /* Prepare transmit descriptors and buffers */
 861         if (em_setup_transmit_structures(adapter)) {
 862                 if_printf(ifp, "Could not setup transmit structures\n");
 863                 em_stop(adapter);
 864                 return;
 865         }
 866         em_initialize_transmit_unit(adapter);
 867
 868         /* Setup Multicast table */
 869         em_set_multi(adapter);
 870
 871         /* Prepare receive descriptors and buffers */
 872         if (em_setup_receive_structures(adapter)) {
 873                 if_printf(ifp, "Could not setup receive structures\n");
 874                 em_stop(adapter);
 875                 return;
 876         }
 877         em_initialize_receive_unit(adapter);
 878
 879         /* Don't loose promiscuous settings */
 880         em_set_promisc(adapter);
 881
 882         ifp->if_flags |= IFF_RUNNING;
 883         ifp->if_flags &= ~IFF_OACTIVE;
 884
 885         if (adapter->hw.mac_type >= em_82543) {
 886                 if (ifp->if_capenable & IFCAP_TXCSUM)
 887                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
 888                 else
 889                         ifp->if_hwassist = 0;
 890         }
 891
 892         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
 893         em_clear_hw_cntrs(&adapter->hw);
 894         em_enable_intr(adapter);
 895
 896         /* Don't reset the phy next time init gets called */
 897         adapter->hw.phy_reset_disable = TRUE;
 898 }
 899
 900 #ifdef DEVICE_POLLING
 901
 902 static void
 903 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 904 {
 905         struct adapter *adapter = ifp->if_softc;
 906         uint32_t reg_icr;
 907
 908         lwkt_serialize_enter(&adapter->serializer);
 909         switch(cmd) {
 910         case POLL_REGISTER:
 911                 em_disable_intr(adapter);
 912                 break;
 913         case POLL_DEREGISTER:
 914                 em_enable_intr(adapter);
 915                 break;
 916         case POLL_AND_CHECK_STATUS:
 917                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 918                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 919                         callout_stop(&adapter->timer);
 920                         adapter->hw.get_link_status = 1;
 921                         em_check_for_link(&adapter->hw);
 922                         em_print_link_status(adapter);
 923                         callout_reset(&adapter->timer, hz, em_local_timer,
 924                                       adapter);
 925                 }
 926                 /* fall through */
 927         case POLL_ONLY:
 928                 if (ifp->if_flags & IFF_RUNNING) {
 929                         em_process_receive_interrupts(adapter, count);
 930                         em_clean_transmit_interrupts(adapter);
 931                 }
 932                 if (ifp->if_flags & IFF_RUNNING) {
 933                         if (!ifq_is_empty(&ifp->if_snd))
 934                                 em_start_serialized(ifp);
 935                 }
 936                 break;
 937         }
 938         lwkt_serialize_exit(&adapter->serializer);
 939 }
 940
 941 #endif /* DEVICE_POLLING */
 942
 943 /*********************************************************************
 944  *
 945  *  Interrupt Service routine
 946  *
 947  **********************************************************************/
 948 static void
 949 em_intr(void *arg)
 950 {
 951         uint32_t reg_icr;
 952         struct ifnet *ifp;
 953         struct adapter *adapter = arg;
 954
 955         ifp = &adapter->interface_data.ac_if;
 956
 957         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
 958         if (!reg_icr)
 959                 return;
 960
 961         /* Link status change */
 962         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
 963                 callout_stop(&adapter->timer);
 964                 adapter->hw.get_link_status = 1;
 965                 em_check_for_link(&adapter->hw);
 966                 em_print_link_status(adapter);
 967                 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
 968         }
 969
 970         /*
 971          * note: do not attempt to improve efficiency by looping.  This
 972          * only results in unnecessary piecemeal collection of received
 973          * packets and unnecessary piecemeal cleanups of the transmit ring.
 974          */
 975         if (ifp->if_flags & IFF_RUNNING) {
 976                 em_process_receive_interrupts(adapter, -1);
 977                 em_clean_transmit_interrupts(adapter);
 978         }
 979
 980         if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd))
 981                 em_start_serialized(ifp);
 982 }
 983
 984 /*********************************************************************
 985  *
 986  *  Media Ioctl callback
 987  *
 988  *  This routine is called whenever the user queries the status of
 989  *  the interface using ifconfig.
 990  *
 991  **********************************************************************/
 992 static void
 993 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
 994 {
 995         struct adapter * adapter = ifp->if_softc;
 996
 997         INIT_DEBUGOUT("em_media_status: begin");
 998
 999         em_check_for_link(&adapter->hw);
1000         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1001                 if (adapter->link_active == 0) {
1002                         em_get_speed_and_duplex(&adapter->hw,
1003                                                 &adapter->link_speed,
1004                                                 &adapter->link_duplex);
1005                         adapter->link_active = 1;
1006                 }
1007         } else {
1008                 if (adapter->link_active == 1) {
1009                         adapter->link_speed = 0;
1010                         adapter->link_duplex = 0;
1011                         adapter->link_active = 0;
1012                 }
1013         }
1014
1015         ifmr->ifm_status = IFM_AVALID;
1016         ifmr->ifm_active = IFM_ETHER;
1017
1018         if (!adapter->link_active)
1019                 return;
1020
1021         ifmr->ifm_status |= IFM_ACTIVE;
1022
1023         if (adapter->hw.media_type == em_media_type_fiber) {
1024                 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1025         } else {
1026                 switch (adapter->link_speed) {
1027                 case 10:
1028                         ifmr->ifm_active |= IFM_10_T;
1029                         break;
1030                 case 100:
1031                         ifmr->ifm_active |= IFM_100_TX;
1032                         break;
1033                 case 1000:
1034                         ifmr->ifm_active |= IFM_1000_T;
1035                         break;
1036                 }
1037                 if (adapter->link_duplex == FULL_DUPLEX)
1038                         ifmr->ifm_active |= IFM_FDX;
1039                 else
1040                         ifmr->ifm_active |= IFM_HDX;
1041         }
1042 }
1043
1044 /*********************************************************************
1045  *
1046  *  Media Ioctl callback
1047  *
1048  *  This routine is called when the user changes speed/duplex using
1049  *  media/mediopt option with ifconfig.
1050  *
1051  **********************************************************************/
1052 static int
1053 em_media_change(struct ifnet *ifp)
1054 {
1055         struct adapter * adapter = ifp->if_softc;
1056         struct ifmedia  *ifm = &adapter->media;
1057
1058         INIT_DEBUGOUT("em_media_change: begin");
1059
1060         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1061                 return(EINVAL);
1062
1063         lwkt_serialize_enter(&adapter->serializer);
1064
1065         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1066         case IFM_AUTO:
1067                 adapter->hw.autoneg = DO_AUTO_NEG;
1068                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1069                 break;
1070         case IFM_1000_SX:
1071         case IFM_1000_T:
1072                 adapter->hw.autoneg = DO_AUTO_NEG;
1073                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1074                 break;
1075         case IFM_100_TX:
1076                 adapter->hw.autoneg = FALSE;
1077                 adapter->hw.autoneg_advertised = 0;
1078                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1079                         adapter->hw.forced_speed_duplex = em_100_full;
1080                 else
1081                         adapter->hw.forced_speed_duplex = em_100_half;
1082                 break;
1083         case IFM_10_T:
1084                 adapter->hw.autoneg = FALSE;
1085                 adapter->hw.autoneg_advertised = 0;
1086                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1087                         adapter->hw.forced_speed_duplex = em_10_full;
1088                 else
1089                         adapter->hw.forced_speed_duplex = em_10_half;
1090                 break;
1091         default:
1092                 if_printf(ifp, "Unsupported media type\n");
1093         }
1094         /*
1095          * As the speed/duplex settings may have changed we need to
1096          * reset the PHY.
1097          */
1098         adapter->hw.phy_reset_disable = FALSE;
1099
1100         em_init_serialized(adapter);
1101
1102         lwkt_serialize_exit(&adapter->serializer);
1103         return(0);
1104 }
1105
1106 static void
1107 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1108          int error)
1109 {
1110         struct em_q *q = arg;
1111
1112         if (error)
1113                 return;
1114         KASSERT(nsegs <= EM_MAX_SCATTER,
1115                 ("Too many DMA segments returned when mapping tx packet"));
1116         q->nsegs = nsegs;
1117         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1118 }
1119
1120 /*********************************************************************
1121  *
1122  *  This routine maps the mbufs to tx descriptors.
1123  *
1124  *  return 0 on success, positive on failure
1125  **********************************************************************/
1126 static int
1127 em_encap(struct adapter *adapter, struct mbuf *m_head)
1128 {
1129         uint32_t txd_upper;
1130         uint32_t txd_lower, txd_used = 0, txd_saved = 0;
1131         int i, j, error;
1132         uint64_t address;
1133
1134         /* For 82544 Workaround */
1135         DESC_ARRAY desc_array;
1136         uint32_t array_elements;
1137         uint32_t counter;
1138
1139         struct ifvlan *ifv = NULL;
1140         struct em_q q;
1141         struct em_buffer *tx_buffer = NULL;
1142         struct em_tx_desc *current_tx_desc = NULL;
1143         struct ifnet *ifp = &adapter->interface_data.ac_if;
1144
1145         /*
1146          * Force a cleanup if number of TX descriptors
1147          * available hits the threshold
1148          */
1149         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1150                 em_clean_transmit_interrupts(adapter);
1151                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1152                         adapter->no_tx_desc_avail1++;
1153                         return(ENOBUFS);
1154                 }
1155         }
1156         /*
1157          * Map the packet for DMA.
1158          */
1159         if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q.map)) {
1160                 adapter->no_tx_map_avail++;
1161                 return(ENOMEM);
1162         }
1163         error = bus_dmamap_load_mbuf(adapter->txtag, q.map, m_head, em_tx_cb,
1164                                      &q, BUS_DMA_NOWAIT);
1165         if (error != 0) {
1166                 adapter->no_tx_dma_setup++;
1167                 bus_dmamap_destroy(adapter->txtag, q.map);
1168                 return(error);
1169         }
1170         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1171
1172         if (q.nsegs > adapter->num_tx_desc_avail) {
1173                 adapter->no_tx_desc_avail2++;
1174                 bus_dmamap_unload(adapter->txtag, q.map);
1175                 bus_dmamap_destroy(adapter->txtag, q.map);
1176                 return(ENOBUFS);
1177         }
1178
1179         if (ifp->if_hwassist > 0) {
1180                 em_transmit_checksum_setup(adapter,  m_head,
1181                                            &txd_upper, &txd_lower);
1182         }
1183         else
1184                 txd_upper = txd_lower = 0;
1185
1186         /* Find out if we are in vlan mode */
1187         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1188             m_head->m_pkthdr.rcvif != NULL &&
1189             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1190                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1191
1192         i = adapter->next_avail_tx_desc;
1193         if (adapter->pcix_82544) {
1194                 txd_saved = i;
1195                 txd_used = 0;
1196         }
1197         for (j = 0; j < q.nsegs; j++) {
1198                 /* If adapter is 82544 and on PCIX bus */
1199                 if(adapter->pcix_82544) {
1200                         array_elements = 0;
1201                         address = htole64(q.segs[j].ds_addr);
1202                         /*
1203                          * Check the Address and Length combination and
1204                          * split the data accordingly
1205                          */
1206                         array_elements = em_fill_descriptors(address,
1207                                                              htole32(q.segs[j].ds_len),
1208                                                              &desc_array);
1209                         for (counter = 0; counter < array_elements; counter++) {
1210                                 if (txd_used == adapter->num_tx_desc_avail) {
1211                                         adapter->next_avail_tx_desc = txd_saved;
1212                                         adapter->no_tx_desc_avail2++;
1213                                         bus_dmamap_unload(adapter->txtag, q.map);
1214                                         bus_dmamap_destroy(adapter->txtag, q.map);
1215                                         return(ENOBUFS);
1216                                 }
1217                                 tx_buffer = &adapter->tx_buffer_area[i];
1218                                 current_tx_desc = &adapter->tx_desc_base[i];
1219                                 current_tx_desc->buffer_addr = htole64(
1220                                 desc_array.descriptor[counter].address);
1221                                 current_tx_desc->lower.data = htole32(
1222                                 (adapter->txd_cmd | txd_lower |
1223                                 (uint16_t)desc_array.descriptor[counter].length));
1224                                 current_tx_desc->upper.data = htole32((txd_upper));
1225                                 if (++i == adapter->num_tx_desc)
1226                                         i = 0;
1227
1228                                 tx_buffer->m_head = NULL;
1229                                 txd_used++;
1230                         }
1231                 } else {
1232                         tx_buffer = &adapter->tx_buffer_area[i];
1233                         current_tx_desc = &adapter->tx_desc_base[i];
1234
1235                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1236                         current_tx_desc->lower.data = htole32(
1237                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1238                         current_tx_desc->upper.data = htole32(txd_upper);
1239
1240                         if (++i == adapter->num_tx_desc)
1241                                 i = 0;
1242
1243                         tx_buffer->m_head = NULL;
1244                 }
1245         }
1246
1247         adapter->next_avail_tx_desc = i;
1248         if (adapter->pcix_82544)
1249                 adapter->num_tx_desc_avail -= txd_used;
1250         else
1251                 adapter->num_tx_desc_avail -= q.nsegs;
1252
1253         if (ifv != NULL) {
1254                 /* Set the vlan id */
1255                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1256
1257                 /* Tell hardware to add tag */
1258                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1259         }
1260
1261         tx_buffer->m_head = m_head;
1262         tx_buffer->map = q.map;
1263         bus_dmamap_sync(adapter->txtag, q.map, BUS_DMASYNC_PREWRITE);
1264
1265         /*
1266          * Last Descriptor of Packet needs End Of Packet (EOP)
1267          */
1268         current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1269
1270         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1271                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1272
1273         /*
1274          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1275          * that this frame is available to transmit.
1276          */
1277         if (adapter->hw.mac_type == em_82547 &&
1278             adapter->link_duplex == HALF_DUPLEX) {
1279                 em_82547_move_tail_serialized(adapter);
1280         } else {
1281                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1282                 if (adapter->hw.mac_type == em_82547) {
1283                         em_82547_update_fifo_head(adapter,
1284                                                   m_head->m_pkthdr.len);
1285                 }
1286         }
1287
1288         return(0);
1289 }
1290
1291 /*********************************************************************
1292  *
1293  * 82547 workaround to avoid controller hang in half-duplex environment.
1294  * The workaround is to avoid queuing a large packet that would span
1295  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1296  * in this case. We do that only when FIFO is quiescent.
1297  *
1298  **********************************************************************/
1299 static void
1300 em_82547_move_tail(void *arg)
1301 {
1302         struct adapter *adapter = arg;
1303
1304         lwkt_serialize_enter(&adapter->serializer);
1305         em_82547_move_tail_serialized(arg);
1306         lwkt_serialize_exit(&adapter->serializer);
1307 }
1308
1309 static void
1310 em_82547_move_tail_serialized(void *arg)
1311 {
1312         struct adapter *adapter = arg;
1313         uint16_t hw_tdt;
1314         uint16_t sw_tdt;
1315         struct em_tx_desc *tx_desc;
1316         uint16_t length = 0;
1317         boolean_t eop = 0;
1318
1319         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1320         sw_tdt = adapter->next_avail_tx_desc;
1321
1322         while (hw_tdt != sw_tdt) {
1323                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1324                 length += tx_desc->lower.flags.length;
1325                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1326                 if(++hw_tdt == adapter->num_tx_desc)
1327                         hw_tdt = 0;
1328
1329                 if(eop) {
1330                         if (em_82547_fifo_workaround(adapter, length)) {
1331                                 adapter->tx_fifo_wrk_cnt++;
1332                                 callout_reset(&adapter->tx_fifo_timer, 1,
1333                                         em_82547_move_tail, adapter);
1334                                 break;
1335                         }
1336                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1337                         em_82547_update_fifo_head(adapter, length);
1338                         length = 0;
1339                 }
1340         }
1341 }
1342
1343 static int
1344 em_82547_fifo_workaround(struct adapter *adapter, int len)
1345 {
1346         int fifo_space, fifo_pkt_len;
1347
1348         fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1349
1350         if (adapter->link_duplex == HALF_DUPLEX) {
1351                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1352
1353                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1354                         if (em_82547_tx_fifo_reset(adapter))
1355                                 return(0);
1356                         else
1357                                 return(1);
1358                 }
1359         }
1360
1361         return(0);
1362 }
1363
1364 static void
1365 em_82547_update_fifo_head(struct adapter *adapter, int len)
1366 {
1367         int fifo_pkt_len = EM_ROUNDUP(len + EM_FIFO_HDR, EM_FIFO_HDR);
1368
1369         /* tx_fifo_head is always 16 byte aligned */
1370         adapter->tx_fifo_head += fifo_pkt_len;
1371         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
1372                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1373 }
1374
1375 static int
1376 em_82547_tx_fifo_reset(struct adapter *adapter)
1377 {
1378         uint32_t tctl;
1379
1380         if ( (E1000_READ_REG(&adapter->hw, TDT) ==
1381               E1000_READ_REG(&adapter->hw, TDH)) &&
1382              (E1000_READ_REG(&adapter->hw, TDFT) ==
1383               E1000_READ_REG(&adapter->hw, TDFH)) &&
1384              (E1000_READ_REG(&adapter->hw, TDFTS) ==
1385               E1000_READ_REG(&adapter->hw, TDFHS)) &&
1386              (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1387
1388                 /* Disable TX unit */
1389                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1390                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1391
1392                 /* Reset FIFO pointers */
1393                 E1000_WRITE_REG(&adapter->hw, TDFT,  adapter->tx_head_addr);
1394                 E1000_WRITE_REG(&adapter->hw, TDFH,  adapter->tx_head_addr);
1395                 E1000_WRITE_REG(&adapter->hw, TDFTS, adapter->tx_head_addr);
1396                 E1000_WRITE_REG(&adapter->hw, TDFHS, adapter->tx_head_addr);
1397
1398                 /* Re-enable TX unit */
1399                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1400                 E1000_WRITE_FLUSH(&adapter->hw);
1401
1402                 adapter->tx_fifo_head = 0;
1403                 adapter->tx_fifo_reset_cnt++;
1404
1405                 return(TRUE);
1406         } else {
1407                 return(FALSE);
1408         }
1409 }
1410
1411 static void
1412 em_set_promisc(struct adapter *adapter)
1413 {
1414         uint32_t reg_rctl, ctrl;
1415         struct ifnet *ifp = &adapter->interface_data.ac_if;
1416
1417         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1418         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
1419
1420         if (ifp->if_flags & IFF_PROMISC) {
1421                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1422                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1423
1424                 /*
1425                  * Disable VLAN stripping in promiscous mode.
1426                  * This enables bridging of vlan tagged frames to occur
1427                  * and also allows vlan tags to be seen in tcpdump.
1428                  */
1429                 ctrl &= ~E1000_CTRL_VME;
1430                 E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
1431         } else if (ifp->if_flags & IFF_ALLMULTI) {
1432                 reg_rctl |= E1000_RCTL_MPE;
1433                 reg_rctl &= ~E1000_RCTL_UPE;
1434                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1435         }
1436 }
1437
1438 static void
1439 em_disable_promisc(struct adapter *adapter)
1440 {
1441         uint32_t reg_rctl;
1442
1443         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1444
1445         reg_rctl &=  (~E1000_RCTL_UPE);
1446         reg_rctl &=  (~E1000_RCTL_MPE);
1447         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1448
1449         em_enable_vlans(adapter);
1450 }
1451
1452 /*********************************************************************
1453  *  Multicast Update
1454  *
1455  *  This routine is called whenever multicast address list is updated.
1456  *
1457  **********************************************************************/
1458
1459 static void
1460 em_set_multi(struct adapter *adapter)
1461 {
1462         uint32_t reg_rctl = 0;
1463         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1464         struct ifmultiaddr *ifma;
1465         int mcnt = 0;
1466         struct ifnet *ifp = &adapter->interface_data.ac_if;
1467
1468         IOCTL_DEBUGOUT("em_set_multi: begin");
1469
1470         if (adapter->hw.mac_type == em_82542_rev2_0) {
1471                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1472                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1473                         em_pci_clear_mwi(&adapter->hw);
1474                 reg_rctl |= E1000_RCTL_RST;
1475                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1476                 msec_delay(5);
1477         }
1478
1479         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1480                 if (ifma->ifma_addr->sa_family != AF_LINK)
1481                         continue;
1482
1483                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1484                         break;
1485
1486                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1487                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1488                 mcnt++;
1489         }
1490
1491         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1492                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1493                 reg_rctl |= E1000_RCTL_MPE;
1494                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1495         } else
1496                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1497
1498         if (adapter->hw.mac_type == em_82542_rev2_0) {
1499                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1500                 reg_rctl &= ~E1000_RCTL_RST;
1501                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1502                 msec_delay(5);
1503                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1504                         em_pci_set_mwi(&adapter->hw);
1505         }
1506 }
1507
1508 /*********************************************************************
1509  *  Timer routine
1510  *
1511  *  This routine checks for link status and updates statistics.
1512  *
1513  **********************************************************************/
1514
1515 static void
1516 em_local_timer(void *arg)
1517 {
1518         struct ifnet *ifp;
1519         struct adapter *adapter = arg;
1520         ifp = &adapter->interface_data.ac_if;
1521
1522         lwkt_serialize_enter(&adapter->serializer);
1523
1524         em_check_for_link(&adapter->hw);
1525         em_print_link_status(adapter);
1526         em_update_stats_counters(adapter);
1527         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1528                 em_print_hw_stats(adapter);
1529         em_smartspeed(adapter);
1530
1531         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1532
1533         lwkt_serialize_exit(&adapter->serializer);
1534 }
1535
1536 static void
1537 em_print_link_status(struct adapter *adapter)
1538 {
1539         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1540                 if (adapter->link_active == 0) {
1541                         em_get_speed_and_duplex(&adapter->hw,
1542                                                 &adapter->link_speed,
1543                                                 &adapter->link_duplex);
1544                         device_printf(adapter->dev, "Link is up %d Mbps %s\n",
1545                                adapter->link_speed,
1546                                ((adapter->link_duplex == FULL_DUPLEX) ?
1547                                 "Full Duplex" : "Half Duplex"));
1548                         adapter->link_active = 1;
1549                         adapter->smartspeed = 0;
1550                 }
1551         } else {
1552                 if (adapter->link_active == 1) {
1553                         adapter->link_speed = 0;
1554                         adapter->link_duplex = 0;
1555                         device_printf(adapter->dev, "Link is Down\n");
1556                         adapter->link_active = 0;
1557                 }
1558         }
1559 }
1560
1561 /*********************************************************************
1562  *
1563  *  This routine disables all traffic on the adapter by issuing a
1564  *  global reset on the MAC and deallocates TX/RX buffers.
1565  *
1566  **********************************************************************/
1567
1568 static void
1569 em_stop(void *arg)
1570 {
1571         struct ifnet   *ifp;
1572         struct adapter * adapter = arg;
1573         ifp = &adapter->interface_data.ac_if;
1574
1575         INIT_DEBUGOUT("em_stop: begin");
1576         em_disable_intr(adapter);
1577         em_reset_hw(&adapter->hw);
1578         callout_stop(&adapter->timer);
1579         callout_stop(&adapter->tx_fifo_timer);
1580         em_free_transmit_structures(adapter);
1581         em_free_receive_structures(adapter);
1582
1583         /* Tell the stack that the interface is no longer active */
1584         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1585         ifp->if_timer = 0;
1586 }
1587
1588 /*********************************************************************
1589  *
1590  *  Determine hardware revision.
1591  *
1592  **********************************************************************/
1593 static void
1594 em_identify_hardware(struct adapter * adapter)
1595 {
1596         device_t dev = adapter->dev;
1597
1598         /* Make sure our PCI config space has the necessary stuff set */
1599         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1600         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1601               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1602                 device_printf(dev, "Memory Access and/or Bus Master bits were not set!\n");
1603                 adapter->hw.pci_cmd_word |=
1604                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1605                 pci_write_config(dev, PCIR_COMMAND, adapter->hw.pci_cmd_word, 2);
1606         }
1607
1608         /* Save off the information about this board */
1609         adapter->hw.vendor_id = pci_get_vendor(dev);
1610         adapter->hw.device_id = pci_get_device(dev);
1611         adapter->hw.revision_id = pci_get_revid(dev);
1612         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1613         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1614
1615         /* Identify the MAC */
1616         if (em_set_mac_type(&adapter->hw))
1617                 device_printf(dev, "Unknown MAC Type\n");
1618
1619         if (adapter->hw.mac_type == em_82541 ||
1620             adapter->hw.mac_type == em_82541_rev_2 ||
1621             adapter->hw.mac_type == em_82547 ||
1622             adapter->hw.mac_type == em_82547_rev_2)
1623                 adapter->hw.phy_init_script = TRUE;
1624 }
1625
1626 /*********************************************************************
1627  *
1628  *  Initialize the hardware to a configuration as specified by the
1629  *  adapter structure. The controller is reset, the EEPROM is
1630  *  verified, the MAC address is set, then the shared initialization
1631  *  routines are called.
1632  *
1633  **********************************************************************/
1634 static int
1635 em_hardware_init(struct adapter *adapter)
1636 {
1637         INIT_DEBUGOUT("em_hardware_init: begin");
1638         /* Issue a global reset */
1639         em_reset_hw(&adapter->hw);
1640
1641         /* When hardware is reset, fifo_head is also reset */
1642         adapter->tx_fifo_head = 0;
1643
1644         /* Make sure we have a good EEPROM before we read from it */
1645         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
1646                 device_printf(adapter->dev, "The EEPROM Checksum Is Not Valid\n");
1647                 return(EIO);
1648         }
1649
1650         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
1651                 device_printf(adapter->dev, "EEPROM read error while reading part number\n");
1652                 return(EIO);
1653         }
1654
1655         if (em_init_hw(&adapter->hw) < 0) {
1656                 device_printf(adapter->dev, "Hardware Initialization Failed");
1657                 return(EIO);
1658         }
1659
1660         em_check_for_link(&adapter->hw);
1661         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)
1662                 adapter->link_active = 1;
1663         else
1664                 adapter->link_active = 0;
1665
1666         if (adapter->link_active) {
1667                 em_get_speed_and_duplex(&adapter->hw,
1668                                         &adapter->link_speed,
1669                                         &adapter->link_duplex);
1670         } else {
1671                 adapter->link_speed = 0;
1672                 adapter->link_duplex = 0;
1673         }
1674
1675         return(0);
1676 }
1677
1678 /*********************************************************************
1679  *
1680  *  Setup networking device structure and register an interface.
1681  *
1682  **********************************************************************/
1683 static void
1684 em_setup_interface(device_t dev, struct adapter *adapter)
1685 {
1686         struct ifnet   *ifp;
1687         INIT_DEBUGOUT("em_setup_interface: begin");
1688
1689         ifp = &adapter->interface_data.ac_if;
1690         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1691         ifp->if_mtu = ETHERMTU;
1692         ifp->if_baudrate = 1000000000;
1693         ifp->if_init =  em_init;
1694         ifp->if_softc = adapter;
1695         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1696         ifp->if_ioctl = em_ioctl;
1697         ifp->if_start = em_start;
1698 #ifdef DEVICE_POLLING
1699         ifp->if_poll = em_poll;
1700 #endif
1701         ifp->if_watchdog = em_watchdog;
1702         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
1703         ifq_set_ready(&ifp->if_snd);
1704
1705         if (adapter->hw.mac_type >= em_82543)
1706                 ifp->if_capabilities |= IFCAP_HWCSUM;
1707
1708         ifp->if_capenable = ifp->if_capabilities;
1709
1710         ether_ifattach(ifp, adapter->hw.mac_addr);
1711
1712         /*
1713          * Tell the upper layer(s) we support long frames.
1714          */
1715         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
1716         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1717
1718         /*
1719          * Specify the media types supported by this adapter and register
1720          * callbacks to update media and link information
1721          */
1722         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
1723                      em_media_status);
1724         if (adapter->hw.media_type == em_media_type_fiber) {
1725                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX,
1726                             0, NULL);
1727                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX,
1728                             0, NULL);
1729         } else {
1730                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
1731                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
1732                             0, NULL);
1733                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
1734                             0, NULL);
1735                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
1736                             0, NULL);
1737                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
1738                             0, NULL);
1739                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1740         }
1741         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
1742         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
1743 }
1744
1745 /*********************************************************************
1746  *
1747  *  Workaround for SmartSpeed on 82541 and 82547 controllers
1748  *
1749  **********************************************************************/
1750 static void
1751 em_smartspeed(struct adapter *adapter)
1752 {
1753         uint16_t phy_tmp;
1754
1755         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
1756             !adapter->hw.autoneg ||
1757             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
1758                 return;
1759
1760         if (adapter->smartspeed == 0) {
1761                 /*
1762                  * If Master/Slave config fault is asserted twice,
1763                  * we assume back-to-back.
1764                  */
1765                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1766                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
1767                         return;
1768                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
1769                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
1770                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL,
1771                                         &phy_tmp);
1772                         if (phy_tmp & CR_1000T_MS_ENABLE) {
1773                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
1774                                 em_write_phy_reg(&adapter->hw,
1775                                                  PHY_1000T_CTRL, phy_tmp);
1776                                 adapter->smartspeed++;
1777                                 if (adapter->hw.autoneg &&
1778                                     !em_phy_setup_autoneg(&adapter->hw) &&
1779                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
1780                                                      &phy_tmp)) {
1781                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1782                                                     MII_CR_RESTART_AUTO_NEG);
1783                                         em_write_phy_reg(&adapter->hw,
1784                                                          PHY_CTRL, phy_tmp);
1785                                 }
1786                         }
1787                 }
1788                 return;
1789         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
1790                 /* If still no link, perhaps using 2/3 pair cable */
1791                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
1792                 phy_tmp |= CR_1000T_MS_ENABLE;
1793                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
1794                 if (adapter->hw.autoneg &&
1795                     !em_phy_setup_autoneg(&adapter->hw) &&
1796                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
1797                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
1798                                     MII_CR_RESTART_AUTO_NEG);
1799                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
1800                 }
1801         }
1802         /* Restart process after EM_SMARTSPEED_MAX iterations */
1803         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
1804                 adapter->smartspeed = 0;
1805 }
1806
1807 /*
1808  * Manage DMA'able memory.
1809  */
1810 static void
1811 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1812 {
1813         if (error)
1814                 return;
1815         *(bus_addr_t*) arg = segs->ds_addr;
1816 }
1817
1818 static int
1819 em_dma_malloc(struct adapter *adapter, bus_size_t size,
1820               struct em_dma_alloc *dma, int mapflags)
1821 {
1822         int r;
1823         device_t dev = adapter->dev;
1824
1825         r = bus_dma_tag_create(NULL,                    /* parent */
1826                                PAGE_SIZE, 0,            /* alignment, bounds */
1827                                BUS_SPACE_MAXADDR,       /* lowaddr */
1828                                BUS_SPACE_MAXADDR,       /* highaddr */
1829                                NULL, NULL,              /* filter, filterarg */
1830                                size,                    /* maxsize */
1831                                1,                       /* nsegments */
1832                                size,                    /* maxsegsize */
1833                                BUS_DMA_ALLOCNOW,        /* flags */
1834                                &dma->dma_tag);
1835         if (r != 0) {
1836                 device_printf(dev, "em_dma_malloc: bus_dma_tag_create failed; "
1837                               "error %u\n", r);
1838                 goto fail_0;
1839         }
1840
1841         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1842                              BUS_DMA_NOWAIT, &dma->dma_map);
1843         if (r != 0) {
1844                 device_printf(dev, "em_dma_malloc: bus_dmammem_alloc failed; "
1845                               "size %llu, error %d\n", (uintmax_t)size, r);
1846                 goto fail_2;
1847         }
1848
1849         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1850                             size,
1851                             em_dmamap_cb,
1852                             &dma->dma_paddr,
1853                             mapflags | BUS_DMA_NOWAIT);
1854         if (r != 0) {
1855                 device_printf(dev, "em_dma_malloc: bus_dmamap_load failed; "
1856                               "error %u\n", r);
1857                 goto fail_3;
1858         }
1859
1860         dma->dma_size = size;
1861         return(0);
1862
1863 fail_3:
1864         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1865 fail_2:
1866         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1867         bus_dma_tag_destroy(dma->dma_tag);
1868 fail_0:
1869         dma->dma_map = NULL;
1870         dma->dma_tag = NULL;
1871         return(r);
1872 }
1873
1874 static void
1875 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
1876 {
1877         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1878         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1879         bus_dma_tag_destroy(dma->dma_tag);
1880 }
1881
1882 /*********************************************************************
1883  *
1884  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
1885  *  the information needed to transmit a packet on the wire.
1886  *
1887  **********************************************************************/
1888 static int
1889 em_allocate_transmit_structures(struct adapter * adapter)
1890 {
1891         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
1892             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
1893         if (adapter->tx_buffer_area == NULL) {
1894                 device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
1895                 return(ENOMEM);
1896         }
1897
1898         return(0);
1899 }
1900
1901 /*********************************************************************
1902  *
1903  *  Allocate and initialize transmit structures.
1904  *
1905  **********************************************************************/
1906 static int
1907 em_setup_transmit_structures(struct adapter * adapter)
1908 {
1909         /*
1910          * Setup DMA descriptor areas.
1911          */
1912         if (bus_dma_tag_create(NULL,                    /* parent */
1913                                1, 0,                    /* alignment, bounds */
1914                                BUS_SPACE_MAXADDR,       /* lowaddr */
1915                                BUS_SPACE_MAXADDR,       /* highaddr */
1916                                NULL, NULL,              /* filter, filterarg */
1917                                MCLBYTES * 8,            /* maxsize */
1918                                EM_MAX_SCATTER,          /* nsegments */
1919                                MCLBYTES * 8,            /* maxsegsize */
1920                                BUS_DMA_ALLOCNOW,        /* flags */
1921                                &adapter->txtag)) {
1922                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
1923                 return(ENOMEM);
1924         }
1925
1926         if (em_allocate_transmit_structures(adapter))
1927                 return(ENOMEM);
1928
1929         bzero((void *) adapter->tx_desc_base,
1930               (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
1931
1932         adapter->next_avail_tx_desc = 0;
1933         adapter->oldest_used_tx_desc = 0;
1934
1935         /* Set number of descriptors available */
1936         adapter->num_tx_desc_avail = adapter->num_tx_desc;
1937
1938         /* Set checksum context */
1939         adapter->active_checksum_context = OFFLOAD_NONE;
1940
1941         return(0);
1942 }
1943
1944 /*********************************************************************
1945  *
1946  *  Enable transmit unit.
1947  *
1948  **********************************************************************/
1949 static void
1950 em_initialize_transmit_unit(struct adapter * adapter)
1951 {
1952         uint32_t reg_tctl;
1953         uint32_t reg_tipg = 0;
1954         uint64_t bus_addr;
1955
1956         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
1957
1958         /* Setup the Base and Length of the Tx Descriptor Ring */
1959         bus_addr = adapter->txdma.dma_paddr;
1960         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
1961         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
1962         E1000_WRITE_REG(&adapter->hw, TDLEN,
1963                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
1964
1965         /* Setup the HW Tx Head and Tail descriptor pointers */
1966         E1000_WRITE_REG(&adapter->hw, TDH, 0);
1967         E1000_WRITE_REG(&adapter->hw, TDT, 0);
1968
1969         HW_DEBUGOUT2("Base = %x, Length = %x\n",
1970                      E1000_READ_REG(&adapter->hw, TDBAL),
1971                      E1000_READ_REG(&adapter->hw, TDLEN));
1972
1973         /* Set the default values for the Tx Inter Packet Gap timer */
1974         switch (adapter->hw.mac_type) {
1975         case em_82542_rev2_0:
1976         case em_82542_rev2_1:
1977                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
1978                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1979                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1980                 break;
1981         default:
1982                 if (adapter->hw.media_type == em_media_type_fiber)
1983                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
1984                 else
1985                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
1986                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
1987                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
1988         }
1989
1990         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
1991         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
1992         if (adapter->hw.mac_type >= em_82540)
1993                 E1000_WRITE_REG(&adapter->hw, TADV,
1994                                 adapter->tx_abs_int_delay.value);
1995
1996         /* Program the Transmit Control Register */
1997         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
1998                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
1999         if (adapter->link_duplex == 1)
2000                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2001         else
2002                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2003         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
2004
2005         /* Setup Transmit Descriptor Settings for this adapter */
2006         adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
2007
2008         if (adapter->tx_int_delay.value > 0)
2009                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2010 }
2011
2012 /*********************************************************************
2013  *
2014  *  Free all transmit related data structures.
2015  *
2016  **********************************************************************/
2017 static void
2018 em_free_transmit_structures(struct adapter * adapter)
2019 {
2020         struct em_buffer *tx_buffer;
2021         int i;
2022
2023         INIT_DEBUGOUT("free_transmit_structures: begin");
2024
2025         if (adapter->tx_buffer_area != NULL) {
2026                 tx_buffer = adapter->tx_buffer_area;
2027                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2028                         if (tx_buffer->m_head != NULL) {
2029                                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2030                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2031                                 m_freem(tx_buffer->m_head);
2032                         }
2033                         tx_buffer->m_head = NULL;
2034                 }
2035         }
2036         if (adapter->tx_buffer_area != NULL) {
2037                 free(adapter->tx_buffer_area, M_DEVBUF);
2038                 adapter->tx_buffer_area = NULL;
2039         }
2040         if (adapter->txtag != NULL) {
2041                 bus_dma_tag_destroy(adapter->txtag);
2042                 adapter->txtag = NULL;
2043         }
2044 }
2045
2046 /*********************************************************************
2047  *
2048  *  The offload context needs to be set when we transfer the first
2049  *  packet of a particular protocol (TCP/UDP). We change the
2050  *  context only if the protocol type changes.
2051  *
2052  **********************************************************************/
2053 static void
2054 em_transmit_checksum_setup(struct adapter * adapter,
2055                            struct mbuf *mp,
2056                            uint32_t *txd_upper,
2057                            uint32_t *txd_lower)
2058 {
2059         struct em_context_desc *TXD;
2060         struct em_buffer *tx_buffer;
2061         int curr_txd;
2062
2063         if (mp->m_pkthdr.csum_flags) {
2064                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2065                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2066                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2067                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2068                                 return;
2069                         else
2070                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2071                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2072                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2073                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2074                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2075                                 return;
2076                         else
2077                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2078                 } else {
2079                         *txd_upper = 0;
2080                         *txd_lower = 0;
2081                         return;
2082                 }
2083         } else {
2084                 *txd_upper = 0;
2085                 *txd_lower = 0;
2086                 return;
2087         }
2088
2089         /* If we reach this point, the checksum offload context
2090          * needs to be reset.
2091          */
2092         curr_txd = adapter->next_avail_tx_desc;
2093         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2094         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2095
2096         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2097         TXD->lower_setup.ip_fields.ipcso =
2098             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2099         TXD->lower_setup.ip_fields.ipcse =
2100             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2101
2102         TXD->upper_setup.tcp_fields.tucss =
2103             ETHER_HDR_LEN + sizeof(struct ip);
2104         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2105
2106         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2107                 TXD->upper_setup.tcp_fields.tucso =
2108                     ETHER_HDR_LEN + sizeof(struct ip) +
2109                     offsetof(struct tcphdr, th_sum);
2110         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2111                 TXD->upper_setup.tcp_fields.tucso =
2112                         ETHER_HDR_LEN + sizeof(struct ip) +
2113                         offsetof(struct udphdr, uh_sum);
2114         }
2115
2116         TXD->tcp_seg_setup.data = htole32(0);
2117         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2118
2119         tx_buffer->m_head = NULL;
2120
2121         if (++curr_txd == adapter->num_tx_desc)
2122                 curr_txd = 0;
2123
2124         adapter->num_tx_desc_avail--;
2125         adapter->next_avail_tx_desc = curr_txd;
2126 }
2127
2128 /**********************************************************************
2129  *
2130  *  Examine each tx_buffer in the used queue. If the hardware is done
2131  *  processing the packet then free associated resources. The
2132  *  tx_buffer is put back on the free queue.
2133  *
2134  **********************************************************************/
2135
2136 static void
2137 em_clean_transmit_interrupts(struct adapter *adapter)
2138 {
2139         int i, num_avail;
2140         struct em_buffer *tx_buffer;
2141         struct em_tx_desc *tx_desc;
2142         struct ifnet *ifp = &adapter->interface_data.ac_if;
2143
2144         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2145                 return;
2146
2147 #ifdef DBG_STATS
2148         adapter->clean_tx_interrupts++;
2149 #endif
2150         num_avail = adapter->num_tx_desc_avail;
2151         i = adapter->oldest_used_tx_desc;
2152
2153         tx_buffer = &adapter->tx_buffer_area[i];
2154         tx_desc = &adapter->tx_desc_base[i];
2155
2156         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2157                         BUS_DMASYNC_POSTREAD);
2158
2159         while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2160                 tx_desc->upper.data = 0;
2161                 num_avail++;
2162
2163                 if (tx_buffer->m_head) {
2164                         ifp->if_opackets++;
2165                         bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2166                                         BUS_DMASYNC_POSTWRITE);
2167                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2168                         bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2169
2170                         m_freem(tx_buffer->m_head);
2171                         tx_buffer->m_head = NULL;
2172                 }
2173
2174                 if (++i == adapter->num_tx_desc)
2175                         i = 0;
2176
2177                 tx_buffer = &adapter->tx_buffer_area[i];
2178                 tx_desc = &adapter->tx_desc_base[i];
2179         }
2180
2181         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2182                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2183
2184         adapter->oldest_used_tx_desc = i;
2185
2186         /*
2187          * If we have enough room, clear IFF_OACTIVE to tell the stack
2188          * that it is OK to send packets.
2189          * If there are no pending descriptors, clear the timeout. Otherwise,
2190          * if some descriptors have been freed, restart the timeout.
2191          */
2192         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2193                 ifp->if_flags &= ~IFF_OACTIVE;
2194                 if (num_avail == adapter->num_tx_desc)
2195                         ifp->if_timer = 0;
2196                 else if (num_avail == adapter->num_tx_desc_avail)
2197                         ifp->if_timer = EM_TX_TIMEOUT;
2198         }
2199         adapter->num_tx_desc_avail = num_avail;
2200 }
2201
2202 /*********************************************************************
2203  *
2204  *  Get a buffer from system mbuf buffer pool.
2205  *
2206  **********************************************************************/
2207 static int
2208 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp, int how)
2209 {
2210         struct mbuf *mp = nmp;
2211         struct em_buffer *rx_buffer;
2212         struct ifnet *ifp;
2213         bus_addr_t paddr;
2214         int error;
2215
2216         ifp = &adapter->interface_data.ac_if;
2217
2218         if (mp == NULL) {
2219                 mp = m_getcl(how, MT_DATA, M_PKTHDR);
2220                 if (mp == NULL) {
2221                         adapter->mbuf_cluster_failed++;
2222                         return(ENOBUFS);
2223                 }
2224                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2225         } else {
2226                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2227                 mp->m_data = mp->m_ext.ext_buf;
2228                 mp->m_next = NULL;
2229         }
2230         if (ifp->if_mtu <= ETHERMTU)
2231                 m_adj(mp, ETHER_ALIGN);
2232
2233         rx_buffer = &adapter->rx_buffer_area[i];
2234
2235         /*
2236          * Using memory from the mbuf cluster pool, invoke the
2237          * bus_dma machinery to arrange the memory mapping.
2238          */
2239         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2240                                 mtod(mp, void *), mp->m_len,
2241                                 em_dmamap_cb, &paddr, 0);
2242         if (error) {
2243                 m_free(mp);
2244                 return(error);
2245         }
2246         rx_buffer->m_head = mp;
2247         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2248         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2249
2250         return(0);
2251 }
2252
2253 /*********************************************************************
2254  *
2255  *  Allocate memory for rx_buffer structures. Since we use one
2256  *  rx_buffer per received packet, the maximum number of rx_buffer's
2257  *  that we'll need is equal to the number of receive descriptors
2258  *  that we've allocated.
2259  *
2260  **********************************************************************/
2261 static int
2262 em_allocate_receive_structures(struct adapter *adapter)
2263 {
2264         int i, error, size;
2265         struct em_buffer *rx_buffer;
2266
2267         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2268         adapter->rx_buffer_area = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2269
2270         error = bus_dma_tag_create(NULL,                /* parent */
2271                                    1, 0,                /* alignment, bounds */
2272                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2273                                    BUS_SPACE_MAXADDR,   /* highaddr */
2274                                    NULL, NULL,          /* filter, filterarg */
2275                                    MCLBYTES,            /* maxsize */
2276                                    1,                   /* nsegments */
2277                                    MCLBYTES,            /* maxsegsize */
2278                                    BUS_DMA_ALLOCNOW,    /* flags */
2279                                    &adapter->rxtag);
2280         if (error != 0) {
2281                 device_printf(adapter->dev, "em_allocate_receive_structures: "
2282                               "bus_dma_tag_create failed; error %u\n", error);
2283                 goto fail_0;
2284         }
2285
2286         rx_buffer = adapter->rx_buffer_area;
2287         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2288                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2289                                           &rx_buffer->map);
2290                 if (error != 0) {
2291                         device_printf(adapter->dev,
2292                                       "em_allocate_receive_structures: "
2293                                       "bus_dmamap_create failed; error %u\n",
2294                                       error);
2295                         goto fail_1;
2296                 }
2297         }
2298
2299         for (i = 0; i < adapter->num_rx_desc; i++) {
2300                 error = em_get_buf(i, adapter, NULL, MB_WAIT);
2301                 if (error != 0) {
2302                         adapter->rx_buffer_area[i].m_head = NULL;
2303                         adapter->rx_desc_base[i].buffer_addr = 0;
2304                         return(error);
2305                 }
2306         }
2307
2308         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2309                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2310
2311         return(0);
2312
2313 fail_1:
2314         bus_dma_tag_destroy(adapter->rxtag);
2315 fail_0:
2316         adapter->rxtag = NULL;
2317         free(adapter->rx_buffer_area, M_DEVBUF);
2318         adapter->rx_buffer_area = NULL;
2319         return(error);
2320 }
2321
2322 /*********************************************************************
2323  *
2324  *  Allocate and initialize receive structures.
2325  *
2326  **********************************************************************/
2327 static int
2328 em_setup_receive_structures(struct adapter *adapter)
2329 {
2330         bzero((void *) adapter->rx_desc_base,
2331               (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2332
2333         if (em_allocate_receive_structures(adapter))
2334                 return(ENOMEM);
2335
2336         /* Setup our descriptor pointers */
2337         adapter->next_rx_desc_to_check = 0;
2338         return(0);
2339 }
2340
2341 /*********************************************************************
2342  *
2343  *  Enable receive unit.
2344  *
2345  **********************************************************************/
2346 static void
2347 em_initialize_receive_unit(struct adapter *adapter)
2348 {
2349         uint32_t reg_rctl;
2350         uint32_t reg_rxcsum;
2351         struct ifnet *ifp;
2352         uint64_t bus_addr;
2353
2354         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2355
2356         ifp = &adapter->interface_data.ac_if;
2357
2358         /* Make sure receives are disabled while setting up the descriptor ring */
2359         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2360
2361         /* Set the Receive Delay Timer Register */
2362         E1000_WRITE_REG(&adapter->hw, RDTR,
2363                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2364
2365         if(adapter->hw.mac_type >= em_82540) {
2366                 E1000_WRITE_REG(&adapter->hw, RADV,
2367                                 adapter->rx_abs_int_delay.value);
2368
2369                 /* Set the interrupt throttling rate in 256ns increments */
2370                 if (em_int_throttle_ceil) {
2371                         E1000_WRITE_REG(&adapter->hw, ITR,
2372                                 1000000000 / 256 / em_int_throttle_ceil);
2373                 } else {
2374                         E1000_WRITE_REG(&adapter->hw, ITR, 0);
2375                 }
2376         }
2377
2378         /* Setup the Base and Length of the Rx Descriptor Ring */
2379         bus_addr = adapter->rxdma.dma_paddr;
2380         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2381         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2382         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2383                         sizeof(struct em_rx_desc));
2384
2385         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2386         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2387         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2388
2389         /* Setup the Receive Control Register */
2390         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2391                    E1000_RCTL_RDMTS_HALF |
2392                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2393
2394         if (adapter->hw.tbi_compatibility_on == TRUE)
2395                 reg_rctl |= E1000_RCTL_SBP;
2396
2397         switch (adapter->rx_buffer_len) {
2398         default:
2399         case EM_RXBUFFER_2048:
2400                 reg_rctl |= E1000_RCTL_SZ_2048;
2401                 break;
2402         case EM_RXBUFFER_4096:
2403                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2404                 break;
2405         case EM_RXBUFFER_8192:
2406                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2407                 break;
2408         case EM_RXBUFFER_16384:
2409                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
2410                 break;
2411         }
2412
2413         if (ifp->if_mtu > ETHERMTU)
2414                 reg_rctl |= E1000_RCTL_LPE;
2415
2416         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2417         if ((adapter->hw.mac_type >= em_82543) &&
2418             (ifp->if_capenable & IFCAP_RXCSUM)) {
2419                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2420                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2421                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2422         }
2423
2424         /* Enable Receives */
2425         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2426 }
2427
2428 /*********************************************************************
2429  *
2430  *  Free receive related data structures.
2431  *
2432  **********************************************************************/
2433 static void
2434 em_free_receive_structures(struct adapter *adapter)
2435 {
2436         struct em_buffer *rx_buffer;
2437         int i;
2438
2439         INIT_DEBUGOUT("free_receive_structures: begin");
2440
2441         if (adapter->rx_buffer_area != NULL) {
2442                 rx_buffer = adapter->rx_buffer_area;
2443                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2444                         if (rx_buffer->map != NULL) {
2445                                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2446                                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
2447                         }
2448                         if (rx_buffer->m_head != NULL)
2449                                 m_freem(rx_buffer->m_head);
2450                         rx_buffer->m_head = NULL;
2451                 }
2452         }
2453         if (adapter->rx_buffer_area != NULL) {
2454                 free(adapter->rx_buffer_area, M_DEVBUF);
2455                 adapter->rx_buffer_area = NULL;
2456         }
2457         if (adapter->rxtag != NULL) {
2458                 bus_dma_tag_destroy(adapter->rxtag);
2459                 adapter->rxtag = NULL;
2460         }
2461 }
2462
2463 /*********************************************************************
2464  *
2465  *  This routine executes in interrupt context. It replenishes
2466  *  the mbufs in the descriptor and sends data which has been
2467  *  dma'ed into host memory to upper layer.
2468  *
2469  *  We loop at most count times if count is > 0, or until done if
2470  *  count < 0.
2471  *
2472  *********************************************************************/
2473 static void
2474 em_process_receive_interrupts(struct adapter *adapter, int count)
2475 {
2476         struct ifnet *ifp;
2477         struct mbuf *mp;
2478         uint8_t accept_frame = 0;
2479         uint8_t eop = 0;
2480         uint16_t len, desc_len, prev_len_adj;
2481         int i;
2482
2483         /* Pointer to the receive descriptor being examined. */
2484         struct em_rx_desc *current_desc;
2485
2486         ifp = &adapter->interface_data.ac_if;
2487         i = adapter->next_rx_desc_to_check;
2488         current_desc = &adapter->rx_desc_base[i];
2489
2490         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2491                         BUS_DMASYNC_POSTREAD);
2492
2493         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
2494 #ifdef DBG_STATS
2495                 adapter->no_pkts_avail++;
2496 #endif
2497                 return;
2498         }
2499         while ((current_desc->status & E1000_RXD_STAT_DD) && (count != 0)) {
2500                 mp = adapter->rx_buffer_area[i].m_head;
2501                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2502                                 BUS_DMASYNC_POSTREAD);
2503
2504                 accept_frame = 1;
2505                 prev_len_adj = 0;
2506                 desc_len = le16toh(current_desc->length);
2507                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2508                         count--;
2509                         eop = 1;
2510                         if (desc_len < ETHER_CRC_LEN) {
2511                                 len = 0;
2512                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2513                         }
2514                         else {
2515                                 len = desc_len - ETHER_CRC_LEN;
2516                         }
2517                 } else {
2518                         eop = 0;
2519                         len = desc_len;
2520                 }
2521
2522                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2523                         uint8_t last_byte;
2524                         uint32_t pkt_len = desc_len;
2525
2526                         if (adapter->fmp != NULL)
2527                                 pkt_len += adapter->fmp->m_pkthdr.len;
2528
2529                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
2530
2531                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
2532                                        current_desc->errors,
2533                                        pkt_len, last_byte)) {
2534                                 em_tbi_adjust_stats(&adapter->hw,
2535                                                     &adapter->stats,
2536                                                     pkt_len,
2537                                                     adapter->hw.mac_addr);
2538                                 if (len > 0)
2539                                         len--;
2540                         }
2541                         else {
2542                                 accept_frame = 0;
2543                         }
2544                 }
2545
2546                 if (accept_frame) {
2547                         if (em_get_buf(i, adapter, NULL, MB_DONTWAIT) == ENOBUFS) {
2548                                 adapter->dropped_pkts++;
2549                                 em_get_buf(i, adapter, mp, MB_DONTWAIT);
2550                                 if (adapter->fmp != NULL)
2551                                         m_freem(adapter->fmp);
2552                                 adapter->fmp = NULL;
2553                                 adapter->lmp = NULL;
2554                                 break;
2555                         }
2556
2557                         /* Assign correct length to the current fragment */
2558                         mp->m_len = len;
2559
2560                         if (adapter->fmp == NULL) {
2561                                 mp->m_pkthdr.len = len;
2562                                 adapter->fmp = mp;       /* Store the first mbuf */
2563                                 adapter->lmp = mp;
2564                         } else {
2565                                 /* Chain mbuf's together */
2566                                 /*
2567                                  * Adjust length of previous mbuf in chain if we
2568                                  * received less than 4 bytes in the last descriptor.
2569                                  */
2570                                 if (prev_len_adj > 0) {
2571                                         adapter->lmp->m_len -= prev_len_adj;
2572                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
2573                                 }
2574                                 adapter->lmp->m_next = mp;
2575                                 adapter->lmp = adapter->lmp->m_next;
2576                                 adapter->fmp->m_pkthdr.len += len;
2577                         }
2578
2579                         if (eop) {
2580                                 adapter->fmp->m_pkthdr.rcvif = ifp;
2581                                 ifp->if_ipackets++;
2582
2583                                 em_receive_checksum(adapter, current_desc,
2584                                                     adapter->fmp);
2585                                 if (current_desc->status & E1000_RXD_STAT_VP)
2586                                         VLAN_INPUT_TAG(adapter->fmp,
2587                                                        (current_desc->special &
2588                                                         E1000_RXD_SPC_VLAN_MASK));
2589                                 else
2590                                         (*ifp->if_input)(ifp, adapter->fmp);
2591                                 adapter->fmp = NULL;
2592                                 adapter->lmp = NULL;
2593                         }
2594                 } else {
2595                         adapter->dropped_pkts++;
2596                         em_get_buf(i, adapter, mp, MB_DONTWAIT);
2597                         if (adapter->fmp != NULL)
2598                                 m_freem(adapter->fmp);
2599                         adapter->fmp = NULL;
2600                         adapter->lmp = NULL;
2601                 }
2602
2603                 /* Zero out the receive descriptors status  */
2604                 current_desc->status = 0;
2605
2606                 /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
2607                 E1000_WRITE_REG(&adapter->hw, RDT, i);
2608
2609                 /* Advance our pointers to the next descriptor */
2610                 if (++i == adapter->num_rx_desc) {
2611                         i = 0;
2612                         current_desc = adapter->rx_desc_base;
2613                 } else
2614                         current_desc++;
2615         }
2616
2617         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2618                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2619
2620         adapter->next_rx_desc_to_check = i;
2621 }
2622
2623 /*********************************************************************
2624  *
2625  *  Verify that the hardware indicated that the checksum is valid.
2626  *  Inform the stack about the status of checksum so that stack
2627  *  doesn't spend time verifying the checksum.
2628  *
2629  *********************************************************************/
2630 static void
2631 em_receive_checksum(struct adapter *adapter,
2632                     struct em_rx_desc *rx_desc,
2633                     struct mbuf *mp)
2634 {
2635         /* 82543 or newer only */
2636         if ((adapter->hw.mac_type < em_82543) ||
2637             /* Ignore Checksum bit is set */
2638             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
2639                 mp->m_pkthdr.csum_flags = 0;
2640                 return;
2641         }
2642
2643         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
2644                 /* Did it pass? */
2645                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
2646                         /* IP Checksum Good */
2647                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2648                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2649                 } else {
2650                         mp->m_pkthdr.csum_flags = 0;
2651                 }
2652         }
2653
2654         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
2655                 /* Did it pass? */
2656                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
2657                         mp->m_pkthdr.csum_flags |=
2658                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2659                         mp->m_pkthdr.csum_data = htons(0xffff);
2660                 }
2661         }
2662 }
2663
2664
2665 static void
2666 em_enable_vlans(struct adapter *adapter)
2667 {
2668         uint32_t ctrl;
2669
2670         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
2671
2672         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
2673         ctrl |= E1000_CTRL_VME;
2674         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
2675 }
2676
2677 /*
2678  * note: we must call bus_enable_intr() prior to enabling the hardware
2679  * interrupt and bus_disable_intr() after disabling the hardware interrupt
2680  * in order to avoid handler execution races from scheduled interrupt
2681  * threads.
2682  */
2683 static void
2684 em_enable_intr(struct adapter *adapter)
2685 {
2686         struct ifnet *ifp = &adapter->interface_data.ac_if;
2687
2688         if ((ifp->if_flags & IFF_POLLING) == 0) {
2689                 lwkt_serialize_handler_enable(&adapter->serializer);
2690                 E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
2691         }
2692 }
2693
2694 static void
2695 em_disable_intr(struct adapter *adapter)
2696 {
2697         /*
2698          * The first version of 82542 had an errata where when link was
2699          * forced it would stay up even up even if the cable was disconnected.
2700          * Sequence errors were used to detect the disconnect and then the
2701          * driver would unforce the link.  This code in the in the ISR.  For
2702          * this to work correctly the Sequence error interrupt had to be
2703          * enabled all the time.
2704          */
2705         if (adapter->hw.mac_type == em_82542_rev2_0) {
2706                 E1000_WRITE_REG(&adapter->hw, IMC,
2707                                 (0xffffffff & ~E1000_IMC_RXSEQ));
2708         } else {
2709                 E1000_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
2710         }
2711
2712         lwkt_serialize_handler_disable(&adapter->serializer);
2713 }
2714
2715 static int
2716 em_is_valid_ether_addr(uint8_t *addr)
2717 {
2718         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
2719
2720         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN)))
2721                 return(FALSE);
2722         else
2723                 return(TRUE);
2724 }
2725
2726 void
2727 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2728 {
2729         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
2730 }
2731
2732 void
2733 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
2734 {
2735         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
2736 }
2737
2738 void
2739 em_pci_set_mwi(struct em_hw *hw)
2740 {
2741         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2742                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
2743 }
2744
2745 void
2746 em_pci_clear_mwi(struct em_hw *hw)
2747 {
2748         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
2749                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
2750 }
2751
2752 uint32_t
2753 em_read_reg_io(struct em_hw *hw, uint32_t offset)
2754 {
2755         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2756         return(bus_space_read_4(hw->reg_io_tag, hw->reg_io_handle, 4));
2757 }
2758
2759 void
2760 em_write_reg_io(struct em_hw *hw, uint32_t offset, uint32_t value)
2761 {
2762         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 0, offset);
2763         bus_space_write_4(hw->reg_io_tag, hw->reg_io_handle, 4, value);
2764 }
2765
2766 /*********************************************************************
2767  * 82544 Coexistence issue workaround.
2768  *    There are 2 issues.
2769  *      1. Transmit Hang issue.
2770  *    To detect this issue, following equation can be used...
2771  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2772  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
2773  *
2774  *      2. DAC issue.
2775  *    To detect this issue, following equation can be used...
2776  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
2777  *          If SUM[3:0] is in between 9 to c, we will have this issue.
2778  *
2779  *
2780  *    WORKAROUND:
2781  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
2782  *          9,a,b,c (DAC)
2783  *
2784 *************************************************************************/
2785 static uint32_t
2786 em_fill_descriptors(uint64_t address, uint32_t length, PDESC_ARRAY desc_array)
2787 {
2788         /* Since issue is sensitive to length and address.*/
2789         /* Let us first check the address...*/
2790         uint32_t safe_terminator;
2791         if (length <= 4) {
2792                 desc_array->descriptor[0].address = address;
2793                 desc_array->descriptor[0].length = length;
2794                 desc_array->elements = 1;
2795                 return(desc_array->elements);
2796         }
2797         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
2798         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
2799         if (safe_terminator == 0 ||
2800             (safe_terminator > 4 && safe_terminator < 9) ||
2801             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
2802                 desc_array->descriptor[0].address = address;
2803                 desc_array->descriptor[0].length = length;
2804                 desc_array->elements = 1;
2805                 return(desc_array->elements);
2806         }
2807
2808         desc_array->descriptor[0].address = address;
2809         desc_array->descriptor[0].length = length - 4;
2810         desc_array->descriptor[1].address = address + (length - 4);
2811         desc_array->descriptor[1].length = 4;
2812         desc_array->elements = 2;
2813         return(desc_array->elements);
2814 }
2815
2816 /**********************************************************************
2817  *
2818  *  Update the board statistics counters.
2819  *
2820  **********************************************************************/
2821 static void
2822 em_update_stats_counters(struct adapter *adapter)
2823 {
2824         struct ifnet   *ifp;
2825
2826         if (adapter->hw.media_type == em_media_type_copper ||
2827             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
2828                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
2829                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
2830         }
2831         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
2832         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
2833         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
2834         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
2835
2836         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
2837         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
2838         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
2839         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
2840         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
2841         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
2842         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
2843         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
2844         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
2845         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
2846         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
2847         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
2848         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
2849         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
2850         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
2851         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
2852         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
2853         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
2854         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
2855         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
2856
2857         /* For the 64-bit byte counters the low dword must be read first. */
2858         /* Both registers clear on the read of the high dword */
2859
2860         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
2861         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
2862         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
2863         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
2864
2865         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
2866         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
2867         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
2868         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
2869         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
2870
2871         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
2872         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
2873         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
2874         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
2875
2876         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
2877         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
2878         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
2879         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
2880         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
2881         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
2882         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
2883         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
2884         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
2885         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
2886
2887         if (adapter->hw.mac_type >= em_82543) {
2888                 adapter->stats.algnerrc +=
2889                     E1000_READ_REG(&adapter->hw, ALGNERRC);
2890                 adapter->stats.rxerrc +=
2891                     E1000_READ_REG(&adapter->hw, RXERRC);
2892                 adapter->stats.tncrs +=
2893                     E1000_READ_REG(&adapter->hw, TNCRS);
2894                 adapter->stats.cexterr +=
2895                     E1000_READ_REG(&adapter->hw, CEXTERR);
2896                 adapter->stats.tsctc +=
2897                     E1000_READ_REG(&adapter->hw, TSCTC);
2898                 adapter->stats.tsctfc +=
2899                     E1000_READ_REG(&adapter->hw, TSCTFC);
2900         }
2901         ifp = &adapter->interface_data.ac_if;
2902
2903         /* Fill out the OS statistics structure */
2904         ifp->if_ibytes = adapter->stats.gorcl;
2905         ifp->if_obytes = adapter->stats.gotcl;
2906         ifp->if_imcasts = adapter->stats.mprc;
2907         ifp->if_collisions = adapter->stats.colc;
2908
2909         /* Rx Errors */
2910         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
2911             adapter->stats.crcerrs + adapter->stats.algnerrc +
2912             adapter->stats.rlec + adapter->stats.mpc + adapter->stats.cexterr;
2913
2914         /* Tx Errors */
2915         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol;
2916 }
2917
2918
2919 /**********************************************************************
2920  *
2921  *  This routine is called only when em_display_debug_stats is enabled.
2922  *  This routine provides a way to take a look at important statistics
2923  *  maintained by the driver and hardware.
2924  *
2925  **********************************************************************/
2926 static void
2927 em_print_debug_info(struct adapter *adapter)
2928 {
2929         device_t dev= adapter->dev;
2930         uint8_t *hw_addr = adapter->hw.hw_addr;
2931
2932         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
2933         device_printf(dev, "CTRL  = 0x%x\n",
2934                       E1000_READ_REG(&adapter->hw, CTRL));
2935         device_printf(dev, "RCTL  = 0x%x PS=(0x8402)\n",
2936                       E1000_READ_REG(&adapter->hw, RCTL));
2937         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
2938                       E1000_READ_REG(&adapter->hw, TIDV),
2939                       E1000_READ_REG(&adapter->hw, TADV));
2940         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
2941                       E1000_READ_REG(&adapter->hw, RDTR),
2942                       E1000_READ_REG(&adapter->hw, RADV));
2943 #ifdef DBG_STATS
2944         device_printf(dev, "Packets not Avail = %ld\n", adapter->no_pkts_avail);
2945         device_printf(dev, "CleanTxInterrupts = %ld\n",
2946                       adapter->clean_tx_interrupts);
2947 #endif
2948         device_printf(dev, "fifo workaround = %lld, fifo_reset = %lld\n",
2949                       (long long)adapter->tx_fifo_wrk_cnt,
2950                       (long long)adapter->tx_fifo_reset_cnt);
2951         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
2952                       E1000_READ_REG(&adapter->hw, TDH),
2953                       E1000_READ_REG(&adapter->hw, TDT));
2954         device_printf(dev, "Num Tx descriptors avail = %d\n",
2955                       adapter->num_tx_desc_avail);
2956         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
2957                       adapter->no_tx_desc_avail1);
2958         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
2959                       adapter->no_tx_desc_avail2);
2960         device_printf(dev, "Std mbuf failed = %ld\n",
2961                       adapter->mbuf_alloc_failed);
2962         device_printf(dev, "Std mbuf cluster failed = %ld\n",
2963                       adapter->mbuf_cluster_failed);
2964         device_printf(dev, "Driver dropped packets = %ld\n",
2965                       adapter->dropped_pkts);
2966 }
2967
2968 static void
2969 em_print_hw_stats(struct adapter *adapter)
2970 {
2971         device_t dev= adapter->dev;
2972
2973         device_printf(dev, "Adapter: %p\n", adapter);
2974
2975         device_printf(dev, "Excessive collisions = %lld\n",
2976                       (long long)adapter->stats.ecol);
2977         device_printf(dev, "Symbol errors = %lld\n",
2978                       (long long)adapter->stats.symerrs);
2979         device_printf(dev, "Sequence errors = %lld\n",
2980                       (long long)adapter->stats.sec);
2981         device_printf(dev, "Defer count = %lld\n",
2982                       (long long)adapter->stats.dc);
2983
2984         device_printf(dev, "Missed Packets = %lld\n",
2985                       (long long)adapter->stats.mpc);
2986         device_printf(dev, "Receive No Buffers = %lld\n",
2987                       (long long)adapter->stats.rnbc);
2988         device_printf(dev, "Receive length errors = %lld\n",
2989                       (long long)adapter->stats.rlec);
2990         device_printf(dev, "Receive errors = %lld\n",
2991                       (long long)adapter->stats.rxerrc);
2992         device_printf(dev, "Crc errors = %lld\n",
2993                       (long long)adapter->stats.crcerrs);
2994         device_printf(dev, "Alignment errors = %lld\n",
2995                       (long long)adapter->stats.algnerrc);
2996         device_printf(dev, "Carrier extension errors = %lld\n",
2997                       (long long)adapter->stats.cexterr);
2998
2999         device_printf(dev, "XON Rcvd = %lld\n",
3000                       (long long)adapter->stats.xonrxc);
3001         device_printf(dev, "XON Xmtd = %lld\n",
3002                       (long long)adapter->stats.xontxc);
3003         device_printf(dev, "XOFF Rcvd = %lld\n",
3004                       (long long)adapter->stats.xoffrxc);
3005         device_printf(dev, "XOFF Xmtd = %lld\n",
3006                       (long long)adapter->stats.xofftxc);
3007
3008         device_printf(dev, "Good Packets Rcvd = %lld\n",
3009                       (long long)adapter->stats.gprc);
3010         device_printf(dev, "Good Packets Xmtd = %lld\n",
3011                       (long long)adapter->stats.gptc);
3012 }
3013
3014 static int
3015 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
3016 {
3017         int error;
3018         int result;
3019         struct adapter *adapter;
3020
3021         result = -1;
3022         error = sysctl_handle_int(oidp, &result, 0, req);
3023
3024         if (error || !req->newptr)
3025                 return(error);
3026
3027         if (result == 1) {
3028                 adapter = (struct adapter *)arg1;
3029                 em_print_debug_info(adapter);
3030         }
3031
3032         return(error);
3033 }
3034
3035 static int
3036 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
3037 {
3038         int error;
3039         int result;
3040         struct adapter *adapter;
3041
3042         result = -1;
3043         error = sysctl_handle_int(oidp, &result, 0, req);
3044
3045         if (error || !req->newptr)
3046                 return(error);
3047
3048         if (result == 1) {
3049                 adapter = (struct adapter *)arg1;
3050                 em_print_hw_stats(adapter);
3051         }
3052
3053         return(error);
3054 }
3055
3056 static int
3057 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3058 {
3059         struct em_int_delay_info *info;
3060         struct adapter *adapter;
3061         uint32_t regval;
3062         int error;
3063         int usecs;
3064         int ticks;
3065
3066         info = (struct em_int_delay_info *)arg1;
3067         adapter = info->adapter;
3068         usecs = info->value;
3069         error = sysctl_handle_int(oidp, &usecs, 0, req);
3070         if (error != 0 || req->newptr == NULL)
3071                 return(error);
3072         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3073                 return(EINVAL);
3074         info->value = usecs;
3075         ticks = E1000_USECS_TO_TICKS(usecs);
3076
3077         lwkt_serialize_enter(&adapter->serializer);
3078         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3079         regval = (regval & ~0xffff) | (ticks & 0xffff);
3080         /* Handle a few special cases. */
3081         switch (info->offset) {
3082         case E1000_RDTR:
3083         case E1000_82542_RDTR:
3084                 regval |= E1000_RDT_FPDB;
3085                 break;
3086         case E1000_TIDV:
3087         case E1000_82542_TIDV:
3088                 if (ticks == 0) {
3089                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3090                         /* Don't write 0 into the TIDV register. */
3091                         regval++;
3092                 } else
3093                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3094                 break;
3095         }
3096         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3097         lwkt_serialize_exit(&adapter->serializer);
3098         return(0);
3099 }
3100
3101 static void
3102 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3103                         const char *description, struct em_int_delay_info *info,
3104                         int offset, int value)
3105 {
3106         info->adapter = adapter;
3107         info->offset = offset;
3108         info->value = value;
3109         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3110                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3111                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3112                         info, 0, em_sysctl_int_delay, "I", description);
3113 }
3114
3115 static int
3116 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
3117 {
3118         struct adapter *adapter = (void *)arg1;
3119         int error;
3120         int throttle;
3121
3122         throttle = em_int_throttle_ceil;
3123         error = sysctl_handle_int(oidp, &throttle, 0, req);
3124         if (error || req->newptr == NULL)
3125                 return error;
3126         if (throttle < 0 || throttle > 1000000000 / 256)
3127                 return EINVAL;
3128         if (throttle) {
3129                 /*
3130                  * Set the interrupt throttling rate in 256ns increments,
3131                  * recalculate sysctl value assignment to get exact frequency.
3132                  */
3133                 throttle = 1000000000 / 256 / throttle;
3134                 lwkt_serialize_enter(&adapter->serializer);
3135                 em_int_throttle_ceil = 1000000000 / 256 / throttle;
3136                 E1000_WRITE_REG(&adapter->hw, ITR, throttle);
3137                 lwkt_serialize_exit(&adapter->serializer);
3138         } else {
3139                 lwkt_serialize_enter(&adapter->serializer);
3140                 em_int_throttle_ceil = 0;
3141                 E1000_WRITE_REG(&adapter->hw, ITR, 0);
3142                 lwkt_serialize_exit(&adapter->serializer);
3143         }
3144         device_printf(adapter->dev, "Interrupt moderation set to %d/sec\n",
3145                         em_int_throttle_ceil);
3146         return 0;
3147 }
3148