sys/dev/netif/em/if_em.c

   1 /*
   2  *
   3  * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4  *
   5  * Copyright (c) 2001-2006, Intel Corporation
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  *  1. Redistributions of source code must retain the above copyright notice,
  12  *     this list of conditions and the following disclaimer.
  13  *
  14  *  2. Redistributions in binary form must reproduce the above copyright
  15  *     notice, this list of conditions and the following disclaimer in the
  16  *     documentation and/or other materials provided with the distribution.
  17  *
  18  *  3. Neither the name of the Intel Corporation nor the names of its
  19  *     contributors may be used to endorse or promote products derived from
  20  *     this software without specific prior written permission.
  21  *
  22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  32  * POSSIBILITY OF SUCH DAMAGE.
  33  *
  34  *
  35  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
  36  *
  37  * This code is derived from software contributed to The DragonFly Project
  38  * by Matthew Dillon <dillon@backplane.com>
  39  *
  40  * Redistribution and use in source and binary forms, with or without
  41  * modification, are permitted provided that the following conditions
  42  * are met:
  43  *
  44  * 1. Redistributions of source code must retain the above copyright
  45  *    notice, this list of conditions and the following disclaimer.
  46  * 2. Redistributions in binary form must reproduce the above copyright
  47  *    notice, this list of conditions and the following disclaimer in
  48  *    the documentation and/or other materials provided with the
  49  *    distribution.
  50  * 3. Neither the name of The DragonFly Project nor the names of its
  51  *    contributors may be used to endorse or promote products derived
  52  *    from this software without specific, prior written permission.
  53  *
  54  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  55  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  56  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  57  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  58  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  59  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  60  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  61  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  62  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  63  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  64  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  65  * SUCH DAMAGE.
  66  *
  67  * $DragonFly: src/sys/dev/netif/em/if_em.c,v 1.53 2006/12/23 10:39:16 sephe Exp $
  68  * $FreeBSD$
  69  */
  70 /*
  71  * SERIALIZATION API RULES:
  72  *
  73  * - If the driver uses the same serializer for the interrupt as for the
  74  *   ifnet, most of the serialization will be done automatically for the
  75  *   driver.
  76  *
  77  * - ifmedia entry points will be serialized by the ifmedia code using the
  78  *   ifnet serializer.
  79  *
  80  * - if_* entry points except for if_input will be serialized by the IF
  81  *   and protocol layers.
  82  *
  83  * - The device driver must be sure to serialize access from timeout code
  84  *   installed by the device driver.
  85  *
  86  * - The device driver typically holds the serializer at the time it wishes
  87  *   to call if_input.  If so, it should pass the serializer to if_input and
  88  *   note that the serializer might be dropped temporarily by if_input
  89  *   (e.g. in case it has to bridge the packet to another interface).
  90  *
  91  *   NOTE!  Since callers into the device driver hold the ifnet serializer,
  92  *   the device driver may be holding a serializer at the time it calls
  93  *   if_input even if it is not serializer-aware.
  94  */
  95
  96 #include "opt_polling.h"
  97 #include "opt_inet.h"
  98
  99 #include <sys/param.h>
 100 #include <sys/bus.h>
 101 #include <sys/endian.h>
 102 #include <sys/kernel.h>
 103 #include <sys/ktr.h>
 104 #include <sys/malloc.h>
 105 #include <sys/mbuf.h>
 106 #include <sys/module.h>
 107 #include <sys/rman.h>
 108 #include <sys/serialize.h>
 109 #include <sys/socket.h>
 110 #include <sys/sockio.h>
 111 #include <sys/sysctl.h>
 112
 113 #include <net/bpf.h>
 114 #include <net/ethernet.h>
 115 #include <net/if.h>
 116 #include <net/if_arp.h>
 117 #include <net/if_dl.h>
 118 #include <net/if_media.h>
 119 #include <net/if_types.h>
 120 #include <net/ifq_var.h>
 121 #include <net/vlan/if_vlan_var.h>
 122
 123 #ifdef INET
 124 #include <netinet/in.h>
 125 #include <netinet/in_systm.h>
 126 #include <netinet/in_var.h>
 127 #include <netinet/ip.h>
 128 #include <netinet/tcp.h>
 129 #include <netinet/udp.h>
 130 #endif
 131
 132 #include <dev/netif/em/if_em_hw.h>
 133 #include <dev/netif/em/if_em.h>
 134
 135 /*********************************************************************
 136  *  Set this to one to display debug statistics
 137  *********************************************************************/
 138 int     em_display_debug_stats = 0;
 139
 140 /*********************************************************************
 141  *  Driver version
 142  *********************************************************************/
 143
 144 char em_driver_version[] = "6.2.9";
 145
 146
 147 /*********************************************************************
 148  *  PCI Device ID Table
 149  *
 150  *  Used by probe to select devices to load on
 151  *  Last field stores an index into em_strings
 152  *  Last entry must be all 0s
 153  *
 154  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 155  *********************************************************************/
 156
 157 static em_vendor_info_t em_vendor_info_array[] =
 158 {
 159         /* Intel(R) PRO/1000 Network Connection */
 160         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 162         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 163         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 164         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 165
 166         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 167         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 168         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 169         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 170         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 171         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 172         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 173
 174         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 175
 176         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 177         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 178
 179         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 180         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 181         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 182         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 183
 184         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 185         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 186         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 187         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 188         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 189
 190         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 191         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 192         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 193         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 194         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 195         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 196         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 197         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 198         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 199                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 200
 201         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 202         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 203         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 204
 205         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 206         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 207         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 208         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 209                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 210         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LOWPROFILE,
 211                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 212
 213         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 214         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 215         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 216         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 217
 218         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 219         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 220         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 221
 222         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 223                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 224         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 225                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 226         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 227                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 228         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 229                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 230
 231         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 232         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 233         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 234         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 235         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 236         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 237         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 238
 239         { 0x8086, 0x101A, PCI_ANY_ID, PCI_ANY_ID, 0},
 240         { 0x8086, 0x1014, PCI_ANY_ID, PCI_ANY_ID, 0},
 241         /* required last entry */
 242         { 0, 0, 0, 0, 0}
 243 };
 244
 245 /*********************************************************************
 246  *  Table of branding strings for all supported NICs.
 247  *********************************************************************/
 248
 249 static const char *em_strings[] = {
 250         "Intel(R) PRO/1000 Network Connection"
 251 };
 252
 253 /*********************************************************************
 254  *  Function prototypes
 255  *********************************************************************/
 256 static int      em_probe(device_t);
 257 static int      em_attach(device_t);
 258 static int      em_detach(device_t);
 259 static int      em_shutdown(device_t);
 260 static void     em_intr(void *);
 261 static int      em_suspend(device_t);
 262 static int      em_resume(device_t);
 263 static void     em_start(struct ifnet *);
 264 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 265 static void     em_watchdog(struct ifnet *);
 266 static void     em_init(void *);
 267 static void     em_stop(void *);
 268 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 269 static int      em_media_change(struct ifnet *);
 270 static void     em_identify_hardware(struct adapter *);
 271 static int      em_allocate_pci_resources(device_t);
 272 static void     em_free_pci_resources(device_t);
 273 static void     em_local_timer(void *);
 274 static int      em_hardware_init(struct adapter *);
 275 static void     em_setup_interface(device_t, struct adapter *);
 276 static int      em_setup_transmit_structures(struct adapter *);
 277 static void     em_initialize_transmit_unit(struct adapter *);
 278 static int      em_setup_receive_structures(struct adapter *);
 279 static void     em_initialize_receive_unit(struct adapter *);
 280 static void     em_enable_intr(struct adapter *);
 281 static void     em_disable_intr(struct adapter *);
 282 static void     em_free_transmit_structures(struct adapter *);
 283 static void     em_free_receive_structures(struct adapter *);
 284 static void     em_update_stats_counters(struct adapter *);
 285 static void     em_txeof(struct adapter *);
 286 static int      em_allocate_receive_structures(struct adapter *);
 287 static void     em_rxeof(struct adapter *, int);
 288 static void     em_receive_checksum(struct adapter *, struct em_rx_desc *,
 289                                     struct mbuf *);
 290 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 291                                            uint32_t *, uint32_t *);
 292 static void     em_set_promisc(struct adapter *);
 293 static void     em_disable_promisc(struct adapter *);
 294 static void     em_set_multi(struct adapter *);
 295 static void     em_print_hw_stats(struct adapter *);
 296 static void     em_update_link_status(struct adapter *);
 297 static int      em_get_buf(int i, struct adapter *, struct mbuf *, int how);
 298 static void     em_enable_vlans(struct adapter *);
 299 static void     em_disable_vlans(struct adapter *);
 300 static int      em_encap(struct adapter *, struct mbuf *);
 301 static void     em_smartspeed(struct adapter *);
 302 static int      em_82547_fifo_workaround(struct adapter *, int);
 303 static void     em_82547_update_fifo_head(struct adapter *, int);
 304 static int      em_82547_tx_fifo_reset(struct adapter *);
 305 static void     em_82547_move_tail(void *);
 306 static void     em_82547_move_tail_serialized(struct adapter *);
 307 static int      em_dma_malloc(struct adapter *, bus_size_t,
 308                               struct em_dma_alloc *);
 309 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 310 static void     em_print_debug_info(struct adapter *);
 311 static int      em_is_valid_ether_addr(uint8_t *);
 312 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 313 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 314 static uint32_t em_fill_descriptors(bus_addr_t address, uint32_t length,
 315                                    PDESC_ARRAY desc_array);
 316 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 317 static int      em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 318 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 319                                         const char *,
 320                                         struct em_int_delay_info *, int, int);
 321
 322 /*********************************************************************
 323  *  FreeBSD Device Interface Entry Points
 324  *********************************************************************/
 325
 326 static device_method_t em_methods[] = {
 327         /* Device interface */
 328         DEVMETHOD(device_probe, em_probe),
 329         DEVMETHOD(device_attach, em_attach),
 330         DEVMETHOD(device_detach, em_detach),
 331         DEVMETHOD(device_shutdown, em_shutdown),
 332         DEVMETHOD(device_suspend, em_suspend),
 333         DEVMETHOD(device_resume, em_resume),
 334         {0, 0}
 335 };
 336
 337 static driver_t em_driver = {
 338         "em", em_methods, sizeof(struct adapter),
 339 };
 340
 341 static devclass_t em_devclass;
 342
 343 DECLARE_DUMMY_MODULE(if_em);
 344 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, 0, 0);
 345
 346 /*********************************************************************
 347  *  Tunable default values.
 348  *********************************************************************/
 349
 350 #define E1000_TICKS_TO_USECS(ticks)     ((1024 * (ticks) + 500) / 1000)
 351 #define E1000_USECS_TO_TICKS(usecs)     ((1000 * (usecs) + 512) / 1024)
 352
 353 static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
 354 static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
 355 static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
 356 static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
 357 static int em_int_throttle_ceil = 10000;
 358 static int em_rxd = EM_DEFAULT_RXD;
 359 static int em_txd = EM_DEFAULT_TXD;
 360 static int em_smart_pwr_down = FALSE;
 361
 362 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 363 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 364 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 365 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 366 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
 367 TUNABLE_INT("hw.em.rxd", &em_rxd);
 368 TUNABLE_INT("hw.em.txd", &em_txd);
 369 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 370
 371 /*
 372  * Kernel trace for characterization of operations
 373  */
 374 #if !defined(KTR_IF_EM)
 375 #define KTR_IF_EM       KTR_ALL
 376 #endif
 377 KTR_INFO_MASTER(if_em);
 378 KTR_INFO(KTR_IF_EM, if_em, intr_beg, 0, "intr begin", 0);
 379 KTR_INFO(KTR_IF_EM, if_em, intr_end, 1, "intr end", 0);
 380 #ifdef DEVICE_POLLING
 381 KTR_INFO(KTR_IF_EM, if_em, poll_beg, 2, "poll begin", 0);
 382 KTR_INFO(KTR_IF_EM, if_em, poll_end, 3, "poll end", 0);
 383 #endif
 384 KTR_INFO(KTR_IF_EM, if_em, pkt_receive, 4, "rx packet", 0);
 385 KTR_INFO(KTR_IF_EM, if_em, pkt_txqueue, 5, "tx packet", 0);
 386 KTR_INFO(KTR_IF_EM, if_em, pkt_txclean, 6, "tx clean", 0);
 387 #define logif(name)     KTR_LOG(if_em_ ## name)
 388
 389 /*********************************************************************
 390  *  Device identification routine
 391  *
 392  *  em_probe determines if the driver should be loaded on
 393  *  adapter based on PCI vendor/device id of the adapter.
 394  *
 395  *  return 0 on success, positive on failure
 396  *********************************************************************/
 397
 398 static int
 399 em_probe(device_t dev)
 400 {
 401         em_vendor_info_t *ent;
 402
 403         uint16_t pci_vendor_id = 0;
 404         uint16_t pci_device_id = 0;
 405         uint16_t pci_subvendor_id = 0;
 406         uint16_t pci_subdevice_id = 0;
 407         char adapter_name[60];
 408
 409         INIT_DEBUGOUT("em_probe: begin");
 410
 411         pci_vendor_id = pci_get_vendor(dev);
 412         if (pci_vendor_id != EM_VENDOR_ID)
 413                 return (ENXIO);
 414
 415         pci_device_id = pci_get_device(dev);
 416         pci_subvendor_id = pci_get_subvendor(dev);
 417         pci_subdevice_id = pci_get_subdevice(dev);
 418
 419         ent = em_vendor_info_array;
 420         while (ent->vendor_id != 0) {
 421                 if ((pci_vendor_id == ent->vendor_id) &&
 422                     (pci_device_id == ent->device_id) &&
 423
 424                     ((pci_subvendor_id == ent->subvendor_id) ||
 425                      (ent->subvendor_id == PCI_ANY_ID)) &&
 426
 427                     ((pci_subdevice_id == ent->subdevice_id) ||
 428                      (ent->subdevice_id == PCI_ANY_ID))) {
 429                         ksnprintf(adapter_name, sizeof(adapter_name),
 430                                  "%s, Version - %s",  em_strings[ent->index],
 431                                  em_driver_version);
 432                         device_set_desc_copy(dev, adapter_name);
 433                         return (0);
 434                 }
 435                 ent++;
 436         }
 437
 438         return (ENXIO);
 439 }
 440
 441 /*********************************************************************
 442  *  Device initialization routine
 443  *
 444  *  The attach entry point is called when the driver is being loaded.
 445  *  This routine identifies the type of hardware, allocates all resources
 446  *  and initializes the hardware.
 447  *
 448  *  return 0 on success, positive on failure
 449  *********************************************************************/
 450
 451 static int
 452 em_attach(device_t dev)
 453 {
 454         struct adapter *adapter;
 455         int tsize, rsize;
 456         int error = 0;
 457
 458         INIT_DEBUGOUT("em_attach: begin");
 459
 460         adapter = device_get_softc(dev);
 461
 462         callout_init(&adapter->timer);
 463         callout_init(&adapter->tx_fifo_timer);
 464
 465         adapter->dev = dev;
 466         adapter->osdep.dev = dev;
 467
 468         /* SYSCTL stuff */
 469         sysctl_ctx_init(&adapter->sysctl_ctx);
 470         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
 471                                                SYSCTL_STATIC_CHILDREN(_hw),
 472                                                OID_AUTO,
 473                                                device_get_nameunit(dev),
 474                                                CTLFLAG_RD,
 475                                                0, "");
 476
 477         if (adapter->sysctl_tree == NULL) {
 478                 device_printf(dev, "Unable to create sysctl tree\n");
 479                 return EIO;
 480         }
 481
 482         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 483                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 484                         OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW,
 485                         (void *)adapter, 0,
 486                         em_sysctl_debug_info, "I", "Debug Information");
 487
 488         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 489                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 490                         OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW,
 491                         (void *)adapter, 0,
 492                         em_sysctl_stats, "I", "Statistics");
 493
 494         /* Determine hardware revision */
 495         em_identify_hardware(adapter);
 496
 497         /* Set up some sysctls for the tunable interrupt delays */
 498         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 499                                 "receive interrupt delay in usecs",
 500                                 &adapter->rx_int_delay,
 501                                 E1000_REG_OFFSET(&adapter->hw, RDTR),
 502                                 em_rx_int_delay_dflt);
 503         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 504                                 "transmit interrupt delay in usecs",
 505                                 &adapter->tx_int_delay,
 506                                 E1000_REG_OFFSET(&adapter->hw, TIDV),
 507                                 em_tx_int_delay_dflt);
 508         if (adapter->hw.mac_type >= em_82540) {
 509                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 510                                         "receive interrupt delay limit in usecs",
 511                                         &adapter->rx_abs_int_delay,
 512                                         E1000_REG_OFFSET(&adapter->hw, RADV),
 513                                         em_rx_abs_int_delay_dflt);
 514                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 515                                         "transmit interrupt delay limit in usecs",
 516                                         &adapter->tx_abs_int_delay,
 517                                         E1000_REG_OFFSET(&adapter->hw, TADV),
 518                                         em_tx_abs_int_delay_dflt);
 519                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
 520                         SYSCTL_CHILDREN(adapter->sysctl_tree),
 521                         OID_AUTO, "int_throttle_ceil", CTLTYPE_INT|CTLFLAG_RW,
 522                         adapter, 0, em_sysctl_int_throttle, "I", NULL);
 523         }
 524
 525         /*
 526          * Validate number of transmit and receive descriptors. It
 527          * must not exceed hardware maximum, and must be multiple
 528          * of EM_DBA_ALIGN.
 529          */
 530         if (((em_txd * sizeof(struct em_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 531             (adapter->hw.mac_type >= em_82544 && em_txd > EM_MAX_TXD) ||
 532             (adapter->hw.mac_type < em_82544 && em_txd > EM_MAX_TXD_82543) ||
 533             (em_txd < EM_MIN_TXD)) {
 534                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 535                               EM_DEFAULT_TXD, em_txd);
 536                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 537         } else {
 538                 adapter->num_tx_desc = em_txd;
 539         }
 540
 541         if (((em_rxd * sizeof(struct em_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 542             (adapter->hw.mac_type >= em_82544 && em_rxd > EM_MAX_RXD) ||
 543             (adapter->hw.mac_type < em_82544 && em_rxd > EM_MAX_RXD_82543) ||
 544             (em_rxd < EM_MIN_RXD)) {
 545                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 546                               EM_DEFAULT_RXD, em_rxd);
 547                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 548         } else {
 549                 adapter->num_rx_desc = em_rxd;
 550         }
 551
 552         adapter->hw.autoneg = DO_AUTO_NEG;
 553         adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
 554         adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 555         adapter->hw.tbi_compatibility_en = TRUE;
 556         adapter->rx_buffer_len = EM_RXBUFFER_2048;
 557
 558         adapter->hw.phy_init_script = 1;
 559         adapter->hw.phy_reset_disable = FALSE;
 560
 561 #ifndef EM_MASTER_SLAVE
 562         adapter->hw.master_slave = em_ms_hw_default;
 563 #else
 564         adapter->hw.master_slave = EM_MASTER_SLAVE;
 565 #endif
 566
 567         /*
 568          * Set the max frame size assuming standard ethernet
 569          * sized frames.
 570          */
 571         adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 572
 573         adapter->hw.min_frame_size =
 574             MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
 575
 576         /*
 577          * This controls when hardware reports transmit completion
 578          * status.
 579          */
 580         adapter->hw.report_tx_early = 1;
 581
 582         error = em_allocate_pci_resources(dev);
 583         if (error)
 584                 goto fail;
 585
 586         /* Initialize eeprom parameters */
 587         em_init_eeprom_params(&adapter->hw);
 588
 589         tsize = roundup2(adapter->num_tx_desc * sizeof(struct em_tx_desc),
 590                          EM_DBA_ALIGN);
 591
 592         /* Allocate Transmit Descriptor ring */
 593         error = em_dma_malloc(adapter, tsize, &adapter->txdma);
 594         if (error) {
 595                 device_printf(dev, "Unable to allocate TxDescriptor memory\n");
 596                 goto fail;
 597         }
 598         adapter->tx_desc_base = (struct em_tx_desc *)adapter->txdma.dma_vaddr;
 599
 600         rsize = roundup2(adapter->num_rx_desc * sizeof(struct em_rx_desc),
 601                          EM_DBA_ALIGN);
 602
 603         /* Allocate Receive Descriptor ring */
 604         error = em_dma_malloc(adapter, rsize, &adapter->rxdma);
 605         if (error) {
 606                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 607                 goto fail;
 608         }
 609         adapter->rx_desc_base = (struct em_rx_desc *)adapter->rxdma.dma_vaddr;
 610
 611         /* Initialize the hardware */
 612         if (em_hardware_init(adapter)) {
 613                 device_printf(dev, "Unable to initialize the hardware\n");
 614                 error = EIO;
 615                 goto fail;
 616         }
 617
 618         /* Copy the permanent MAC address out of the EEPROM */
 619         if (em_read_mac_addr(&adapter->hw) < 0) {
 620                 device_printf(dev,
 621                               "EEPROM read error while reading MAC address\n");
 622                 error = EIO;
 623                 goto fail;
 624         }
 625
 626         if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
 627                 device_printf(dev, "Invalid MAC address\n");
 628                 error = EIO;
 629                 goto fail;
 630         }
 631
 632         /* Setup OS specific network interface */
 633         em_setup_interface(dev, adapter);
 634
 635         /* Initialize statistics */
 636         em_clear_hw_cntrs(&adapter->hw);
 637         em_update_stats_counters(adapter);
 638         adapter->hw.get_link_status = 1;
 639         em_update_link_status(adapter);
 640
 641         /* Indicate SOL/IDER usage */
 642         if (em_check_phy_reset_block(&adapter->hw)) {
 643                 device_printf(dev, "PHY reset is blocked due to "
 644                               "SOL/IDER session.\n");
 645         }
 646
 647         /* Identify 82544 on PCIX */
 648         em_get_bus_info(&adapter->hw);
 649         if (adapter->hw.bus_type == em_bus_type_pcix &&
 650             adapter->hw.mac_type == em_82544)
 651                 adapter->pcix_82544 = TRUE;
 652         else
 653                 adapter->pcix_82544 = FALSE;
 654
 655         error = bus_setup_intr(dev, adapter->res_interrupt, INTR_NETSAFE,
 656                            em_intr, adapter,
 657                            &adapter->int_handler_tag,
 658                            adapter->interface_data.ac_if.if_serializer);
 659         if (error) {
 660                 device_printf(dev, "Error registering interrupt handler!\n");
 661                 ether_ifdetach(&adapter->interface_data.ac_if);
 662                 goto fail;
 663         }
 664
 665         INIT_DEBUGOUT("em_attach: end");
 666         return(0);
 667
 668 fail:
 669         em_detach(dev);
 670         return(error);
 671 }
 672
 673 /*********************************************************************
 674  *  Device removal routine
 675  *
 676  *  The detach entry point is called when the driver is being removed.
 677  *  This routine stops the adapter and deallocates all the resources
 678  *  that were allocated for driver operation.
 679  *
 680  *  return 0 on success, positive on failure
 681  *********************************************************************/
 682
 683 static int
 684 em_detach(device_t dev)
 685 {
 686         struct adapter *adapter = device_get_softc(dev);
 687
 688         INIT_DEBUGOUT("em_detach: begin");
 689
 690         if (device_is_attached(dev)) {
 691                 struct ifnet *ifp = &adapter->interface_data.ac_if;
 692
 693                 lwkt_serialize_enter(ifp->if_serializer);
 694                 adapter->in_detach = 1;
 695                 em_stop(adapter);
 696                 em_phy_hw_reset(&adapter->hw);
 697                 bus_teardown_intr(dev, adapter->res_interrupt,
 698                                   adapter->int_handler_tag);
 699                 lwkt_serialize_exit(ifp->if_serializer);
 700
 701                 ether_ifdetach(ifp);
 702         }
 703         bus_generic_detach(dev);
 704
 705         em_free_pci_resources(dev);
 706
 707         /* Free Transmit Descriptor ring */
 708         if (adapter->tx_desc_base != NULL) {
 709                 em_dma_free(adapter, &adapter->txdma);
 710                 adapter->tx_desc_base = NULL;
 711         }
 712
 713         /* Free Receive Descriptor ring */
 714         if (adapter->rx_desc_base != NULL) {
 715                 em_dma_free(adapter, &adapter->rxdma);
 716                 adapter->rx_desc_base = NULL;
 717         }
 718
 719         /* Free sysctl tree */
 720         if (adapter->sysctl_tree != NULL) {
 721                 adapter->sysctl_tree = NULL;
 722                 sysctl_ctx_free(&adapter->sysctl_ctx);
 723         }
 724
 725         return (0);
 726 }
 727
 728 /*********************************************************************
 729  *
 730  *  Shutdown entry point
 731  *
 732  **********************************************************************/
 733
 734 static int
 735 em_shutdown(device_t dev)
 736 {
 737         struct adapter *adapter = device_get_softc(dev);
 738         struct ifnet *ifp = &adapter->interface_data.ac_if;
 739
 740         lwkt_serialize_enter(ifp->if_serializer);
 741         em_stop(adapter);
 742         lwkt_serialize_exit(ifp->if_serializer);
 743
 744         return (0);
 745 }
 746
 747 /*
 748  * Suspend/resume device methods.
 749  */
 750 static int
 751 em_suspend(device_t dev)
 752 {
 753         struct adapter *adapter = device_get_softc(dev);
 754         struct ifnet *ifp = &adapter->interface_data.ac_if;
 755
 756         lwkt_serialize_enter(ifp->if_serializer);
 757         em_stop(adapter);
 758         lwkt_serialize_exit(ifp->if_serializer);
 759         return (0);
 760 }
 761
 762 static int
 763 em_resume(device_t dev)
 764 {
 765         struct adapter *adapter = device_get_softc(dev);
 766         struct ifnet *ifp = &adapter->interface_data.ac_if;
 767
 768         lwkt_serialize_enter(ifp->if_serializer);
 769         ifp->if_flags &= ~IFF_RUNNING;
 770         em_init(adapter);
 771         if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == (IFF_UP | IFF_RUNNING))
 772                 em_start(ifp);
 773         lwkt_serialize_exit(ifp->if_serializer);
 774
 775         return bus_generic_resume(dev);
 776 }
 777
 778 /*********************************************************************
 779  *  Transmit entry point
 780  *
 781  *  em_start is called by the stack to initiate a transmit.
 782  *  The driver will remain in this routine as long as there are
 783  *  packets to transmit and transmit resources are available.
 784  *  In case resources are not available stack is notified and
 785  *  the packet is requeued.
 786  **********************************************************************/
 787
 788 static void
 789 em_start(struct ifnet *ifp)
 790 {
 791         struct mbuf *m_head;
 792         struct adapter *adapter = ifp->if_softc;
 793
 794         ASSERT_SERIALIZED(ifp->if_serializer);
 795
 796         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
 797                 return;
 798         if (!adapter->link_active)
 799                 return;
 800         while (!ifq_is_empty(&ifp->if_snd)) {
 801                 m_head = ifq_poll(&ifp->if_snd);
 802
 803                 if (m_head == NULL)
 804                         break;
 805
 806                 logif(pkt_txqueue);
 807                 if (em_encap(adapter, m_head)) {
 808                         ifp->if_flags |= IFF_OACTIVE;
 809                         break;
 810                 }
 811                 ifq_dequeue(&ifp->if_snd, m_head);
 812
 813                 /* Send a copy of the frame to the BPF listener */
 814                 BPF_MTAP(ifp, m_head);
 815
 816                 /* Set timeout in case hardware has problems transmitting. */
 817                 ifp->if_timer = EM_TX_TIMEOUT;
 818         }
 819 }
 820
 821 /*********************************************************************
 822  *  Ioctl entry point
 823  *
 824  *  em_ioctl is called when the user wants to configure the
 825  *  interface.
 826  *
 827  *  return 0 on success, positive on failure
 828  **********************************************************************/
 829
 830 static int
 831 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
 832 {
 833         int max_frame_size, mask, error = 0, reinit = 0;
 834         struct ifreq *ifr = (struct ifreq *) data;
 835         struct adapter *adapter = ifp->if_softc;
 836         uint16_t eeprom_data = 0;
 837
 838         ASSERT_SERIALIZED(ifp->if_serializer);
 839
 840         if (adapter->in_detach)
 841                 return 0;
 842
 843         switch (command) {
 844         case SIOCSIFMTU:
 845                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
 846                 switch (adapter->hw.mac_type) {
 847                 case em_82573:
 848                         /*
 849                          * 82573 only supports jumbo frames
 850                          * if ASPM is disabled.
 851                          */
 852                         em_read_eeprom(&adapter->hw, EEPROM_INIT_3GIO_3,
 853                             1, &eeprom_data);
 854                         if (eeprom_data & EEPROM_WORD1A_ASPM_MASK) {
 855                                 max_frame_size = ETHER_MAX_LEN;
 856                                 break;
 857                         }
 858                         /* Allow Jumbo frames */
 859                         /* FALLTHROUGH */
 860                 case em_82571:
 861                 case em_82572:
 862                 case em_80003es2lan:    /* Limit Jumbo Frame size */
 863                         max_frame_size = 9234;
 864                         break;
 865                 case em_ich8lan:
 866                         /* ICH8 does not support jumbo frames */
 867                         max_frame_size = ETHER_MAX_LEN;
 868                         break;
 869                 default:
 870                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
 871                         break;
 872                 }
 873                 if (ifr->ifr_mtu >
 874                         max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN) {
 875                         error = EINVAL;
 876                 } else {
 877                         ifp->if_mtu = ifr->ifr_mtu;
 878                         adapter->hw.max_frame_size =
 879                         ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
 880                         ifp->if_flags &= ~IFF_RUNNING;
 881                         em_init(adapter);
 882                 }
 883                 break;
 884         case SIOCSIFFLAGS:
 885                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS "
 886                                "(Set Interface Flags)");
 887                 if (ifp->if_flags & IFF_UP) {
 888                         if (!(ifp->if_flags & IFF_RUNNING)) {
 889                                 em_init(adapter);
 890                         } else if ((ifp->if_flags ^ adapter->if_flags) &
 891                                    IFF_PROMISC) {
 892                                 em_disable_promisc(adapter);
 893                                 em_set_promisc(adapter);
 894                         }
 895                 } else {
 896                         if (ifp->if_flags & IFF_RUNNING)
 897                                 em_stop(adapter);
 898                 }
 899                 adapter->if_flags = ifp->if_flags;
 900                 break;
 901         case SIOCADDMULTI:
 902         case SIOCDELMULTI:
 903                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
 904                 if (ifp->if_flags & IFF_RUNNING) {
 905                         em_disable_intr(adapter);
 906                         em_set_multi(adapter);
 907                         if (adapter->hw.mac_type == em_82542_rev2_0)
 908                                 em_initialize_receive_unit(adapter);
 909 #ifdef DEVICE_POLLING
 910                         /* Do not enable interrupt if polling(4) is enabled */
 911                         if ((ifp->if_flags & IFF_POLLING) == 0)
 912 #endif
 913                         em_enable_intr(adapter);
 914                 }
 915                 break;
 916         case SIOCSIFMEDIA:
 917                 /* Check SOL/IDER usage */
 918                 if (em_check_phy_reset_block(&adapter->hw)) {
 919                         if_printf(ifp, "Media change is blocked due to "
 920                                   "SOL/IDER session.\n");
 921                         break;
 922                 }
 923                 /* FALLTHROUGH */
 924         case SIOCGIFMEDIA:
 925                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA "
 926                                "(Get/Set Interface Media)");
 927                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
 928                 break;
 929         case SIOCSIFCAP:
 930                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
 931                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 932                 if (mask & IFCAP_HWCSUM) {
 933                         ifp->if_capenable ^= IFCAP_HWCSUM;
 934                         reinit = 1;
 935                 }
 936                 if (mask & IFCAP_VLAN_HWTAGGING) {
 937                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
 938                         reinit = 1;
 939                 }
 940                 if (reinit && (ifp->if_flags & IFF_RUNNING)) {
 941                         ifp->if_flags &= ~IFF_RUNNING;
 942                         em_init(adapter);
 943                 }
 944                 break;
 945         default:
 946                 error = ether_ioctl(ifp, command, data);
 947                 break;
 948         }
 949
 950         return (error);
 951 }
 952
 953 /*********************************************************************
 954  *  Watchdog entry point
 955  *
 956  *  This routine is called whenever hardware quits transmitting.
 957  *
 958  **********************************************************************/
 959
 960 static void
 961 em_watchdog(struct ifnet *ifp)
 962 {
 963         struct adapter *adapter = ifp->if_softc;
 964
 965         /*
 966          * If we are in this routine because of pause frames, then
 967          * don't reset the hardware.
 968          */
 969         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
 970                 ifp->if_timer = EM_TX_TIMEOUT;
 971                 return;
 972         }
 973
 974         if (em_check_for_link(&adapter->hw) == 0)
 975                 if_printf(ifp, "watchdog timeout -- resetting\n");
 976
 977         ifp->if_flags &= ~IFF_RUNNING;
 978         em_init(adapter);
 979
 980         adapter->watchdog_timeouts++;
 981 }
 982
 983 /*********************************************************************
 984  *  Init entry point
 985  *
 986  *  This routine is used in two ways. It is used by the stack as
 987  *  init entry point in network interface structure. It is also used
 988  *  by the driver as a hw/sw initialization routine to get to a
 989  *  consistent state.
 990  *
 991  *  return 0 on success, positive on failure
 992  **********************************************************************/
 993
 994 static void
 995 em_init(void *arg)
 996 {
 997         struct adapter *adapter = arg;
 998         uint32_t pba;
 999         struct ifnet *ifp = &adapter->interface_data.ac_if;
1000
1001         ASSERT_SERIALIZED(ifp->if_serializer);
1002
1003         INIT_DEBUGOUT("em_init: begin");
1004
1005         if (ifp->if_flags & IFF_RUNNING)
1006                 return;
1007
1008         em_stop(adapter);
1009
1010         /*
1011          * Packet Buffer Allocation (PBA)
1012          * Writing PBA sets the receive portion of the buffer
1013          * the remainder is used for the transmit buffer.
1014          *
1015          * Devices before the 82547 had a Packet Buffer of 64K.
1016          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1017          * After the 82547 the buffer was reduced to 40K.
1018          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1019          *   Note: default does not leave enough room for Jumbo Frame >10k.
1020          */
1021         switch (adapter->hw.mac_type) {
1022         case em_82547:
1023         case em_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1024                 if (adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1025                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1026                 else
1027                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1028
1029                 adapter->tx_fifo_head = 0;
1030                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1031                 adapter->tx_fifo_size =
1032                         (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1033                 break;
1034         /* Total Packet Buffer on these is 48K */
1035         case em_82571:
1036         case em_82572:
1037         case em_80003es2lan:
1038                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1039                 break;
1040         case em_82573: /* 82573: Total Packet Buffer is 32K */
1041                 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1042                 break;
1043         case em_ich8lan:
1044                 pba = E1000_PBA_8K;
1045                 break;
1046         default:
1047                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1048                 if(adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1049                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1050                 else
1051                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1052         }
1053
1054         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1055         E1000_WRITE_REG(&adapter->hw, PBA, pba);
1056
1057         /* Get the latest mac address, User can use a LAA */
1058         bcopy(adapter->interface_data.ac_enaddr, adapter->hw.mac_addr,
1059               ETHER_ADDR_LEN);
1060
1061         /* Initialize the hardware */
1062         if (em_hardware_init(adapter)) {
1063                 if_printf(ifp, "Unable to initialize the hardware\n");
1064                 return;
1065         }
1066         em_update_link_status(adapter);
1067
1068         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1069                 em_enable_vlans(adapter);
1070
1071         /* Set hardware offload abilities */
1072         if (adapter->hw.mac_type >= em_82543) {
1073                 if (ifp->if_capenable & IFCAP_TXCSUM)
1074                         ifp->if_hwassist = EM_CHECKSUM_FEATURES;
1075                 else
1076                         ifp->if_hwassist = 0;
1077         }
1078
1079         /* Prepare transmit descriptors and buffers */
1080         if (em_setup_transmit_structures(adapter)) {
1081                 if_printf(ifp, "Could not setup transmit structures\n");
1082                 em_stop(adapter);
1083                 return;
1084         }
1085         em_initialize_transmit_unit(adapter);
1086
1087         /* Setup Multicast table */
1088         em_set_multi(adapter);
1089
1090         /* Prepare receive descriptors and buffers */
1091         if (em_setup_receive_structures(adapter)) {
1092                 if_printf(ifp, "Could not setup receive structures\n");
1093                 em_stop(adapter);
1094                 return;
1095         }
1096         em_initialize_receive_unit(adapter);
1097
1098         /* Don't lose promiscuous settings */
1099         em_set_promisc(adapter);
1100
1101         ifp->if_flags |= IFF_RUNNING;
1102         ifp->if_flags &= ~IFF_OACTIVE;
1103
1104         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1105         em_clear_hw_cntrs(&adapter->hw);
1106
1107 #ifdef DEVICE_POLLING
1108         /* Do not enable interrupt if polling(4) is enabled */
1109         if (ifp->if_flags & IFF_POLLING)
1110                 em_disable_intr(adapter);
1111         else
1112 #endif
1113         em_enable_intr(adapter);
1114
1115         /* Don't reset the phy next time init gets called */
1116         adapter->hw.phy_reset_disable = TRUE;
1117 }
1118
1119 #ifdef DEVICE_POLLING
1120
1121 static void
1122 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1123 {
1124         struct adapter *adapter = ifp->if_softc;
1125         uint32_t reg_icr;
1126
1127         logif(poll_beg);
1128
1129         ASSERT_SERIALIZED(ifp->if_serializer);
1130
1131         switch(cmd) {
1132         case POLL_REGISTER:
1133                 em_disable_intr(adapter);
1134                 break;
1135         case POLL_DEREGISTER:
1136                 em_enable_intr(adapter);
1137                 break;
1138         case POLL_AND_CHECK_STATUS:
1139                 reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1140                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1141                         callout_stop(&adapter->timer);
1142                         adapter->hw.get_link_status = 1;
1143                         em_check_for_link(&adapter->hw);
1144                         em_update_link_status(adapter);
1145                         callout_reset(&adapter->timer, hz, em_local_timer,
1146                                       adapter);
1147                 }
1148                 /* fall through */
1149         case POLL_ONLY:
1150                 if (ifp->if_flags & IFF_RUNNING) {
1151                         em_rxeof(adapter, count);
1152                         em_txeof(adapter);
1153
1154                         if (!ifq_is_empty(&ifp->if_snd))
1155                                 em_start(ifp);
1156                 }
1157                 break;
1158         }
1159         logif(poll_end);
1160 }
1161
1162 #endif /* DEVICE_POLLING */
1163
1164 /*********************************************************************
1165  *
1166  *  Interrupt Service routine
1167  *
1168  *********************************************************************/
1169 static void
1170 em_intr(void *arg)
1171 {
1172         uint32_t reg_icr;
1173         struct ifnet *ifp;
1174         struct adapter *adapter = arg;
1175
1176         ifp = &adapter->interface_data.ac_if;
1177
1178         logif(intr_beg);
1179         ASSERT_SERIALIZED(ifp->if_serializer);
1180
1181         reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1182         if ((adapter->hw.mac_type >= em_82571 &&
1183              (reg_icr & E1000_ICR_INT_ASSERTED) == 0) ||
1184             reg_icr == 0) {
1185                 logif(intr_end);
1186                 return;
1187         }
1188
1189         /*
1190          * XXX: some laptops trigger several spurious interrupts on em(4)
1191          * when in the resume cycle. The ICR register reports all-ones
1192          * value in this case. Processing such interrupts would lead to
1193          * a freeze. I don't know why.
1194          */
1195         if (reg_icr == 0xffffffff) {
1196                 logif(intr_end);
1197                 return;
1198         }
1199
1200         /*
1201          * note: do not attempt to improve efficiency by looping.  This
1202          * only results in unnecessary piecemeal collection of received
1203          * packets and unnecessary piecemeal cleanups of the transmit ring.
1204          */
1205         if (ifp->if_flags & IFF_RUNNING) {
1206                 em_rxeof(adapter, -1);
1207                 em_txeof(adapter);
1208         }
1209
1210         /* Link status change */
1211         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1212                 callout_stop(&adapter->timer);
1213                 adapter->hw.get_link_status = 1;
1214                 em_check_for_link(&adapter->hw);
1215                 em_update_link_status(adapter);
1216                 callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1217         }
1218
1219         if (reg_icr & E1000_ICR_RXO)
1220                 adapter->rx_overruns++;
1221
1222         if ((ifp->if_flags & IFF_RUNNING) && !ifq_is_empty(&ifp->if_snd))
1223                 em_start(ifp);
1224         logif(intr_end);
1225 }
1226
1227 /*********************************************************************
1228  *
1229  *  Media Ioctl callback
1230  *
1231  *  This routine is called whenever the user queries the status of
1232  *  the interface using ifconfig.
1233  *
1234  **********************************************************************/
1235 static void
1236 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1237 {
1238         struct adapter *adapter = ifp->if_softc;
1239         u_char fiber_type = IFM_1000_SX;
1240
1241         INIT_DEBUGOUT("em_media_status: begin");
1242
1243         ASSERT_SERIALIZED(ifp->if_serializer);
1244
1245         em_check_for_link(&adapter->hw);
1246         em_update_link_status(adapter);
1247
1248         ifmr->ifm_status = IFM_AVALID;
1249         ifmr->ifm_active = IFM_ETHER;
1250
1251         if (!adapter->link_active)
1252                 return;
1253
1254         ifmr->ifm_status |= IFM_ACTIVE;
1255
1256         if (adapter->hw.media_type == em_media_type_fiber ||
1257             adapter->hw.media_type == em_media_type_internal_serdes) {
1258                 if (adapter->hw.mac_type == em_82545)
1259                         fiber_type = IFM_1000_LX;
1260                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1261         } else {
1262                 switch (adapter->link_speed) {
1263                 case 10:
1264                         ifmr->ifm_active |= IFM_10_T;
1265                         break;
1266                 case 100:
1267                         ifmr->ifm_active |= IFM_100_TX;
1268                         break;
1269                 case 1000:
1270                         ifmr->ifm_active |= IFM_1000_T;
1271                         break;
1272                 }
1273                 if (adapter->link_duplex == FULL_DUPLEX)
1274                         ifmr->ifm_active |= IFM_FDX;
1275                 else
1276                         ifmr->ifm_active |= IFM_HDX;
1277         }
1278 }
1279
1280 /*********************************************************************
1281  *
1282  *  Media Ioctl callback
1283  *
1284  *  This routine is called when the user changes speed/duplex using
1285  *  media/mediopt option with ifconfig.
1286  *
1287  **********************************************************************/
1288 static int
1289 em_media_change(struct ifnet *ifp)
1290 {
1291         struct adapter *adapter = ifp->if_softc;
1292         struct ifmedia *ifm = &adapter->media;
1293
1294         INIT_DEBUGOUT("em_media_change: begin");
1295
1296         ASSERT_SERIALIZED(ifp->if_serializer);
1297
1298         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1299                 return (EINVAL);
1300
1301         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1302         case IFM_AUTO:
1303                 adapter->hw.autoneg = DO_AUTO_NEG;
1304                 adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1305                 break;
1306         case IFM_1000_LX:
1307         case IFM_1000_SX:
1308         case IFM_1000_T:
1309                 adapter->hw.autoneg = DO_AUTO_NEG;
1310                 adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1311                 break;
1312         case IFM_100_TX:
1313                 adapter->hw.autoneg = FALSE;
1314                 adapter->hw.autoneg_advertised = 0;
1315                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1316                         adapter->hw.forced_speed_duplex = em_100_full;
1317                 else
1318                         adapter->hw.forced_speed_duplex = em_100_half;
1319                 break;
1320         case IFM_10_T:
1321                 adapter->hw.autoneg = FALSE;
1322                 adapter->hw.autoneg_advertised = 0;
1323                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1324                         adapter->hw.forced_speed_duplex = em_10_full;
1325                 else
1326                         adapter->hw.forced_speed_duplex = em_10_half;
1327                 break;
1328         default:
1329                 if_printf(ifp, "Unsupported media type\n");
1330         }
1331         /*
1332          * As the speed/duplex settings may have changed we need to
1333          * reset the PHY.
1334          */
1335         adapter->hw.phy_reset_disable = FALSE;
1336
1337         ifp->if_flags &= ~IFF_RUNNING;
1338         em_init(adapter);
1339
1340         return(0);
1341 }
1342
1343 static void
1344 em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize,
1345          int error)
1346 {
1347         struct em_q *q = arg;
1348
1349         if (error)
1350                 return;
1351         KASSERT(nsegs <= EM_MAX_SCATTER,
1352                 ("Too many DMA segments returned when mapping tx packet"));
1353         q->nsegs = nsegs;
1354         bcopy(seg, q->segs, nsegs * sizeof(seg[0]));
1355 }
1356
1357 /*********************************************************************
1358  *
1359  *  This routine maps the mbufs to tx descriptors.
1360  *
1361  *  return 0 on success, positive on failure
1362  **********************************************************************/
1363 static int
1364 em_encap(struct adapter *adapter, struct mbuf *m_head)
1365 {
1366         uint32_t txd_upper = 0, txd_lower = 0, txd_used = 0, txd_saved = 0;
1367         int i, j, error, last = 0;
1368
1369         struct ifvlan *ifv = NULL;
1370         struct em_q q;
1371         struct em_buffer *tx_buffer = NULL, *tx_buffer_first;
1372         bus_dmamap_t map;
1373         struct em_tx_desc *current_tx_desc = NULL;
1374         struct ifnet *ifp = &adapter->interface_data.ac_if;
1375
1376         /*
1377          * Force a cleanup if number of TX descriptors
1378          * available hits the threshold
1379          */
1380         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1381                 em_txeof(adapter);
1382                 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1383                         adapter->no_tx_desc_avail1++;
1384                         return (ENOBUFS);
1385                 }
1386         }
1387
1388         /*
1389          * Capture the first descriptor index, this descriptor will have
1390          * the index of the EOP which is the only one that now gets a
1391          * DONE bit writeback.
1392          */
1393         tx_buffer_first = &adapter->tx_buffer_area[adapter->next_avail_tx_desc];
1394
1395         /*
1396          * Map the packet for DMA.
1397          */
1398         map = tx_buffer_first->map;
1399         error = bus_dmamap_load_mbuf(adapter->txtag, map, m_head, em_tx_cb,
1400                                      &q, BUS_DMA_NOWAIT);
1401         if (error != 0) {
1402                 adapter->no_tx_dma_setup++;
1403                 return (error);
1404         }
1405         KASSERT(q.nsegs != 0, ("em_encap: empty packet"));
1406
1407         if (q.nsegs > (adapter->num_tx_desc_avail - 2)) {
1408                 adapter->no_tx_desc_avail2++;
1409                 error = ENOBUFS;
1410                 goto fail;
1411         }
1412
1413         if (ifp->if_hwassist > 0) {
1414                 em_transmit_checksum_setup(adapter,  m_head,
1415                                            &txd_upper, &txd_lower);
1416         }
1417
1418         /* Find out if we are in vlan mode */
1419         if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) &&
1420             m_head->m_pkthdr.rcvif != NULL &&
1421             m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN)
1422                 ifv = m_head->m_pkthdr.rcvif->if_softc;
1423
1424         i = adapter->next_avail_tx_desc;
1425         if (adapter->pcix_82544)
1426                 txd_saved = i;
1427
1428         /* Set up our transmit descriptors */
1429         for (j = 0; j < q.nsegs; j++) {
1430                 /* If adapter is 82544 and on PCIX bus */
1431                 if(adapter->pcix_82544) {
1432                         DESC_ARRAY desc_array;
1433                         uint32_t array_elements, counter;
1434
1435                         /*
1436                          * Check the Address and Length combination and
1437                          * split the data accordingly
1438                          */
1439                         array_elements = em_fill_descriptors(q.segs[j].ds_addr,
1440                                                 q.segs[j].ds_len, &desc_array);
1441                         for (counter = 0; counter < array_elements; counter++) {
1442                                 if (txd_used == adapter->num_tx_desc_avail) {
1443                                         adapter->next_avail_tx_desc = txd_saved;
1444                                         adapter->no_tx_desc_avail2++;
1445                                         error = ENOBUFS;
1446                                         goto fail;
1447                                 }
1448                                 tx_buffer = &adapter->tx_buffer_area[i];
1449                                 current_tx_desc = &adapter->tx_desc_base[i];
1450                                 current_tx_desc->buffer_addr = htole64(
1451                                         desc_array.descriptor[counter].address);
1452                                 current_tx_desc->lower.data = htole32(
1453                                         adapter->txd_cmd | txd_lower |
1454                                         (uint16_t)desc_array.descriptor[counter].length);
1455                                 current_tx_desc->upper.data = htole32(txd_upper);
1456
1457                                 last = i;
1458                                 if (++i == adapter->num_tx_desc)
1459                                         i = 0;
1460
1461                                 tx_buffer->m_head = NULL;
1462                                 tx_buffer->next_eop = -1;
1463                                 txd_used++;
1464                         }
1465                 } else {
1466                         tx_buffer = &adapter->tx_buffer_area[i];
1467                         current_tx_desc = &adapter->tx_desc_base[i];
1468
1469                         current_tx_desc->buffer_addr = htole64(q.segs[j].ds_addr);
1470                         current_tx_desc->lower.data = htole32(
1471                                 adapter->txd_cmd | txd_lower | q.segs[j].ds_len);
1472                         current_tx_desc->upper.data = htole32(txd_upper);
1473
1474                         last = i;
1475                         if (++i == adapter->num_tx_desc)
1476                                 i = 0;
1477
1478                         tx_buffer->m_head = NULL;
1479                         tx_buffer->next_eop = -1;
1480                 }
1481         }
1482
1483         adapter->next_avail_tx_desc = i;
1484         if (adapter->pcix_82544)
1485                 adapter->num_tx_desc_avail -= txd_used;
1486         else
1487                 adapter->num_tx_desc_avail -= q.nsegs;
1488
1489         if (ifv != NULL) {
1490                 /* Set the vlan id */
1491                 current_tx_desc->upper.fields.special = htole16(ifv->ifv_tag);
1492
1493                 /* Tell hardware to add tag */
1494                 current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1495         }
1496
1497         tx_buffer->m_head = m_head;
1498         tx_buffer_first->map = tx_buffer->map;
1499         tx_buffer->map = map;
1500         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1501
1502         /*
1503          * Last Descriptor of Packet needs End Of Packet (EOP)
1504          * and Report Status (RS)
1505          */
1506         current_tx_desc->lower.data |=
1507                 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1508
1509         /*
1510          * Keep track in the first buffer which descriptor will be
1511          * written back.
1512          */
1513         tx_buffer_first->next_eop = last;
1514
1515         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1516                         BUS_DMASYNC_PREWRITE);
1517
1518         /*
1519          * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1520          * that this frame is available to transmit.
1521          */
1522         if (adapter->hw.mac_type == em_82547 &&
1523             adapter->link_duplex == HALF_DUPLEX) {
1524                 em_82547_move_tail_serialized(adapter);
1525         } else {
1526                 E1000_WRITE_REG(&adapter->hw, TDT, i);
1527                 if (adapter->hw.mac_type == em_82547) {
1528                         em_82547_update_fifo_head(adapter,
1529                                                   m_head->m_pkthdr.len);
1530                 }
1531         }
1532
1533         return (0);
1534 fail:
1535         bus_dmamap_unload(adapter->txtag, map);
1536         return error;
1537 }
1538
1539 /*********************************************************************
1540  *
1541  * 82547 workaround to avoid controller hang in half-duplex environment.
1542  * The workaround is to avoid queuing a large packet that would span
1543  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1544  * in this case. We do that only when FIFO is quiescent.
1545  *
1546  **********************************************************************/
1547 static void
1548 em_82547_move_tail(void *arg)
1549 {
1550         struct adapter *adapter = arg;
1551         struct ifnet *ifp = &adapter->interface_data.ac_if;
1552
1553         lwkt_serialize_enter(ifp->if_serializer);
1554         em_82547_move_tail_serialized(adapter);
1555         lwkt_serialize_exit(ifp->if_serializer);
1556 }
1557
1558 static void
1559 em_82547_move_tail_serialized(struct adapter *adapter)
1560 {
1561         uint16_t hw_tdt;
1562         uint16_t sw_tdt;
1563         struct em_tx_desc *tx_desc;
1564         uint16_t length = 0;
1565         boolean_t eop = 0;
1566
1567         hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1568         sw_tdt = adapter->next_avail_tx_desc;
1569
1570         while (hw_tdt != sw_tdt) {
1571                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1572                 length += tx_desc->lower.flags.length;
1573                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1574                 if (++hw_tdt == adapter->num_tx_desc)
1575                         hw_tdt = 0;
1576
1577                 if (eop) {
1578                         if (em_82547_fifo_workaround(adapter, length)) {
1579                                 adapter->tx_fifo_wrk_cnt++;
1580                                 callout_reset(&adapter->tx_fifo_timer, 1,
1581                                         em_82547_move_tail, adapter);
1582                                 break;
1583                         }
1584                         E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1585                         em_82547_update_fifo_head(adapter, length);
1586                         length = 0;
1587                 }
1588         }
1589 }
1590
1591 static int
1592 em_82547_fifo_workaround(struct adapter *adapter, int len)
1593 {
1594         int fifo_space, fifo_pkt_len;
1595
1596         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1597
1598         if (adapter->link_duplex == HALF_DUPLEX) {
1599                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1600
1601                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1602                         if (em_82547_tx_fifo_reset(adapter))
1603                                 return (0);
1604                         else
1605                                 return (1);
1606                 }
1607         }
1608
1609         return (0);
1610 }
1611
1612 static void
1613 em_82547_update_fifo_head(struct adapter *adapter, int len)
1614 {
1615         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1616
1617         /* tx_fifo_head is always 16 byte aligned */
1618         adapter->tx_fifo_head += fifo_pkt_len;
1619         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
1620                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1621 }
1622
1623 static int
1624 em_82547_tx_fifo_reset(struct adapter *adapter)
1625 {
1626         uint32_t tctl;
1627
1628         if (E1000_READ_REG(&adapter->hw, TDT) == E1000_READ_REG(&adapter->hw, TDH) &&
1629             E1000_READ_REG(&adapter->hw, TDFT) == E1000_READ_REG(&adapter->hw, TDFH) &&
1630             E1000_READ_REG(&adapter->hw, TDFTS) == E1000_READ_REG(&adapter->hw, TDFHS) &&
1631             E1000_READ_REG(&adapter->hw, TDFPC) == 0) {
1632                 /* Disable TX unit */
1633                 tctl = E1000_READ_REG(&adapter->hw, TCTL);
1634                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1635
1636                 /* Reset FIFO pointers */
1637                 E1000_WRITE_REG(&adapter->hw, TDFT,  adapter->tx_head_addr);
1638                 E1000_WRITE_REG(&adapter->hw, TDFH,  adapter->tx_head_addr);
1639                 E1000_WRITE_REG(&adapter->hw, TDFTS, adapter->tx_head_addr);
1640                 E1000_WRITE_REG(&adapter->hw, TDFHS, adapter->tx_head_addr);
1641
1642                 /* Re-enable TX unit */
1643                 E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1644                 E1000_WRITE_FLUSH(&adapter->hw);
1645
1646                 adapter->tx_fifo_head = 0;
1647                 adapter->tx_fifo_reset_cnt++;
1648
1649                 return (TRUE);
1650         } else {
1651                 return (FALSE);
1652         }
1653 }
1654
1655 static void
1656 em_set_promisc(struct adapter *adapter)
1657 {
1658         uint32_t reg_rctl;
1659         struct ifnet *ifp = &adapter->interface_data.ac_if;
1660
1661         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1662
1663         adapter->em_insert_vlan_header = 0;
1664         if (ifp->if_flags & IFF_PROMISC) {
1665                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1666                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1667
1668                 /*
1669                  * Disable VLAN stripping in promiscous mode.
1670                  * This enables bridging of vlan tagged frames to occur
1671                  * and also allows vlan tags to be seen in tcpdump.
1672                  */
1673                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1674                         em_disable_vlans(adapter);
1675                 adapter->em_insert_vlan_header = 1;
1676         } else if (ifp->if_flags & IFF_ALLMULTI) {
1677                 reg_rctl |= E1000_RCTL_MPE;
1678                 reg_rctl &= ~E1000_RCTL_UPE;
1679                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1680         }
1681 }
1682
1683 static void
1684 em_disable_promisc(struct adapter *adapter)
1685 {
1686         struct ifnet *ifp = &adapter->interface_data.ac_if;
1687
1688         uint32_t reg_rctl;
1689
1690         reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1691
1692         reg_rctl &= (~E1000_RCTL_UPE);
1693         reg_rctl &= (~E1000_RCTL_MPE);
1694         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1695
1696         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1697                 em_enable_vlans(adapter);
1698         adapter->em_insert_vlan_header = 0;
1699 }
1700
1701 /*********************************************************************
1702  *  Multicast Update
1703  *
1704  *  This routine is called whenever multicast address list is updated.
1705  *
1706  **********************************************************************/
1707
1708 static void
1709 em_set_multi(struct adapter *adapter)
1710 {
1711         uint32_t reg_rctl = 0;
1712         uint8_t mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1713         struct ifmultiaddr *ifma;
1714         int mcnt = 0;
1715         struct ifnet *ifp = &adapter->interface_data.ac_if;
1716
1717         IOCTL_DEBUGOUT("em_set_multi: begin");
1718
1719         if (adapter->hw.mac_type == em_82542_rev2_0) {
1720                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1721                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1722                         em_pci_clear_mwi(&adapter->hw);
1723                 reg_rctl |= E1000_RCTL_RST;
1724                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1725                 msec_delay(5);
1726         }
1727
1728         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1729                 if (ifma->ifma_addr->sa_family != AF_LINK)
1730                         continue;
1731
1732                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1733                         break;
1734
1735                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1736                       &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1737                 mcnt++;
1738         }
1739
1740         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1741                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1742                 reg_rctl |= E1000_RCTL_MPE;
1743                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1744         } else {
1745                 em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1746         }
1747
1748         if (adapter->hw.mac_type == em_82542_rev2_0) {
1749                 reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1750                 reg_rctl &= ~E1000_RCTL_RST;
1751                 E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1752                 msec_delay(5);
1753                 if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1754                         em_pci_set_mwi(&adapter->hw);
1755         }
1756 }
1757
1758 /*********************************************************************
1759  *  Timer routine
1760  *
1761  *  This routine checks for link status and updates statistics.
1762  *
1763  **********************************************************************/
1764
1765 static void
1766 em_local_timer(void *arg)
1767 {
1768         struct ifnet *ifp;
1769         struct adapter *adapter = arg;
1770         ifp = &adapter->interface_data.ac_if;
1771
1772         lwkt_serialize_enter(ifp->if_serializer);
1773
1774         em_check_for_link(&adapter->hw);
1775         em_update_link_status(adapter);
1776         em_update_stats_counters(adapter);
1777         if (em_display_debug_stats && ifp->if_flags & IFF_RUNNING)
1778                 em_print_hw_stats(adapter);
1779         em_smartspeed(adapter);
1780
1781         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1782
1783         lwkt_serialize_exit(ifp->if_serializer);
1784 }
1785
1786 static void
1787 em_update_link_status(struct adapter *adapter)
1788 {
1789         if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1790                 if (adapter->link_active == 0) {
1791                         em_get_speed_and_duplex(&adapter->hw,
1792                                                 &adapter->link_speed,
1793                                                 &adapter->link_duplex);
1794                         /* Check if we may set SPEED_MODE bit on PCI-E */
1795                         if (adapter->link_speed == SPEED_1000 &&
1796                             (adapter->hw.mac_type == em_82571 ||
1797                              adapter->hw.mac_type == em_82572)) {
1798                                 int tarc0;
1799
1800                                 tarc0 = E1000_READ_REG(&adapter->hw, TARC0);
1801                                 tarc0 |= SPEED_MODE_BIT;
1802                                 E1000_WRITE_REG(&adapter->hw, TARC0, tarc0);
1803                         }
1804                         if (bootverbose) {
1805                                 if_printf(&adapter->interface_data.ac_if,
1806                                           "Link is up %d Mbps %s\n",
1807                                           adapter->link_speed,
1808                                           adapter->link_duplex == FULL_DUPLEX ?
1809                                                 "Full Duplex" : "Half Duplex");
1810                         }
1811                         adapter->link_active = 1;
1812                         adapter->smartspeed = 0;
1813 #ifdef notyet
1814                         ifp->if_baudrate = adapter->link_speed * 1000000;
1815                         if_link_state_change(ifp, LINK_STATE_UP);
1816 #endif
1817                 }
1818         } else {
1819                 if (adapter->link_active == 1) {
1820 #ifdef notyet
1821                         ifp->if_baudrate = 0;
1822 #endif
1823                         adapter->link_speed = 0;
1824                         adapter->link_duplex = 0;
1825                         if (bootverbose) {
1826                                 if_printf(&adapter->interface_data.ac_if,
1827                                           "Link is Down\n");
1828                         }
1829                         adapter->link_active = 0;
1830 #ifdef notyet
1831                         if_link_state_change(ifp, LINK_STATE_DOWN);
1832 #endif
1833                 }
1834         }
1835 }
1836
1837 /*********************************************************************
1838  *
1839  *  This routine disables all traffic on the adapter by issuing a
1840  *  global reset on the MAC and deallocates TX/RX buffers.
1841  *
1842  **********************************************************************/
1843
1844 static void
1845 em_stop(void *arg)
1846 {
1847         struct ifnet   *ifp;
1848         struct adapter * adapter = arg;
1849         ifp = &adapter->interface_data.ac_if;
1850
1851         ASSERT_SERIALIZED(ifp->if_serializer);
1852
1853         INIT_DEBUGOUT("em_stop: begin");
1854         em_disable_intr(adapter);
1855         em_reset_hw(&adapter->hw);
1856         callout_stop(&adapter->timer);
1857         callout_stop(&adapter->tx_fifo_timer);
1858         em_free_transmit_structures(adapter);
1859         em_free_receive_structures(adapter);
1860
1861         /* Tell the stack that the interface is no longer active */
1862         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1863         ifp->if_timer = 0;
1864 }
1865
1866 /*********************************************************************
1867  *
1868  *  Determine hardware revision.
1869  *
1870  **********************************************************************/
1871 static void
1872 em_identify_hardware(struct adapter *adapter)
1873 {
1874         device_t dev = adapter->dev;
1875
1876         /* Make sure our PCI config space has the necessary stuff set */
1877         adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1878         if (!((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
1879               (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN))) {
1880                 device_printf(dev, "Memory Access and/or Bus Master bits "
1881                               "were not set!\n");
1882                 adapter->hw.pci_cmd_word |= PCIM_CMD_BUSMASTEREN |
1883                                             PCIM_CMD_MEMEN;
1884                 pci_write_config(dev, PCIR_COMMAND,
1885                                  adapter->hw.pci_cmd_word, 2);
1886         }
1887
1888         /* Save off the information about this board */
1889         adapter->hw.vendor_id = pci_get_vendor(dev);
1890         adapter->hw.device_id = pci_get_device(dev);
1891         adapter->hw.revision_id = pci_get_revid(dev);
1892         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1893         adapter->hw.subsystem_id = pci_get_subdevice(dev);
1894
1895         /* Identify the MAC */
1896         if (em_set_mac_type(&adapter->hw))
1897                 device_printf(dev, "Unknown MAC Type\n");
1898
1899         if (adapter->hw.mac_type == em_82541 ||
1900             adapter->hw.mac_type == em_82541_rev_2 ||
1901             adapter->hw.mac_type == em_82547 ||
1902             adapter->hw.mac_type == em_82547_rev_2)
1903                 adapter->hw.phy_init_script = TRUE;
1904 }
1905
1906 static int
1907 em_allocate_pci_resources(device_t dev)
1908 {
1909         struct adapter *adapter = device_get_softc(dev);
1910         int rid;
1911
1912         rid = PCIR_BAR(0);
1913         adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1914                                                      &rid, RF_ACTIVE);
1915         if (adapter->res_memory == NULL) {
1916                 device_printf(dev, "Unable to allocate bus resource: memory\n");
1917                 return ENXIO;
1918         }
1919         adapter->osdep.mem_bus_space_tag =
1920                 rman_get_bustag(adapter->res_memory);
1921         adapter->osdep.mem_bus_space_handle =
1922             rman_get_bushandle(adapter->res_memory);
1923         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
1924
1925         if (adapter->hw.mac_type > em_82543) {
1926                 /* Figure our where our IO BAR is ? */
1927                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
1928                         uint32_t val;
1929
1930                         val = pci_read_config(dev, rid, 4);
1931                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
1932                                 adapter->io_rid = rid;
1933                                 break;
1934                         }
1935                         rid += 4;
1936                         /* check for 64bit BAR */
1937                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
1938                                 rid += 4;
1939                 }
1940                 if (rid >= PCIR_CIS) {
1941                         device_printf(dev, "Unable to locate IO BAR\n");
1942                         return (ENXIO);
1943                 }
1944
1945                 adapter->res_ioport = bus_alloc_resource_any(dev,
1946                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
1947                 if (!(adapter->res_ioport)) {
1948                         device_printf(dev, "Unable to allocate bus resource: "
1949                                       "ioport\n");
1950                         return ENXIO;
1951                 }
1952                 adapter->hw.io_base = 0;
1953                 adapter->osdep.io_bus_space_tag =
1954                         rman_get_bustag(adapter->res_ioport);
1955                 adapter->osdep.io_bus_space_handle =
1956                         rman_get_bushandle(adapter->res_ioport);
1957         }
1958
1959         /* For ICH8 we need to find the flash memory. */
1960         if (adapter->hw.mac_type == em_ich8lan) {
1961                 rid = EM_FLASH;
1962                 adapter->flash_mem = bus_alloc_resource_any(dev,
1963                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
1964                 if (adapter->flash_mem == NULL) {
1965                         device_printf(dev, "Unable to allocate bus resource: "
1966                                       "flash memory\n");
1967                         return ENXIO;
1968                 }
1969                 adapter->osdep.flash_bus_space_tag =
1970                     rman_get_bustag(adapter->flash_mem);
1971                 adapter->osdep.flash_bus_space_handle =
1972                     rman_get_bushandle(adapter->flash_mem);
1973         }
1974
1975         rid = 0x0;
1976         adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1977             &rid, RF_SHAREABLE | RF_ACTIVE);
1978         if (adapter->res_interrupt == NULL) {
1979                 device_printf(dev, "Unable to allocate bus resource: "
1980                               "interrupt\n");
1981                 return ENXIO;
1982         }
1983
1984         adapter->hw.back = &adapter->osdep;
1985
1986         return 0;
1987 }
1988
1989 static void
1990 em_free_pci_resources(device_t dev)
1991 {
1992         struct adapter *adapter = device_get_softc(dev);
1993
1994         if (adapter->res_interrupt != NULL) {
1995                 bus_release_resource(dev, SYS_RES_IRQ, 0,
1996                                      adapter->res_interrupt);
1997         }
1998         if (adapter->res_memory != NULL) {
1999                 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0),
2000                                      adapter->res_memory);
2001         }
2002
2003         if (adapter->res_ioport != NULL) {
2004                 bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
2005                                      adapter->res_ioport);
2006         }
2007
2008         if (adapter->flash_mem != NULL) {
2009                 bus_release_resource(dev, SYS_RES_MEMORY, EM_FLASH,
2010                                      adapter->flash_mem);
2011         }
2012 }
2013
2014 /*********************************************************************
2015  *
2016  *  Initialize the hardware to a configuration as specified by the
2017  *  adapter structure. The controller is reset, the EEPROM is
2018  *  verified, the MAC address is set, then the shared initialization
2019  *  routines are called.
2020  *
2021  **********************************************************************/
2022 static int
2023 em_hardware_init(struct adapter *adapter)
2024 {
2025         uint16_t        rx_buffer_size;
2026
2027         INIT_DEBUGOUT("em_hardware_init: begin");
2028         /* Issue a global reset */
2029         em_reset_hw(&adapter->hw);
2030
2031         /* When hardware is reset, fifo_head is also reset */
2032         adapter->tx_fifo_head = 0;
2033
2034         /* Make sure we have a good EEPROM before we read from it */
2035         if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
2036                 device_printf(adapter->dev,
2037                               "The EEPROM Checksum Is Not Valid\n");
2038                 return (EIO);
2039         }
2040
2041         if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
2042                 device_printf(adapter->dev,
2043                               "EEPROM read error while reading part number\n");
2044                 return (EIO);
2045         }
2046
2047         /* Set up smart power down as default off on newer adapters. */
2048         if (!em_smart_pwr_down &&
2049             (adapter->hw.mac_type == em_82571 ||
2050              adapter->hw.mac_type == em_82572)) {
2051                 uint16_t phy_tmp = 0;
2052
2053                 /* Speed up time to link by disabling smart power down. */
2054                 em_read_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT,
2055                                 &phy_tmp);
2056                 phy_tmp &= ~IGP02E1000_PM_SPD;
2057                 em_write_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT,
2058                                  phy_tmp);
2059         }
2060
2061         /*
2062          * These parameters control the automatic generation (Tx) and
2063          * response (Rx) to Ethernet PAUSE frames.
2064          * - High water mark should allow for at least two frames to be
2065          *   received after sending an XOFF.
2066          * - Low water mark works best when it is very near the high water mark.
2067          *   This allows the receiver to restart by sending XON when it has
2068          *   drained a bit.  Here we use an arbitary value of 1500 which will
2069          *   restart after one full frame is pulled from the buffer.  There
2070          *   could be several smaller frames in the buffer and if so they will
2071          *   not trigger the XON until their total number reduces the buffer
2072          *   by 1500.
2073          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2074          */
2075         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff) << 10);
2076
2077         adapter->hw.fc_high_water =
2078             rx_buffer_size - roundup2(adapter->hw.max_frame_size, 1024);
2079         adapter->hw.fc_low_water = adapter->hw.fc_high_water - 1500;
2080         if (adapter->hw.mac_type == em_80003es2lan)
2081                 adapter->hw.fc_pause_time = 0xFFFF;
2082         else
2083                 adapter->hw.fc_pause_time = 0x1000;
2084         adapter->hw.fc_send_xon = TRUE;
2085         adapter->hw.fc = E1000_FC_FULL;
2086
2087         if (em_init_hw(&adapter->hw) < 0) {
2088                 device_printf(adapter->dev, "Hardware Initialization Failed");
2089                 return (EIO);
2090         }
2091
2092         em_check_for_link(&adapter->hw);
2093
2094         return (0);
2095 }
2096
2097 /*********************************************************************
2098  *
2099  *  Setup networking device structure and register an interface.
2100  *
2101  **********************************************************************/
2102 static void
2103 em_setup_interface(device_t dev, struct adapter *adapter)
2104 {
2105         struct ifnet *ifp;
2106         u_char fiber_type = IFM_1000_SX;        /* default type */
2107         INIT_DEBUGOUT("em_setup_interface: begin");
2108
2109         ifp = &adapter->interface_data.ac_if;
2110         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2111         ifp->if_mtu = ETHERMTU;
2112         ifp->if_baudrate = 1000000000;
2113         ifp->if_init =  em_init;
2114         ifp->if_softc = adapter;
2115         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2116         ifp->if_ioctl = em_ioctl;
2117         ifp->if_start = em_start;
2118 #ifdef DEVICE_POLLING
2119         ifp->if_poll = em_poll;
2120 #endif
2121         ifp->if_watchdog = em_watchdog;
2122         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
2123         ifq_set_ready(&ifp->if_snd);
2124
2125         if (adapter->hw.mac_type >= em_82543)
2126                 ifp->if_capabilities |= IFCAP_HWCSUM;
2127
2128         ifp->if_capenable = ifp->if_capabilities;
2129
2130         ether_ifattach(ifp, adapter->hw.mac_addr, NULL);
2131
2132         /*
2133          * Tell the upper layer(s) we support long frames.
2134          */
2135         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2136         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2137 #if 0
2138         ifp->if_capenable |= IFCAP_VLAN_MTU;
2139 #endif
2140
2141         /*
2142          * Specify the media types supported by this adapter and register
2143          * callbacks to update media and link information
2144          */
2145         ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
2146                      em_media_status);
2147         if (adapter->hw.media_type == em_media_type_fiber ||
2148             adapter->hw.media_type == em_media_type_internal_serdes) {
2149                 if (adapter->hw.mac_type == em_82545)
2150                         fiber_type = IFM_1000_LX;
2151                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2152                             0, NULL);
2153                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2154         } else {
2155                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2156                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2157                             0, NULL);
2158                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2159                             0, NULL);
2160                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2161                             0, NULL);
2162                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
2163                             0, NULL);
2164                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2165         }
2166         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2167         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2168 }
2169
2170 /*********************************************************************
2171  *
2172  *  Workaround for SmartSpeed on 82541 and 82547 controllers
2173  *
2174  **********************************************************************/
2175 static void
2176 em_smartspeed(struct adapter *adapter)
2177 {
2178         uint16_t phy_tmp;
2179
2180         if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
2181             !adapter->hw.autoneg ||
2182             !(adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL))
2183                 return;
2184
2185         if (adapter->smartspeed == 0) {
2186                 /*
2187                  * If Master/Slave config fault is asserted twice,
2188                  * we assume back-to-back.
2189                  */
2190                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2191                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2192                         return;
2193                 em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2194                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2195                         em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2196                         if (phy_tmp & CR_1000T_MS_ENABLE) {
2197                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
2198                                 em_write_phy_reg(&adapter->hw,
2199                                                  PHY_1000T_CTRL, phy_tmp);
2200                                 adapter->smartspeed++;
2201                                 if (adapter->hw.autoneg &&
2202                                     !em_phy_setup_autoneg(&adapter->hw) &&
2203                                     !em_read_phy_reg(&adapter->hw, PHY_CTRL,
2204                                                      &phy_tmp)) {
2205                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
2206                                                     MII_CR_RESTART_AUTO_NEG);
2207                                         em_write_phy_reg(&adapter->hw,
2208                                                          PHY_CTRL, phy_tmp);
2209                                 }
2210                         }
2211                 }
2212                 return;
2213         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2214                 /* If still no link, perhaps using 2/3 pair cable */
2215                 em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2216                 phy_tmp |= CR_1000T_MS_ENABLE;
2217                 em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2218                 if (adapter->hw.autoneg &&
2219                     !em_phy_setup_autoneg(&adapter->hw) &&
2220                     !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
2221                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
2222                                     MII_CR_RESTART_AUTO_NEG);
2223                         em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
2224                 }
2225         }
2226         /* Restart process after EM_SMARTSPEED_MAX iterations */
2227         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2228                 adapter->smartspeed = 0;
2229 }
2230
2231 /*
2232  * Manage DMA'able memory.
2233  */
2234 static void
2235 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2236 {
2237         if (error)
2238                 return;
2239         *(bus_addr_t *)arg = segs->ds_addr;
2240 }
2241
2242 static int
2243 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2244               struct em_dma_alloc *dma)
2245 {
2246         device_t dev = adapter->dev;
2247         int error;
2248
2249         error = bus_dma_tag_create(NULL,                /* parent */
2250                                    EM_DBA_ALIGN, 0,     /* alignment, bounds */
2251                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2252                                    BUS_SPACE_MAXADDR,   /* highaddr */
2253                                    NULL, NULL,          /* filter, filterarg */
2254                                    size,                /* maxsize */
2255                                    1,                   /* nsegments */
2256                                    size,                /* maxsegsize */
2257                                    0,                   /* flags */
2258                                    &dma->dma_tag);
2259         if (error) {
2260                 device_printf(dev, "%s: bus_dma_tag_create failed; error %d\n",
2261                               __func__, error);
2262                 return error;
2263         }
2264
2265         error = bus_dmamem_alloc(dma->dma_tag, (void**)&dma->dma_vaddr,
2266                                  BUS_DMA_WAITOK, &dma->dma_map);
2267         if (error) {
2268                 device_printf(dev, "%s: bus_dmammem_alloc failed; "
2269                               "size %llu, error %d\n",
2270                               __func__, (uintmax_t)size, error);
2271                 goto fail;
2272         }
2273
2274         error = bus_dmamap_load(dma->dma_tag, dma->dma_map,
2275                                 dma->dma_vaddr, size,
2276                                 em_dmamap_cb, &dma->dma_paddr,
2277                                 BUS_DMA_WAITOK);
2278         if (error) {
2279                 device_printf(dev, "%s: bus_dmamap_load failed; error %u\n",
2280                               __func__, error);
2281                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2282                 goto fail;
2283         }
2284
2285         return 0;
2286 fail:
2287         bus_dma_tag_destroy(dma->dma_tag);
2288         dma->dma_tag = NULL;
2289         return error;
2290 }
2291
2292 static void
2293 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2294 {
2295         if (dma->dma_tag != NULL) {
2296                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2297                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2298                 bus_dma_tag_destroy(dma->dma_tag);
2299                 dma->dma_tag = NULL;
2300         }
2301 }
2302
2303 /*********************************************************************
2304  *
2305  *  Allocate and initialize transmit structures.
2306  *
2307  **********************************************************************/
2308 static int
2309 em_setup_transmit_structures(struct adapter *adapter)
2310 {
2311         struct em_buffer *tx_buffer;
2312         bus_size_t size;
2313         int error, i;
2314
2315         /*
2316          * Setup DMA descriptor areas.
2317          */
2318         size = roundup2(adapter->hw.max_frame_size, MCLBYTES);
2319         if (bus_dma_tag_create(NULL,                    /* parent */
2320                                1, 0,                    /* alignment, bounds */
2321                                BUS_SPACE_MAXADDR,       /* lowaddr */
2322                                BUS_SPACE_MAXADDR,       /* highaddr */
2323                                NULL, NULL,              /* filter, filterarg */
2324                                size,                    /* maxsize */
2325                                EM_MAX_SCATTER,          /* nsegments */
2326                                size,                    /* maxsegsize */
2327                                0,                       /* flags */
2328                                &adapter->txtag)) {
2329                 device_printf(adapter->dev, "Unable to allocate TX DMA tag\n");
2330                 return(ENOMEM);
2331         }
2332
2333         adapter->tx_buffer_area =
2334                 kmalloc(sizeof(struct em_buffer) * adapter->num_tx_desc,
2335                         M_DEVBUF, M_WAITOK | M_ZERO);
2336
2337         bzero(adapter->tx_desc_base,
2338               sizeof(struct em_tx_desc) * adapter->num_tx_desc);
2339         tx_buffer = adapter->tx_buffer_area;
2340         for (i = 0; i < adapter->num_tx_desc; i++) {
2341                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
2342                 if (error) {
2343                         device_printf(adapter->dev,
2344                                       "Unable to create TX DMA map\n");
2345                         goto fail;
2346                 }
2347                 tx_buffer++;
2348         }
2349
2350         adapter->next_avail_tx_desc = 0;
2351         adapter->next_tx_to_clean = 0;
2352
2353         /* Set number of descriptors available */
2354         adapter->num_tx_desc_avail = adapter->num_tx_desc;
2355
2356         /* Set checksum context */
2357         adapter->active_checksum_context = OFFLOAD_NONE;
2358
2359         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2360                         BUS_DMASYNC_PREWRITE);
2361
2362         return (0);
2363 fail:
2364         em_free_transmit_structures(adapter);
2365         return (error);
2366 }
2367
2368 /*********************************************************************
2369  *
2370  *  Enable transmit unit.
2371  *
2372  **********************************************************************/
2373 static void
2374 em_initialize_transmit_unit(struct adapter *adapter)
2375 {
2376         uint32_t reg_tctl;
2377         uint32_t reg_tipg = 0;
2378         uint64_t bus_addr;
2379
2380         INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
2381
2382         /* Setup the Base and Length of the Tx Descriptor Ring */
2383         bus_addr = adapter->txdma.dma_paddr;
2384         E1000_WRITE_REG(&adapter->hw, TDLEN,
2385                         adapter->num_tx_desc * sizeof(struct em_tx_desc));
2386         E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
2387         E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
2388
2389         /* Setup the HW Tx Head and Tail descriptor pointers */
2390         E1000_WRITE_REG(&adapter->hw, TDT, 0);
2391         E1000_WRITE_REG(&adapter->hw, TDH, 0);
2392
2393         HW_DEBUGOUT2("Base = %x, Length = %x\n",
2394                      E1000_READ_REG(&adapter->hw, TDBAL),
2395                      E1000_READ_REG(&adapter->hw, TDLEN));
2396
2397         /* Set the default values for the Tx Inter Packet Gap timer */
2398         switch (adapter->hw.mac_type) {
2399         case em_82542_rev2_0:
2400         case em_82542_rev2_1:
2401                 reg_tipg = DEFAULT_82542_TIPG_IPGT;
2402                 reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2403                 reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2404                 break;
2405         case em_80003es2lan:
2406                 reg_tipg = DEFAULT_82543_TIPG_IPGR1;
2407                 reg_tipg |=
2408                     DEFAULT_80003ES2LAN_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2409                 break;
2410         default:
2411                 if (adapter->hw.media_type == em_media_type_fiber ||
2412                     adapter->hw.media_type == em_media_type_internal_serdes)
2413                         reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2414                 else
2415                         reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2416                 reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2417                 reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2418         }
2419
2420         E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
2421         E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
2422         if (adapter->hw.mac_type >= em_82540) {
2423                 E1000_WRITE_REG(&adapter->hw, TADV,
2424                                 adapter->tx_abs_int_delay.value);
2425         }
2426
2427         /* Program the Transmit Control Register */
2428         reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
2429                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2430         if (adapter->hw.mac_type >= em_82571)
2431                 reg_tctl |= E1000_TCTL_MULR;
2432         if (adapter->link_duplex == 1)
2433                 reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2434         else
2435                 reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2436
2437         /* This write will effectively turn on the transmit unit. */
2438         E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
2439
2440         /* Setup Transmit Descriptor Base Settings */
2441         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
2442
2443         if (adapter->tx_int_delay.value > 0)
2444                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2445 }
2446
2447 /*********************************************************************
2448  *
2449  *  Free all transmit related data structures.
2450  *
2451  **********************************************************************/
2452 static void
2453 em_free_transmit_structures(struct adapter *adapter)
2454 {
2455         struct em_buffer *tx_buffer;
2456         int i;
2457
2458         INIT_DEBUGOUT("free_transmit_structures: begin");
2459
2460         if (adapter->tx_buffer_area != NULL) {
2461                 tx_buffer = adapter->tx_buffer_area;
2462                 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2463                         if (tx_buffer->m_head != NULL) {
2464                                 bus_dmamap_unload(adapter->txtag,
2465                                                   tx_buffer->map);
2466                                 m_freem(tx_buffer->m_head);
2467                         }
2468
2469                         if (tx_buffer->map != NULL) {
2470                                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2471                                 tx_buffer->map = NULL;
2472                         }
2473                         tx_buffer->m_head = NULL;
2474                 }
2475         }
2476         if (adapter->tx_buffer_area != NULL) {
2477                 kfree(adapter->tx_buffer_area, M_DEVBUF);
2478                 adapter->tx_buffer_area = NULL;
2479         }
2480         if (adapter->txtag != NULL) {
2481                 bus_dma_tag_destroy(adapter->txtag);
2482                 adapter->txtag = NULL;
2483         }
2484 }
2485
2486 /*********************************************************************
2487  *
2488  *  The offload context needs to be set when we transfer the first
2489  *  packet of a particular protocol (TCP/UDP). We change the
2490  *  context only if the protocol type changes.
2491  *
2492  **********************************************************************/
2493 static void
2494 em_transmit_checksum_setup(struct adapter *adapter,
2495                            struct mbuf *mp,
2496                            uint32_t *txd_upper,
2497                            uint32_t *txd_lower)
2498 {
2499         struct em_context_desc *TXD;
2500         struct em_buffer *tx_buffer;
2501         int curr_txd;
2502
2503         if (mp->m_pkthdr.csum_flags) {
2504                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2505                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2506                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2507                         if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2508                                 return;
2509                         else
2510                                 adapter->active_checksum_context = OFFLOAD_TCP_IP;
2511                 } else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2512                         *txd_upper = E1000_TXD_POPTS_TXSM << 8;
2513                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2514                         if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2515                                 return;
2516                         else
2517                                 adapter->active_checksum_context = OFFLOAD_UDP_IP;
2518                 } else {
2519                         *txd_upper = 0;
2520                         *txd_lower = 0;
2521                         return;
2522                 }
2523         } else {
2524                 *txd_upper = 0;
2525                 *txd_lower = 0;
2526                 return;
2527         }
2528
2529         /*
2530          * If we reach this point, the checksum offload context
2531          * needs to be reset.
2532          */
2533         curr_txd = adapter->next_avail_tx_desc;
2534         tx_buffer = &adapter->tx_buffer_area[curr_txd];
2535         TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2536
2537         TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2538         TXD->lower_setup.ip_fields.ipcso =
2539             ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2540         TXD->lower_setup.ip_fields.ipcse =
2541             htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2542
2543         TXD->upper_setup.tcp_fields.tucss =
2544             ETHER_HDR_LEN + sizeof(struct ip);
2545         TXD->upper_setup.tcp_fields.tucse = htole16(0);
2546
2547         if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2548                 TXD->upper_setup.tcp_fields.tucso =
2549                         ETHER_HDR_LEN + sizeof(struct ip) +
2550                         offsetof(struct tcphdr, th_sum);
2551         } else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2552                 TXD->upper_setup.tcp_fields.tucso =
2553                         ETHER_HDR_LEN + sizeof(struct ip) +
2554                         offsetof(struct udphdr, uh_sum);
2555         }
2556
2557         TXD->tcp_seg_setup.data = htole32(0);
2558         TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2559
2560         tx_buffer->m_head = NULL;
2561         tx_buffer->next_eop = -1;
2562
2563         if (++curr_txd == adapter->num_tx_desc)
2564                 curr_txd = 0;
2565
2566         adapter->num_tx_desc_avail--;
2567         adapter->next_avail_tx_desc = curr_txd;
2568 }
2569
2570 /**********************************************************************
2571  *
2572  *  Examine each tx_buffer in the used queue. If the hardware is done
2573  *  processing the packet then free associated resources. The
2574  *  tx_buffer is put back on the free queue.
2575  *
2576  **********************************************************************/
2577
2578 static void
2579 em_txeof(struct adapter *adapter)
2580 {
2581         int first, last, done, num_avail;
2582         struct em_buffer *tx_buffer;
2583         struct em_tx_desc *tx_desc, *eop_desc;
2584         struct ifnet *ifp = &adapter->interface_data.ac_if;
2585
2586         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2587                 return;
2588
2589         num_avail = adapter->num_tx_desc_avail;
2590         first = adapter->next_tx_to_clean;
2591         tx_desc = &adapter->tx_desc_base[first];
2592         tx_buffer = &adapter->tx_buffer_area[first];
2593         last = tx_buffer->next_eop;
2594         KKASSERT(last >= 0 && last < adapter->num_tx_desc);
2595         eop_desc = &adapter->tx_desc_base[last];
2596
2597         /*
2598          * Now caculate the terminating index for the cleanup loop below
2599          */
2600         if (++last == adapter->num_tx_desc)
2601                 last = 0;
2602         done = last;
2603
2604         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2605                         BUS_DMASYNC_POSTREAD);
2606
2607         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2608                 while (first != done) {
2609                         tx_desc->upper.data = 0;
2610                         tx_desc->lower.data = 0;
2611                         num_avail++;
2612
2613                         logif(pkt_txclean);
2614
2615                         if (tx_buffer->m_head) {
2616                                 ifp->if_opackets++;
2617                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2618                                                 BUS_DMASYNC_POSTWRITE);
2619                                 bus_dmamap_unload(adapter->txtag,
2620                                                   tx_buffer->map);
2621
2622                                 m_freem(tx_buffer->m_head);
2623                                 tx_buffer->m_head = NULL;
2624                         }
2625                         tx_buffer->next_eop = -1;
2626
2627                         if (++first == adapter->num_tx_desc)
2628                                 first = 0;
2629
2630                         tx_buffer = &adapter->tx_buffer_area[first];
2631                         tx_desc = &adapter->tx_desc_base[first];
2632                 }
2633                 /* See if we can continue to the next packet */
2634                 last = tx_buffer->next_eop;
2635                 if (last != -1) {
2636                         KKASSERT(last >= 0 && last < adapter->num_tx_desc);
2637                         eop_desc = &adapter->tx_desc_base[last];
2638                         if (++last == adapter->num_tx_desc)
2639                                 last = 0;
2640                         done = last;
2641                 } else {
2642                         break;
2643                 }
2644         }
2645
2646         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2647                         BUS_DMASYNC_PREWRITE);
2648
2649         adapter->next_tx_to_clean = first;
2650
2651         /*
2652          * If we have enough room, clear IFF_OACTIVE to tell the stack
2653          * that it is OK to send packets.
2654          * If there are no pending descriptors, clear the timeout. Otherwise,
2655          * if some descriptors have been freed, restart the timeout.
2656          */
2657         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2658                 ifp->if_flags &= ~IFF_OACTIVE;
2659                 if (num_avail == adapter->num_tx_desc)
2660                         ifp->if_timer = 0;
2661                 else if (num_avail == adapter->num_tx_desc_avail)
2662                         ifp->if_timer = EM_TX_TIMEOUT;
2663         }
2664         adapter->num_tx_desc_avail = num_avail;
2665 }
2666
2667 /*********************************************************************
2668  *
2669  *  Get a buffer from system mbuf buffer pool.
2670  *
2671  **********************************************************************/
2672 static int
2673 em_get_buf(int i, struct adapter *adapter, struct mbuf *nmp, int how)
2674 {
2675         struct mbuf *mp = nmp;
2676         struct em_buffer *rx_buffer;
2677         struct ifnet *ifp;
2678         bus_addr_t paddr;
2679         int error;
2680
2681         ifp = &adapter->interface_data.ac_if;
2682
2683         if (mp == NULL) {
2684                 mp = m_getcl(how, MT_DATA, M_PKTHDR);
2685                 if (mp == NULL) {
2686                         adapter->mbuf_cluster_failed++;
2687                         return (ENOBUFS);
2688                 }
2689                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2690         } else {
2691                 mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2692                 mp->m_data = mp->m_ext.ext_buf;
2693                 mp->m_next = NULL;
2694         }
2695
2696         if (ifp->if_mtu <= ETHERMTU)
2697                 m_adj(mp, ETHER_ALIGN);
2698
2699         rx_buffer = &adapter->rx_buffer_area[i];
2700
2701         /*
2702          * Using memory from the mbuf cluster pool, invoke the
2703          * bus_dma machinery to arrange the memory mapping.
2704          */
2705         error = bus_dmamap_load(adapter->rxtag, rx_buffer->map,
2706                                 mtod(mp, void *), mp->m_len,
2707                                 em_dmamap_cb, &paddr, 0);
2708         if (error) {
2709                 m_free(mp);
2710                 return (error);
2711         }
2712         rx_buffer->m_head = mp;
2713         adapter->rx_desc_base[i].buffer_addr = htole64(paddr);
2714         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2715
2716         return (0);
2717 }
2718
2719 /*********************************************************************
2720  *
2721  *  Allocate memory for rx_buffer structures. Since we use one
2722  *  rx_buffer per received packet, the maximum number of rx_buffer's
2723  *  that we'll need is equal to the number of receive descriptors
2724  *  that we've allocated.
2725  *
2726  **********************************************************************/
2727 static int
2728 em_allocate_receive_structures(struct adapter *adapter)
2729 {
2730         int i, error, size;
2731         struct em_buffer *rx_buffer;
2732
2733         size = adapter->num_rx_desc * sizeof(struct em_buffer);
2734         adapter->rx_buffer_area = kmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
2735
2736         error = bus_dma_tag_create(NULL,                /* parent */
2737                                    1, 0,                /* alignment, bounds */
2738                                    BUS_SPACE_MAXADDR,   /* lowaddr */
2739                                    BUS_SPACE_MAXADDR,   /* highaddr */
2740                                    NULL, NULL,          /* filter, filterarg */
2741                                    MCLBYTES,            /* maxsize */
2742                                    1,                   /* nsegments */
2743                                    MCLBYTES,            /* maxsegsize */
2744                                    0,                   /* flags */
2745                                    &adapter->rxtag);
2746         if (error) {
2747                 device_printf(adapter->dev, "%s: bus_dma_tag_create failed; "
2748                               "error %u\n", __func__, error);
2749                 goto fail;
2750         }
2751
2752         rx_buffer = adapter->rx_buffer_area;
2753         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2754                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2755                                           &rx_buffer->map);
2756                 if (error) {
2757                         device_printf(adapter->dev,
2758                                       "%s: bus_dmamap_create failed; "
2759                                       "error %u\n", __func__, error);
2760                         goto fail;
2761                 }
2762         }
2763
2764         for (i = 0; i < adapter->num_rx_desc; i++) {
2765                 error = em_get_buf(i, adapter, NULL, MB_WAIT);
2766                 if (error)
2767                         goto fail;
2768         }
2769
2770         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2771                         BUS_DMASYNC_PREWRITE);
2772
2773         return (0);
2774 fail:
2775         em_free_receive_structures(adapter);
2776         return (error);
2777 }
2778
2779 /*********************************************************************
2780  *
2781  *  Allocate and initialize receive structures.
2782  *
2783  **********************************************************************/
2784 static int
2785 em_setup_receive_structures(struct adapter *adapter)
2786 {
2787         int error;
2788
2789         bzero(adapter->rx_desc_base,
2790               sizeof(struct em_rx_desc) * adapter->num_rx_desc);
2791
2792         error = em_allocate_receive_structures(adapter);
2793         if (error)
2794                 return (error);
2795
2796         /* Setup our descriptor pointers */
2797         adapter->next_rx_desc_to_check = 0;
2798
2799         return (0);
2800 }
2801
2802 /*********************************************************************
2803  *
2804  *  Enable receive unit.
2805  *
2806  **********************************************************************/
2807 static void
2808 em_initialize_receive_unit(struct adapter *adapter)
2809 {
2810         uint32_t reg_rctl;
2811         uint32_t reg_rxcsum;
2812         struct ifnet *ifp;
2813         uint64_t bus_addr;
2814
2815         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2816
2817         ifp = &adapter->interface_data.ac_if;
2818
2819         /*
2820          * Make sure receives are disabled while setting
2821          * up the descriptor ring
2822          */
2823         E1000_WRITE_REG(&adapter->hw, RCTL, 0);
2824
2825         /* Set the Receive Delay Timer Register */
2826         E1000_WRITE_REG(&adapter->hw, RDTR,
2827                         adapter->rx_int_delay.value | E1000_RDT_FPDB);
2828
2829         if(adapter->hw.mac_type >= em_82540) {
2830                 E1000_WRITE_REG(&adapter->hw, RADV,
2831                                 adapter->rx_abs_int_delay.value);
2832
2833                 /* Set the interrupt throttling rate in 256ns increments */
2834                 if (em_int_throttle_ceil) {
2835                         E1000_WRITE_REG(&adapter->hw, ITR,
2836                                 1000000000 / 256 / em_int_throttle_ceil);
2837                 } else {
2838                         E1000_WRITE_REG(&adapter->hw, ITR, 0);
2839                 }
2840         }
2841
2842         /* Setup the Base and Length of the Rx Descriptor Ring */
2843         bus_addr = adapter->rxdma.dma_paddr;
2844         E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
2845                         sizeof(struct em_rx_desc));
2846         E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
2847         E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
2848
2849         /* Setup the Receive Control Register */
2850         reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
2851                    E1000_RCTL_RDMTS_HALF |
2852                    (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
2853
2854         if (adapter->hw.tbi_compatibility_on == TRUE)
2855                 reg_rctl |= E1000_RCTL_SBP;
2856
2857         switch (adapter->rx_buffer_len) {
2858         default:
2859         case EM_RXBUFFER_2048:
2860                 reg_rctl |= E1000_RCTL_SZ_2048;
2861                 break;
2862         case EM_RXBUFFER_4096:
2863                 reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX |
2864                             E1000_RCTL_LPE;
2865                 break;
2866         case EM_RXBUFFER_8192:
2867                 reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX |
2868                             E1000_RCTL_LPE;
2869                 break;
2870         case EM_RXBUFFER_16384:
2871                 reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX |
2872                             E1000_RCTL_LPE;
2873                 break;
2874         }
2875
2876         if (ifp->if_mtu > ETHERMTU)
2877                 reg_rctl |= E1000_RCTL_LPE;
2878
2879         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
2880         if ((adapter->hw.mac_type >= em_82543) &&
2881             (ifp->if_capenable & IFCAP_RXCSUM)) {
2882                 reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
2883                 reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
2884                 E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
2885         }
2886
2887         /* Enable Receives */
2888         E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
2889
2890         /* Setup the HW Rx Head and Tail Descriptor Pointers */
2891         E1000_WRITE_REG(&adapter->hw, RDH, 0);
2892         E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
2893 }
2894
2895 /*********************************************************************
2896  *
2897  *  Free receive related data structures.
2898  *
2899  **********************************************************************/
2900 static void
2901 em_free_receive_structures(struct adapter *adapter)
2902 {
2903         struct em_buffer *rx_buffer;
2904         int i;
2905
2906         INIT_DEBUGOUT("free_receive_structures: begin");
2907
2908         if (adapter->rx_buffer_area != NULL) {
2909                 rx_buffer = adapter->rx_buffer_area;
2910                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2911                         if (rx_buffer->m_head != NULL) {
2912                                 bus_dmamap_unload(adapter->rxtag,
2913                                                   rx_buffer->map);
2914                                 m_freem(rx_buffer->m_head);
2915                                 rx_buffer->m_head = NULL;
2916                         }
2917                         if (rx_buffer->map != NULL) {
2918                                 bus_dmamap_destroy(adapter->rxtag,
2919                                                    rx_buffer->map);
2920                                 rx_buffer->map = NULL;
2921                         }
2922                 }
2923         }
2924         if (adapter->rx_buffer_area != NULL) {
2925                 kfree(adapter->rx_buffer_area, M_DEVBUF);
2926                 adapter->rx_buffer_area = NULL;
2927         }
2928         if (adapter->rxtag != NULL) {
2929                 bus_dma_tag_destroy(adapter->rxtag);
2930                 adapter->rxtag = NULL;
2931         }
2932 }
2933
2934 /*********************************************************************
2935  *
2936  *  This routine executes in interrupt context. It replenishes
2937  *  the mbufs in the descriptor and sends data which has been
2938  *  dma'ed into host memory to upper layer.
2939  *
2940  *  We loop at most count times if count is > 0, or until done if
2941  *  count < 0.
2942  *
2943  *********************************************************************/
2944 static void
2945 em_rxeof(struct adapter *adapter, int count)
2946 {
2947         struct ifnet *ifp;
2948         struct mbuf *mp;
2949         uint8_t accept_frame = 0;
2950         uint8_t eop = 0;
2951         uint16_t len, desc_len, prev_len_adj;
2952         int i;
2953
2954         /* Pointer to the receive descriptor being examined. */
2955         struct em_rx_desc *current_desc;
2956
2957         ifp = &adapter->interface_data.ac_if;
2958         i = adapter->next_rx_desc_to_check;
2959         current_desc = &adapter->rx_desc_base[i];
2960
2961         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2962                         BUS_DMASYNC_POSTREAD);
2963
2964         if (!(current_desc->status & E1000_RXD_STAT_DD))
2965                 return;
2966
2967         while ((current_desc->status & E1000_RXD_STAT_DD) && count != 0) {
2968                 logif(pkt_receive);
2969                 mp = adapter->rx_buffer_area[i].m_head;
2970                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
2971                                 BUS_DMASYNC_POSTREAD);
2972                 bus_dmamap_unload(adapter->rxtag,
2973                                   adapter->rx_buffer_area[i].map);
2974
2975                 accept_frame = 1;
2976                 prev_len_adj = 0;
2977                 desc_len = le16toh(current_desc->length);
2978                 if (current_desc->status & E1000_RXD_STAT_EOP) {
2979                         count--;
2980                         eop = 1;
2981                         if (desc_len < ETHER_CRC_LEN) {
2982                                 len = 0;
2983                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
2984                         } else {
2985                                 len = desc_len - ETHER_CRC_LEN;
2986                         }
2987                 } else {
2988                         eop = 0;
2989                         len = desc_len;
2990                 }
2991
2992                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
2993                         uint8_t last_byte;
2994                         uint32_t pkt_len = desc_len;
2995
2996                         if (adapter->fmp != NULL)
2997                                 pkt_len += adapter->fmp->m_pkthdr.len;
2998
2999                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3000
3001                         if (TBI_ACCEPT(&adapter->hw, current_desc->status,
3002                                        current_desc->errors,
3003                                        pkt_len, last_byte)) {
3004                                 em_tbi_adjust_stats(&adapter->hw,
3005                                                     &adapter->stats,
3006                                                     pkt_len,
3007                                                     adapter->hw.mac_addr);
3008                                 if (len > 0)
3009                                         len--;
3010                         } else {
3011                                 accept_frame = 0;
3012                         }
3013                 }
3014
3015                 if (accept_frame) {
3016                         if (em_get_buf(i, adapter, NULL, MB_DONTWAIT) == ENOBUFS) {
3017                                 adapter->dropped_pkts++;
3018                                 em_get_buf(i, adapter, mp, MB_DONTWAIT);
3019                                 if (adapter->fmp != NULL)
3020                                         m_freem(adapter->fmp);
3021                                 adapter->fmp = NULL;
3022                                 adapter->lmp = NULL;
3023                                 goto skip;
3024                         }
3025
3026                         /* Assign correct length to the current fragment */
3027                         mp->m_len = len;
3028
3029                         if (adapter->fmp == NULL) {
3030                                 mp->m_pkthdr.len = len;
3031                                 adapter->fmp = mp;       /* Store the first mbuf */
3032                                 adapter->lmp = mp;
3033                         } else {
3034                                 /* Chain mbuf's together */
3035                                 /*
3036                                  * Adjust length of previous mbuf in chain if
3037                                  * we received less than 4 bytes in the last
3038                                  * descriptor.
3039                                  */
3040                                 if (prev_len_adj > 0) {
3041                                         adapter->lmp->m_len -= prev_len_adj;
3042                                         adapter->fmp->m_pkthdr.len -= prev_len_adj;
3043                                 }
3044                                 adapter->lmp->m_next = mp;
3045                                 adapter->lmp = adapter->lmp->m_next;
3046                                 adapter->fmp->m_pkthdr.len += len;
3047                         }
3048
3049                         if (eop) {
3050                                 adapter->fmp->m_pkthdr.rcvif = ifp;
3051                                 ifp->if_ipackets++;
3052
3053                                 em_receive_checksum(adapter, current_desc,
3054                                                     adapter->fmp);
3055                                 if (current_desc->status & E1000_RXD_STAT_VP) {
3056                                         VLAN_INPUT_TAG(adapter->fmp,
3057                                                        (current_desc->special &
3058                                                         E1000_RXD_SPC_VLAN_MASK));
3059                                 } else {
3060                                         ifp->if_input(ifp, adapter->fmp);
3061                                 }
3062                                 adapter->fmp = NULL;
3063                                 adapter->lmp = NULL;
3064                         }
3065                 } else {
3066                         adapter->dropped_pkts++;
3067                         em_get_buf(i, adapter, mp, MB_DONTWAIT);
3068                         if (adapter->fmp != NULL)
3069                                 m_freem(adapter->fmp);
3070                         adapter->fmp = NULL;
3071                         adapter->lmp = NULL;
3072                 }
3073
3074 skip:
3075                 /* Zero out the receive descriptors status. */
3076                 current_desc->status = 0;
3077
3078                 /* Advance our pointers to the next descriptor. */
3079                 if (++i == adapter->num_rx_desc) {
3080                         i = 0;
3081                         current_desc = adapter->rx_desc_base;
3082                 } else {
3083                         current_desc++;
3084                 }
3085         }
3086
3087         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3088                         BUS_DMASYNC_PREWRITE);
3089
3090         adapter->next_rx_desc_to_check = i;
3091
3092         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3093         if (--i < 0)
3094                 i = adapter->num_rx_desc - 1;
3095
3096         E1000_WRITE_REG(&adapter->hw, RDT, i);
3097 }
3098
3099 /*********************************************************************
3100  *
3101  *  Verify that the hardware indicated that the checksum is valid.
3102  *  Inform the stack about the status of checksum so that stack
3103  *  doesn't spend time verifying the checksum.
3104  *
3105  *********************************************************************/
3106 static void
3107 em_receive_checksum(struct adapter *adapter,
3108                     struct em_rx_desc *rx_desc,
3109                     struct mbuf *mp)
3110 {
3111         /* 82543 or newer only */
3112         if ((adapter->hw.mac_type < em_82543) ||
3113             /* Ignore Checksum bit is set */
3114             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3115                 mp->m_pkthdr.csum_flags = 0;
3116                 return;
3117         }
3118
3119         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3120                 /* Did it pass? */
3121                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3122                         /* IP Checksum Good */
3123                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3124                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3125                 } else {
3126                         mp->m_pkthdr.csum_flags = 0;
3127                 }
3128         }
3129
3130         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3131                 /* Did it pass? */
3132                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3133                         mp->m_pkthdr.csum_flags |=
3134                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3135                         mp->m_pkthdr.csum_data = htons(0xffff);
3136                 }
3137         }
3138 }
3139
3140
3141 static void
3142 em_enable_vlans(struct adapter *adapter)
3143 {
3144         uint32_t ctrl;
3145
3146         E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
3147
3148         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3149         ctrl |= E1000_CTRL_VME;
3150         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3151 }
3152
3153 static void
3154 em_disable_vlans(struct adapter *adapter)
3155 {
3156         uint32_t ctrl;
3157
3158         ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3159         ctrl &= ~E1000_CTRL_VME;
3160         E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3161 }
3162
3163 /*
3164  * note: we must call bus_enable_intr() prior to enabling the hardware
3165  * interrupt and bus_disable_intr() after disabling the hardware interrupt
3166  * in order to avoid handler execution races from scheduled interrupt
3167  * threads.
3168  */
3169 static void
3170 em_enable_intr(struct adapter *adapter)
3171 {
3172         struct ifnet *ifp = &adapter->interface_data.ac_if;
3173
3174         if ((ifp->if_flags & IFF_POLLING) == 0) {
3175                 lwkt_serialize_handler_enable(ifp->if_serializer);
3176                 E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
3177         }
3178 }
3179
3180 static void
3181 em_disable_intr(struct adapter *adapter)
3182 {
3183         /*
3184          * The first version of 82542 had an errata where when link was forced
3185          * it would stay up even up even if the cable was disconnected.
3186          * Sequence errors were used to detect the disconnect and then the
3187          * driver would unforce the link.  This code in the in the ISR.  For
3188          * this to work correctly the Sequence error interrupt had to be
3189          * enabled all the time.
3190          */
3191         if (adapter->hw.mac_type == em_82542_rev2_0) {
3192                 E1000_WRITE_REG(&adapter->hw, IMC,
3193                                 (0xffffffff & ~E1000_IMC_RXSEQ));
3194         } else {
3195                 E1000_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
3196         }
3197
3198         lwkt_serialize_handler_disable(adapter->interface_data.ac_if.if_serializer);
3199 }
3200
3201 static int
3202 em_is_valid_ether_addr(uint8_t *addr)
3203 {
3204         static const char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
3205
3206         if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN))
3207                 return (FALSE);
3208         else
3209                 return (TRUE);
3210 }
3211
3212 void
3213 em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3214 {
3215         pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
3216 }
3217
3218 void
3219 em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3220 {
3221         *value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
3222 }
3223
3224 void
3225 em_pci_set_mwi(struct em_hw *hw)
3226 {
3227         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3228                          (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
3229 }
3230
3231 void
3232 em_pci_clear_mwi(struct em_hw *hw)
3233 {
3234         pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3235                          (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
3236 }
3237
3238 uint32_t
3239 em_io_read(struct em_hw *hw, unsigned long port)
3240 {
3241         struct em_osdep *io = hw->back;
3242
3243         return bus_space_read_4(io->io_bus_space_tag,
3244                                 io->io_bus_space_handle, port);
3245 }
3246
3247 void
3248 em_io_write(struct em_hw *hw, unsigned long port, uint32_t value)
3249 {
3250         struct em_osdep *io = hw->back;
3251
3252         bus_space_write_4(io->io_bus_space_tag,
3253                           io->io_bus_space_handle, port, value);
3254 }
3255
3256 /*
3257  * We may eventually really do this, but its unnecessary
3258  * for now so we just return unsupported.
3259  */
3260 int32_t
3261 em_read_pcie_cap_reg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3262 {
3263         return (0);
3264 }
3265
3266
3267 /*********************************************************************
3268  * 82544 Coexistence issue workaround.
3269  *    There are 2 issues.
3270  *      1. Transmit Hang issue.
3271  *    To detect this issue, following equation can be used...
3272  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3273  *          If SUM[3:0] is in between 1 to 4, we will have this issue.
3274  *
3275  *      2. DAC issue.
3276  *    To detect this issue, following equation can be used...
3277  *          SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3278  *          If SUM[3:0] is in between 9 to c, we will have this issue.
3279  *
3280  *
3281  *    WORKAROUND:
3282  *          Make sure we do not have ending address as 1,2,3,4(Hang) or
3283  *          9,a,b,c (DAC)
3284  *
3285 *************************************************************************/
3286 static uint32_t
3287 em_fill_descriptors(bus_addr_t address, uint32_t length, PDESC_ARRAY desc_array)
3288 {
3289         /* Since issue is sensitive to length and address.*/
3290         /* Let us first check the address...*/
3291         uint32_t safe_terminator;
3292         if (length <= 4) {
3293                 desc_array->descriptor[0].address = address;
3294                 desc_array->descriptor[0].length = length;
3295                 desc_array->elements = 1;
3296                 return (desc_array->elements);
3297         }
3298         safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
3299         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3300         if (safe_terminator == 0 ||
3301             (safe_terminator > 4 && safe_terminator < 9) ||
3302             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
3303                 desc_array->descriptor[0].address = address;
3304                 desc_array->descriptor[0].length = length;
3305                 desc_array->elements = 1;
3306                 return (desc_array->elements);
3307         }
3308
3309         desc_array->descriptor[0].address = address;
3310         desc_array->descriptor[0].length = length - 4;
3311         desc_array->descriptor[1].address = address + (length - 4);
3312         desc_array->descriptor[1].length = 4;
3313         desc_array->elements = 2;
3314         return (desc_array->elements);
3315 }
3316
3317 /**********************************************************************
3318  *
3319  *  Update the board statistics counters.
3320  *
3321  **********************************************************************/
3322 static void
3323 em_update_stats_counters(struct adapter *adapter)
3324 {
3325         struct ifnet   *ifp;
3326
3327         if (adapter->hw.media_type == em_media_type_copper ||
3328             (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
3329                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
3330                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
3331         }
3332         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
3333         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
3334         adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
3335         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
3336
3337         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
3338         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
3339         adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
3340         adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
3341         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
3342         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
3343         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
3344         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
3345         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
3346         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
3347         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
3348         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
3349         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
3350         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
3351         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
3352         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
3353         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
3354         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
3355         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
3356         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
3357
3358         /* For the 64-bit byte counters the low dword must be read first. */
3359         /* Both registers clear on the read of the high dword */
3360
3361         adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
3362         adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
3363         adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
3364         adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
3365
3366         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
3367         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
3368         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
3369         adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
3370         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
3371
3372         adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
3373         adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
3374         adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
3375         adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
3376
3377         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
3378         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
3379         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
3380         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
3381         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
3382         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
3383         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
3384         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
3385         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
3386         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
3387
3388         if (adapter->hw.mac_type >= em_82543) {
3389                 adapter->stats.algnerrc +=
3390                     E1000_READ_REG(&adapter->hw, ALGNERRC);
3391                 adapter->stats.rxerrc +=
3392                     E1000_READ_REG(&adapter->hw, RXERRC);
3393                 adapter->stats.tncrs +=
3394                     E1000_READ_REG(&adapter->hw, TNCRS);
3395                 adapter->stats.cexterr +=
3396                     E1000_READ_REG(&adapter->hw, CEXTERR);
3397                 adapter->stats.tsctc +=
3398                     E1000_READ_REG(&adapter->hw, TSCTC);
3399                 adapter->stats.tsctfc +=
3400                     E1000_READ_REG(&adapter->hw, TSCTFC);
3401         }
3402         ifp = &adapter->interface_data.ac_if;
3403
3404         /* Fill out the OS statistics structure */
3405         ifp->if_collisions = adapter->stats.colc;
3406
3407         /* Rx Errors */
3408         ifp->if_ierrors =
3409                 adapter->dropped_pkts +
3410                 adapter->stats.rxerrc +
3411                 adapter->stats.crcerrs +
3412                 adapter->stats.algnerrc +
3413                 adapter->stats.ruc + adapter->stats.roc +
3414                 adapter->stats.mpc + adapter->stats.cexterr +
3415                 adapter->rx_overruns;
3416
3417         /* Tx Errors */
3418         ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol +
3419                           adapter->watchdog_timeouts;
3420 }
3421
3422
3423 /**********************************************************************
3424  *
3425  *  This routine is called only when em_display_debug_stats is enabled.
3426  *  This routine provides a way to take a look at important statistics
3427  *  maintained by the driver and hardware.
3428  *
3429  **********************************************************************/
3430 static void
3431 em_print_debug_info(struct adapter *adapter)
3432 {
3433         device_t dev= adapter->dev;
3434         uint8_t *hw_addr = adapter->hw.hw_addr;
3435
3436         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
3437         device_printf(dev, "CTRL  = 0x%x RCTL = 0x%x\n",
3438                       E1000_READ_REG(&adapter->hw, CTRL),
3439                       E1000_READ_REG(&adapter->hw, RCTL));
3440         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk\n",
3441                       ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff0000) >> 16),
3442                       (E1000_READ_REG(&adapter->hw, PBA) & 0xffff));
3443         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
3444                       adapter->hw.fc_high_water, adapter->hw.fc_low_water);
3445         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3446                       E1000_READ_REG(&adapter->hw, TIDV),
3447                       E1000_READ_REG(&adapter->hw, TADV));
3448         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3449                       E1000_READ_REG(&adapter->hw, RDTR),
3450                       E1000_READ_REG(&adapter->hw, RADV));
3451         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
3452                       (long long)adapter->tx_fifo_wrk_cnt,
3453                       (long long)adapter->tx_fifo_reset_cnt);
3454         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
3455                       E1000_READ_REG(&adapter->hw, TDH),
3456                       E1000_READ_REG(&adapter->hw, TDT));
3457         device_printf(dev, "Num Tx descriptors avail = %d\n",
3458                       adapter->num_tx_desc_avail);
3459         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
3460                       adapter->no_tx_desc_avail1);
3461         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
3462                       adapter->no_tx_desc_avail2);
3463         device_printf(dev, "Std mbuf failed = %ld\n",
3464                       adapter->mbuf_alloc_failed);
3465         device_printf(dev, "Std mbuf cluster failed = %ld\n",
3466                       adapter->mbuf_cluster_failed);
3467         device_printf(dev, "Driver dropped packets = %ld\n",
3468                       adapter->dropped_pkts);
3469 }
3470
3471 static void
3472 em_print_hw_stats(struct adapter *adapter)
3473 {
3474         device_t dev= adapter->dev;
3475
3476         device_printf(dev, "Excessive collisions = %lld\n",
3477                       (long long)adapter->stats.ecol);
3478         device_printf(dev, "Symbol errors = %lld\n",
3479                       (long long)adapter->stats.symerrs);
3480         device_printf(dev, "Sequence errors = %lld\n",
3481                       (long long)adapter->stats.sec);
3482         device_printf(dev, "Defer count = %lld\n",
3483                       (long long)adapter->stats.dc);
3484
3485         device_printf(dev, "Missed Packets = %lld\n",
3486                       (long long)adapter->stats.mpc);
3487         device_printf(dev, "Receive No Buffers = %lld\n",
3488                       (long long)adapter->stats.rnbc);
3489         /* RLEC is inaccurate on some hardware, calculate our own. */
3490         device_printf(dev, "Receive Length errors = %lld\n",
3491                       (long long)adapter->stats.roc +
3492                       (long long)adapter->stats.ruc);
3493         device_printf(dev, "Receive errors = %lld\n",
3494                       (long long)adapter->stats.rxerrc);
3495         device_printf(dev, "Crc errors = %lld\n",
3496                       (long long)adapter->stats.crcerrs);
3497         device_printf(dev, "Alignment errors = %lld\n",
3498                       (long long)adapter->stats.algnerrc);
3499         device_printf(dev, "Carrier extension errors = %lld\n",
3500                       (long long)adapter->stats.cexterr);
3501         device_printf(dev, "RX overruns = %lu\n", adapter->rx_overruns);
3502         device_printf(dev, "Watchdog timeouts = %lu\n",
3503                       adapter->watchdog_timeouts);
3504
3505         device_printf(dev, "XON Rcvd = %lld\n",
3506                       (long long)adapter->stats.xonrxc);
3507         device_printf(dev, "XON Xmtd = %lld\n",
3508                       (long long)adapter->stats.xontxc);
3509         device_printf(dev, "XOFF Rcvd = %lld\n",
3510                       (long long)adapter->stats.xoffrxc);
3511         device_printf(dev, "XOFF Xmtd = %lld\n",
3512                       (long long)adapter->stats.xofftxc);
3513
3514         device_printf(dev, "Good Packets Rcvd = %lld\n",
3515                       (long long)adapter->stats.gprc);
3516         device_printf(dev, "Good Packets Xmtd = %lld\n",
3517                       (long long)adapter->stats.gptc);
3518 }
3519
3520 static int
3521 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
3522 {
3523         int error;
3524         int result;
3525         struct adapter *adapter;
3526
3527         result = -1;
3528         error = sysctl_handle_int(oidp, &result, 0, req);
3529
3530         if (error || !req->newptr)
3531                 return (error);
3532
3533         if (result == 1) {
3534                 adapter = (struct adapter *)arg1;
3535                 em_print_debug_info(adapter);
3536         }
3537
3538         return (error);
3539 }
3540
3541 static int
3542 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
3543 {
3544         int error;
3545         int result;
3546         struct adapter *adapter;
3547
3548         result = -1;
3549         error = sysctl_handle_int(oidp, &result, 0, req);
3550
3551         if (error || !req->newptr)
3552                 return (error);
3553
3554         if (result == 1) {
3555                 adapter = (struct adapter *)arg1;
3556                 em_print_hw_stats(adapter);
3557         }
3558
3559         return (error);
3560 }
3561
3562 static int
3563 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3564 {
3565         struct em_int_delay_info *info;
3566         struct adapter *adapter;
3567         uint32_t regval;
3568         int error;
3569         int usecs;
3570         int ticks;
3571
3572         info = (struct em_int_delay_info *)arg1;
3573         adapter = info->adapter;
3574         usecs = info->value;
3575         error = sysctl_handle_int(oidp, &usecs, 0, req);
3576         if (error != 0 || req->newptr == NULL)
3577                 return (error);
3578         if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3579                 return (EINVAL);
3580         info->value = usecs;
3581         ticks = E1000_USECS_TO_TICKS(usecs);
3582
3583         lwkt_serialize_enter(adapter->interface_data.ac_if.if_serializer);
3584         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3585         regval = (regval & ~0xffff) | (ticks & 0xffff);
3586         /* Handle a few special cases. */
3587         switch (info->offset) {
3588         case E1000_RDTR:
3589         case E1000_82542_RDTR:
3590                 regval |= E1000_RDT_FPDB;
3591                 break;
3592         case E1000_TIDV:
3593         case E1000_82542_TIDV:
3594                 if (ticks == 0) {
3595                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3596                         /* Don't write 0 into the TIDV register. */
3597                         regval++;
3598                 } else
3599                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3600                 break;
3601         }
3602         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3603         lwkt_serialize_exit(adapter->interface_data.ac_if.if_serializer);
3604         return (0);
3605 }
3606
3607 static void
3608 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3609                         const char *description, struct em_int_delay_info *info,
3610                         int offset, int value)
3611 {
3612         info->adapter = adapter;
3613         info->offset = offset;
3614         info->value = value;
3615         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
3616                         SYSCTL_CHILDREN(adapter->sysctl_tree),
3617                         OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3618                         info, 0, em_sysctl_int_delay, "I", description);
3619 }
3620
3621 static int
3622 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
3623 {
3624         struct adapter *adapter = (void *)arg1;
3625         int error;
3626         int throttle;
3627
3628         throttle = em_int_throttle_ceil;
3629         error = sysctl_handle_int(oidp, &throttle, 0, req);
3630         if (error || req->newptr == NULL)
3631                 return error;
3632         if (throttle < 0 || throttle > 1000000000 / 256)
3633                 return EINVAL;
3634         if (throttle) {
3635                 /*
3636                  * Set the interrupt throttling rate in 256ns increments,
3637                  * recalculate sysctl value assignment to get exact frequency.
3638                  */
3639                 throttle = 1000000000 / 256 / throttle;
3640                 lwkt_serialize_enter(adapter->interface_data.ac_if.if_serializer);
3641                 em_int_throttle_ceil = 1000000000 / 256 / throttle;
3642                 E1000_WRITE_REG(&adapter->hw, ITR, throttle);
3643                 lwkt_serialize_exit(adapter->interface_data.ac_if.if_serializer);
3644         } else {
3645                 lwkt_serialize_enter(adapter->interface_data.ac_if.if_serializer);
3646                 em_int_throttle_ceil = 0;
3647                 E1000_WRITE_REG(&adapter->hw, ITR, 0);
3648                 lwkt_serialize_exit(adapter->interface_data.ac_if.if_serializer);
3649         }
3650         device_printf(adapter->dev, "Interrupt moderation set to %d/sec\n",
3651                         em_int_throttle_ceil);
3652         return 0;
3653 }