sys/dev/netif/e1000/if_em.c

   1 /******************************************************************************
   2
   3   Copyright (c) 2001-2010, Intel Corporation
   4   All rights reserved.
   5
   6   Redistribution and use in source and binary forms, with or without
   7   modification, are permitted provided that the following conditions are met:
   8
   9    1. Redistributions of source code must retain the above copyright notice,
  10       this list of conditions and the following disclaimer.
  11
  12    2. Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in the
  14       documentation and/or other materials provided with the distribution.
  15
  16    3. Neither the name of the Intel Corporation nor the names of its
  17       contributors may be used to endorse or promote products derived from
  18       this software without specific prior written permission.
  19
  20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30   POSSIBILITY OF SUCH DAMAGE.
  31
  32 ******************************************************************************/
  33 /*$FreeBSD$*/
  34
  35 #ifdef HAVE_KERNEL_OPTION_HEADERS
  36 #include "opt_device_polling.h"
  37 #include "opt_inet.h"
  38 #include "opt_altq.h"
  39 #endif
  40
  41 #include <sys/param.h>
  42 #include <sys/systm.h>
  43 #if __FreeBSD_version >= 800000
  44 #include <sys/buf_ring.h>
  45 #endif
  46 #include <sys/bus.h>
  47 #include <sys/endian.h>
  48 #include <sys/kernel.h>
  49 #include <sys/kthread.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/module.h>
  53 #include <sys/rman.h>
  54 #include <sys/socket.h>
  55 #include <sys/sockio.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/taskqueue.h>
  58 #if __FreeBSD_version >= 700029
  59 #include <sys/eventhandler.h>
  60 #endif
  61 #include <machine/bus.h>
  62 #include <machine/resource.h>
  63
  64 #include <net/bpf.h>
  65 #include <net/ethernet.h>
  66 #include <net/if.h>
  67 #include <net/if_arp.h>
  68 #include <net/if_dl.h>
  69 #include <net/if_media.h>
  70
  71 #include <net/if_types.h>
  72 #include <net/if_vlan_var.h>
  73
  74 #include <netinet/in_systm.h>
  75 #include <netinet/in.h>
  76 #include <netinet/if_ether.h>
  77 #include <netinet/ip.h>
  78 #include <netinet/ip6.h>
  79 #include <netinet/tcp.h>
  80 #include <netinet/udp.h>
  81
  82 #include <machine/in_cksum.h>
  83 #include <dev/pci/pcivar.h>
  84 #include <dev/pci/pcireg.h>
  85
  86 #include "e1000_api.h"
  87 #include "e1000_82571.h"
  88 #include "if_em.h"
  89
  90 /*********************************************************************
  91  *  Set this to one to display debug statistics
  92  *********************************************************************/
  93 int     em_display_debug_stats = 0;
  94
  95 /*********************************************************************
  96  *  Driver version:
  97  *********************************************************************/
  98 char em_driver_version[] = "6.9.25";
  99
 100
 101 /*********************************************************************
 102  *  PCI Device ID Table
 103  *
 104  *  Used by probe to select devices to load on
 105  *  Last field stores an index into e1000_strings
 106  *  Last entry must be all 0s
 107  *
 108  *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
 109  *********************************************************************/
 110
 111 static em_vendor_info_t em_vendor_info_array[] =
 112 {
 113         /* Intel(R) PRO/1000 Network Connection */
 114         { 0x8086, E1000_DEV_ID_82540EM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 115         { 0x8086, E1000_DEV_ID_82540EM_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 116         { 0x8086, E1000_DEV_ID_82540EP,         PCI_ANY_ID, PCI_ANY_ID, 0},
 117         { 0x8086, E1000_DEV_ID_82540EP_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 118         { 0x8086, E1000_DEV_ID_82540EP_LP,      PCI_ANY_ID, PCI_ANY_ID, 0},
 119
 120         { 0x8086, E1000_DEV_ID_82541EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 121         { 0x8086, E1000_DEV_ID_82541ER,         PCI_ANY_ID, PCI_ANY_ID, 0},
 122         { 0x8086, E1000_DEV_ID_82541ER_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 123         { 0x8086, E1000_DEV_ID_82541EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 124         { 0x8086, E1000_DEV_ID_82541GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 125         { 0x8086, E1000_DEV_ID_82541GI_LF,      PCI_ANY_ID, PCI_ANY_ID, 0},
 126         { 0x8086, E1000_DEV_ID_82541GI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 127
 128         { 0x8086, E1000_DEV_ID_82542,           PCI_ANY_ID, PCI_ANY_ID, 0},
 129
 130         { 0x8086, E1000_DEV_ID_82543GC_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 131         { 0x8086, E1000_DEV_ID_82543GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 132
 133         { 0x8086, E1000_DEV_ID_82544EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 134         { 0x8086, E1000_DEV_ID_82544EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 135         { 0x8086, E1000_DEV_ID_82544GC_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 136         { 0x8086, E1000_DEV_ID_82544GC_LOM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 137
 138         { 0x8086, E1000_DEV_ID_82545EM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 139         { 0x8086, E1000_DEV_ID_82545EM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 140         { 0x8086, E1000_DEV_ID_82545GM_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 141         { 0x8086, E1000_DEV_ID_82545GM_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 142         { 0x8086, E1000_DEV_ID_82545GM_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 143
 144         { 0x8086, E1000_DEV_ID_82546EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 145         { 0x8086, E1000_DEV_ID_82546EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 146         { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 147         { 0x8086, E1000_DEV_ID_82546GB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 148         { 0x8086, E1000_DEV_ID_82546GB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 149         { 0x8086, E1000_DEV_ID_82546GB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 150         { 0x8086, E1000_DEV_ID_82546GB_PCIE,    PCI_ANY_ID, PCI_ANY_ID, 0},
 151         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
 152         { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
 153                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 154
 155         { 0x8086, E1000_DEV_ID_82547EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 156         { 0x8086, E1000_DEV_ID_82547EI_MOBILE,  PCI_ANY_ID, PCI_ANY_ID, 0},
 157         { 0x8086, E1000_DEV_ID_82547GI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 158
 159         { 0x8086, E1000_DEV_ID_82571EB_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 160         { 0x8086, E1000_DEV_ID_82571EB_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 161         { 0x8086, E1000_DEV_ID_82571EB_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 162         { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
 163                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 164         { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
 165                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 166         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
 167                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 168         { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
 169                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 170         { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
 171                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 172         { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
 173                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 174         { 0x8086, E1000_DEV_ID_82572EI_COPPER,  PCI_ANY_ID, PCI_ANY_ID, 0},
 175         { 0x8086, E1000_DEV_ID_82572EI_FIBER,   PCI_ANY_ID, PCI_ANY_ID, 0},
 176         { 0x8086, E1000_DEV_ID_82572EI_SERDES,  PCI_ANY_ID, PCI_ANY_ID, 0},
 177         { 0x8086, E1000_DEV_ID_82572EI,         PCI_ANY_ID, PCI_ANY_ID, 0},
 178
 179         { 0x8086, E1000_DEV_ID_82573E,          PCI_ANY_ID, PCI_ANY_ID, 0},
 180         { 0x8086, E1000_DEV_ID_82573E_IAMT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 181         { 0x8086, E1000_DEV_ID_82573L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 182         { 0x8086, E1000_DEV_ID_82583V,          PCI_ANY_ID, PCI_ANY_ID, 0},
 183         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
 184                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 185         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
 186                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 187         { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
 188                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 189         { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
 190                                                 PCI_ANY_ID, PCI_ANY_ID, 0},
 191         { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 192         { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 193         { 0x8086, E1000_DEV_ID_ICH8_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 194         { 0x8086, E1000_DEV_ID_ICH8_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 195         { 0x8086, E1000_DEV_ID_ICH8_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 196         { 0x8086, E1000_DEV_ID_ICH8_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 197         { 0x8086, E1000_DEV_ID_ICH8_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 198         { 0x8086, E1000_DEV_ID_ICH8_82567V_3,   PCI_ANY_ID, PCI_ANY_ID, 0},
 199         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,  PCI_ANY_ID, PCI_ANY_ID, 0},
 200         { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,    PCI_ANY_ID, PCI_ANY_ID, 0},
 201         { 0x8086, E1000_DEV_ID_ICH9_IGP_C,      PCI_ANY_ID, PCI_ANY_ID, 0},
 202         { 0x8086, E1000_DEV_ID_ICH9_IGP_M,      PCI_ANY_ID, PCI_ANY_ID, 0},
 203         { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 204         { 0x8086, E1000_DEV_ID_ICH9_IFE,        PCI_ANY_ID, PCI_ANY_ID, 0},
 205         { 0x8086, E1000_DEV_ID_ICH9_IFE_GT,     PCI_ANY_ID, PCI_ANY_ID, 0},
 206         { 0x8086, E1000_DEV_ID_ICH9_IFE_G,      PCI_ANY_ID, PCI_ANY_ID, 0},
 207         { 0x8086, E1000_DEV_ID_ICH9_BM,         PCI_ANY_ID, PCI_ANY_ID, 0},
 208         { 0x8086, E1000_DEV_ID_82574L,          PCI_ANY_ID, PCI_ANY_ID, 0},
 209         { 0x8086, E1000_DEV_ID_82574LA,         PCI_ANY_ID, PCI_ANY_ID, 0},
 210         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 211         { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 212         { 0x8086, E1000_DEV_ID_ICH10_R_BM_V,    PCI_ANY_ID, PCI_ANY_ID, 0},
 213         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,   PCI_ANY_ID, PCI_ANY_ID, 0},
 214         { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,   PCI_ANY_ID, PCI_ANY_ID, 0},
 215         { 0x8086, E1000_DEV_ID_PCH_M_HV_LM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 216         { 0x8086, E1000_DEV_ID_PCH_M_HV_LC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 217         { 0x8086, E1000_DEV_ID_PCH_D_HV_DM,     PCI_ANY_ID, PCI_ANY_ID, 0},
 218         { 0x8086, E1000_DEV_ID_PCH_D_HV_DC,     PCI_ANY_ID, PCI_ANY_ID, 0},
 219         /* required last entry */
 220         { 0, 0, 0, 0, 0}
 221 };
 222
 223 /*********************************************************************
 224  *  Table of branding strings for all supported NICs.
 225  *********************************************************************/
 226
 227 static char *em_strings[] = {
 228         "Intel(R) PRO/1000 Network Connection"
 229 };
 230
 231 /*********************************************************************
 232  *  Function prototypes
 233  *********************************************************************/
 234 static int      em_probe(device_t);
 235 static int      em_attach(device_t);
 236 static int      em_detach(device_t);
 237 static int      em_shutdown(device_t);
 238 static int      em_suspend(device_t);
 239 static int      em_resume(device_t);
 240 static void     em_start(struct ifnet *);
 241 static void     em_start_locked(struct ifnet *ifp);
 242 #if __FreeBSD_version >= 800000
 243 static int      em_mq_start(struct ifnet *, struct mbuf *);
 244 static int      em_mq_start_locked(struct ifnet *, struct mbuf *);
 245 static void     em_qflush(struct ifnet *);
 246 #endif
 247 static int      em_ioctl(struct ifnet *, u_long, caddr_t);
 248 static void     em_init(void *);
 249 static void     em_init_locked(struct adapter *);
 250 static void     em_stop(void *);
 251 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 252 static int      em_media_change(struct ifnet *);
 253 static void     em_identify_hardware(struct adapter *);
 254 static int      em_allocate_pci_resources(struct adapter *);
 255 static int      em_allocate_legacy(struct adapter *adapter);
 256 static int      em_allocate_msix(struct adapter *adapter);
 257 static int      em_setup_msix(struct adapter *);
 258 static void     em_free_pci_resources(struct adapter *);
 259 static void     em_local_timer(void *);
 260 static int      em_hardware_init(struct adapter *);
 261 static void     em_setup_interface(device_t, struct adapter *);
 262 static void     em_setup_transmit_structures(struct adapter *);
 263 static void     em_initialize_transmit_unit(struct adapter *);
 264 static int      em_setup_receive_structures(struct adapter *);
 265 static void     em_initialize_receive_unit(struct adapter *);
 266 static void     em_enable_intr(struct adapter *);
 267 static void     em_disable_intr(struct adapter *);
 268 static void     em_free_transmit_structures(struct adapter *);
 269 static void     em_free_receive_structures(struct adapter *);
 270 static void     em_update_stats_counters(struct adapter *);
 271 static void     em_txeof(struct adapter *);
 272 static void     em_tx_purge(struct adapter *);
 273 static int      em_allocate_receive_structures(struct adapter *);
 274 static int      em_allocate_transmit_structures(struct adapter *);
 275 static int      em_rxeof(struct adapter *, int);
 276 #ifndef __NO_STRICT_ALIGNMENT
 277 static int      em_fixup_rx(struct adapter *);
 278 #endif
 279 static void     em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
 280                     struct mbuf *);
 281 static void     em_transmit_checksum_setup(struct adapter *, struct mbuf *,
 282                     u32 *, u32 *);
 283 #if __FreeBSD_version >= 700000
 284 static bool     em_tso_setup(struct adapter *, struct mbuf *,
 285                     u32 *, u32 *);
 286 #endif /* FreeBSD_version >= 700000 */
 287 static void     em_set_promisc(struct adapter *);
 288 static void     em_disable_promisc(struct adapter *);
 289 static void     em_set_multi(struct adapter *);
 290 static void     em_print_hw_stats(struct adapter *);
 291 static void     em_update_link_status(struct adapter *);
 292 static int      em_get_buf(struct adapter *, int);
 293 #if __FreeBSD_version >= 700029
 294 static void     em_register_vlan(void *, struct ifnet *, u16);
 295 static void     em_unregister_vlan(void *, struct ifnet *, u16);
 296 static void     em_setup_vlan_hw_support(struct adapter *);
 297 #endif
 298 static int      em_xmit(struct adapter *, struct mbuf **);
 299 static void     em_smartspeed(struct adapter *);
 300 static int      em_82547_fifo_workaround(struct adapter *, int);
 301 static void     em_82547_update_fifo_head(struct adapter *, int);
 302 static int      em_82547_tx_fifo_reset(struct adapter *);
 303 static void     em_82547_move_tail(void *);
 304 static int      em_dma_malloc(struct adapter *, bus_size_t,
 305                     struct em_dma_alloc *, int);
 306 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 307 static void     em_print_debug_info(struct adapter *);
 308 static void     em_print_nvm_info(struct adapter *);
 309 static int      em_is_valid_ether_addr(u8 *);
 310 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 311 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 312 static u32      em_fill_descriptors (bus_addr_t address, u32 length,
 313                     PDESC_ARRAY desc_array);
 314 static int      em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
 315 static void     em_add_int_delay_sysctl(struct adapter *, const char *,
 316                     const char *, struct em_int_delay_info *, int, int);
 317 /* Management and WOL Support */
 318 static void     em_init_manageability(struct adapter *);
 319 static void     em_release_manageability(struct adapter *);
 320 static void     em_get_hw_control(struct adapter *);
 321 static void     em_release_hw_control(struct adapter *);
 322 static void     em_get_wakeup(device_t);
 323 static void     em_enable_wakeup(device_t);
 324 static int      em_enable_phy_wakeup(struct adapter *);
 325
 326 #ifdef EM_LEGACY_IRQ
 327 static void     em_intr(void *);
 328 #else /* FAST IRQ */
 329 #if __FreeBSD_version < 700000
 330 static void     em_irq_fast(void *);
 331 #else
 332 static int      em_irq_fast(void *);
 333 #endif
 334
 335 /* MSIX handlers */
 336 static void     em_msix_tx(void *);
 337 static void     em_msix_rx(void *);
 338 static void     em_msix_link(void *);
 339 static void     em_handle_rx(void *context, int pending);
 340 static void     em_handle_tx(void *context, int pending);
 341
 342 static void     em_handle_rxtx(void *context, int pending);
 343 static void     em_handle_link(void *context, int pending);
 344 static void     em_add_rx_process_limit(struct adapter *, const char *,
 345                     const char *, int *, int);
 346 #endif /* ~EM_LEGACY_IRQ */
 347
 348 #ifdef DEVICE_POLLING
 349 static poll_handler_t em_poll;
 350 #endif /* POLLING */
 351
 352 /*********************************************************************
 353  *  FreeBSD Device Interface Entry Points
 354  *********************************************************************/
 355
 356 static device_method_t em_methods[] = {
 357         /* Device interface */
 358         DEVMETHOD(device_probe, em_probe),
 359         DEVMETHOD(device_attach, em_attach),
 360         DEVMETHOD(device_detach, em_detach),
 361         DEVMETHOD(device_shutdown, em_shutdown),
 362         DEVMETHOD(device_suspend, em_suspend),
 363         DEVMETHOD(device_resume, em_resume),
 364         {0, 0}
 365 };
 366
 367 static driver_t em_driver = {
 368         "em", em_methods, sizeof(struct adapter),
 369 };
 370
 371 static devclass_t em_devclass;
 372 DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
 373 MODULE_DEPEND(em, pci, 1, 1, 1);
 374 MODULE_DEPEND(em, ether, 1, 1, 1);
 375
 376 /*********************************************************************
 377  *  Tunable default values.
 378  *********************************************************************/
 379
 380 #define EM_TICKS_TO_USECS(ticks)        ((1024 * (ticks) + 500) / 1000)
 381 #define EM_USECS_TO_TICKS(usecs)        ((1000 * (usecs) + 512) / 1024)
 382 #define M_TSO_LEN                       66
 383
 384 /* Allow common code without TSO */
 385 #ifndef CSUM_TSO
 386 #define CSUM_TSO        0
 387 #endif
 388
 389 static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
 390 static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
 391 static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
 392 static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
 393 static int em_rxd = EM_DEFAULT_RXD;
 394 static int em_txd = EM_DEFAULT_TXD;
 395 static int em_smart_pwr_down = FALSE;
 396 /* Controls whether promiscuous also shows bad packets */
 397 static int em_debug_sbp = FALSE;
 398 /* Local switch for MSI/MSIX */
 399 static int em_enable_msi = TRUE;
 400
 401 TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
 402 TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
 403 TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
 404 TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
 405 TUNABLE_INT("hw.em.rxd", &em_rxd);
 406 TUNABLE_INT("hw.em.txd", &em_txd);
 407 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 408 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 409 TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
 410
 411 #ifndef EM_LEGACY_IRQ
 412 /* How many packets rxeof tries to clean at a time */
 413 static int em_rx_process_limit = 100;
 414 TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
 415 #endif
 416
 417 /* Flow control setting - default to FULL */
 418 static int em_fc_setting = e1000_fc_full;
 419 TUNABLE_INT("hw.em.fc_setting", &em_fc_setting);
 420
 421 /*
 422 ** Shadow VFTA table, this is needed because
 423 ** the real vlan filter table gets cleared during
 424 ** a soft reset and the driver needs to be able
 425 ** to repopulate it.
 426 */
 427 static u32 em_shadow_vfta[EM_VFTA_SIZE];
 428
 429 /* Global used in WOL setup with multiport cards */
 430 static int global_quad_port_a = 0;
 431
 432 /*********************************************************************
 433  *  Device identification routine
 434  *
 435  *  em_probe determines if the driver should be loaded on
 436  *  adapter based on PCI vendor/device id of the adapter.
 437  *
 438  *  return BUS_PROBE_DEFAULT on success, positive on failure
 439  *********************************************************************/
 440
 441 static int
 442 em_probe(device_t dev)
 443 {
 444         char            adapter_name[60];
 445         u16             pci_vendor_id = 0;
 446         u16             pci_device_id = 0;
 447         u16             pci_subvendor_id = 0;
 448         u16             pci_subdevice_id = 0;
 449         em_vendor_info_t *ent;
 450
 451         INIT_DEBUGOUT("em_probe: begin");
 452
 453         pci_vendor_id = pci_get_vendor(dev);
 454         if (pci_vendor_id != EM_VENDOR_ID)
 455                 return (ENXIO);
 456
 457         pci_device_id = pci_get_device(dev);
 458         pci_subvendor_id = pci_get_subvendor(dev);
 459         pci_subdevice_id = pci_get_subdevice(dev);
 460
 461         ent = em_vendor_info_array;
 462         while (ent->vendor_id != 0) {
 463                 if ((pci_vendor_id == ent->vendor_id) &&
 464                     (pci_device_id == ent->device_id) &&
 465
 466                     ((pci_subvendor_id == ent->subvendor_id) ||
 467                     (ent->subvendor_id == PCI_ANY_ID)) &&
 468
 469                     ((pci_subdevice_id == ent->subdevice_id) ||
 470                     (ent->subdevice_id == PCI_ANY_ID))) {
 471                         sprintf(adapter_name, "%s %s",
 472                                 em_strings[ent->index],
 473                                 em_driver_version);
 474                         device_set_desc_copy(dev, adapter_name);
 475                         return (BUS_PROBE_DEFAULT);
 476                 }
 477                 ent++;
 478         }
 479
 480         return (ENXIO);
 481 }
 482
 483 /*********************************************************************
 484  *  Device initialization routine
 485  *
 486  *  The attach entry point is called when the driver is being loaded.
 487  *  This routine identifies the type of hardware, allocates all resources
 488  *  and initializes the hardware.
 489  *
 490  *  return 0 on success, positive on failure
 491  *********************************************************************/
 492
 493 static int
 494 em_attach(device_t dev)
 495 {
 496         struct adapter  *adapter;
 497         int             tsize, rsize;
 498         int             error = 0;
 499
 500         INIT_DEBUGOUT("em_attach: begin");
 501
 502         adapter = device_get_softc(dev);
 503         adapter->dev = adapter->osdep.dev = dev;
 504         EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
 505         EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
 506         EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
 507
 508         /* SYSCTL stuff */
 509         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 510             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 511             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 512             em_sysctl_debug_info, "I", "Debug Information");
 513
 514         SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
 515             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
 516             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
 517             em_sysctl_stats, "I", "Statistics");
 518
 519         callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
 520         callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
 521
 522         /* Determine hardware and mac info */
 523         em_identify_hardware(adapter);
 524
 525         /* Setup PCI resources */
 526         if (em_allocate_pci_resources(adapter)) {
 527                 device_printf(dev, "Allocation of PCI resources failed\n");
 528                 error = ENXIO;
 529                 goto err_pci;
 530         }
 531
 532         /*
 533         ** For ICH8 and family we need to
 534         ** map the flash memory, and this
 535         ** must happen after the MAC is
 536         ** identified
 537         */
 538         if ((adapter->hw.mac.type == e1000_ich8lan) ||
 539             (adapter->hw.mac.type == e1000_pchlan) ||
 540             (adapter->hw.mac.type == e1000_ich9lan) ||
 541             (adapter->hw.mac.type == e1000_ich10lan)) {
 542                 int rid = EM_BAR_TYPE_FLASH;
 543                 adapter->flash = bus_alloc_resource_any(dev,
 544                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
 545                 if (adapter->flash == NULL) {
 546                         device_printf(dev, "Mapping of Flash failed\n");
 547                         error = ENXIO;
 548                         goto err_pci;
 549                 }
 550                 /* This is used in the shared code */
 551                 adapter->hw.flash_address = (u8 *)adapter->flash;
 552                 adapter->osdep.flash_bus_space_tag =
 553                     rman_get_bustag(adapter->flash);
 554                 adapter->osdep.flash_bus_space_handle =
 555                     rman_get_bushandle(adapter->flash);
 556         }
 557
 558         /* Do Shared Code initialization */
 559         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 560                 device_printf(dev, "Setup of Shared code failed\n");
 561                 error = ENXIO;
 562                 goto err_pci;
 563         }
 564
 565         e1000_get_bus_info(&adapter->hw);
 566
 567         /* Set up some sysctls for the tunable interrupt delays */
 568         em_add_int_delay_sysctl(adapter, "rx_int_delay",
 569             "receive interrupt delay in usecs", &adapter->rx_int_delay,
 570             E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
 571         em_add_int_delay_sysctl(adapter, "tx_int_delay",
 572             "transmit interrupt delay in usecs", &adapter->tx_int_delay,
 573             E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
 574         if (adapter->hw.mac.type >= e1000_82540) {
 575                 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
 576                     "receive interrupt delay limit in usecs",
 577                     &adapter->rx_abs_int_delay,
 578                     E1000_REGISTER(&adapter->hw, E1000_RADV),
 579                     em_rx_abs_int_delay_dflt);
 580                 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
 581                     "transmit interrupt delay limit in usecs",
 582                     &adapter->tx_abs_int_delay,
 583                     E1000_REGISTER(&adapter->hw, E1000_TADV),
 584                     em_tx_abs_int_delay_dflt);
 585         }
 586
 587 #ifndef EM_LEGACY_IRQ
 588         /* Sysctls for limiting the amount of work done in the taskqueue */
 589         em_add_rx_process_limit(adapter, "rx_processing_limit",
 590             "max number of rx packets to process", &adapter->rx_process_limit,
 591             em_rx_process_limit);
 592 #endif
 593
 594         /*
 595          * Validate number of transmit and receive descriptors. It
 596          * must not exceed hardware maximum, and must be multiple
 597          * of E1000_DBA_ALIGN.
 598          */
 599         if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
 600             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 601             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 602             (em_txd < EM_MIN_TXD)) {
 603                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 604                     EM_DEFAULT_TXD, em_txd);
 605                 adapter->num_tx_desc = EM_DEFAULT_TXD;
 606         } else
 607                 adapter->num_tx_desc = em_txd;
 608         if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
 609             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 610             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 611             (em_rxd < EM_MIN_RXD)) {
 612                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 613                     EM_DEFAULT_RXD, em_rxd);
 614                 adapter->num_rx_desc = EM_DEFAULT_RXD;
 615         } else
 616                 adapter->num_rx_desc = em_rxd;
 617
 618         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 619         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 620         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 621         adapter->rx_buffer_len = 2048;
 622
 623         e1000_init_script_state_82541(&adapter->hw, TRUE);
 624         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 625
 626         /* Copper options */
 627         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 628                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 629                 adapter->hw.phy.disable_polarity_correction = FALSE;
 630                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 631         }
 632
 633         /*
 634          * Set the frame limits assuming
 635          * standard ethernet sized frames.
 636          */
 637         adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
 638         adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
 639
 640         /*
 641          * This controls when hardware reports transmit completion
 642          * status.
 643          */
 644         adapter->hw.mac.report_tx_early = 1;
 645
 646         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 647             EM_DBA_ALIGN);
 648
 649         /* Allocate Transmit Descriptor ring */
 650         if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
 651                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 652                 error = ENOMEM;
 653                 goto err_tx_desc;
 654         }
 655         adapter->tx_desc_base =
 656             (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
 657
 658         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 659             EM_DBA_ALIGN);
 660
 661         /* Allocate Receive Descriptor ring */
 662         if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
 663                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 664                 error = ENOMEM;
 665                 goto err_rx_desc;
 666         }
 667         adapter->rx_desc_base =
 668             (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
 669
 670         /*
 671         ** Start from a known state, this is
 672         ** important in reading the nvm and
 673         ** mac from that.
 674         */
 675         e1000_reset_hw(&adapter->hw);
 676
 677         /* Make sure we have a good EEPROM before we read from it */
 678         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 679                 /*
 680                 ** Some PCI-E parts fail the first check due to
 681                 ** the link being in sleep state, call it again,
 682                 ** if it fails a second time its a real issue.
 683                 */
 684                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 685                         device_printf(dev,
 686                             "The EEPROM Checksum Is Not Valid\n");
 687                         error = EIO;
 688                         goto err_hw_init;
 689                 }
 690         }
 691
 692         /* Copy the permanent MAC address out of the EEPROM */
 693         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 694                 device_printf(dev, "EEPROM read error while reading MAC"
 695                     " address\n");
 696                 error = EIO;
 697                 goto err_hw_init;
 698         }
 699
 700         if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
 701                 device_printf(dev, "Invalid MAC address\n");
 702                 error = EIO;
 703                 goto err_hw_init;
 704         }
 705
 706         /* Initialize the hardware */
 707         if (em_hardware_init(adapter)) {
 708                 device_printf(dev, "Unable to initialize the hardware\n");
 709                 error = EIO;
 710                 goto err_hw_init;
 711         }
 712
 713         /* Allocate transmit descriptors and buffers */
 714         if (em_allocate_transmit_structures(adapter)) {
 715                 device_printf(dev, "Could not setup transmit structures\n");
 716                 error = ENOMEM;
 717                 goto err_tx_struct;
 718         }
 719
 720         /* Allocate receive descriptors and buffers */
 721         if (em_allocate_receive_structures(adapter)) {
 722                 device_printf(dev, "Could not setup receive structures\n");
 723                 error = ENOMEM;
 724                 goto err_rx_struct;
 725         }
 726
 727         /*
 728         **  Do interrupt configuration
 729         */
 730         if (adapter->msi > 1) /* Do MSI/X */
 731                 error = em_allocate_msix(adapter);
 732         else  /* MSI or Legacy */
 733                 error = em_allocate_legacy(adapter);
 734         if (error)
 735                 goto err_rx_struct;
 736
 737         /*
 738          * Get Wake-on-Lan and Management info for later use
 739          */
 740         em_get_wakeup(dev);
 741
 742         /* Setup OS specific network interface */
 743         em_setup_interface(dev, adapter);
 744
 745         /* Initialize statistics */
 746         em_update_stats_counters(adapter);
 747
 748         adapter->hw.mac.get_link_status = 1;
 749         em_update_link_status(adapter);
 750
 751         /* Indicate SOL/IDER usage */
 752         if (e1000_check_reset_block(&adapter->hw))
 753                 device_printf(dev,
 754                     "PHY reset is blocked due to SOL/IDER session.\n");
 755
 756         /* Do we need workaround for 82544 PCI-X adapter? */
 757         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 758             adapter->hw.mac.type == e1000_82544)
 759                 adapter->pcix_82544 = TRUE;
 760         else
 761                 adapter->pcix_82544 = FALSE;
 762
 763 #if __FreeBSD_version >= 700029
 764         /* Register for VLAN events */
 765         adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 766             em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 767         adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
 768             em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST);
 769 #endif
 770
 771         /* Non-AMT based hardware can now take control from firmware */
 772         if (adapter->has_manage && !adapter->has_amt)
 773                 em_get_hw_control(adapter);
 774
 775         /* Tell the stack that the interface is not active */
 776         adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 777
 778         INIT_DEBUGOUT("em_attach: end");
 779
 780         return (0);
 781
 782 err_rx_struct:
 783         em_free_transmit_structures(adapter);
 784 err_tx_struct:
 785 err_hw_init:
 786         em_release_hw_control(adapter);
 787         em_dma_free(adapter, &adapter->rxdma);
 788 err_rx_desc:
 789         em_dma_free(adapter, &adapter->txdma);
 790 err_tx_desc:
 791 err_pci:
 792         em_free_pci_resources(adapter);
 793         EM_TX_LOCK_DESTROY(adapter);
 794         EM_RX_LOCK_DESTROY(adapter);
 795         EM_CORE_LOCK_DESTROY(adapter);
 796
 797         return (error);
 798 }
 799
 800 /*********************************************************************
 801  *  Device removal routine
 802  *
 803  *  The detach entry point is called when the driver is being removed.
 804  *  This routine stops the adapter and deallocates all the resources
 805  *  that were allocated for driver operation.
 806  *
 807  *  return 0 on success, positive on failure
 808  *********************************************************************/
 809
 810 static int
 811 em_detach(device_t dev)
 812 {
 813         struct adapter  *adapter = device_get_softc(dev);
 814         struct ifnet    *ifp = adapter->ifp;
 815
 816         INIT_DEBUGOUT("em_detach: begin");
 817
 818         /* Make sure VLANS are not using driver */
 819 #if __FreeBSD_version >= 700000
 820         if (adapter->ifp->if_vlantrunk != NULL) {
 821 #else
 822         if (adapter->ifp->if_nvlans != 0) {
 823 #endif
 824                 device_printf(dev,"Vlan in use, detach first\n");
 825                 return (EBUSY);
 826         }
 827
 828 #ifdef DEVICE_POLLING
 829         if (ifp->if_capenable & IFCAP_POLLING)
 830                 ether_poll_deregister(ifp);
 831 #endif
 832
 833         EM_CORE_LOCK(adapter);
 834         EM_TX_LOCK(adapter);
 835         adapter->in_detach = 1;
 836         em_stop(adapter);
 837         e1000_phy_hw_reset(&adapter->hw);
 838
 839         em_release_manageability(adapter);
 840
 841         EM_TX_UNLOCK(adapter);
 842         EM_CORE_UNLOCK(adapter);
 843
 844 #if __FreeBSD_version >= 700029
 845         /* Unregister VLAN events */
 846         if (adapter->vlan_attach != NULL)
 847                 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
 848         if (adapter->vlan_detach != NULL)
 849                 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
 850 #endif
 851
 852         ether_ifdetach(adapter->ifp);
 853         callout_drain(&adapter->timer);
 854         callout_drain(&adapter->tx_fifo_timer);
 855
 856         em_free_pci_resources(adapter);
 857         bus_generic_detach(dev);
 858         if_free(ifp);
 859
 860         em_free_transmit_structures(adapter);
 861         em_free_receive_structures(adapter);
 862
 863         /* Free Transmit Descriptor ring */
 864         if (adapter->tx_desc_base) {
 865                 em_dma_free(adapter, &adapter->txdma);
 866                 adapter->tx_desc_base = NULL;
 867         }
 868
 869         /* Free Receive Descriptor ring */
 870         if (adapter->rx_desc_base) {
 871                 em_dma_free(adapter, &adapter->rxdma);
 872                 adapter->rx_desc_base = NULL;
 873         }
 874
 875         em_release_hw_control(adapter);
 876         EM_TX_LOCK_DESTROY(adapter);
 877         EM_RX_LOCK_DESTROY(adapter);
 878         EM_CORE_LOCK_DESTROY(adapter);
 879
 880         return (0);
 881 }
 882
 883 /*********************************************************************
 884  *
 885  *  Shutdown entry point
 886  *
 887  **********************************************************************/
 888
 889 static int
 890 em_shutdown(device_t dev)
 891 {
 892         return em_suspend(dev);
 893 }
 894
 895 /*
 896  * Suspend/resume device methods.
 897  */
 898 static int
 899 em_suspend(device_t dev)
 900 {
 901         struct adapter *adapter = device_get_softc(dev);
 902
 903         EM_CORE_LOCK(adapter);
 904
 905         em_release_manageability(adapter);
 906         em_release_hw_control(adapter);
 907         em_enable_wakeup(dev);
 908
 909         EM_CORE_UNLOCK(adapter);
 910
 911         return bus_generic_suspend(dev);
 912 }
 913
 914 static int
 915 em_resume(device_t dev)
 916 {
 917         struct adapter *adapter = device_get_softc(dev);
 918         struct ifnet *ifp = adapter->ifp;
 919
 920         EM_CORE_LOCK(adapter);
 921         em_init_locked(adapter);
 922         em_init_manageability(adapter);
 923         EM_CORE_UNLOCK(adapter);
 924         em_start(ifp);
 925
 926         return bus_generic_resume(dev);
 927 }
 928
 929
 930 /*********************************************************************
 931  *  Transmit entry point
 932  *
 933  *  em_start is called by the stack to initiate a transmit.
 934  *  The driver will remain in this routine as long as there are
 935  *  packets to transmit and transmit resources are available.
 936  *  In case resources are not available stack is notified and
 937  *  the packet is requeued.
 938  **********************************************************************/
 939
 940 #if __FreeBSD_version >= 800000
 941 static int
 942 em_mq_start_locked(struct ifnet *ifp, struct mbuf *m)
 943 {
 944         struct adapter  *adapter = ifp->if_softc;
 945         struct mbuf     *next;
 946         int error = E1000_SUCCESS;
 947
 948         EM_TX_LOCK_ASSERT(adapter);
 949         /* To allow being called from a tasklet */
 950         if (m == NULL)
 951                 goto process;
 952
 953         if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
 954             IFF_DRV_RUNNING)
 955             || (!adapter->link_active)) {
 956                 error = drbr_enqueue(ifp, adapter->br, m);
 957                 return (error);
 958         } else if (!drbr_needs_enqueue(ifp, adapter->br) &&
 959             (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) {
 960                 if ((error = em_xmit(adapter, &m)) != 0) {
 961                         if (m)
 962                                 error = drbr_enqueue(ifp, adapter->br, m);
 963                         return (error);
 964                 } else {
 965                         /*
 966                          * We've bypassed the buf ring so we need to update
 967                          * ifp directly
 968                          */
 969                         drbr_stats_update(ifp, m->m_pkthdr.len, m->m_flags);
 970                         /*
 971                         ** Send a copy of the frame to the BPF
 972                         ** listener and set the watchdog on.
 973                         */
 974                         ETHER_BPF_MTAP(ifp, m);
 975                         adapter->watchdog_check = TRUE;
 976                 }
 977         } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0)
 978                 return (error);
 979
 980 process:
 981         if (drbr_empty(ifp, adapter->br))
 982                 return(error);
 983         /* Process the queue */
 984         while (TRUE) {
 985                 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 986                         break;
 987                 next = drbr_dequeue(ifp, adapter->br);
 988                 if (next == NULL)
 989                         break;
 990                 if ((error = em_xmit(adapter, &next)) != 0) {
 991                         if (next != NULL)
 992                                 error = drbr_enqueue(ifp, adapter->br, next);
 993                         break;
 994                 }
 995                 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags);
 996                 ETHER_BPF_MTAP(ifp, next);
 997                 /* Set the watchdog */
 998                 adapter->watchdog_check = TRUE;
 999         }
1000
1001         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1002                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1003
1004         return (error);
1005 }
1006
1007 /*
1008 ** Multiqueue capable stack interface, this is not
1009 ** yet truely multiqueue, but that is coming...
1010 */
1011 static int
1012 em_mq_start(struct ifnet *ifp, struct mbuf *m)
1013 {
1014
1015         struct adapter *adapter = ifp->if_softc;
1016         int error = 0;
1017
1018         if (EM_TX_TRYLOCK(adapter)) {
1019                 if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1020                         error = em_mq_start_locked(ifp, m);
1021                 EM_TX_UNLOCK(adapter);
1022         } else
1023                 error = drbr_enqueue(ifp, adapter->br, m);
1024
1025         return (error);
1026 }
1027
1028 static void
1029 em_qflush(struct ifnet *ifp)
1030 {
1031         struct mbuf *m;
1032         struct adapter *adapter = (struct adapter *)ifp->if_softc;
1033
1034         EM_TX_LOCK(adapter);
1035         while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL)
1036                 m_freem(m);
1037         if_qflush(ifp);
1038         EM_TX_UNLOCK(adapter);
1039 }
1040 #endif /* FreeBSD_version */
1041
1042 static void
1043 em_start_locked(struct ifnet *ifp)
1044 {
1045         struct adapter  *adapter = ifp->if_softc;
1046         struct mbuf     *m_head;
1047
1048         EM_TX_LOCK_ASSERT(adapter);
1049
1050         if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
1051             IFF_DRV_RUNNING)
1052                 return;
1053         if (!adapter->link_active)
1054                 return;
1055
1056         while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1057
1058                 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1059                 if (m_head == NULL)
1060                         break;
1061                 /*
1062                  *  Encapsulation can modify our pointer, and or make it
1063                  *  NULL on failure.  In that event, we can't requeue.
1064                  */
1065                 if (em_xmit(adapter, &m_head)) {
1066                         if (m_head == NULL)
1067                                 break;
1068                         ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1069                         IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1070                         break;
1071                 }
1072
1073                 /* Send a copy of the frame to the BPF listener */
1074                 ETHER_BPF_MTAP(ifp, m_head);
1075
1076                 /* Set timeout in case hardware has problems transmitting. */
1077                 adapter->watchdog_check = TRUE;
1078         }
1079         if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)
1080                 ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1081
1082         return;
1083 }
1084
1085 static void
1086 em_start(struct ifnet *ifp)
1087 {
1088         struct adapter *adapter = ifp->if_softc;
1089
1090         EM_TX_LOCK(adapter);
1091         if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1092                 em_start_locked(ifp);
1093         EM_TX_UNLOCK(adapter);
1094 }
1095
1096 /*********************************************************************
1097  *  Ioctl entry point
1098  *
1099  *  em_ioctl is called when the user wants to configure the
1100  *  interface.
1101  *
1102  *  return 0 on success, positive on failure
1103  **********************************************************************/
1104
1105 static int
1106 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1107 {
1108         struct adapter  *adapter = ifp->if_softc;
1109         struct ifreq *ifr = (struct ifreq *)data;
1110 #ifdef INET
1111         struct ifaddr *ifa = (struct ifaddr *)data;
1112 #endif
1113         int error = 0;
1114
1115         if (adapter->in_detach)
1116                 return (error);
1117
1118         switch (command) {
1119         case SIOCSIFADDR:
1120 #ifdef INET
1121                 if (ifa->ifa_addr->sa_family == AF_INET) {
1122                         /*
1123                          * XXX
1124                          * Since resetting hardware takes a very long time
1125                          * and results in link renegotiation we only
1126                          * initialize the hardware only when it is absolutely
1127                          * required.
1128                          */
1129                         ifp->if_flags |= IFF_UP;
1130                         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1131                                 EM_CORE_LOCK(adapter);
1132                                 em_init_locked(adapter);
1133                                 EM_CORE_UNLOCK(adapter);
1134                         }
1135                         arp_ifinit(ifp, ifa);
1136                 } else
1137 #endif
1138                         error = ether_ioctl(ifp, command, data);
1139                 break;
1140         case SIOCSIFMTU:
1141             {
1142                 int max_frame_size;
1143                 u16 eeprom_data = 0;
1144
1145                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1146
1147                 EM_CORE_LOCK(adapter);
1148                 switch (adapter->hw.mac.type) {
1149                 case e1000_82573:
1150                         /*
1151                          * 82573 only supports jumbo frames
1152                          * if ASPM is disabled.
1153                          */
1154                         e1000_read_nvm(&adapter->hw,
1155                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1156                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1157                                 max_frame_size = ETHER_MAX_LEN;
1158                                 break;
1159                         }
1160                         /* Allow Jumbo frames - fall thru */
1161                 case e1000_82571:
1162                 case e1000_82572:
1163                 case e1000_ich9lan:
1164                 case e1000_ich10lan:
1165                 case e1000_82574:
1166                 case e1000_80003es2lan: /* Limit Jumbo Frame size */
1167                         max_frame_size = 9234;
1168                         break;
1169                 case e1000_pchlan:
1170                         max_frame_size = 4096;
1171                         break;
1172                         /* Adapters that do not support jumbo frames */
1173                 case e1000_82542:
1174                 case e1000_82583:
1175                 case e1000_ich8lan:
1176                         max_frame_size = ETHER_MAX_LEN;
1177                         break;
1178                 default:
1179                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1180                 }
1181                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1182                     ETHER_CRC_LEN) {
1183                         EM_CORE_UNLOCK(adapter);
1184                         error = EINVAL;
1185                         break;
1186                 }
1187
1188                 ifp->if_mtu = ifr->ifr_mtu;
1189                 adapter->max_frame_size =
1190                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1191                 em_init_locked(adapter);
1192                 EM_CORE_UNLOCK(adapter);
1193                 break;
1194             }
1195         case SIOCSIFFLAGS:
1196                 IOCTL_DEBUGOUT("ioctl rcv'd:\
1197                     SIOCSIFFLAGS (Set Interface Flags)");
1198                 EM_CORE_LOCK(adapter);
1199                 if (ifp->if_flags & IFF_UP) {
1200                         if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1201                                 if ((ifp->if_flags ^ adapter->if_flags) &
1202                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1203                                         em_disable_promisc(adapter);
1204                                         em_set_promisc(adapter);
1205                                 }
1206                         } else
1207                                 em_init_locked(adapter);
1208                 } else
1209                         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1210                                 EM_TX_LOCK(adapter);
1211                                 em_stop(adapter);
1212                                 EM_TX_UNLOCK(adapter);
1213                         }
1214                 adapter->if_flags = ifp->if_flags;
1215                 EM_CORE_UNLOCK(adapter);
1216                 break;
1217         case SIOCADDMULTI:
1218         case SIOCDELMULTI:
1219                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1220                 if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1221                         EM_CORE_LOCK(adapter);
1222                         em_disable_intr(adapter);
1223                         em_set_multi(adapter);
1224                         if (adapter->hw.mac.type == e1000_82542 &&
1225                             adapter->hw.revision_id == E1000_REVISION_2) {
1226                                 em_initialize_receive_unit(adapter);
1227                         }
1228 #ifdef DEVICE_POLLING
1229                         if (!(ifp->if_capenable & IFCAP_POLLING))
1230 #endif
1231                                 em_enable_intr(adapter);
1232                         EM_CORE_UNLOCK(adapter);
1233                 }
1234                 break;
1235         case SIOCSIFMEDIA:
1236                 /* Check SOL/IDER usage */
1237                 EM_CORE_LOCK(adapter);
1238                 if (e1000_check_reset_block(&adapter->hw)) {
1239                         EM_CORE_UNLOCK(adapter);
1240                         device_printf(adapter->dev, "Media change is"
1241                             " blocked due to SOL/IDER session.\n");
1242                         break;
1243                 }
1244                 EM_CORE_UNLOCK(adapter);
1245         case SIOCGIFMEDIA:
1246                 IOCTL_DEBUGOUT("ioctl rcv'd: \
1247                     SIOCxIFMEDIA (Get/Set Interface Media)");
1248                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1249                 break;
1250         case SIOCSIFCAP:
1251             {
1252                 int mask, reinit;
1253
1254                 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1255                 reinit = 0;
1256                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1257 #ifdef DEVICE_POLLING
1258                 if (mask & IFCAP_POLLING) {
1259                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
1260                                 error = ether_poll_register(em_poll, ifp);
1261                                 if (error)
1262                                         return (error);
1263                                 EM_CORE_LOCK(adapter);
1264                                 em_disable_intr(adapter);
1265                                 ifp->if_capenable |= IFCAP_POLLING;
1266                                 EM_CORE_UNLOCK(adapter);
1267                         } else {
1268                                 error = ether_poll_deregister(ifp);
1269                                 /* Enable interrupt even in error case */
1270                                 EM_CORE_LOCK(adapter);
1271                                 em_enable_intr(adapter);
1272                                 ifp->if_capenable &= ~IFCAP_POLLING;
1273                                 EM_CORE_UNLOCK(adapter);
1274                         }
1275                 }
1276 #endif
1277                 if (mask & IFCAP_HWCSUM) {
1278                         ifp->if_capenable ^= IFCAP_HWCSUM;
1279                         reinit = 1;
1280                 }
1281 #if __FreeBSD_version >= 700000
1282                 if (mask & IFCAP_TSO4) {
1283                         ifp->if_capenable ^= IFCAP_TSO4;
1284                         reinit = 1;
1285                 }
1286 #endif
1287                 if (mask & IFCAP_VLAN_HWTAGGING) {
1288                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1289                         reinit = 1;
1290                 }
1291
1292                 if (mask & IFCAP_VLAN_HWFILTER) {
1293                         ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1294                         reinit = 1;
1295                 }
1296
1297                 if ((mask & IFCAP_WOL) &&
1298                     (ifp->if_capabilities & IFCAP_WOL) != 0) {
1299                         if (mask & IFCAP_WOL_MCAST)
1300                                 ifp->if_capenable ^= IFCAP_WOL_MCAST;
1301                         if (mask & IFCAP_WOL_MAGIC)
1302                                 ifp->if_capenable ^= IFCAP_WOL_MAGIC;
1303                 }
1304
1305                 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1306                         em_init(adapter);
1307 #if __FreeBSD_version >= 700000
1308                 VLAN_CAPABILITIES(ifp);
1309 #endif
1310                 break;
1311             }
1312
1313         default:
1314                 error = ether_ioctl(ifp, command, data);
1315                 break;
1316         }
1317
1318         return (error);
1319 }
1320
1321
1322 /*********************************************************************
1323  *  Init entry point
1324  *
1325  *  This routine is used in two ways. It is used by the stack as
1326  *  init entry point in network interface structure. It is also used
1327  *  by the driver as a hw/sw initialization routine to get to a
1328  *  consistent state.
1329  *
1330  *  return 0 on success, positive on failure
1331  **********************************************************************/
1332
1333 static void
1334 em_init_locked(struct adapter *adapter)
1335 {
1336         struct ifnet    *ifp = adapter->ifp;
1337         device_t        dev = adapter->dev;
1338         u32             pba;
1339
1340         INIT_DEBUGOUT("em_init: begin");
1341
1342         EM_CORE_LOCK_ASSERT(adapter);
1343
1344         EM_TX_LOCK(adapter);
1345         em_stop(adapter);
1346         EM_TX_UNLOCK(adapter);
1347
1348         /*
1349          * Packet Buffer Allocation (PBA)
1350          * Writing PBA sets the receive portion of the buffer
1351          * the remainder is used for the transmit buffer.
1352          *
1353          * Devices before the 82547 had a Packet Buffer of 64K.
1354          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1355          * After the 82547 the buffer was reduced to 40K.
1356          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1357          *   Note: default does not leave enough room for Jumbo Frame >10k.
1358          */
1359         switch (adapter->hw.mac.type) {
1360         case e1000_82547:
1361         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1362                 if (adapter->max_frame_size > 8192)
1363                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1364                 else
1365                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1366                 adapter->tx_fifo_head = 0;
1367                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1368                 adapter->tx_fifo_size =
1369                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1370                 break;
1371         /* Total Packet Buffer on these is 48K */
1372         case e1000_82571:
1373         case e1000_82572:
1374         case e1000_80003es2lan:
1375                         pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1376                 break;
1377         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1378                         pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1379                 break;
1380         case e1000_82574:
1381         case e1000_82583:
1382                         pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1383                 break;
1384         case e1000_ich9lan:
1385         case e1000_ich10lan:
1386         case e1000_pchlan:
1387                 pba = E1000_PBA_10K;
1388                 break;
1389         case e1000_ich8lan:
1390                 pba = E1000_PBA_8K;
1391                 break;
1392         default:
1393                 /* Devices before 82547 had a Packet Buffer of 64K.   */
1394                 if (adapter->max_frame_size > 8192)
1395                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1396                 else
1397                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1398         }
1399
1400         INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1401         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1402
1403         /* Get the latest mac address, User can use a LAA */
1404         bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1405               ETHER_ADDR_LEN);
1406
1407         /* Put the address into the Receive Address Array */
1408         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1409
1410         /*
1411          * With the 82571 adapter, RAR[0] may be overwritten
1412          * when the other port is reset, we make a duplicate
1413          * in RAR[14] for that eventuality, this assures
1414          * the interface continues to function.
1415          */
1416         if (adapter->hw.mac.type == e1000_82571) {
1417                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1418                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1419                     E1000_RAR_ENTRIES - 1);
1420         }
1421
1422         /* Initialize the hardware */
1423         if (em_hardware_init(adapter)) {
1424                 device_printf(dev, "Unable to initialize the hardware\n");
1425                 return;
1426         }
1427         em_update_link_status(adapter);
1428
1429         /* Setup VLAN support, basic and offload if available */
1430         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1431         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1432                 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
1433                         /* Use real VLAN Filter support */
1434                         em_setup_vlan_hw_support(adapter);
1435                 else {
1436                         u32 ctrl;
1437                         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1438                         ctrl |= E1000_CTRL_VME;
1439                         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1440                 }
1441         }
1442
1443         /* Set hardware offload abilities */
1444         ifp->if_hwassist = 0;
1445         if (adapter->hw.mac.type >= e1000_82543) {
1446                 if (ifp->if_capenable & IFCAP_TXCSUM)
1447                         ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1448 #if __FreeBSD_version >= 700000
1449                 if (ifp->if_capenable & IFCAP_TSO4)
1450                         ifp->if_hwassist |= CSUM_TSO;
1451 #endif
1452         }
1453
1454         /* Configure for OS presence */
1455         em_init_manageability(adapter);
1456
1457         /* Prepare transmit descriptors and buffers */
1458         em_setup_transmit_structures(adapter);
1459         em_initialize_transmit_unit(adapter);
1460
1461         /* Setup Multicast table */
1462         em_set_multi(adapter);
1463
1464         /* Prepare receive descriptors and buffers */
1465         if (em_setup_receive_structures(adapter)) {
1466                 device_printf(dev, "Could not setup receive structures\n");
1467                 EM_TX_LOCK(adapter);
1468                 em_stop(adapter);
1469                 EM_TX_UNLOCK(adapter);
1470                 return;
1471         }
1472         em_initialize_receive_unit(adapter);
1473
1474         /* Don't lose promiscuous settings */
1475         em_set_promisc(adapter);
1476
1477         ifp->if_drv_flags |= IFF_DRV_RUNNING;
1478         ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1479
1480         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1481         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1482
1483         /* MSI/X configuration for 82574 */
1484         if (adapter->hw.mac.type == e1000_82574) {
1485                 int tmp;
1486                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1487                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1488                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1489                 /*
1490                 ** Set the IVAR - interrupt vector routing.
1491                 ** Each nibble represents a vector, high bit
1492                 ** is enable, other 3 bits are the MSIX table
1493                 ** entry, we map RXQ0 to 0, TXQ0 to 1, and
1494                 ** Link (other) to 2, hence the magic number.
1495                 */
1496                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1497         }
1498
1499 #ifdef DEVICE_POLLING
1500         /*
1501          * Only enable interrupts if we are not polling, make sure
1502          * they are off otherwise.
1503          */
1504         if (ifp->if_capenable & IFCAP_POLLING)
1505                 em_disable_intr(adapter);
1506         else
1507 #endif /* DEVICE_POLLING */
1508                 em_enable_intr(adapter);
1509
1510         /* AMT based hardware can now take control from firmware */
1511         if (adapter->has_manage && adapter->has_amt)
1512                 em_get_hw_control(adapter);
1513
1514         /* Don't reset the phy next time init gets called */
1515         adapter->hw.phy.reset_disable = TRUE;
1516 }
1517
1518 static void
1519 em_init(void *arg)
1520 {
1521         struct adapter *adapter = arg;
1522
1523         EM_CORE_LOCK(adapter);
1524         em_init_locked(adapter);
1525         EM_CORE_UNLOCK(adapter);
1526 }
1527
1528
1529 #ifdef DEVICE_POLLING
1530 /*********************************************************************
1531  *
1532  *  Legacy polling routine
1533  *
1534  *********************************************************************/
1535 static int
1536 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1537 {
1538         struct adapter *adapter = ifp->if_softc;
1539         u32             reg_icr, rx_done = 0;
1540
1541         EM_CORE_LOCK(adapter);
1542         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1543                 EM_CORE_UNLOCK(adapter);
1544                 return (rx_done);
1545         }
1546
1547         if (cmd == POLL_AND_CHECK_STATUS) {
1548                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1549                 /* Link status change */
1550                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1551                         adapter->hw.mac.get_link_status = 1;
1552                         em_update_link_status(adapter);
1553                 }
1554                 if (reg_icr & E1000_ICR_RXO)
1555                         adapter->rx_overruns++;
1556         }
1557         EM_CORE_UNLOCK(adapter);
1558
1559         rx_done = em_rxeof(adapter, count);
1560
1561         EM_TX_LOCK(adapter);
1562         em_txeof(adapter);
1563 #if __FreeBSD_version >= 800000
1564         if (!drbr_empty(ifp, adapter->br))
1565                 em_mq_start_locked(ifp, NULL);
1566 #else
1567         if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1568                 em_start_locked(ifp);
1569 #endif
1570         EM_TX_UNLOCK(adapter);
1571         return (rx_done);
1572 }
1573 #endif /* DEVICE_POLLING */
1574
1575 #ifdef EM_LEGACY_IRQ
1576 /*********************************************************************
1577  *
1578  *  Legacy Interrupt Service routine
1579  *
1580  *********************************************************************/
1581
1582 static void
1583 em_intr(void *arg)
1584 {
1585         struct adapter  *adapter = arg;
1586         struct ifnet    *ifp = adapter->ifp;
1587         u32             reg_icr;
1588
1589
1590         if (ifp->if_capenable & IFCAP_POLLING)
1591                 return;
1592
1593         EM_CORE_LOCK(adapter);
1594         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1595         if (reg_icr & E1000_ICR_RXO)
1596                 adapter->rx_overruns++;
1597         if ((reg_icr == 0xffffffff) || (reg_icr == 0)||
1598             (adapter->hw.mac.type >= e1000_82571 &&
1599             (reg_icr & E1000_ICR_INT_ASSERTED) == 0))
1600                         goto out;
1601
1602         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1603                         goto out;
1604
1605         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1606                 callout_stop(&adapter->timer);
1607                 adapter->hw.mac.get_link_status = 1;
1608                 em_update_link_status(adapter);
1609                 /* Deal with TX cruft when link lost */
1610                 em_tx_purge(adapter);
1611                 callout_reset(&adapter->timer, hz,
1612                     em_local_timer, adapter);
1613                 goto out;
1614         }
1615
1616         EM_TX_LOCK(adapter);
1617         em_txeof(adapter);
1618         em_rxeof(adapter, -1);
1619         em_txeof(adapter);
1620         if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1621             !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1622                 em_start_locked(ifp);
1623         EM_TX_UNLOCK(adapter);
1624
1625 out:
1626         EM_CORE_UNLOCK(adapter);
1627         return;
1628 }
1629
1630 #else /* EM_FAST_IRQ, then fast interrupt routines only */
1631
1632 static void
1633 em_handle_link(void *context, int pending)
1634 {
1635         struct adapter  *adapter = context;
1636         struct ifnet *ifp = adapter->ifp;
1637
1638         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1639                 return;
1640
1641         EM_CORE_LOCK(adapter);
1642         callout_stop(&adapter->timer);
1643         em_update_link_status(adapter);
1644         /* Deal with TX cruft when link lost */
1645         em_tx_purge(adapter);
1646         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1647         EM_CORE_UNLOCK(adapter);
1648 }
1649
1650
1651 /* Combined RX/TX handler, used by Legacy and MSI */
1652 static void
1653 em_handle_rxtx(void *context, int pending)
1654 {
1655         struct adapter  *adapter = context;
1656         struct ifnet    *ifp = adapter->ifp;
1657
1658
1659         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1660                 if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1661                         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1662                 EM_TX_LOCK(adapter);
1663                 em_txeof(adapter);
1664
1665 #if __FreeBSD_version >= 800000
1666                 if (!drbr_empty(ifp, adapter->br))
1667                         em_mq_start_locked(ifp, NULL);
1668 #else
1669                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1670                         em_start_locked(ifp);
1671 #endif
1672                 EM_TX_UNLOCK(adapter);
1673         }
1674
1675         em_enable_intr(adapter);
1676 }
1677
1678 /*********************************************************************
1679  *
1680  *  Fast Legacy/MSI Combined Interrupt Service routine
1681  *
1682  *********************************************************************/
1683 #if __FreeBSD_version < 700000
1684 #define FILTER_STRAY
1685 #define FILTER_HANDLED
1686 static void
1687 #else
1688 static int
1689 #endif
1690 em_irq_fast(void *arg)
1691 {
1692         struct adapter  *adapter = arg;
1693         struct ifnet    *ifp;
1694         u32             reg_icr;
1695
1696         ifp = adapter->ifp;
1697
1698         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1699
1700         /* Hot eject?  */
1701         if (reg_icr == 0xffffffff)
1702                 return FILTER_STRAY;
1703
1704         /* Definitely not our interrupt.  */
1705         if (reg_icr == 0x0)
1706                 return FILTER_STRAY;
1707
1708         /*
1709          * Starting with the 82571 chip, bit 31 should be used to
1710          * determine whether the interrupt belongs to us.
1711          */
1712         if (adapter->hw.mac.type >= e1000_82571 &&
1713             (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1714                 return FILTER_STRAY;
1715
1716         /*
1717          * Mask interrupts until the taskqueue is finished running.  This is
1718          * cheap, just assume that it is needed.  This also works around the
1719          * MSI message reordering errata on certain systems.
1720          */
1721         em_disable_intr(adapter);
1722         taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1723
1724         /* Link status change */
1725         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1726                 adapter->hw.mac.get_link_status = 1;
1727                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1728         }
1729
1730         if (reg_icr & E1000_ICR_RXO)
1731                 adapter->rx_overruns++;
1732         return FILTER_HANDLED;
1733 }
1734
1735 /*********************************************************************
1736  *
1737  *  MSIX Interrupt Service Routines
1738  *
1739  **********************************************************************/
1740 #define EM_MSIX_TX      0x00040000
1741 #define EM_MSIX_RX      0x00010000
1742 #define EM_MSIX_LINK    0x00100000
1743
1744 static void
1745 em_msix_tx(void *arg)
1746 {
1747         struct adapter *adapter = arg;
1748         struct ifnet    *ifp = adapter->ifp;
1749
1750         ++adapter->tx_irq;
1751         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1752                 EM_TX_LOCK(adapter);
1753                 em_txeof(adapter);
1754                 EM_TX_UNLOCK(adapter);
1755                 taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1756         }
1757         /* Reenable this interrupt */
1758         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1759         return;
1760 }
1761
1762 /*********************************************************************
1763  *
1764  *  MSIX RX Interrupt Service routine
1765  *
1766  **********************************************************************/
1767
1768 static void
1769 em_msix_rx(void *arg)
1770 {
1771         struct adapter *adapter = arg;
1772         struct ifnet    *ifp = adapter->ifp;
1773
1774         ++adapter->rx_irq;
1775         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1776             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1777                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1778         /* Reenable this interrupt */
1779         E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1780         return;
1781 }
1782
1783 /*********************************************************************
1784  *
1785  *  MSIX Link Fast Interrupt Service routine
1786  *
1787  **********************************************************************/
1788
1789 static void
1790 em_msix_link(void *arg)
1791 {
1792         struct adapter  *adapter = arg;
1793         u32             reg_icr;
1794
1795         ++adapter->link_irq;
1796         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1797
1798         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1799                 adapter->hw.mac.get_link_status = 1;
1800                 taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1801         }
1802         E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1803             EM_MSIX_LINK | E1000_IMS_LSC);
1804         return;
1805 }
1806
1807 static void
1808 em_handle_rx(void *context, int pending)
1809 {
1810         struct adapter  *adapter = context;
1811         struct ifnet    *ifp = adapter->ifp;
1812
1813         if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1814             (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1815                 taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1816
1817 }
1818
1819 static void
1820 em_handle_tx(void *context, int pending)
1821 {
1822         struct adapter  *adapter = context;
1823         struct ifnet    *ifp = adapter->ifp;
1824
1825         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1826                 if (!EM_TX_TRYLOCK(adapter))
1827                         return;
1828                 em_txeof(adapter);
1829 #if __FreeBSD_version >= 800000
1830                 if (!drbr_empty(ifp, adapter->br))
1831                         em_mq_start_locked(ifp, NULL);
1832 #else
1833                 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1834                         em_start_locked(ifp);
1835 #endif
1836                 EM_TX_UNLOCK(adapter);
1837         }
1838 }
1839 #endif /* EM_FAST_IRQ */
1840
1841 /*********************************************************************
1842  *
1843  *  Media Ioctl callback
1844  *
1845  *  This routine is called whenever the user queries the status of
1846  *  the interface using ifconfig.
1847  *
1848  **********************************************************************/
1849 static void
1850 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1851 {
1852         struct adapter *adapter = ifp->if_softc;
1853         u_char fiber_type = IFM_1000_SX;
1854
1855         INIT_DEBUGOUT("em_media_status: begin");
1856
1857         EM_CORE_LOCK(adapter);
1858         em_update_link_status(adapter);
1859
1860         ifmr->ifm_status = IFM_AVALID;
1861         ifmr->ifm_active = IFM_ETHER;
1862
1863         if (!adapter->link_active) {
1864                 EM_CORE_UNLOCK(adapter);
1865                 return;
1866         }
1867
1868         ifmr->ifm_status |= IFM_ACTIVE;
1869
1870         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1871             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1872                 if (adapter->hw.mac.type == e1000_82545)
1873                         fiber_type = IFM_1000_LX;
1874                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1875         } else {
1876                 switch (adapter->link_speed) {
1877                 case 10:
1878                         ifmr->ifm_active |= IFM_10_T;
1879                         break;
1880                 case 100:
1881                         ifmr->ifm_active |= IFM_100_TX;
1882                         break;
1883                 case 1000:
1884                         ifmr->ifm_active |= IFM_1000_T;
1885                         break;
1886                 }
1887                 if (adapter->link_duplex == FULL_DUPLEX)
1888                         ifmr->ifm_active |= IFM_FDX;
1889                 else
1890                         ifmr->ifm_active |= IFM_HDX;
1891         }
1892         EM_CORE_UNLOCK(adapter);
1893 }
1894
1895 /*********************************************************************
1896  *
1897  *  Media Ioctl callback
1898  *
1899  *  This routine is called when the user changes speed/duplex using
1900  *  media/mediopt option with ifconfig.
1901  *
1902  **********************************************************************/
1903 static int
1904 em_media_change(struct ifnet *ifp)
1905 {
1906         struct adapter *adapter = ifp->if_softc;
1907         struct ifmedia  *ifm = &adapter->media;
1908
1909         INIT_DEBUGOUT("em_media_change: begin");
1910
1911         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1912                 return (EINVAL);
1913
1914         EM_CORE_LOCK(adapter);
1915         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1916         case IFM_AUTO:
1917                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1918                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1919                 break;
1920         case IFM_1000_LX:
1921         case IFM_1000_SX:
1922         case IFM_1000_T:
1923                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1924                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1925                 break;
1926         case IFM_100_TX:
1927                 adapter->hw.mac.autoneg = FALSE;
1928                 adapter->hw.phy.autoneg_advertised = 0;
1929                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1930                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1931                 else
1932                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1933                 break;
1934         case IFM_10_T:
1935                 adapter->hw.mac.autoneg = FALSE;
1936                 adapter->hw.phy.autoneg_advertised = 0;
1937                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1938                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1939                 else
1940                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1941                 break;
1942         default:
1943                 device_printf(adapter->dev, "Unsupported media type\n");
1944         }
1945
1946         /* As the speed/duplex settings my have changed we need to
1947          * reset the PHY.
1948          */
1949         adapter->hw.phy.reset_disable = FALSE;
1950
1951         em_init_locked(adapter);
1952         EM_CORE_UNLOCK(adapter);
1953
1954         return (0);
1955 }
1956
1957 /*********************************************************************
1958  *
1959  *  This routine maps the mbufs to tx descriptors.
1960  *
1961  *  return 0 on success, positive on failure
1962  **********************************************************************/
1963
1964 static int
1965 em_xmit(struct adapter *adapter, struct mbuf **m_headp)
1966 {
1967         bus_dma_segment_t       segs[EM_MAX_SCATTER];
1968         bus_dmamap_t            map;
1969         struct em_buffer        *tx_buffer, *tx_buffer_mapped;
1970         struct e1000_tx_desc    *ctxd = NULL;
1971         struct mbuf             *m_head;
1972         u32                     txd_upper, txd_lower, txd_used, txd_saved;
1973         int                     nsegs, i, j, first, last = 0;
1974         int                     error, do_tso, tso_desc = 0;
1975 #if __FreeBSD_version < 700000
1976         struct m_tag            *mtag;
1977 #endif
1978         m_head = *m_headp;
1979         txd_upper = txd_lower = txd_used = txd_saved = 0;
1980
1981 #if __FreeBSD_version >= 700000
1982         do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1983 #else
1984         do_tso = 0;
1985 #endif
1986
1987         /*
1988          * Force a cleanup if number of TX descriptors
1989          * available hits the threshold
1990          */
1991         if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1992                 em_txeof(adapter);
1993                 /* Now do we at least have a minimal? */
1994                 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
1995                         adapter->no_tx_desc_avail1++;
1996                         return (ENOBUFS);
1997                 }
1998         }
1999
2000
2001         /*
2002          * TSO workaround:
2003          *  If an mbuf is only header we need
2004          *     to pull 4 bytes of data into it.
2005          */
2006         if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
2007                 m_head = m_pullup(m_head, M_TSO_LEN + 4);
2008                 *m_headp = m_head;
2009                 if (m_head == NULL)
2010                         return (ENOBUFS);
2011         }
2012
2013         /*
2014          * Map the packet for DMA
2015          *
2016          * Capture the first descriptor index,
2017          * this descriptor will have the index
2018          * of the EOP which is the only one that
2019          * now gets a DONE bit writeback.
2020          */
2021         first = adapter->next_avail_tx_desc;
2022         tx_buffer = &adapter->tx_buffer_area[first];
2023         tx_buffer_mapped = tx_buffer;
2024         map = tx_buffer->map;
2025
2026         error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2027             *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2028
2029         /*
2030          * There are two types of errors we can (try) to handle:
2031          * - EFBIG means the mbuf chain was too long and bus_dma ran
2032          *   out of segments.  Defragment the mbuf chain and try again.
2033          * - ENOMEM means bus_dma could not obtain enough bounce buffers
2034          *   at this point in time.  Defer sending and try again later.
2035          * All other errors, in particular EINVAL, are fatal and prevent the
2036          * mbuf chain from ever going through.  Drop it and report error.
2037          */
2038         if (error == EFBIG) {
2039                 struct mbuf *m;
2040
2041                 m = m_defrag(*m_headp, M_DONTWAIT);
2042                 if (m == NULL) {
2043                         adapter->mbuf_alloc_failed++;
2044                         m_freem(*m_headp);
2045                         *m_headp = NULL;
2046                         return (ENOBUFS);
2047                 }
2048                 *m_headp = m;
2049
2050                 /* Try it again */
2051                 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2052                     *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2053
2054                 if (error) {
2055                         adapter->no_tx_dma_setup++;
2056                         m_freem(*m_headp);
2057                         *m_headp = NULL;
2058                         return (error);
2059                 }
2060         } else if (error != 0) {
2061                 adapter->no_tx_dma_setup++;
2062                 return (error);
2063         }
2064
2065         /*
2066          * TSO Hardware workaround, if this packet is not
2067          * TSO, and is only a single descriptor long, and
2068          * it follows a TSO burst, then we need to add a
2069          * sentinel descriptor to prevent premature writeback.
2070          */
2071         if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2072                 if (nsegs == 1)
2073                         tso_desc = TRUE;
2074                 adapter->tx_tso = FALSE;
2075         }
2076
2077         if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2078                 adapter->no_tx_desc_avail2++;
2079                 bus_dmamap_unload(adapter->txtag, map);
2080                 return (ENOBUFS);
2081         }
2082         m_head = *m_headp;
2083
2084         /* Do hardware assists */
2085 #if __FreeBSD_version >= 700000
2086         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2087                 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2088                 if (error != TRUE)
2089                         return (ENXIO); /* something foobar */
2090                 /* we need to make a final sentinel transmit desc */
2091                 tso_desc = TRUE;
2092         } else
2093 #endif
2094         if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2095                 em_transmit_checksum_setup(adapter,  m_head,
2096                     &txd_upper, &txd_lower);
2097
2098         i = adapter->next_avail_tx_desc;
2099         if (adapter->pcix_82544)
2100                 txd_saved = i;
2101
2102         /* Set up our transmit descriptors */
2103         for (j = 0; j < nsegs; j++) {
2104                 bus_size_t seg_len;
2105                 bus_addr_t seg_addr;
2106                 /* If adapter is 82544 and on PCIX bus */
2107                 if(adapter->pcix_82544) {
2108                         DESC_ARRAY      desc_array;
2109                         u32             array_elements, counter;
2110                         /*
2111                          * Check the Address and Length combination and
2112                          * split the data accordingly
2113                          */
2114                         array_elements = em_fill_descriptors(segs[j].ds_addr,
2115                             segs[j].ds_len, &desc_array);
2116                         for (counter = 0; counter < array_elements; counter++) {
2117                                 if (txd_used == adapter->num_tx_desc_avail) {
2118                                         adapter->next_avail_tx_desc = txd_saved;
2119                                         adapter->no_tx_desc_avail2++;
2120                                         bus_dmamap_unload(adapter->txtag, map);
2121                                         return (ENOBUFS);
2122                                 }
2123                                 tx_buffer = &adapter->tx_buffer_area[i];
2124                                 ctxd = &adapter->tx_desc_base[i];
2125                                 ctxd->buffer_addr = htole64(
2126                                     desc_array.descriptor[counter].address);
2127                                 ctxd->lower.data = htole32(
2128                                     (adapter->txd_cmd | txd_lower | (u16)
2129                                     desc_array.descriptor[counter].length));
2130                                 ctxd->upper.data =
2131                                     htole32((txd_upper));
2132                                 last = i;
2133                                 if (++i == adapter->num_tx_desc)
2134                                          i = 0;
2135                                 tx_buffer->m_head = NULL;
2136                                 tx_buffer->next_eop = -1;
2137                                 txd_used++;
2138                         }
2139                 } else {
2140                         tx_buffer = &adapter->tx_buffer_area[i];
2141                         ctxd = &adapter->tx_desc_base[i];
2142                         seg_addr = segs[j].ds_addr;
2143                         seg_len  = segs[j].ds_len;
2144                         /*
2145                         ** TSO Workaround:
2146                         ** If this is the last descriptor, we want to
2147                         ** split it so we have a small final sentinel
2148                         */
2149                         if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2150                                 seg_len -= 4;
2151                                 ctxd->buffer_addr = htole64(seg_addr);
2152                                 ctxd->lower.data = htole32(
2153                                 adapter->txd_cmd | txd_lower | seg_len);
2154                                 ctxd->upper.data =
2155                                     htole32(txd_upper);
2156                                 if (++i == adapter->num_tx_desc)
2157                                         i = 0;
2158                                 /* Now make the sentinel */
2159                                 ++txd_used; /* using an extra txd */
2160                                 ctxd = &adapter->tx_desc_base[i];
2161                                 tx_buffer = &adapter->tx_buffer_area[i];
2162                                 ctxd->buffer_addr =
2163                                     htole64(seg_addr + seg_len);
2164                                 ctxd->lower.data = htole32(
2165                                 adapter->txd_cmd | txd_lower | 4);
2166                                 ctxd->upper.data =
2167                                     htole32(txd_upper);
2168                                 last = i;
2169                                 if (++i == adapter->num_tx_desc)
2170                                         i = 0;
2171                         } else {
2172                                 ctxd->buffer_addr = htole64(seg_addr);
2173                                 ctxd->lower.data = htole32(
2174                                 adapter->txd_cmd | txd_lower | seg_len);
2175                                 ctxd->upper.data =
2176                                     htole32(txd_upper);
2177                                 last = i;
2178                                 if (++i == adapter->num_tx_desc)
2179                                         i = 0;
2180                         }
2181                         tx_buffer->m_head = NULL;
2182                         tx_buffer->next_eop = -1;
2183                 }
2184         }
2185
2186         adapter->next_avail_tx_desc = i;
2187         if (adapter->pcix_82544)
2188                 adapter->num_tx_desc_avail -= txd_used;
2189         else {
2190                 adapter->num_tx_desc_avail -= nsegs;
2191                 if (tso_desc) /* TSO used an extra for sentinel */
2192                         adapter->num_tx_desc_avail -= txd_used;
2193         }
2194
2195         /*
2196         ** Handle VLAN tag, this is the
2197         ** biggest difference between
2198         ** 6.x and 7
2199         */
2200 #if __FreeBSD_version < 700000
2201         /* Find out if we are in vlan mode. */
2202         mtag = VLAN_OUTPUT_TAG(ifp, m_head);
2203         if (mtag != NULL) {
2204                 ctxd->upper.fields.special =
2205                     htole16(VLAN_TAG_VALUE(mtag));
2206 #else /* FreeBSD 7 */
2207         if (m_head->m_flags & M_VLANTAG) {
2208                 /* Set the vlan id. */
2209                 ctxd->upper.fields.special =
2210                     htole16(m_head->m_pkthdr.ether_vtag);
2211 #endif
2212                 /* Tell hardware to add tag */
2213                 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2214         }
2215
2216         tx_buffer->m_head = m_head;
2217         tx_buffer_mapped->map = tx_buffer->map;
2218         tx_buffer->map = map;
2219         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2220
2221         /*
2222          * Last Descriptor of Packet
2223          * needs End Of Packet (EOP)
2224          * and Report Status (RS)
2225          */
2226         ctxd->lower.data |=
2227             htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2228         /*
2229          * Keep track in the first buffer which
2230          * descriptor will be written back
2231          */
2232         tx_buffer = &adapter->tx_buffer_area[first];
2233         tx_buffer->next_eop = last;
2234         adapter->watchdog_time = ticks;
2235
2236         /*
2237          * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2238          * that this frame is available to transmit.
2239          */
2240         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2241             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2242         if (adapter->hw.mac.type == e1000_82547 &&
2243             adapter->link_duplex == HALF_DUPLEX)
2244                 em_82547_move_tail(adapter);
2245         else {
2246                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2247                 if (adapter->hw.mac.type == e1000_82547)
2248                         em_82547_update_fifo_head(adapter,
2249                             m_head->m_pkthdr.len);
2250         }
2251
2252         return (0);
2253 }
2254
2255 /*********************************************************************
2256  *
2257  * 82547 workaround to avoid controller hang in half-duplex environment.
2258  * The workaround is to avoid queuing a large packet that would span
2259  * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2260  * in this case. We do that only when FIFO is quiescent.
2261  *
2262  **********************************************************************/
2263 static void
2264 em_82547_move_tail(void *arg)
2265 {
2266         struct adapter *adapter = arg;
2267         struct e1000_tx_desc *tx_desc;
2268         u16     hw_tdt, sw_tdt, length = 0;
2269         bool    eop = 0;
2270
2271         EM_TX_LOCK_ASSERT(adapter);
2272
2273         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2274         sw_tdt = adapter->next_avail_tx_desc;
2275
2276         while (hw_tdt != sw_tdt) {
2277                 tx_desc = &adapter->tx_desc_base[hw_tdt];
2278                 length += tx_desc->lower.flags.length;
2279                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2280                 if (++hw_tdt == adapter->num_tx_desc)
2281                         hw_tdt = 0;
2282
2283                 if (eop) {
2284                         if (em_82547_fifo_workaround(adapter, length)) {
2285                                 adapter->tx_fifo_wrk_cnt++;
2286                                 callout_reset(&adapter->tx_fifo_timer, 1,
2287                                         em_82547_move_tail, adapter);
2288                                 break;
2289                         }
2290                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2291                         em_82547_update_fifo_head(adapter, length);
2292                         length = 0;
2293                 }
2294         }
2295 }
2296
2297 static int
2298 em_82547_fifo_workaround(struct adapter *adapter, int len)
2299 {
2300         int fifo_space, fifo_pkt_len;
2301
2302         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2303
2304         if (adapter->link_duplex == HALF_DUPLEX) {
2305                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2306
2307                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2308                         if (em_82547_tx_fifo_reset(adapter))
2309                                 return (0);
2310                         else
2311                                 return (1);
2312                 }
2313         }
2314
2315         return (0);
2316 }
2317
2318 static void
2319 em_82547_update_fifo_head(struct adapter *adapter, int len)
2320 {
2321         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2322
2323         /* tx_fifo_head is always 16 byte aligned */
2324         adapter->tx_fifo_head += fifo_pkt_len;
2325         if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2326                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
2327         }
2328 }
2329
2330
2331 static int
2332 em_82547_tx_fifo_reset(struct adapter *adapter)
2333 {
2334         u32 tctl;
2335
2336         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2337             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2338             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2339             E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2340             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2341             E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2342             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2343                 /* Disable TX unit */
2344                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2345                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2346                     tctl & ~E1000_TCTL_EN);
2347
2348                 /* Reset FIFO pointers */
2349                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2350                     adapter->tx_head_addr);
2351                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2352                     adapter->tx_head_addr);
2353                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2354                     adapter->tx_head_addr);
2355                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2356                     adapter->tx_head_addr);
2357
2358                 /* Re-enable TX unit */
2359                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2360                 E1000_WRITE_FLUSH(&adapter->hw);
2361
2362                 adapter->tx_fifo_head = 0;
2363                 adapter->tx_fifo_reset_cnt++;
2364
2365                 return (TRUE);
2366         }
2367         else {
2368                 return (FALSE);
2369         }
2370 }
2371
2372 static void
2373 em_set_promisc(struct adapter *adapter)
2374 {
2375         struct ifnet    *ifp = adapter->ifp;
2376         u32             reg_rctl;
2377
2378         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2379
2380         if (ifp->if_flags & IFF_PROMISC) {
2381                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2382                 /* Turn this on if you want to see bad packets */
2383                 if (em_debug_sbp)
2384                         reg_rctl |= E1000_RCTL_SBP;
2385                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2386         } else if (ifp->if_flags & IFF_ALLMULTI) {
2387                 reg_rctl |= E1000_RCTL_MPE;
2388                 reg_rctl &= ~E1000_RCTL_UPE;
2389                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2390         }
2391 }
2392
2393 static void
2394 em_disable_promisc(struct adapter *adapter)
2395 {
2396         u32     reg_rctl;
2397
2398         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2399
2400         reg_rctl &=  (~E1000_RCTL_UPE);
2401         reg_rctl &=  (~E1000_RCTL_MPE);
2402         reg_rctl &=  (~E1000_RCTL_SBP);
2403         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2404 }
2405
2406
2407 /*********************************************************************
2408  *  Multicast Update
2409  *
2410  *  This routine is called whenever multicast address list is updated.
2411  *
2412  **********************************************************************/
2413
2414 static void
2415 em_set_multi(struct adapter *adapter)
2416 {
2417         struct ifnet    *ifp = adapter->ifp;
2418         struct ifmultiaddr *ifma;
2419         u32 reg_rctl = 0;
2420         u8  *mta; /* Multicast array memory */
2421         int mcnt = 0;
2422
2423         IOCTL_DEBUGOUT("em_set_multi: begin");
2424
2425         if (adapter->hw.mac.type == e1000_82542 &&
2426             adapter->hw.revision_id == E1000_REVISION_2) {
2427                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2428                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2429                         e1000_pci_clear_mwi(&adapter->hw);
2430                 reg_rctl |= E1000_RCTL_RST;
2431                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2432                 msec_delay(5);
2433         }
2434
2435         /* Allocate temporary memory to setup array */
2436         mta = malloc(sizeof(u8) *
2437             (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES),
2438             M_DEVBUF, M_NOWAIT | M_ZERO);
2439         if (mta == NULL)
2440                 panic("em_set_multi memory failure\n");
2441
2442 #if __FreeBSD_version < 800000
2443         IF_ADDR_LOCK(ifp);
2444 #else
2445         if_maddr_rlock(ifp);
2446 #endif
2447         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2448                 if (ifma->ifma_addr->sa_family != AF_LINK)
2449                         continue;
2450
2451                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2452                         break;
2453
2454                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2455                     &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2456                 mcnt++;
2457         }
2458 #if __FreeBSD_version < 800000
2459         IF_ADDR_UNLOCK(ifp);
2460 #else
2461         if_maddr_runlock(ifp);
2462 #endif
2463         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2464                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2465                 reg_rctl |= E1000_RCTL_MPE;
2466                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2467         } else
2468                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
2469
2470         if (adapter->hw.mac.type == e1000_82542 &&
2471             adapter->hw.revision_id == E1000_REVISION_2) {
2472                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2473                 reg_rctl &= ~E1000_RCTL_RST;
2474                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2475                 msec_delay(5);
2476                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2477                         e1000_pci_set_mwi(&adapter->hw);
2478         }
2479         free(mta, M_DEVBUF);
2480 }
2481
2482
2483 /*********************************************************************
2484  *  Timer routine
2485  *
2486  *  This routine checks for link status and updates statistics.
2487  *
2488  **********************************************************************/
2489
2490 static void
2491 em_local_timer(void *arg)
2492 {
2493         struct adapter  *adapter = arg;
2494         struct ifnet    *ifp = adapter->ifp;
2495
2496         EM_CORE_LOCK_ASSERT(adapter);
2497
2498 #ifndef DEVICE_POLLING
2499         taskqueue_enqueue(adapter->tq,
2500             &adapter->rxtx_task);
2501 #endif
2502         em_update_link_status(adapter);
2503         em_update_stats_counters(adapter);
2504
2505         /* Reset LAA into RAR[0] on 82571 */
2506         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2507                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2508
2509         if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2510                 em_print_hw_stats(adapter);
2511
2512         em_smartspeed(adapter);
2513
2514         /*
2515          * We check the watchdog: the time since
2516          * the last TX descriptor was cleaned.
2517          * This implies a functional TX engine.
2518          */
2519         if ((adapter->watchdog_check == TRUE) &&
2520             (ticks - adapter->watchdog_time > EM_WATCHDOG))
2521                 goto hung;
2522
2523         callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2524         return;
2525 hung:
2526         device_printf(adapter->dev, "Watchdog timeout -- resetting\n");
2527         adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2528         adapter->watchdog_events++;
2529         em_init_locked(adapter);
2530 }
2531
2532 static void
2533 em_update_link_status(struct adapter *adapter)
2534 {
2535         struct e1000_hw *hw = &adapter->hw;
2536         struct ifnet *ifp = adapter->ifp;
2537         device_t dev = adapter->dev;
2538         u32 link_check = 0;
2539
2540         /* Get the cached link value or read phy for real */
2541         switch (hw->phy.media_type) {
2542         case e1000_media_type_copper:
2543                 if (hw->mac.get_link_status) {
2544                         /* Do the work to read phy */
2545                         e1000_check_for_link(hw);
2546                         link_check = !hw->mac.get_link_status;
2547                         if (link_check) /* ESB2 fix */
2548                                 e1000_cfg_on_link_up(hw);
2549                 } else
2550                         link_check = TRUE;
2551                 break;
2552         case e1000_media_type_fiber:
2553                 e1000_check_for_link(hw);
2554                 link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2555                                  E1000_STATUS_LU);
2556                 break;
2557         case e1000_media_type_internal_serdes:
2558                 e1000_check_for_link(hw);
2559                 link_check = adapter->hw.mac.serdes_has_link;
2560                 break;
2561         default:
2562         case e1000_media_type_unknown:
2563                 break;
2564         }
2565
2566         /* Now check for a transition */
2567         if (link_check && (adapter->link_active == 0)) {
2568                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2569                     &adapter->link_duplex);
2570                 /* Check if we must disable SPEED_MODE bit on PCI-E */
2571                 if ((adapter->link_speed != SPEED_1000) &&
2572                     ((hw->mac.type == e1000_82571) ||
2573                     (hw->mac.type == e1000_82572))) {
2574                         int tarc0;
2575                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2576                         tarc0 &= ~SPEED_MODE_BIT;
2577                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2578                 }
2579                 if (bootverbose)
2580                         device_printf(dev, "Link is up %d Mbps %s\n",
2581                             adapter->link_speed,
2582                             ((adapter->link_duplex == FULL_DUPLEX) ?
2583                             "Full Duplex" : "Half Duplex"));
2584                 adapter->link_active = 1;
2585                 adapter->smartspeed = 0;
2586                 ifp->if_baudrate = adapter->link_speed * 1000000;
2587                 if_link_state_change(ifp, LINK_STATE_UP);
2588         } else if (!link_check && (adapter->link_active == 1)) {
2589                 ifp->if_baudrate = adapter->link_speed = 0;
2590                 adapter->link_duplex = 0;
2591                 if (bootverbose)
2592                         device_printf(dev, "Link is Down\n");
2593                 adapter->link_active = 0;
2594                 /* Link down, disable watchdog */
2595                 adapter->watchdog_check = FALSE;
2596                 if_link_state_change(ifp, LINK_STATE_DOWN);
2597         }
2598 }
2599
2600 /*********************************************************************
2601  *
2602  *  This routine disables all traffic on the adapter by issuing a
2603  *  global reset on the MAC and deallocates TX/RX buffers.
2604  *
2605  *  This routine should always be called with BOTH the CORE
2606  *  and TX locks.
2607  **********************************************************************/
2608
2609 static void
2610 em_stop(void *arg)
2611 {
2612         struct adapter  *adapter = arg;
2613         struct ifnet    *ifp = adapter->ifp;
2614
2615         EM_CORE_LOCK_ASSERT(adapter);
2616         EM_TX_LOCK_ASSERT(adapter);
2617
2618         INIT_DEBUGOUT("em_stop: begin");
2619
2620         em_disable_intr(adapter);
2621         callout_stop(&adapter->timer);
2622         callout_stop(&adapter->tx_fifo_timer);
2623
2624         /* Tell the stack that the interface is no longer active */
2625         ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2626
2627         e1000_reset_hw(&adapter->hw);
2628         if (adapter->hw.mac.type >= e1000_82544)
2629                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2630 }
2631
2632
2633 /*********************************************************************
2634  *
2635  *  Determine hardware revision.
2636  *
2637  **********************************************************************/
2638 static void
2639 em_identify_hardware(struct adapter *adapter)
2640 {
2641         device_t dev = adapter->dev;
2642
2643         /* Make sure our PCI config space has the necessary stuff set */
2644         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2645         if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2646             (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2647                 device_printf(dev, "Memory Access and/or Bus Master bits "
2648                     "were not set!\n");
2649                 adapter->hw.bus.pci_cmd_word |=
2650                 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2651                 pci_write_config(dev, PCIR_COMMAND,
2652                     adapter->hw.bus.pci_cmd_word, 2);
2653         }
2654
2655         /* Save off the information about this board */
2656         adapter->hw.vendor_id = pci_get_vendor(dev);
2657         adapter->hw.device_id = pci_get_device(dev);
2658         adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2659         adapter->hw.subsystem_vendor_id =
2660             pci_read_config(dev, PCIR_SUBVEND_0, 2);
2661         adapter->hw.subsystem_device_id =
2662             pci_read_config(dev, PCIR_SUBDEV_0, 2);
2663
2664         /* Do Shared Code Init and Setup */
2665         if (e1000_set_mac_type(&adapter->hw)) {
2666                 device_printf(dev, "Setup init failure\n");
2667                 return;
2668         }
2669 }
2670
2671 static int
2672 em_allocate_pci_resources(struct adapter *adapter)
2673 {
2674         device_t        dev = adapter->dev;
2675         int             val, rid, error = E1000_SUCCESS;
2676
2677         rid = PCIR_BAR(0);
2678         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2679             &rid, RF_ACTIVE);
2680         if (adapter->memory == NULL) {
2681                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2682                 return (ENXIO);
2683         }
2684         adapter->osdep.mem_bus_space_tag =
2685             rman_get_bustag(adapter->memory);
2686         adapter->osdep.mem_bus_space_handle =
2687             rman_get_bushandle(adapter->memory);
2688         adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2689
2690         /* Only older adapters use IO mapping */
2691         if ((adapter->hw.mac.type > e1000_82543) &&
2692             (adapter->hw.mac.type < e1000_82571)) {
2693                 /* Figure our where our IO BAR is ? */
2694                 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2695                         val = pci_read_config(dev, rid, 4);
2696                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2697                                 adapter->io_rid = rid;
2698                                 break;
2699                         }
2700                         rid += 4;
2701                         /* check for 64bit BAR */
2702                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2703                                 rid += 4;
2704                 }
2705                 if (rid >= PCIR_CIS) {
2706                         device_printf(dev, "Unable to locate IO BAR\n");
2707                         return (ENXIO);
2708                 }
2709                 adapter->ioport = bus_alloc_resource_any(dev,
2710                     SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2711                 if (adapter->ioport == NULL) {
2712                         device_printf(dev, "Unable to allocate bus resource: "
2713                             "ioport\n");
2714                         return (ENXIO);
2715                 }
2716                 adapter->hw.io_base = 0;
2717                 adapter->osdep.io_bus_space_tag =
2718                     rman_get_bustag(adapter->ioport);
2719                 adapter->osdep.io_bus_space_handle =
2720                     rman_get_bushandle(adapter->ioport);
2721         }
2722
2723         /*
2724         ** Init the resource arrays
2725         **  used by MSIX setup
2726         */
2727         for (int i = 0; i < 3; i++) {
2728                 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2729                 adapter->tag[i] = NULL;
2730                 adapter->res[i] = NULL;
2731         }
2732
2733         /*
2734          * Setup MSI/X or MSI if PCI Express
2735          */
2736         if (em_enable_msi)
2737                 adapter->msi = em_setup_msix(adapter);
2738
2739         adapter->hw.back = &adapter->osdep;
2740
2741         return (error);
2742 }
2743
2744 /*********************************************************************
2745  *
2746  *  Setup the Legacy or MSI Interrupt handler
2747  *
2748  **********************************************************************/
2749 int
2750 em_allocate_legacy(struct adapter *adapter)
2751 {
2752         device_t dev = adapter->dev;
2753         int error;
2754
2755         /* Manually turn off all interrupts */
2756         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2757
2758         /* Legacy RID is 0 */
2759         if (adapter->msi == 0)
2760                 adapter->rid[0] = 0;
2761
2762         /* We allocate a single interrupt resource */
2763         adapter->res[0] = bus_alloc_resource_any(dev,
2764             SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2765         if (adapter->res[0] == NULL) {
2766                 device_printf(dev, "Unable to allocate bus resource: "
2767                     "interrupt\n");
2768                 return (ENXIO);
2769         }
2770
2771 #ifdef EM_LEGACY_IRQ
2772         /* We do Legacy setup */
2773         if ((error = bus_setup_intr(dev, adapter->res[0],
2774 #if __FreeBSD_version > 700000
2775             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter,
2776 #else /* 6.X */
2777             INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2778 #endif
2779             &adapter->tag[0])) != 0) {
2780                 device_printf(dev, "Failed to register interrupt handler");
2781                 return (error);
2782         }
2783
2784 #else /* FAST_IRQ */
2785         /*
2786          * Try allocating a fast interrupt and the associated deferred
2787          * processing contexts.
2788          */
2789         TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2790         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2791         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2792             taskqueue_thread_enqueue, &adapter->tq);
2793         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2794             device_get_nameunit(adapter->dev));
2795 #if __FreeBSD_version < 700000
2796         if ((error = bus_setup_intr(dev, adapter->res[0],
2797             INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter,
2798 #else
2799         if ((error = bus_setup_intr(dev, adapter->res[0],
2800             INTR_TYPE_NET, em_irq_fast, NULL, adapter,
2801 #endif
2802             &adapter->tag[0])) != 0) {
2803                 device_printf(dev, "Failed to register fast interrupt "
2804                             "handler: %d\n", error);
2805                 taskqueue_free(adapter->tq);
2806                 adapter->tq = NULL;
2807                 return (error);
2808         }
2809 #endif  /* EM_LEGACY_IRQ */
2810
2811         return (0);
2812 }
2813
2814 /*********************************************************************
2815  *
2816  *  Setup the MSIX Interrupt handlers
2817  *   This is not really Multiqueue, rather
2818  *   its just multiple interrupt vectors.
2819  *
2820  **********************************************************************/
2821 int
2822 em_allocate_msix(struct adapter *adapter)
2823 {
2824         device_t dev = adapter->dev;
2825         int error;
2826
2827         /* Make sure all interrupts are disabled */
2828         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2829
2830         /* First get the resources */
2831         for (int i = 0; i < adapter->msi; i++) {
2832                 adapter->res[i] = bus_alloc_resource_any(dev,
2833                     SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2834                 if (adapter->res[i] == NULL) {
2835                         device_printf(dev,
2836                             "Unable to allocate bus resource: "
2837                             "MSIX Interrupt\n");
2838                         return (ENXIO);
2839                 }
2840         }
2841
2842         /*
2843          * Now allocate deferred processing contexts.
2844          */
2845         TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2846         TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2847         /*
2848          * Handle compatibility for msi case for deferral due to
2849          * trylock failure
2850          */
2851         TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter);
2852         TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2853         adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2854             taskqueue_thread_enqueue, &adapter->tq);
2855         taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2856             device_get_nameunit(adapter->dev));
2857
2858         /*
2859          * And setup the interrupt handlers
2860          */
2861
2862         /* First slot to RX */
2863         if ((error = bus_setup_intr(dev, adapter->res[0],
2864 #if __FreeBSD_version > 700000
2865             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter,
2866 #else /* 6.X */
2867             INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter,
2868 #endif
2869             &adapter->tag[0])) != 0) {
2870                 device_printf(dev, "Failed to register RX handler");
2871                 return (error);
2872         }
2873
2874         /* Next TX */
2875         if ((error = bus_setup_intr(dev, adapter->res[1],
2876 #if __FreeBSD_version > 700000
2877             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter,
2878 #else /* 6.X */
2879             INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter,
2880 #endif
2881             &adapter->tag[1])) != 0) {
2882                 device_printf(dev, "Failed to register TX handler");
2883                 return (error);
2884         }
2885
2886         /* And Link */
2887         if ((error = bus_setup_intr(dev, adapter->res[2],
2888 #if __FreeBSD_version > 700000
2889             INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter,
2890 #else /* 6.X */
2891             INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter,
2892 #endif
2893             &adapter->tag[2])) != 0) {
2894                 device_printf(dev, "Failed to register TX handler");
2895                 return (error);
2896         }
2897
2898         return (0);
2899 }
2900
2901
2902 static void
2903 em_free_pci_resources(struct adapter *adapter)
2904 {
2905         device_t dev = adapter->dev;
2906
2907         /* Make sure the for loop below runs once */
2908         if (adapter->msi == 0)
2909                 adapter->msi = 1;
2910
2911         /*
2912          * First release all the interrupt resources:
2913          *      notice that since these are just kept
2914          *      in an array we can do the same logic
2915          *      whether its MSIX or just legacy.
2916          */
2917         for (int i = 0; i < adapter->msi; i++) {
2918                 if (adapter->tag[i] != NULL) {
2919                         bus_teardown_intr(dev, adapter->res[i],
2920                             adapter->tag[i]);
2921                         adapter->tag[i] = NULL;
2922                 }
2923                 if (adapter->res[i] != NULL) {
2924                         bus_release_resource(dev, SYS_RES_IRQ,
2925                             adapter->rid[i], adapter->res[i]);
2926                 }
2927         }
2928
2929         if (adapter->msi)
2930                 pci_release_msi(dev);
2931
2932         if (adapter->msix != NULL)
2933                 bus_release_resource(dev, SYS_RES_MEMORY,
2934                     PCIR_BAR(EM_MSIX_BAR), adapter->msix);
2935
2936         if (adapter->memory != NULL)
2937                 bus_release_resource(dev, SYS_RES_MEMORY,
2938                     PCIR_BAR(0), adapter->memory);
2939
2940         if (adapter->flash != NULL)
2941                 bus_release_resource(dev, SYS_RES_MEMORY,
2942                     EM_FLASH, adapter->flash);
2943
2944         if (adapter->ioport != NULL)
2945                 bus_release_resource(dev, SYS_RES_IOPORT,
2946                     adapter->io_rid, adapter->ioport);
2947 }
2948
2949 /*
2950  * Setup MSI or MSI/X
2951  */
2952 static int
2953 em_setup_msix(struct adapter *adapter)
2954 {
2955         device_t dev = adapter->dev;
2956         int val = 0;
2957
2958         if (adapter->hw.mac.type < e1000_82571)
2959                 return (0);
2960
2961         /* Setup MSI/X for Hartwell */
2962         if (adapter->hw.mac.type == e1000_82574) {
2963                 /* Map the MSIX BAR */
2964                 int rid = PCIR_BAR(EM_MSIX_BAR);
2965                 adapter->msix = bus_alloc_resource_any(dev,
2966                     SYS_RES_MEMORY, &rid, RF_ACTIVE);
2967                 if (!adapter->msix) {
2968                         /* May not be enabled */
2969                         device_printf(adapter->dev,
2970                             "Unable to map MSIX table \n");
2971                         goto msi;
2972                 }
2973                 val = pci_msix_count(dev);
2974                 /*
2975                 ** 82574 can be configured for 5 but
2976                 ** we limit use to 3.
2977                 */
2978                 if (val > 3) val = 3;
2979                 if ((val) && pci_alloc_msix(dev, &val) == 0) {
2980                         device_printf(adapter->dev,"Using MSIX interrupts\n");
2981                         return (val);
2982                 }
2983         }
2984 msi:
2985         val = pci_msi_count(dev);
2986         if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
2987                 adapter->msi = 1;
2988                 device_printf(adapter->dev,"Using MSI interrupt\n");
2989                 return (val);
2990         }
2991         return (0);
2992 }
2993
2994 /*********************************************************************
2995  *
2996  *  Initialize the hardware to a configuration
2997  *  as specified by the adapter structure.
2998  *
2999  **********************************************************************/
3000 static int
3001 em_hardware_init(struct adapter *adapter)
3002 {
3003         device_t dev = adapter->dev;
3004         u16     rx_buffer_size;
3005
3006         INIT_DEBUGOUT("em_hardware_init: begin");
3007
3008         /* Issue a global reset */
3009         e1000_reset_hw(&adapter->hw);
3010
3011         /* When hardware is reset, fifo_head is also reset */
3012         adapter->tx_fifo_head = 0;
3013
3014         /* Set up smart power down as default off on newer adapters. */
3015         if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3016             adapter->hw.mac.type == e1000_82572)) {
3017                 u16 phy_tmp = 0;
3018
3019                 /* Speed up time to link by disabling smart power down. */
3020                 e1000_read_phy_reg(&adapter->hw,
3021                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3022                 phy_tmp &= ~IGP02E1000_PM_SPD;
3023                 e1000_write_phy_reg(&adapter->hw,
3024                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3025         }
3026
3027         /*
3028          * These parameters control the automatic generation (Tx) and
3029          * response (Rx) to Ethernet PAUSE frames.
3030          * - High water mark should allow for at least two frames to be
3031          *   received after sending an XOFF.
3032          * - Low water mark works best when it is very near the high water mark.
3033          *   This allows the receiver to restart by sending XON when it has
3034          *   drained a bit. Here we use an arbitary value of 1500 which will
3035          *   restart after one full frame is pulled from the buffer. There
3036          *   could be several smaller frames in the buffer and if so they will
3037          *   not trigger the XON until their total number reduces the buffer
3038          *   by 1500.
3039          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3040          */
3041         rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3042             0xffff) << 10 );
3043
3044         adapter->hw.fc.high_water = rx_buffer_size -
3045             roundup2(adapter->max_frame_size, 1024);
3046         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3047
3048         if (adapter->hw.mac.type == e1000_80003es2lan)
3049                 adapter->hw.fc.pause_time = 0xFFFF;
3050         else
3051                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3052         adapter->hw.fc.send_xon = TRUE;
3053
3054         /* Set Flow control, use the tunable location if sane */
3055         if ((em_fc_setting >= 0) || (em_fc_setting < 4))
3056                 adapter->hw.fc.requested_mode = em_fc_setting;
3057         else
3058                 adapter->hw.fc.requested_mode = e1000_fc_none;
3059
3060         /* Override - workaround for PCHLAN issue */
3061         if (adapter->hw.mac.type == e1000_pchlan)
3062                 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
3063
3064         if (e1000_init_hw(&adapter->hw) < 0) {
3065                 device_printf(dev, "Hardware Initialization Failed\n");
3066                 return (EIO);
3067         }
3068
3069         e1000_check_for_link(&adapter->hw);
3070
3071         return (0);
3072 }
3073
3074 /*********************************************************************
3075  *
3076  *  Setup networking device structure and register an interface.
3077  *
3078  **********************************************************************/
3079 static void
3080 em_setup_interface(device_t dev, struct adapter *adapter)
3081 {
3082         struct ifnet   *ifp;
3083
3084         INIT_DEBUGOUT("em_setup_interface: begin");
3085
3086         ifp = adapter->ifp = if_alloc(IFT_ETHER);
3087         if (ifp == NULL)
3088                 panic("%s: can not if_alloc()", device_get_nameunit(dev));
3089         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3090         ifp->if_mtu = ETHERMTU;
3091         ifp->if_init =  em_init;
3092         ifp->if_softc = adapter;
3093         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3094         ifp->if_ioctl = em_ioctl;
3095         ifp->if_start = em_start;
3096         IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3097         ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3098         IFQ_SET_READY(&ifp->if_snd);
3099
3100         ether_ifattach(ifp, adapter->hw.mac.addr);
3101
3102         ifp->if_capabilities = ifp->if_capenable = 0;
3103
3104 #if __FreeBSD_version >= 800000
3105         /* Multiqueue tx functions */
3106         ifp->if_transmit = em_mq_start;
3107         ifp->if_qflush = em_qflush;
3108         adapter->br = buf_ring_alloc(4096, M_DEVBUF, M_WAITOK, &adapter->tx_mtx);
3109 #endif
3110         if (adapter->hw.mac.type >= e1000_82543) {
3111                 int version_cap;
3112 #if __FreeBSD_version < 700000
3113                 version_cap = IFCAP_HWCSUM;
3114 #else
3115                 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3116 #endif
3117                 ifp->if_capabilities |= version_cap;
3118                 ifp->if_capenable |= version_cap;
3119         }
3120
3121 #if __FreeBSD_version >= 700000
3122         /* Identify TSO capable adapters */
3123         if ((adapter->hw.mac.type > e1000_82544) &&
3124             (adapter->hw.mac.type != e1000_82547))
3125                 ifp->if_capabilities |= IFCAP_TSO4;
3126         /*
3127          * By default only enable on PCI-E, this
3128          * can be overriden by ifconfig.
3129          */
3130         if (adapter->hw.mac.type >= e1000_82571)
3131                 ifp->if_capenable |= IFCAP_TSO4;
3132 #endif
3133         /*
3134          * Tell the upper layer(s) we
3135          * support full VLAN capability
3136          */
3137         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3138         ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3139         ifp->if_capenable |= (IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING);
3140
3141         /*
3142         ** Dont turn this on by default, if vlans are
3143         ** created on another pseudo device (eg. lagg)
3144         ** then vlan events are not passed thru, breaking
3145         ** operation, but with HW FILTER off it works. If
3146         ** using vlans directly on the em driver you can
3147         ** enable this and get full hardware tag filtering.
3148         */
3149         ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
3150
3151 #ifdef DEVICE_POLLING
3152         ifp->if_capabilities |= IFCAP_POLLING;
3153 #endif
3154
3155         /* Limit WOL to MAGIC, not clear others are used */
3156         if (adapter->wol) {
3157                 ifp->if_capabilities |= IFCAP_WOL_MAGIC;
3158                 ifp->if_capenable |= IFCAP_WOL_MAGIC;
3159         }
3160
3161         /*
3162          * Specify the media types supported by this adapter and register
3163          * callbacks to update media and link information
3164          */
3165         ifmedia_init(&adapter->media, IFM_IMASK,
3166             em_media_change, em_media_status);
3167         if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3168             (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3169                 u_char fiber_type = IFM_1000_SX;        /* default type */
3170
3171                 if (adapter->hw.mac.type == e1000_82545)
3172                         fiber_type = IFM_1000_LX;
3173                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3174                             0, NULL);
3175                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3176         } else {
3177                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3178                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3179                             0, NULL);
3180                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3181                             0, NULL);
3182                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3183                             0, NULL);
3184                 if (adapter->hw.phy.type != e1000_phy_ife) {
3185                         ifmedia_add(&adapter->media,
3186                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3187                         ifmedia_add(&adapter->media,
3188                                 IFM_ETHER | IFM_1000_T, 0, NULL);
3189                 }
3190         }
3191         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3192         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3193 }
3194
3195
3196 /*********************************************************************
3197  *
3198  *  Workaround for SmartSpeed on 82541 and 82547 controllers
3199  *
3200  **********************************************************************/
3201 static void
3202 em_smartspeed(struct adapter *adapter)
3203 {
3204         u16 phy_tmp;
3205
3206         if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3207             adapter->hw.mac.autoneg == 0 ||
3208             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3209                 return;
3210
3211         if (adapter->smartspeed == 0) {
3212                 /* If Master/Slave config fault is asserted twice,
3213                  * we assume back-to-back */
3214                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3215                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3216                         return;
3217                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3218                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3219                         e1000_read_phy_reg(&adapter->hw,
3220                             PHY_1000T_CTRL, &phy_tmp);
3221                         if(phy_tmp & CR_1000T_MS_ENABLE) {
3222                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
3223                                 e1000_write_phy_reg(&adapter->hw,
3224                                     PHY_1000T_CTRL, phy_tmp);
3225                                 adapter->smartspeed++;
3226                                 if(adapter->hw.mac.autoneg &&
3227                                    !e1000_copper_link_autoneg(&adapter->hw) &&
3228                                    !e1000_read_phy_reg(&adapter->hw,
3229                                     PHY_CONTROL, &phy_tmp)) {
3230                                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3231                                                     MII_CR_RESTART_AUTO_NEG);
3232                                         e1000_write_phy_reg(&adapter->hw,
3233                                             PHY_CONTROL, phy_tmp);
3234                                 }
3235                         }
3236                 }
3237                 return;
3238         } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3239                 /* If still no link, perhaps using 2/3 pair cable */
3240                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3241                 phy_tmp |= CR_1000T_MS_ENABLE;
3242                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3243                 if(adapter->hw.mac.autoneg &&
3244                    !e1000_copper_link_autoneg(&adapter->hw) &&
3245                    !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3246                         phy_tmp |= (MII_CR_AUTO_NEG_EN |
3247                                     MII_CR_RESTART_AUTO_NEG);
3248                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3249                 }
3250         }
3251         /* Restart process after EM_SMARTSPEED_MAX iterations */
3252         if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3253                 adapter->smartspeed = 0;
3254 }
3255
3256
3257 /*
3258  * Manage DMA'able memory.
3259  */
3260 static void
3261 em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3262 {
3263         if (error)
3264                 return;
3265         *(bus_addr_t *) arg = segs[0].ds_addr;
3266 }
3267
3268 static int
3269 em_dma_malloc(struct adapter *adapter, bus_size_t size,
3270         struct em_dma_alloc *dma, int mapflags)
3271 {
3272         int error;
3273
3274 #if __FreeBSD_version >= 700000
3275         error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3276 #else
3277         error = bus_dma_tag_create(NULL,                 /* parent */
3278 #endif
3279                                 EM_DBA_ALIGN, 0,        /* alignment, bounds */
3280                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3281                                 BUS_SPACE_MAXADDR,      /* highaddr */
3282                                 NULL, NULL,             /* filter, filterarg */
3283                                 size,                   /* maxsize */
3284                                 1,                      /* nsegments */
3285                                 size,                   /* maxsegsize */
3286                                 0,                      /* flags */
3287                                 NULL,                   /* lockfunc */
3288                                 NULL,                   /* lockarg */
3289                                 &dma->dma_tag);
3290         if (error) {
3291                 device_printf(adapter->dev,
3292                     "%s: bus_dma_tag_create failed: %d\n",
3293                     __func__, error);
3294                 goto fail_0;
3295         }
3296
3297         error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3298             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3299         if (error) {
3300                 device_printf(adapter->dev,
3301                     "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3302                     __func__, (uintmax_t)size, error);
3303                 goto fail_2;
3304         }
3305
3306         dma->dma_paddr = 0;
3307         error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3308             size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3309         if (error || dma->dma_paddr == 0) {
3310                 device_printf(adapter->dev,
3311                     "%s: bus_dmamap_load failed: %d\n",
3312                     __func__, error);
3313                 goto fail_3;
3314         }
3315
3316         return (0);
3317
3318 fail_3:
3319         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3320 fail_2:
3321         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3322         bus_dma_tag_destroy(dma->dma_tag);
3323 fail_0:
3324         dma->dma_map = NULL;
3325         dma->dma_tag = NULL;
3326
3327         return (error);
3328 }
3329
3330 static void
3331 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3332 {
3333         if (dma->dma_tag == NULL)
3334                 return;
3335         if (dma->dma_map != NULL) {
3336                 bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3337                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3338                 bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3339                 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3340                 dma->dma_map = NULL;
3341         }
3342         bus_dma_tag_destroy(dma->dma_tag);
3343         dma->dma_tag = NULL;
3344 }
3345
3346
3347 /*********************************************************************
3348  *
3349  *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3350  *  the information needed to transmit a packet on the wire.
3351  *
3352  **********************************************************************/
3353 static int
3354 em_allocate_transmit_structures(struct adapter *adapter)
3355 {
3356         device_t dev = adapter->dev;
3357         struct em_buffer *tx_buffer;
3358         int error;
3359
3360         /*
3361          * Create DMA tags for tx descriptors
3362          */
3363 #if __FreeBSD_version >= 700000
3364         if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3365 #else
3366         if ((error = bus_dma_tag_create(NULL,            /* parent */
3367 #endif
3368                                 1, 0,                   /* alignment, bounds */
3369                                 BUS_SPACE_MAXADDR,      /* lowaddr */
3370                                 BUS_SPACE_MAXADDR,      /* highaddr */
3371                                 NULL, NULL,             /* filter, filterarg */
3372                                 EM_TSO_SIZE,            /* maxsize */
3373                                 EM_MAX_SCATTER,         /* nsegments */
3374                                 EM_TSO_SEG_SIZE,        /* maxsegsize */
3375                                 0,                      /* flags */
3376                                 NULL,           /* lockfunc */
3377                                 NULL,           /* lockarg */
3378                                 &adapter->txtag)) != 0) {
3379                 device_printf(dev, "Unable to allocate TX DMA tag\n");
3380                 goto fail;
3381         }
3382
3383         adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
3384             adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3385         if (adapter->tx_buffer_area == NULL) {
3386                 device_printf(dev, "Unable to allocate tx_buffer memory\n");
3387                 error = ENOMEM;
3388                 goto fail;
3389         }
3390
3391         /* Create the descriptor buffer dma maps */
3392         for (int i = 0; i < adapter->num_tx_desc; i++) {
3393                 tx_buffer = &adapter->tx_buffer_area[i];
3394                 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3395                 if (error != 0) {
3396                         device_printf(dev, "Unable to create TX DMA map\n");
3397                         goto fail;
3398                 }
3399                 tx_buffer->next_eop = -1;
3400         }
3401
3402         return (0);
3403 fail:
3404         em_free_transmit_structures(adapter);
3405         return (error);
3406 }
3407
3408 /*********************************************************************
3409  *
3410  *  (Re)Initialize transmit structures.
3411  *
3412  **********************************************************************/
3413 static void
3414 em_setup_transmit_structures(struct adapter *adapter)
3415 {
3416         struct em_buffer *tx_buffer;
3417
3418         /* Clear the old ring contents */
3419         bzero(adapter->tx_desc_base,
3420             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3421
3422         /* Free any existing TX buffers */
3423         for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3424                 tx_buffer = &adapter->tx_buffer_area[i];
3425                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3426                     BUS_DMASYNC_POSTWRITE);
3427                 bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3428                 m_freem(tx_buffer->m_head);
3429                 tx_buffer->m_head = NULL;
3430                 tx_buffer->next_eop = -1;
3431         }
3432
3433         /* Reset state */
3434         adapter->next_avail_tx_desc = 0;
3435         adapter->next_tx_to_clean = 0;
3436         adapter->num_tx_desc_avail = adapter->num_tx_desc;
3437
3438         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3439             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3440
3441         return;
3442 }
3443
3444 /*********************************************************************
3445  *
3446  *  Enable transmit unit.
3447  *
3448  **********************************************************************/
3449 static void
3450 em_initialize_transmit_unit(struct adapter *adapter)
3451 {
3452         u32     tctl, tarc, tipg = 0;
3453         u64     bus_addr;
3454
3455          INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3456         /* Setup the Base and Length of the Tx Descriptor Ring */
3457         bus_addr = adapter->txdma.dma_paddr;
3458         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3459             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3460         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3461             (u32)(bus_addr >> 32));
3462         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3463             (u32)bus_addr);
3464         /* Setup the HW Tx Head and Tail descriptor pointers */
3465         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3466         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3467
3468         HW_DEBUGOUT2("Base = %x, Length = %x\n",
3469             E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3470             E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3471
3472         /* Set the default values for the Tx Inter Packet Gap timer */
3473         switch (adapter->hw.mac.type) {
3474         case e1000_82542:
3475                 tipg = DEFAULT_82542_TIPG_IPGT;
3476                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3477                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3478                 break;
3479         case e1000_80003es2lan:
3480                 tipg = DEFAULT_82543_TIPG_IPGR1;
3481                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3482                     E1000_TIPG_IPGR2_SHIFT;
3483                 break;
3484         default:
3485                 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3486                     (adapter->hw.phy.media_type ==
3487                     e1000_media_type_internal_serdes))
3488                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3489                 else
3490                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3491                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3492                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3493         }
3494
3495         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3496         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3497         if(adapter->hw.mac.type >= e1000_82540)
3498                 E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3499                     adapter->tx_abs_int_delay.value);
3500
3501         if ((adapter->hw.mac.type == e1000_82571) ||
3502             (adapter->hw.mac.type == e1000_82572)) {
3503                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3504                 tarc |= SPEED_MODE_BIT;
3505                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3506         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
3507                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3508                 tarc |= 1;
3509                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3510                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3511                 tarc |= 1;
3512                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3513         }
3514
3515         /* Program the Transmit Control Register */
3516         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3517         tctl &= ~E1000_TCTL_CT;
3518         tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3519                    (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3520
3521         if (adapter->hw.mac.type >= e1000_82571)
3522                 tctl |= E1000_TCTL_MULR;
3523
3524         /* This write will effectively turn on the transmit unit. */
3525         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3526
3527         /* Setup Transmit Descriptor Base Settings */
3528         adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3529
3530         if (adapter->tx_int_delay.value > 0)
3531                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3532 }
3533
3534 /*********************************************************************
3535  *
3536  *  Free all transmit related data structures.
3537  *
3538  **********************************************************************/
3539 static void
3540 em_free_transmit_structures(struct adapter *adapter)
3541 {
3542         struct em_buffer *tx_buffer;
3543
3544         INIT_DEBUGOUT("free_transmit_structures: begin");
3545
3546         if (adapter->tx_buffer_area != NULL) {
3547                 for (int i = 0; i < adapter->num_tx_desc; i++) {
3548                         tx_buffer = &adapter->tx_buffer_area[i];
3549                         if (tx_buffer->m_head != NULL) {
3550                                 bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3551                                     BUS_DMASYNC_POSTWRITE);
3552                                 bus_dmamap_unload(adapter->txtag,
3553                                     tx_buffer->map);
3554                                 m_freem(tx_buffer->m_head);
3555                                 tx_buffer->m_head = NULL;
3556                         } else if (tx_buffer->map != NULL)
3557                                 bus_dmamap_unload(adapter->txtag,
3558                                     tx_buffer->map);
3559                         if (tx_buffer->map != NULL) {
3560                                 bus_dmamap_destroy(adapter->txtag,
3561                                     tx_buffer->map);
3562                                 tx_buffer->map = NULL;
3563                         }
3564                 }
3565         }
3566         if (adapter->tx_buffer_area != NULL) {
3567                 free(adapter->tx_buffer_area, M_DEVBUF);
3568                 adapter->tx_buffer_area = NULL;
3569         }
3570         if (adapter->txtag != NULL) {
3571                 bus_dma_tag_destroy(adapter->txtag);
3572                 adapter->txtag = NULL;
3573         }
3574 #if __FreeBSD_version >= 800000
3575         if (adapter->br != NULL)
3576                 buf_ring_free(adapter->br, M_DEVBUF);
3577 #endif
3578 }
3579
3580 /*********************************************************************
3581  *
3582  *  The offload context needs to be set when we transfer the first
3583  *  packet of a particular protocol (TCP/UDP). This routine has been
3584  *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3585  *
3586  *  Added back the old method of keeping the current context type
3587  *  and not setting if unnecessary, as this is reported to be a
3588  *  big performance win.  -jfv
3589  **********************************************************************/
3590 static void
3591 em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3592     u32 *txd_upper, u32 *txd_lower)
3593 {
3594         struct e1000_context_desc *TXD = NULL;
3595         struct em_buffer *tx_buffer;
3596         struct ether_vlan_header *eh;
3597         struct ip *ip = NULL;
3598         struct ip6_hdr *ip6;
3599         int curr_txd, ehdrlen;
3600         u32 cmd, hdr_len, ip_hlen;
3601         u16 etype;
3602         u8 ipproto;
3603
3604
3605         cmd = hdr_len = ipproto = 0;
3606         curr_txd = adapter->next_avail_tx_desc;
3607
3608         /*
3609          * Determine where frame payload starts.
3610          * Jump over vlan headers if already present,
3611          * helpful for QinQ too.
3612          */
3613         eh = mtod(mp, struct ether_vlan_header *);
3614         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3615                 etype = ntohs(eh->evl_proto);
3616                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3617         } else {
3618                 etype = ntohs(eh->evl_encap_proto);
3619                 ehdrlen = ETHER_HDR_LEN;
3620         }
3621
3622         /*
3623          * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3624          * TODO: Support SCTP too when it hits the tree.
3625          */
3626         switch (etype) {
3627         case ETHERTYPE_IP:
3628                 ip = (struct ip *)(mp->m_data + ehdrlen);
3629                 ip_hlen = ip->ip_hl << 2;
3630
3631                 /* Setup of IP header checksum. */
3632                 if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3633                         /*
3634                          * Start offset for header checksum calculation.
3635                          * End offset for header checksum calculation.
3636                          * Offset of place to put the checksum.
3637                          */
3638                         TXD = (struct e1000_context_desc *)
3639                             &adapter->tx_desc_base[curr_txd];
3640                         TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3641                         TXD->lower_setup.ip_fields.ipcse =
3642                             htole16(ehdrlen + ip_hlen);
3643                         TXD->lower_setup.ip_fields.ipcso =
3644                             ehdrlen + offsetof(struct ip, ip_sum);
3645                         cmd |= E1000_TXD_CMD_IP;
3646                         *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3647                 }
3648
3649                 if (mp->m_len < ehdrlen + ip_hlen)
3650                         return; /* failure */
3651
3652                 hdr_len = ehdrlen + ip_hlen;
3653                 ipproto = ip->ip_p;
3654
3655                 break;
3656         case ETHERTYPE_IPV6:
3657                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3658                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3659
3660                 if (mp->m_len < ehdrlen + ip_hlen)
3661                         return; /* failure */
3662
3663                 /* IPv6 doesn't have a header checksum. */
3664
3665                 hdr_len = ehdrlen + ip_hlen;
3666                 ipproto = ip6->ip6_nxt;
3667
3668                 break;
3669         default:
3670                 *txd_upper = 0;
3671                 *txd_lower = 0;
3672                 return;
3673         }
3674
3675         switch (ipproto) {
3676         case IPPROTO_TCP:
3677                 if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3678                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3679                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3680                         /* no need for context if already set */
3681                         if (adapter->last_hw_offload == CSUM_TCP)
3682                                 return;
3683                         adapter->last_hw_offload = CSUM_TCP;
3684                         /*
3685                          * Start offset for payload checksum calculation.
3686                          * End offset for payload checksum calculation.
3687                          * Offset of place to put the checksum.
3688                          */
3689                         TXD = (struct e1000_context_desc *)
3690                             &adapter->tx_desc_base[curr_txd];
3691                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3692                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3693                         TXD->upper_setup.tcp_fields.tucso =
3694                             hdr_len + offsetof(struct tcphdr, th_sum);
3695                         cmd |= E1000_TXD_CMD_TCP;
3696                 }
3697                 break;
3698         case IPPROTO_UDP:
3699         {
3700                 if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3701                         *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
3702                         *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3703                         /* no need for context if already set */
3704                         if (adapter->last_hw_offload == CSUM_UDP)
3705                                 return;
3706                         adapter->last_hw_offload = CSUM_UDP;
3707                         /*
3708                          * Start offset for header checksum calculation.
3709                          * End offset for header checksum calculation.
3710                          * Offset of place to put the checksum.
3711                          */
3712                         TXD = (struct e1000_context_desc *)
3713                             &adapter->tx_desc_base[curr_txd];
3714                         TXD->upper_setup.tcp_fields.tucss = hdr_len;
3715                         TXD->upper_setup.tcp_fields.tucse = htole16(0);
3716                         TXD->upper_setup.tcp_fields.tucso =
3717                             hdr_len + offsetof(struct udphdr, uh_sum);
3718                 }
3719                 /* Fall Thru */
3720         }
3721         default:
3722                 break;
3723         }
3724
3725         TXD->tcp_seg_setup.data = htole32(0);
3726         TXD->cmd_and_length =
3727             htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3728         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3729         tx_buffer->m_head = NULL;
3730         tx_buffer->next_eop = -1;
3731
3732         if (++curr_txd == adapter->num_tx_desc)
3733                 curr_txd = 0;
3734
3735         adapter->num_tx_desc_avail--;
3736         adapter->next_avail_tx_desc = curr_txd;
3737 }
3738
3739
3740 #if __FreeBSD_version >= 700000
3741 /**********************************************************************
3742  *
3743  *  Setup work for hardware segmentation offload (TSO)
3744  *
3745  **********************************************************************/
3746 static bool
3747 em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3748    u32 *txd_lower)
3749 {
3750         struct e1000_context_desc *TXD;
3751         struct em_buffer *tx_buffer;
3752         struct ether_vlan_header *eh;
3753         struct ip *ip;
3754         struct ip6_hdr *ip6;
3755         struct tcphdr *th;
3756         int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3757         u16 etype;
3758
3759         /*
3760          * This function could/should be extended to support IP/IPv6
3761          * fragmentation as well.  But as they say, one step at a time.
3762          */
3763
3764         /*
3765          * Determine where frame payload starts.
3766          * Jump over vlan headers if already present,
3767          * helpful for QinQ too.
3768          */
3769         eh = mtod(mp, struct ether_vlan_header *);
3770         if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3771                 etype = ntohs(eh->evl_proto);
3772                 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3773         } else {
3774                 etype = ntohs(eh->evl_encap_proto);
3775                 ehdrlen = ETHER_HDR_LEN;
3776         }
3777
3778         /* Ensure we have at least the IP+TCP header in the first mbuf. */
3779         if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3780                 return FALSE;   /* -1 */
3781
3782         /*
3783          * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3784          * TODO: Support SCTP too when it hits the tree.
3785          */
3786         switch (etype) {
3787         case ETHERTYPE_IP:
3788                 isip6 = 0;
3789                 ip = (struct ip *)(mp->m_data + ehdrlen);
3790                 if (ip->ip_p != IPPROTO_TCP)
3791                         return FALSE;   /* 0 */
3792                 ip->ip_len = 0;
3793                 ip->ip_sum = 0;
3794                 ip_hlen = ip->ip_hl << 2;
3795                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3796                         return FALSE;   /* -1 */
3797                 th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3798 #if 1
3799                 th->th_sum = in_pseudo(ip->ip_src.s_addr,
3800                     ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3801 #else
3802                 th->th_sum = mp->m_pkthdr.csum_data;
3803 #endif
3804                 break;
3805         case ETHERTYPE_IPV6:
3806                 isip6 = 1;
3807                 return FALSE;                   /* Not supported yet. */
3808                 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3809                 if (ip6->ip6_nxt != IPPROTO_TCP)
3810                         return FALSE;   /* 0 */
3811                 ip6->ip6_plen = 0;
3812                 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3813                 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3814                         return FALSE;   /* -1 */
3815                 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3816 #if 0
3817                 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3818                     htons(IPPROTO_TCP));        /* XXX: function notyet. */
3819 #else
3820                 th->th_sum = mp->m_pkthdr.csum_data;
3821 #endif
3822                 break;
3823         default:
3824                 return FALSE;
3825         }
3826         hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3827
3828         *txd_lower = (E1000_TXD_CMD_DEXT |      /* Extended descr type */
3829                       E1000_TXD_DTYP_D |        /* Data descr type */
3830                       E1000_TXD_CMD_TSE);       /* Do TSE on this packet */
3831
3832         /* IP and/or TCP header checksum calculation and insertion. */
3833         *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3834                       E1000_TXD_POPTS_TXSM) << 8;
3835
3836         curr_txd = adapter->next_avail_tx_desc;
3837         tx_buffer = &adapter->tx_buffer_area[curr_txd];
3838         TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3839
3840         /* IPv6 doesn't have a header checksum. */
3841         if (!isip6) {
3842                 /*
3843                  * Start offset for header checksum calculation.
3844                  * End offset for header checksum calculation.
3845                  * Offset of place put the checksum.
3846                  */
3847                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3848                 TXD->lower_setup.ip_fields.ipcse =
3849                     htole16(ehdrlen + ip_hlen - 1);
3850                 TXD->lower_setup.ip_fields.ipcso =
3851                     ehdrlen + offsetof(struct ip, ip_sum);
3852         }
3853         /*
3854          * Start offset for payload checksum calculation.
3855          * End offset for payload checksum calculation.
3856          * Offset of place to put the checksum.
3857          */
3858         TXD->upper_setup.tcp_fields.tucss =
3859             ehdrlen + ip_hlen;
3860         TXD->upper_setup.tcp_fields.tucse = 0;
3861         TXD->upper_setup.tcp_fields.tucso =
3862             ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3863         /*
3864          * Payload size per packet w/o any headers.
3865          * Length of all headers up to payload.
3866          */
3867         TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3868         TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3869
3870         TXD->cmd_and_length = htole32(adapter->txd_cmd |
3871                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
3872                                 E1000_TXD_CMD_TSE |     /* TSE context */
3873                                 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3874                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
3875                                 (mp->m_pkthdr.len - (hdr_len))); /* Total len */
3876
3877         tx_buffer->m_head = NULL;
3878         tx_buffer->next_eop = -1;
3879
3880         if (++curr_txd == adapter->num_tx_desc)
3881                 curr_txd = 0;
3882
3883         adapter->num_tx_desc_avail--;
3884         adapter->next_avail_tx_desc = curr_txd;
3885         adapter->tx_tso = TRUE;
3886
3887         return TRUE;
3888 }
3889
3890 #endif /* __FreeBSD_version >= 700000 */
3891
3892 /**********************************************************************
3893  *
3894  *  Examine each tx_buffer in the used queue. If the hardware is done
3895  *  processing the packet then free associated resources. The
3896  *  tx_buffer is put back on the free queue.
3897  *
3898  **********************************************************************/
3899 static void
3900 em_txeof(struct adapter *adapter)
3901 {
3902         int first, last, done, num_avail;
3903         struct em_buffer *tx_buffer;
3904         struct e1000_tx_desc   *tx_desc, *eop_desc;
3905         struct ifnet   *ifp = adapter->ifp;
3906
3907         EM_TX_LOCK_ASSERT(adapter);
3908
3909         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3910                 return;
3911
3912         num_avail = adapter->num_tx_desc_avail;
3913         first = adapter->next_tx_to_clean;
3914         tx_desc = &adapter->tx_desc_base[first];
3915         tx_buffer = &adapter->tx_buffer_area[first];
3916         last = tx_buffer->next_eop;
3917         eop_desc = &adapter->tx_desc_base[last];
3918
3919         /*
3920          * What this does is get the index of the
3921          * first descriptor AFTER the EOP of the
3922          * first packet, that way we can do the
3923          * simple comparison on the inner while loop.
3924          */
3925         if (++last == adapter->num_tx_desc)
3926                 last = 0;
3927         done = last;
3928
3929         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3930             BUS_DMASYNC_POSTREAD);
3931
3932         while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3933                 /* We clean the range of the packet */
3934                 while (first != done) {
3935                         tx_desc->upper.data = 0;
3936                         tx_desc->lower.data = 0;
3937                         tx_desc->buffer_addr = 0;
3938                         ++num_avail;
3939
3940                         if (tx_buffer->m_head) {
3941                                 ifp->if_opackets++;
3942                                 bus_dmamap_sync(adapter->txtag,
3943                                     tx_buffer->map,
3944                                     BUS_DMASYNC_POSTWRITE);
3945                                 bus_dmamap_unload(adapter->txtag,
3946                                     tx_buffer->map);
3947
3948                                 m_freem(tx_buffer->m_head);
3949                                 tx_buffer->m_head = NULL;
3950                         }
3951                         tx_buffer->next_eop = -1;
3952                         adapter->watchdog_time = ticks;
3953
3954                         if (++first == adapter->num_tx_desc)
3955                                 first = 0;
3956
3957                         tx_buffer = &adapter->tx_buffer_area[first];
3958                         tx_desc = &adapter->tx_desc_base[first];
3959                 }
3960                 /* See if we can continue to the next packet */
3961                 last = tx_buffer->next_eop;
3962                 if (last != -1) {
3963                         eop_desc = &adapter->tx_desc_base[last];
3964                         /* Get new done point */
3965                         if (++last == adapter->num_tx_desc) last = 0;
3966                         done = last;
3967                 } else
3968                         break;
3969         }
3970         bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3971             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3972
3973         adapter->next_tx_to_clean = first;
3974
3975         /*
3976          * If we have enough room, clear IFF_DRV_OACTIVE to
3977          * tell the stack that it is OK to send packets.
3978          * If there are no pending descriptors, clear the watchdog.
3979          */
3980         if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
3981                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3982                 if (num_avail == adapter->num_tx_desc) {
3983                         adapter->watchdog_check = FALSE;
3984                         adapter->num_tx_desc_avail = num_avail;
3985                         return;
3986                 }
3987         }
3988
3989         adapter->num_tx_desc_avail = num_avail;
3990         return;
3991 }
3992
3993 /*********************************************************************
3994  *
3995  *  When Link is lost sometimes there is work still in the TX ring
3996  *  which may result in a watchdog, rather than allow that we do an
3997  *  attempted cleanup and then reinit here. Note that this has been
3998  *  seens mostly with fiber adapters.
3999  *
4000  **********************************************************************/
4001 static void
4002 em_tx_purge(struct adapter *adapter)
4003 {
4004         if ((!adapter->link_active) && (adapter->watchdog_check)) {
4005                 EM_TX_LOCK(adapter);
4006                 em_txeof(adapter);
4007                 EM_TX_UNLOCK(adapter);
4008                 if (adapter->watchdog_check) /* Still outstanding? */
4009                         em_init_locked(adapter);
4010         }
4011 }
4012
4013 /*********************************************************************
4014  *
4015  *  Get a buffer from system mbuf buffer pool.
4016  *
4017  **********************************************************************/
4018 static int
4019 em_get_buf(struct adapter *adapter, int i)
4020 {
4021         struct mbuf             *m;
4022         bus_dma_segment_t       segs[1];
4023         bus_dmamap_t            map;
4024         struct em_buffer        *rx_buffer;
4025         int                     error, nsegs;
4026
4027         m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4028         if (m == NULL) {
4029                 adapter->mbuf_cluster_failed++;
4030                 return (ENOBUFS);
4031         }
4032         m->m_len = m->m_pkthdr.len = MCLBYTES;
4033
4034         if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4035                 m_adj(m, ETHER_ALIGN);
4036
4037         /*
4038          * Using memory from the mbuf cluster pool, invoke the
4039          * bus_dma machinery to arrange the memory mapping.
4040          */
4041         error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
4042             adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
4043         if (error != 0) {
4044                 m_free(m);
4045                 return (error);
4046         }
4047
4048         /* If nsegs is wrong then the stack is corrupt. */
4049         KASSERT(nsegs == 1, ("Too many segments returned!"));
4050
4051         rx_buffer = &adapter->rx_buffer_area[i];
4052         if (rx_buffer->m_head != NULL)
4053                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4054
4055         map = rx_buffer->map;
4056         rx_buffer->map = adapter->rx_sparemap;
4057         adapter->rx_sparemap = map;
4058         bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
4059         rx_buffer->m_head = m;
4060
4061         adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
4062         return (0);
4063 }
4064
4065 /*********************************************************************
4066  *
4067  *  Allocate memory for rx_buffer structures. Since we use one
4068  *  rx_buffer per received packet, the maximum number of rx_buffer's
4069  *  that we'll need is equal to the number of receive descriptors
4070  *  that we've allocated.
4071  *
4072  **********************************************************************/
4073 static int
4074 em_allocate_receive_structures(struct adapter *adapter)
4075 {
4076         device_t dev = adapter->dev;
4077         struct em_buffer *rx_buffer;
4078         int i, error;
4079
4080         adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
4081             adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4082         if (adapter->rx_buffer_area == NULL) {
4083                 device_printf(dev, "Unable to allocate rx_buffer memory\n");
4084                 return (ENOMEM);
4085         }
4086
4087 #if __FreeBSD_version >= 700000
4088         error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4089 #else
4090         error = bus_dma_tag_create(NULL,                 /* parent */
4091 #endif
4092                                 1, 0,                   /* alignment, bounds */
4093                                 BUS_SPACE_MAXADDR,      /* lowaddr */
4094                                 BUS_SPACE_MAXADDR,      /* highaddr */
4095                                 NULL, NULL,             /* filter, filterarg */
4096                                 MCLBYTES,               /* maxsize */
4097                                 1,                      /* nsegments */
4098                                 MCLBYTES,               /* maxsegsize */
4099                                 0,                      /* flags */
4100                                 NULL,                   /* lockfunc */
4101                                 NULL,                   /* lockarg */
4102                                 &adapter->rxtag);
4103         if (error) {
4104                 device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4105                     __func__, error);
4106                 goto fail;
4107         }
4108
4109         /* Create the spare map (used by getbuf) */
4110         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4111              &adapter->rx_sparemap);
4112         if (error) {
4113                 device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4114                     __func__, error);
4115                 goto fail;
4116         }
4117
4118         rx_buffer = adapter->rx_buffer_area;
4119         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4120                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4121                     &rx_buffer->map);
4122                 if (error) {
4123                         device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4124                             __func__, error);
4125                         goto fail;
4126                 }
4127         }
4128
4129         return (0);
4130
4131 fail:
4132         em_free_receive_structures(adapter);
4133         return (error);
4134 }
4135
4136 /*********************************************************************
4137  *
4138  *  (Re)initialize receive structures.
4139  *
4140  **********************************************************************/
4141 static int
4142 em_setup_receive_structures(struct adapter *adapter)
4143 {
4144         struct em_buffer *rx_buffer;
4145         int i, error;
4146
4147         /* Reset descriptor ring */
4148         bzero(adapter->rx_desc_base,
4149             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
4150
4151         /* Free current RX buffers. */
4152         rx_buffer = adapter->rx_buffer_area;
4153         for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4154                 if (rx_buffer->m_head != NULL) {
4155                         bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4156                             BUS_DMASYNC_POSTREAD);
4157                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4158                         m_freem(rx_buffer->m_head);
4159                         rx_buffer->m_head = NULL;
4160                 }
4161         }
4162
4163         /* Allocate new ones. */
4164         for (i = 0; i < adapter->num_rx_desc; i++) {
4165                 error = em_get_buf(adapter, i);
4166                 if (error)
4167                         return (error);
4168         }
4169
4170         /* Setup our descriptor pointers */
4171         adapter->next_rx_desc_to_check = 0;
4172         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4173             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4174
4175         return (0);
4176 }
4177
4178 /*********************************************************************
4179  *
4180  *  Enable receive unit.
4181  *
4182  **********************************************************************/
4183 #define MAX_INTS_PER_SEC        8000
4184 #define DEFAULT_ITR          1000000000/(MAX_INTS_PER_SEC * 256)
4185
4186 static void
4187 em_initialize_receive_unit(struct adapter *adapter)
4188 {
4189         struct ifnet    *ifp = adapter->ifp;
4190         u64     bus_addr;
4191         u32     rctl, rxcsum;
4192
4193         INIT_DEBUGOUT("em_initialize_receive_unit: begin");
4194
4195         /*
4196          * Make sure receives are disabled while setting
4197          * up the descriptor ring
4198          */
4199         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4200         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4201
4202         if (adapter->hw.mac.type >= e1000_82540) {
4203                 E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4204                     adapter->rx_abs_int_delay.value);
4205                 /*
4206                  * Set the interrupt throttling rate. Value is calculated
4207                  * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4208                  */
4209                 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
4210         }
4211
4212         /*
4213         ** When using MSIX interrupts we need to throttle
4214         ** using the EITR register (82574 only)
4215         */
4216         if (adapter->msix)
4217                 for (int i = 0; i < 4; i++)
4218                         E1000_WRITE_REG(&adapter->hw,
4219                             E1000_EITR_82574(i), DEFAULT_ITR);
4220
4221         /* Disable accelerated ackknowledge */
4222         if (adapter->hw.mac.type == e1000_82574)
4223                 E1000_WRITE_REG(&adapter->hw,
4224                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4225
4226         /* Setup the Base and Length of the Rx Descriptor Ring */
4227         bus_addr = adapter->rxdma.dma_paddr;
4228         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
4229             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4230         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
4231             (u32)(bus_addr >> 32));
4232         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
4233             (u32)bus_addr);
4234
4235         /* Setup the Receive Control Register */
4236         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4237         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4238                    E1000_RCTL_RDMTS_HALF |
4239                    (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4240
4241         /* Make sure VLAN Filters are off */
4242         rctl &= ~E1000_RCTL_VFE;
4243
4244         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
4245                 rctl |= E1000_RCTL_SBP;
4246         else
4247                 rctl &= ~E1000_RCTL_SBP;
4248
4249         switch (adapter->rx_buffer_len) {
4250         default:
4251         case 2048:
4252                 rctl |= E1000_RCTL_SZ_2048;
4253                 break;
4254         case 4096:
4255                 rctl |= E1000_RCTL_SZ_4096 |
4256                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4257                 break;
4258         case 8192:
4259                 rctl |= E1000_RCTL_SZ_8192 |
4260                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4261                 break;
4262         case 16384:
4263                 rctl |= E1000_RCTL_SZ_16384 |
4264                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
4265                 break;
4266         }
4267
4268         if (ifp->if_mtu > ETHERMTU)
4269                 rctl |= E1000_RCTL_LPE;
4270         else
4271                 rctl &= ~E1000_RCTL_LPE;
4272
4273         /* Enable 82543 Receive Checksum Offload for TCP and UDP */
4274         if ((adapter->hw.mac.type >= e1000_82543) &&
4275             (ifp->if_capenable & IFCAP_RXCSUM)) {
4276                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
4277                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4278                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
4279         }
4280
4281         /*
4282         ** XXX TEMPORARY WORKAROUND: on some systems with 82573
4283         ** long latencies are observed, like Lenovo X60. This
4284         ** change eliminates the problem, but since having positive
4285         ** values in RDTR is a known source of problems on other
4286         ** platforms another solution is being sought.
4287         */
4288         if (adapter->hw.mac.type == e1000_82573)
4289                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20);
4290
4291         /* Enable Receives */
4292         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4293
4294         /*
4295          * Setup the HW Rx Head and
4296          * Tail Descriptor Pointers
4297          */
4298         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
4299         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
4300
4301         return;
4302 }
4303
4304 /*********************************************************************
4305  *
4306  *  Free receive related data structures.
4307  *
4308  **********************************************************************/
4309 static void
4310 em_free_receive_structures(struct adapter *adapter)
4311 {
4312         struct em_buffer *rx_buffer;
4313         int i;
4314
4315         INIT_DEBUGOUT("free_receive_structures: begin");
4316
4317         if (adapter->rx_sparemap) {
4318                 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
4319                 adapter->rx_sparemap = NULL;
4320         }
4321
4322         /* Cleanup any existing buffers */
4323         if (adapter->rx_buffer_area != NULL) {
4324                 rx_buffer = adapter->rx_buffer_area;
4325                 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4326                         if (rx_buffer->m_head != NULL) {
4327                                 bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4328                                     BUS_DMASYNC_POSTREAD);
4329                                 bus_dmamap_unload(adapter->rxtag,
4330                                     rx_buffer->map);
4331                                 m_freem(rx_buffer->m_head);
4332                                 rx_buffer->m_head = NULL;
4333                         } else if (rx_buffer->map != NULL)
4334                                 bus_dmamap_unload(adapter->rxtag,
4335                                     rx_buffer->map);
4336                         if (rx_buffer->map != NULL) {
4337                                 bus_dmamap_destroy(adapter->rxtag,
4338                                     rx_buffer->map);
4339                                 rx_buffer->map = NULL;
4340                         }
4341                 }
4342         }
4343
4344         if (adapter->rx_buffer_area != NULL) {
4345                 free(adapter->rx_buffer_area, M_DEVBUF);
4346                 adapter->rx_buffer_area = NULL;
4347         }
4348
4349         if (adapter->rxtag != NULL) {
4350                 bus_dma_tag_destroy(adapter->rxtag);
4351                 adapter->rxtag = NULL;
4352         }
4353 }
4354
4355 /*********************************************************************
4356  *
4357  *  This routine executes in interrupt context. It replenishes
4358  *  the mbufs in the descriptor and sends data which has been
4359  *  dma'ed into host memory to upper layer.
4360  *
4361  *  We loop at most count times if count is > 0, or until done if
4362  *  count < 0.
4363  *
4364  *  For polling we also now return the number of cleaned packets
4365  *********************************************************************/
4366 static int
4367 em_rxeof(struct adapter *adapter, int count)
4368 {
4369         struct ifnet    *ifp = adapter->ifp;;
4370         struct mbuf     *mp;
4371         u8              status, accept_frame = 0, eop = 0;
4372         u16             len, desc_len, prev_len_adj;
4373         int             i, rx_sent = 0;
4374         struct e1000_rx_desc   *current_desc;
4375
4376         EM_RX_LOCK(adapter);
4377         i = adapter->next_rx_desc_to_check;
4378         current_desc = &adapter->rx_desc_base[i];
4379         bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4380             BUS_DMASYNC_POSTREAD);
4381
4382         if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
4383                 EM_RX_UNLOCK(adapter);
4384                 return (rx_sent);
4385         }
4386
4387         while ((current_desc->status & E1000_RXD_STAT_DD) &&
4388             (count != 0) &&
4389             (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4390                 struct mbuf *m = NULL;
4391
4392                 mp = adapter->rx_buffer_area[i].m_head;
4393                 /*
4394                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
4395                  * needs to access the last received byte in the mbuf.
4396                  */
4397                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
4398                     BUS_DMASYNC_POSTREAD);
4399
4400                 accept_frame = 1;
4401                 prev_len_adj = 0;
4402                 desc_len = le16toh(current_desc->length);
4403                 status = current_desc->status;
4404                 if (status & E1000_RXD_STAT_EOP) {
4405                         count--;
4406                         eop = 1;
4407                         if (desc_len < ETHER_CRC_LEN) {
4408                                 len = 0;
4409                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
4410                         } else
4411                                 len = desc_len - ETHER_CRC_LEN;
4412                 } else {
4413                         eop = 0;
4414                         len = desc_len;
4415                 }
4416
4417                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
4418                         u8      last_byte;
4419                         u32     pkt_len = desc_len;
4420
4421                         if (adapter->fmp != NULL)
4422                                 pkt_len += adapter->fmp->m_pkthdr.len;
4423
4424                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
4425                         if (TBI_ACCEPT(&adapter->hw, status,
4426                             current_desc->errors, pkt_len, last_byte,
4427                             adapter->min_frame_size, adapter->max_frame_size)) {
4428                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
4429                                     &adapter->stats, pkt_len,
4430                                     adapter->hw.mac.addr,
4431                                     adapter->max_frame_size);
4432                                 if (len > 0)
4433                                         len--;
4434                         } else
4435                                 accept_frame = 0;
4436                 }
4437
4438                 if (accept_frame) {
4439                         if (em_get_buf(adapter, i) != 0) {
4440                                 ifp->if_iqdrops++;
4441                                 goto discard;
4442                         }
4443
4444                         /* Assign correct length to the current fragment */
4445                         mp->m_len = len;
4446
4447                         if (adapter->fmp == NULL) {
4448                                 mp->m_pkthdr.len = len;
4449                                 adapter->fmp = mp; /* Store the first mbuf */
4450                                 adapter->lmp = mp;
4451                         } else {
4452                                 /* Chain mbuf's together */
4453                                 mp->m_flags &= ~M_PKTHDR;
4454                                 /*
4455                                  * Adjust length of previous mbuf in chain if
4456                                  * we received less than 4 bytes in the last
4457                                  * descriptor.
4458                                  */
4459                                 if (prev_len_adj > 0) {
4460                                         adapter->lmp->m_len -= prev_len_adj;
4461                                         adapter->fmp->m_pkthdr.len -=
4462                                             prev_len_adj;
4463                                 }
4464                                 adapter->lmp->m_next = mp;
4465                                 adapter->lmp = adapter->lmp->m_next;
4466                                 adapter->fmp->m_pkthdr.len += len;
4467                         }
4468
4469                         if (eop) {
4470                                 adapter->fmp->m_pkthdr.rcvif = ifp;
4471                                 ifp->if_ipackets++;
4472                                 em_receive_checksum(adapter, current_desc,
4473                                     adapter->fmp);
4474 #ifndef __NO_STRICT_ALIGNMENT
4475                                 if (adapter->max_frame_size >
4476                                     (MCLBYTES - ETHER_ALIGN) &&
4477                                     em_fixup_rx(adapter) != 0)
4478                                         goto skip;
4479 #endif
4480                                 if (status & E1000_RXD_STAT_VP) {
4481 #if __FreeBSD_version < 700000
4482                                         VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
4483                                             (le16toh(current_desc->special) &
4484                                             E1000_RXD_SPC_VLAN_MASK));
4485 #else
4486                                         adapter->fmp->m_pkthdr.ether_vtag =
4487                                             (le16toh(current_desc->special) &
4488                                             E1000_RXD_SPC_VLAN_MASK);
4489                                         adapter->fmp->m_flags |= M_VLANTAG;
4490 #endif
4491                                 }
4492 #ifndef __NO_STRICT_ALIGNMENT
4493 skip:
4494 #endif
4495                                 m = adapter->fmp;
4496                                 adapter->fmp = NULL;
4497                                 adapter->lmp = NULL;
4498                         }
4499                 } else {
4500                         ifp->if_ierrors++;
4501 discard:
4502                         /* Reuse loaded DMA map and just update mbuf chain */
4503                         mp = adapter->rx_buffer_area[i].m_head;
4504                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4505                         mp->m_data = mp->m_ext.ext_buf;
4506                         mp->m_next = NULL;
4507                         if (adapter->max_frame_size <=
4508                             (MCLBYTES - ETHER_ALIGN))
4509                                 m_adj(mp, ETHER_ALIGN);
4510                         if (adapter->fmp != NULL) {
4511                                 m_freem(adapter->fmp);
4512                                 adapter->fmp = NULL;
4513                                 adapter->lmp = NULL;
4514                         }
4515                         m = NULL;
4516                 }
4517
4518                 /* Zero out the receive descriptors status. */
4519                 current_desc->status = 0;
4520                 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4521                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4522
4523                 /* Advance our pointers to the next descriptor. */
4524                 if (++i == adapter->num_rx_desc)
4525                         i = 0;
4526                 /* Call into the stack */
4527                 if (m != NULL) {
4528                         adapter->next_rx_desc_to_check = i;
4529                         EM_RX_UNLOCK(adapter);
4530                         (*ifp->if_input)(ifp, m);
4531                         EM_RX_LOCK(adapter);
4532                         rx_sent++;
4533                         i = adapter->next_rx_desc_to_check;
4534                 }
4535                 current_desc = &adapter->rx_desc_base[i];
4536         }
4537         adapter->next_rx_desc_to_check = i;
4538
4539         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
4540         if (--i < 0)
4541                 i = adapter->num_rx_desc - 1;
4542         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
4543         EM_RX_UNLOCK(adapter);
4544         return (rx_sent);
4545 }
4546
4547 #ifndef __NO_STRICT_ALIGNMENT
4548 /*
4549  * When jumbo frames are enabled we should realign entire payload on
4550  * architecures with strict alignment. This is serious design mistake of 8254x
4551  * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4552  * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4553  * payload. On architecures without strict alignment restrictions 8254x still
4554  * performs unaligned memory access which would reduce the performance too.
4555  * To avoid copying over an entire frame to align, we allocate a new mbuf and
4556  * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4557  * existing mbuf chain.
4558  *
4559  * Be aware, best performance of the 8254x is achived only when jumbo frame is
4560  * not used at all on architectures with strict alignment.
4561  */
4562 static int
4563 em_fixup_rx(struct adapter *adapter)
4564 {
4565         struct mbuf *m, *n;
4566         int error;
4567
4568         error = 0;
4569         m = adapter->fmp;
4570         if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4571                 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4572                 m->m_data += ETHER_HDR_LEN;
4573         } else {
4574                 MGETHDR(n, M_DONTWAIT, MT_DATA);
4575                 if (n != NULL) {
4576                         bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4577                         m->m_data += ETHER_HDR_LEN;
4578                         m->m_len -= ETHER_HDR_LEN;
4579                         n->m_len = ETHER_HDR_LEN;
4580                         M_MOVE_PKTHDR(n, m);
4581                         n->m_next = m;
4582                         adapter->fmp = n;
4583                 } else {
4584                         adapter->dropped_pkts++;
4585                         m_freem(adapter->fmp);
4586                         adapter->fmp = NULL;
4587                         error = ENOMEM;
4588                 }
4589         }
4590
4591         return (error);
4592 }
4593 #endif
4594
4595 /*********************************************************************
4596  *
4597  *  Verify that the hardware indicated that the checksum is valid.
4598  *  Inform the stack about the status of checksum so that stack
4599  *  doesn't spend time verifying the checksum.
4600  *
4601  *********************************************************************/
4602 static void
4603 em_receive_checksum(struct adapter *adapter,
4604             struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4605 {
4606         /* 82543 or newer only */
4607         if ((adapter->hw.mac.type < e1000_82543) ||
4608             /* Ignore Checksum bit is set */
4609             (rx_desc->status & E1000_RXD_STAT_IXSM)) {
4610                 mp->m_pkthdr.csum_flags = 0;
4611                 return;
4612         }
4613
4614         if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4615                 /* Did it pass? */
4616                 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4617                         /* IP Checksum Good */
4618                         mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4619                         mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4620
4621                 } else {
4622                         mp->m_pkthdr.csum_flags = 0;
4623                 }
4624         }
4625
4626         if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4627                 /* Did it pass? */
4628                 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4629                         mp->m_pkthdr.csum_flags |=
4630                         (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4631                         mp->m_pkthdr.csum_data = htons(0xffff);
4632                 }
4633         }
4634 }
4635
4636 #if __FreeBSD_version >= 700029
4637 /*
4638  * This routine is run via an vlan
4639  * config EVENT
4640  */
4641 static void
4642 em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4643 {
4644         struct adapter  *adapter = ifp->if_softc;
4645         u32             index, bit;
4646
4647         if (ifp->if_softc !=  arg)   /* Not our event */
4648                 return;
4649
4650         if ((vtag == 0) || (vtag > 4095))       /* Invalid ID */
4651                 return;
4652
4653         index = (vtag >> 5) & 0x7F;
4654         bit = vtag & 0x1F;
4655         em_shadow_vfta[index] |= (1 << bit);
4656         ++adapter->num_vlans;
4657         /* Re-init to load the changes */
4658         em_init(adapter);
4659 }
4660
4661 /*
4662  * This routine is run via an vlan
4663  * unconfig EVENT
4664  */
4665 static void
4666 em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
4667 {
4668         struct adapter  *adapter = ifp->if_softc;
4669         u32             index, bit;
4670
4671         if (ifp->if_softc !=  arg)
4672                 return;
4673
4674         if ((vtag == 0) || (vtag > 4095))       /* Invalid */
4675                 return;
4676
4677         index = (vtag >> 5) & 0x7F;
4678         bit = vtag & 0x1F;
4679         em_shadow_vfta[index] &= ~(1 << bit);
4680         --adapter->num_vlans;
4681         /* Re-init to load the changes */
4682         em_init(adapter);
4683 }
4684
4685 static void
4686 em_setup_vlan_hw_support(struct adapter *adapter)
4687 {
4688         struct e1000_hw *hw = &adapter->hw;
4689         u32             reg;
4690
4691         /*
4692         ** We get here thru init_locked, meaning
4693         ** a soft reset, this has already cleared
4694         ** the VFTA and other state, so if there
4695         ** have been no vlan's registered do nothing.
4696         */
4697         if (adapter->num_vlans == 0)
4698                 return;
4699
4700         /*
4701         ** A soft reset zero's out the VFTA, so
4702         ** we need to repopulate it now.
4703         */
4704         for (int i = 0; i < EM_VFTA_SIZE; i++)
4705                 if (em_shadow_vfta[i] != 0)
4706                         E1000_WRITE_REG_ARRAY(hw, E1000_VFTA,
4707                             i, em_shadow_vfta[i]);
4708
4709         reg = E1000_READ_REG(hw, E1000_CTRL);
4710         reg |= E1000_CTRL_VME;
4711         E1000_WRITE_REG(hw, E1000_CTRL, reg);
4712
4713         /* Enable the Filter Table */
4714         reg = E1000_READ_REG(hw, E1000_RCTL);
4715         reg &= ~E1000_RCTL_CFIEN;
4716         reg |= E1000_RCTL_VFE;
4717         E1000_WRITE_REG(hw, E1000_RCTL, reg);
4718
4719         /* Update the frame size */
4720         E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4721             adapter->max_frame_size + VLAN_TAG_SIZE);
4722 }
4723 #endif
4724
4725 static void
4726 em_enable_intr(struct adapter *adapter)
4727 {
4728         struct e1000_hw *hw = &adapter->hw;
4729         u32 ims_mask = IMS_ENABLE_MASK;
4730
4731         if (adapter->msix) {
4732                 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4733                 ims_mask |= EM_MSIX_MASK;
4734         }
4735         E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4736 }
4737
4738 static void
4739 em_disable_intr(struct adapter *adapter)
4740 {
4741         struct e1000_hw *hw = &adapter->hw;
4742
4743         if (adapter->msix)
4744                 E1000_WRITE_REG(hw, EM_EIAC, 0);
4745         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4746 }
4747
4748 /*
4749  * Bit of a misnomer, what this really means is
4750  * to enable OS management of the system... aka
4751  * to disable special hardware management features
4752  */
4753 static void
4754 em_init_manageability(struct adapter *adapter)
4755 {
4756         /* A shared code workaround */
4757 #define E1000_82542_MANC2H E1000_MANC2H
4758         if (adapter->has_manage) {
4759                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4760                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4761
4762                 /* disable hardware interception of ARP */
4763                 manc &= ~(E1000_MANC_ARP_EN);
4764
4765                 /* enable receiving management packets to the host */
4766                 if (adapter->hw.mac.type >= e1000_82571) {
4767                         manc |= E1000_MANC_EN_MNG2HOST;
4768 #define E1000_MNG2HOST_PORT_623 (1 << 5)
4769 #define E1000_MNG2HOST_PORT_664 (1 << 6)
4770                         manc2h |= E1000_MNG2HOST_PORT_623;
4771                         manc2h |= E1000_MNG2HOST_PORT_664;
4772                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4773                 }
4774
4775                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4776         }
4777 }
4778
4779 /*
4780  * Give control back to hardware management
4781  * controller if there is one.
4782  */
4783 static void
4784 em_release_manageability(struct adapter *adapter)
4785 {
4786         if (adapter->has_manage) {
4787                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4788
4789                 /* re-enable hardware interception of ARP */
4790                 manc |= E1000_MANC_ARP_EN;
4791
4792                 if (adapter->hw.mac.type >= e1000_82571)
4793                         manc &= ~E1000_MANC_EN_MNG2HOST;
4794
4795                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4796         }
4797 }
4798
4799 /*
4800  * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit.
4801  * For ASF and Pass Through versions of f/w this means
4802  * that the driver is loaded. For AMT version type f/w
4803  * this means that the network i/f is open.
4804  */
4805 static void
4806 em_get_hw_control(struct adapter *adapter)
4807 {
4808         u32 ctrl_ext, swsm;
4809
4810         if (adapter->hw.mac.type == e1000_82573) {
4811                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4812                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4813                     swsm | E1000_SWSM_DRV_LOAD);
4814                 return;
4815         }
4816         /* else */
4817         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4818         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4819             ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4820         return;
4821 }
4822
4823 /*
4824  * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4825  * For ASF and Pass Through versions of f/w this means that
4826  * the driver is no longer loaded. For AMT versions of the
4827  * f/w this means that the network i/f is closed.
4828  */
4829 static void
4830 em_release_hw_control(struct adapter *adapter)
4831 {
4832         u32 ctrl_ext, swsm;
4833
4834         if (!adapter->has_manage)
4835                 return;
4836
4837         if (adapter->hw.mac.type == e1000_82573) {
4838                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4839                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4840                     swsm & ~E1000_SWSM_DRV_LOAD);
4841                 return;
4842         }
4843         /* else */
4844         ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4845         E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4846             ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4847         return;
4848 }
4849
4850 static int
4851 em_is_valid_ether_addr(u8 *addr)
4852 {
4853         char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4854
4855         if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4856                 return (FALSE);
4857         }
4858
4859         return (TRUE);
4860 }
4861
4862 /*
4863 ** Parse the interface capabilities with regard
4864 ** to both system management and wake-on-lan for
4865 ** later use.
4866 */
4867 static void
4868 em_get_wakeup(device_t dev)
4869 {
4870         struct adapter  *adapter = device_get_softc(dev);
4871         u16             eeprom_data = 0, device_id, apme_mask;
4872
4873         adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
4874         apme_mask = EM_EEPROM_APME;
4875
4876         switch (adapter->hw.mac.type) {
4877         case e1000_82542:
4878         case e1000_82543:
4879                 break;
4880         case e1000_82544:
4881                 e1000_read_nvm(&adapter->hw,
4882                     NVM_INIT_CONTROL2_REG, 1, &eeprom_data);
4883                 apme_mask = EM_82544_APME;
4884                 break;
4885         case e1000_82573:
4886         case e1000_82583:
4887                 adapter->has_amt = TRUE;
4888                 /* Falls thru */
4889         case e1000_82546:
4890         case e1000_82546_rev_3:
4891         case e1000_82571:
4892         case e1000_82572:
4893         case e1000_80003es2lan:
4894                 if (adapter->hw.bus.func == 1) {
4895                         e1000_read_nvm(&adapter->hw,
4896                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
4897                         break;
4898                 } else
4899                         e1000_read_nvm(&adapter->hw,
4900                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4901                 break;
4902         case e1000_ich8lan:
4903         case e1000_ich9lan:
4904         case e1000_ich10lan:
4905         case e1000_pchlan:
4906                 apme_mask = E1000_WUC_APME;
4907                 adapter->has_amt = TRUE;
4908                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
4909                 break;
4910         default:
4911                 e1000_read_nvm(&adapter->hw,
4912                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
4913                 break;
4914         }
4915         if (eeprom_data & apme_mask)
4916                 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC);
4917         /*
4918          * We have the eeprom settings, now apply the special cases
4919          * where the eeprom may be wrong or the board won't support
4920          * wake on lan on a particular port
4921          */
4922         device_id = pci_get_device(dev);
4923         switch (device_id) {
4924         case E1000_DEV_ID_82546GB_PCIE:
4925                 adapter->wol = 0;
4926                 break;
4927         case E1000_DEV_ID_82546EB_FIBER:
4928         case E1000_DEV_ID_82546GB_FIBER:
4929         case E1000_DEV_ID_82571EB_FIBER:
4930                 /* Wake events only supported on port A for dual fiber
4931                  * regardless of eeprom setting */
4932                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
4933                     E1000_STATUS_FUNC_1)
4934                         adapter->wol = 0;
4935                 break;
4936         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
4937         case E1000_DEV_ID_82571EB_QUAD_COPPER:
4938         case E1000_DEV_ID_82571EB_QUAD_FIBER:
4939         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
4940                 /* if quad port adapter, disable WoL on all but port A */
4941                 if (global_quad_port_a != 0)
4942                         adapter->wol = 0;
4943                 /* Reset for multiple quad port adapters */
4944                 if (++global_quad_port_a == 4)
4945                         global_quad_port_a = 0;
4946                 break;
4947         }
4948         return;
4949 }
4950
4951
4952 /*
4953  * Enable PCI Wake On Lan capability
4954  */
4955 void
4956 em_enable_wakeup(device_t dev)
4957 {
4958         struct adapter  *adapter = device_get_softc(dev);
4959         struct ifnet    *ifp = adapter->ifp;
4960         u32             pmc, ctrl, ctrl_ext, rctl;
4961         u16             status;
4962
4963         if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0))
4964                 return;
4965
4966         /* Advertise the wakeup capability */
4967         ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4968         ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3);
4969         E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4970         E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
4971
4972         /* ICH workaround code */
4973         if ((adapter->hw.mac.type == e1000_ich8lan) ||
4974             (adapter->hw.mac.type == e1000_pchlan) ||
4975             (adapter->hw.mac.type == e1000_ich9lan) ||
4976             (adapter->hw.mac.type == e1000_ich10lan)) {
4977                 e1000_disable_gig_wol_ich8lan(&adapter->hw);
4978                 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw);
4979         }
4980
4981         /* Keep the laser running on Fiber adapters */
4982         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
4983             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
4984                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4985                 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
4986                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext);
4987         }
4988
4989         /*
4990         ** Determine type of Wakeup: note that wol
4991         ** is set with all bits on by default.
4992         */
4993         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0)
4994                 adapter->wol &= ~E1000_WUFC_MAG;
4995
4996         if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0)
4997                 adapter->wol &= ~E1000_WUFC_MC;
4998         else {
4999                 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
5000                 rctl |= E1000_RCTL_MPE;
5001                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
5002         }
5003
5004         if (adapter->hw.mac.type == e1000_pchlan) {
5005                 if (em_enable_phy_wakeup(adapter))
5006                         return;
5007         } else {
5008                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
5009                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
5010         }
5011
5012         if (adapter->hw.phy.type == e1000_phy_igp_3)
5013                 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5014
5015         /* Request PME */
5016         status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2);
5017         status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
5018         if (ifp->if_capenable & IFCAP_WOL)
5019                 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
5020         pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2);
5021
5022         return;
5023 }
5024
5025 /*
5026 ** WOL in the newer chipset interfaces (pchlan)
5027 ** require thing to be copied into the phy
5028 */
5029 static int
5030 em_enable_phy_wakeup(struct adapter *adapter)
5031 {
5032         struct e1000_hw *hw = &adapter->hw;
5033         u32 mreg, ret = 0;
5034         u16 preg;
5035
5036         /* copy MAC RARs to PHY RARs */
5037         for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) {
5038                 mreg = E1000_READ_REG(hw, E1000_RAL(i));
5039                 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF));
5040                 e1000_write_phy_reg(hw, BM_RAR_M(i),
5041                     (u16)((mreg >> 16) & 0xFFFF));
5042                 mreg = E1000_READ_REG(hw, E1000_RAH(i));
5043                 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF));
5044                 e1000_write_phy_reg(hw, BM_RAR_CTRL(i),
5045                     (u16)((mreg >> 16) & 0xFFFF));
5046         }
5047
5048         /* copy MAC MTA to PHY MTA */
5049         for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5050                 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5051                 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF));
5052                 e1000_write_phy_reg(hw, BM_MTA(i) + 1,
5053                     (u16)((mreg >> 16) & 0xFFFF));
5054         }
5055
5056         /* configure PHY Rx Control register */
5057         e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg);
5058         mreg = E1000_READ_REG(hw, E1000_RCTL);
5059         if (mreg & E1000_RCTL_UPE)
5060                 preg |= BM_RCTL_UPE;
5061         if (mreg & E1000_RCTL_MPE)
5062                 preg |= BM_RCTL_MPE;
5063         preg &= ~(BM_RCTL_MO_MASK);
5064         if (mreg & E1000_RCTL_MO_3)
5065                 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5066                                 << BM_RCTL_MO_SHIFT);
5067         if (mreg & E1000_RCTL_BAM)
5068                 preg |= BM_RCTL_BAM;
5069         if (mreg & E1000_RCTL_PMCF)
5070                 preg |= BM_RCTL_PMCF;
5071         mreg = E1000_READ_REG(hw, E1000_CTRL);
5072         if (mreg & E1000_CTRL_RFCE)
5073                 preg |= BM_RCTL_RFCE;
5074         e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg);
5075
5076         /* enable PHY wakeup in MAC register */
5077         E1000_WRITE_REG(hw, E1000_WUC,
5078             E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5079         E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol);
5080
5081         /* configure and enable PHY wakeup in PHY registers */
5082         e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol);
5083         e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5084
5085         /* activate PHY wakeup */
5086         ret = hw->phy.ops.acquire(hw);
5087         if (ret) {
5088                 printf("Could not acquire PHY\n");
5089                 return ret;
5090         }
5091         e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5092                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5093         ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg);
5094         if (ret) {
5095                 printf("Could not read PHY page 769\n");
5096                 goto out;
5097         }
5098         preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5099         ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg);
5100         if (ret)
5101                 printf("Could not set PHY Host Wakeup bit\n");
5102 out:
5103         hw->phy.ops.release(hw);
5104
5105         return ret;
5106 }
5107
5108
5109 /*********************************************************************
5110 * 82544 Coexistence issue workaround.
5111 *    There are 2 issues.
5112 *       1. Transmit Hang issue.
5113 *    To detect this issue, following equation can be used...
5114 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
5115 *         If SUM[3:0] is in between 1 to 4, we will have this issue.
5116 *
5117 *       2. DAC issue.
5118 *    To detect this issue, following equation can be used...
5119 *         SIZE[3:0] + ADDR[2:0] = SUM[3:0].
5120 *         If SUM[3:0] is in between 9 to c, we will have this issue.
5121 *
5122 *
5123 *    WORKAROUND:
5124 *         Make sure we do not have ending address
5125 *         as 1,2,3,4(Hang) or 9,a,b,c (DAC)
5126 *
5127 *************************************************************************/
5128 static u32
5129 em_fill_descriptors (bus_addr_t address, u32 length,
5130                 PDESC_ARRAY desc_array)
5131 {
5132         u32 safe_terminator;
5133
5134         /* Since issue is sensitive to length and address.*/
5135         /* Let us first check the address...*/
5136         if (length <= 4) {
5137                 desc_array->descriptor[0].address = address;
5138                 desc_array->descriptor[0].length = length;
5139                 desc_array->elements = 1;
5140                 return (desc_array->elements);
5141         }
5142         safe_terminator = (u32)((((u32)address & 0x7) +
5143             (length & 0xF)) & 0xF);
5144         /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
5145         if (safe_terminator == 0   ||
5146         (safe_terminator > 4   &&
5147         safe_terminator < 9)   ||
5148         (safe_terminator > 0xC &&
5149         safe_terminator <= 0xF)) {
5150                 desc_array->descriptor[0].address = address;
5151                 desc_array->descriptor[0].length = length;
5152                 desc_array->elements = 1;
5153                 return (desc_array->elements);
5154         }
5155
5156         desc_array->descriptor[0].address = address;
5157         desc_array->descriptor[0].length = length - 4;
5158         desc_array->descriptor[1].address = address + (length - 4);
5159         desc_array->descriptor[1].length = 4;
5160         desc_array->elements = 2;
5161         return (desc_array->elements);
5162 }
5163
5164 /**********************************************************************
5165  *
5166  *  Update the board statistics counters.
5167  *
5168  **********************************************************************/
5169 static void
5170 em_update_stats_counters(struct adapter *adapter)
5171 {
5172         struct ifnet   *ifp;
5173
5174         if(adapter->hw.phy.media_type == e1000_media_type_copper ||
5175            (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
5176                 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
5177                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
5178         }
5179         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
5180         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
5181         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
5182         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
5183
5184         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
5185         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
5186         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
5187         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
5188         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
5189         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
5190         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
5191         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
5192         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
5193         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
5194         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
5195         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
5196         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
5197         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
5198         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
5199         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
5200         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
5201         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
5202         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
5203         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
5204
5205         /* For the 64-bit byte counters the low dword must be read first. */
5206         /* Both registers clear on the read of the high dword */
5207
5208         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
5209         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
5210
5211         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
5212         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
5213         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
5214         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
5215         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
5216
5217         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
5218         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
5219
5220         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5221         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5222         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5223         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5224         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5225         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5226         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5227         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5228         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5229         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5230
5231         if (adapter->hw.mac.type >= e1000_82543) {
5232                 adapter->stats.algnerrc +=
5233                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5234                 adapter->stats.rxerrc +=
5235                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5236                 adapter->stats.tncrs +=
5237                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5238                 adapter->stats.cexterr +=
5239                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5240                 adapter->stats.tsctc +=
5241                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5242                 adapter->stats.tsctfc +=
5243                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5244         }
5245         ifp = adapter->ifp;
5246
5247         ifp->if_collisions = adapter->stats.colc;
5248
5249         /* Rx Errors */
5250         ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5251             adapter->stats.crcerrs + adapter->stats.algnerrc +
5252             adapter->stats.ruc + adapter->stats.roc +
5253             adapter->stats.mpc + adapter->stats.cexterr;
5254
5255         /* Tx Errors */
5256         ifp->if_oerrors = adapter->stats.ecol +
5257             adapter->stats.latecol + adapter->watchdog_events;
5258 }
5259
5260
5261 /**********************************************************************
5262  *
5263  *  This routine is called only when em_display_debug_stats is enabled.
5264  *  This routine provides a way to take a look at important statistics
5265  *  maintained by the driver and hardware.
5266  *
5267  **********************************************************************/
5268 static void
5269 em_print_debug_info(struct adapter *adapter)
5270 {
5271         device_t dev = adapter->dev;
5272         u8 *hw_addr = adapter->hw.hw_addr;
5273
5274         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
5275         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
5276             E1000_READ_REG(&adapter->hw, E1000_CTRL),
5277             E1000_READ_REG(&adapter->hw, E1000_RCTL));
5278         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
5279             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
5280             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
5281         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
5282             adapter->hw.fc.high_water,
5283             adapter->hw.fc.low_water);
5284         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
5285             E1000_READ_REG(&adapter->hw, E1000_TIDV),
5286             E1000_READ_REG(&adapter->hw, E1000_TADV));
5287         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
5288             E1000_READ_REG(&adapter->hw, E1000_RDTR),
5289             E1000_READ_REG(&adapter->hw, E1000_RADV));
5290         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
5291             (long long)adapter->tx_fifo_wrk_cnt,
5292             (long long)adapter->tx_fifo_reset_cnt);
5293         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5294             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5295             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5296         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5297             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5298             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5299         device_printf(dev, "Num Tx descriptors avail = %d\n",
5300             adapter->num_tx_desc_avail);
5301         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
5302             adapter->no_tx_desc_avail1);
5303         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
5304             adapter->no_tx_desc_avail2);
5305         device_printf(dev, "Std mbuf failed = %ld\n",
5306             adapter->mbuf_alloc_failed);
5307         device_printf(dev, "Std mbuf cluster failed = %ld\n",
5308             adapter->mbuf_cluster_failed);
5309         device_printf(dev, "Driver dropped packets = %ld\n",
5310             adapter->dropped_pkts);
5311         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
5312                 adapter->no_tx_dma_setup);
5313 }
5314
5315 static void
5316 em_print_hw_stats(struct adapter *adapter)
5317 {
5318         device_t dev = adapter->dev;
5319
5320         device_printf(dev, "Excessive collisions = %lld\n",
5321             (long long)adapter->stats.ecol);
5322 #if     (DEBUG_HW > 0)  /* Dont output these errors normally */
5323         device_printf(dev, "Symbol errors = %lld\n",
5324             (long long)adapter->stats.symerrs);
5325 #endif
5326         device_printf(dev, "Sequence errors = %lld\n",
5327             (long long)adapter->stats.sec);
5328         device_printf(dev, "Defer count = %lld\n",
5329             (long long)adapter->stats.dc);
5330         device_printf(dev, "Missed Packets = %lld\n",
5331             (long long)adapter->stats.mpc);
5332         device_printf(dev, "Receive No Buffers = %lld\n",
5333             (long long)adapter->stats.rnbc);
5334         /* RLEC is inaccurate on some hardware, calculate our own. */
5335         device_printf(dev, "Receive Length Errors = %lld\n",
5336             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
5337         device_printf(dev, "Receive errors = %lld\n",
5338             (long long)adapter->stats.rxerrc);
5339         device_printf(dev, "Crc errors = %lld\n",
5340             (long long)adapter->stats.crcerrs);
5341         device_printf(dev, "Alignment errors = %lld\n",
5342             (long long)adapter->stats.algnerrc);
5343         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
5344             (long long)adapter->stats.cexterr);
5345         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
5346         device_printf(dev, "watchdog timeouts = %ld\n",
5347             adapter->watchdog_events);
5348         device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld"
5349             " LINK MSIX IRQ = %ld\n", adapter->rx_irq,
5350             adapter->tx_irq , adapter->link_irq);
5351         device_printf(dev, "XON Rcvd = %lld\n",
5352             (long long)adapter->stats.xonrxc);
5353         device_printf(dev, "XON Xmtd = %lld\n",
5354             (long long)adapter->stats.xontxc);
5355         device_printf(dev, "XOFF Rcvd = %lld\n",
5356             (long long)adapter->stats.xoffrxc);
5357         device_printf(dev, "XOFF Xmtd = %lld\n",
5358             (long long)adapter->stats.xofftxc);
5359         device_printf(dev, "Good Packets Rcvd = %lld\n",
5360             (long long)adapter->stats.gprc);
5361         device_printf(dev, "Good Packets Xmtd = %lld\n",
5362             (long long)adapter->stats.gptc);
5363         device_printf(dev, "TSO Contexts Xmtd = %lld\n",
5364             (long long)adapter->stats.tsctc);
5365         device_printf(dev, "TSO Contexts Failed = %lld\n",
5366             (long long)adapter->stats.tsctfc);
5367 }
5368
5369 /**********************************************************************
5370  *
5371  *  This routine provides a way to dump out the adapter eeprom,
5372  *  often a useful debug/service tool. This only dumps the first
5373  *  32 words, stuff that matters is in that extent.
5374  *
5375  **********************************************************************/
5376 static void
5377 em_print_nvm_info(struct adapter *adapter)
5378 {
5379         u16     eeprom_data;
5380         int     i, j, row = 0;
5381
5382         /* Its a bit crude, but it gets the job done */
5383         printf("\nInterface EEPROM Dump:\n");
5384         printf("Offset\n0x0000  ");
5385         for (i = 0, j = 0; i < 32; i++, j++) {
5386                 if (j == 8) { /* Make the offset block */
5387                         j = 0; ++row;
5388                         printf("\n0x00%x0  ",row);
5389                 }
5390                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5391                 printf("%04x ", eeprom_data);
5392         }
5393         printf("\n");
5394 }
5395
5396 static int
5397 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5398 {
5399         struct adapter *adapter;
5400         int error;
5401         int result;
5402
5403         result = -1;
5404         error = sysctl_handle_int(oidp, &result, 0, req);
5405
5406         if (error || !req->newptr)
5407                 return (error);
5408
5409         if (result == 1) {
5410                 adapter = (struct adapter *)arg1;
5411                 em_print_debug_info(adapter);
5412         }
5413         /*
5414          * This value will cause a hex dump of the
5415          * first 32 16-bit words of the EEPROM to
5416          * the screen.
5417          */
5418         if (result == 2) {
5419                 adapter = (struct adapter *)arg1;
5420                 em_print_nvm_info(adapter);
5421         }
5422
5423         return (error);
5424 }
5425
5426
5427 static int
5428 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5429 {
5430         struct adapter *adapter;
5431         int error;
5432         int result;
5433
5434         result = -1;
5435         error = sysctl_handle_int(oidp, &result, 0, req);
5436
5437         if (error || !req->newptr)
5438                 return (error);
5439
5440         if (result == 1) {
5441                 adapter = (struct adapter *)arg1;
5442                 em_print_hw_stats(adapter);
5443         }
5444
5445         return (error);
5446 }
5447
5448 static int
5449 em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5450 {
5451         struct em_int_delay_info *info;
5452         struct adapter *adapter;
5453         u32 regval;
5454         int error;
5455         int usecs;
5456         int ticks;
5457
5458         info = (struct em_int_delay_info *)arg1;
5459         usecs = info->value;
5460         error = sysctl_handle_int(oidp, &usecs, 0, req);
5461         if (error != 0 || req->newptr == NULL)
5462                 return (error);
5463         if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5464                 return (EINVAL);
5465         info->value = usecs;
5466         ticks = EM_USECS_TO_TICKS(usecs);
5467
5468         adapter = info->adapter;
5469
5470         EM_CORE_LOCK(adapter);
5471         regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5472         regval = (regval & ~0xffff) | (ticks & 0xffff);
5473         /* Handle a few special cases. */
5474         switch (info->offset) {
5475         case E1000_RDTR:
5476                 break;
5477         case E1000_TIDV:
5478                 if (ticks == 0) {
5479                         adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5480                         /* Don't write 0 into the TIDV register. */
5481                         regval++;
5482                 } else
5483                         adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5484                 break;
5485         }
5486         E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5487         EM_CORE_UNLOCK(adapter);
5488         return (0);
5489 }
5490
5491 static void
5492 em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5493         const char *description, struct em_int_delay_info *info,
5494         int offset, int value)
5495 {
5496         info->adapter = adapter;
5497         info->offset = offset;
5498         info->value = value;
5499         SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5500             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5501             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5502             info, 0, em_sysctl_int_delay, "I", description);
5503 }
5504
5505 #ifndef EM_LEGACY_IRQ
5506 static void
5507 em_add_rx_process_limit(struct adapter *adapter, const char *name,
5508         const char *description, int *limit, int value)
5509 {
5510         *limit = value;
5511         SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5512             SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5513             OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5514 }
5515 #endif
5516
5517