sys/dev/netif/em/if_em.c

   1 /*
   2  * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   3  *
   4  * Copyright (c) 2001-2008, Intel Corporation
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  *  1. Redistributions of source code must retain the above copyright notice,
  11  *     this list of conditions and the following disclaimer.
  12  *
  13  *  2. Redistributions in binary form must reproduce the above copyright
  14  *     notice, this list of conditions and the following disclaimer in the
  15  *     documentation and/or other materials provided with the distribution.
  16  *
  17  *  3. Neither the name of the Intel Corporation nor the names of its
  18  *     contributors may be used to endorse or promote products derived from
  19  *     this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31  * POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  *
  34  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
  35  *
  36  * This code is derived from software contributed to The DragonFly Project
  37  * by Matthew Dillon <dillon@backplane.com>
  38  *
  39  * Redistribution and use in source and binary forms, with or without
  40  * modification, are permitted provided that the following conditions
  41  * are met:
  42  *
  43  * 1. Redistributions of source code must retain the above copyright
  44  *    notice, this list of conditions and the following disclaimer.
  45  * 2. Redistributions in binary form must reproduce the above copyright
  46  *    notice, this list of conditions and the following disclaimer in
  47  *    the documentation and/or other materials provided with the
  48  *    distribution.
  49  * 3. Neither the name of The DragonFly Project nor the names of its
  50  *    contributors may be used to endorse or promote products derived
  51  *    from this software without specific, prior written permission.
  52  *
  53  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  54  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  55  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  56  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  57  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  58  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  59  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  60  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  61  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  62  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  63  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  64  * SUCH DAMAGE.
  65  *
  66  */
  67 /*
  68  * SERIALIZATION API RULES:
  69  *
  70  * - We must call lwkt_serialize_handler_enable() prior to enabling the
  71  *   hardware interrupt and lwkt_serialize_handler_disable() after disabling
  72  *   the hardware interrupt in order to avoid handler execution races from
  73  *   scheduled interrupt threads.
  74  */
  75
  76 #include "opt_ifpoll.h"
  77
  78 #include <sys/param.h>
  79 #include <sys/bus.h>
  80 #include <sys/endian.h>
  81 #include <sys/interrupt.h>
  82 #include <sys/kernel.h>
  83 #include <sys/ktr.h>
  84 #include <sys/malloc.h>
  85 #include <sys/mbuf.h>
  86 #include <sys/proc.h>
  87 #include <sys/rman.h>
  88 #include <sys/serialize.h>
  89 #include <sys/socket.h>
  90 #include <sys/sockio.h>
  91 #include <sys/sysctl.h>
  92 #include <sys/systm.h>
  93
  94 #include <net/bpf.h>
  95 #include <net/ethernet.h>
  96 #include <net/if.h>
  97 #include <net/if_arp.h>
  98 #include <net/if_dl.h>
  99 #include <net/if_media.h>
 100 #include <net/if_poll.h>
 101 #include <net/ifq_var.h>
 102 #include <net/vlan/if_vlan_var.h>
 103 #include <net/vlan/if_vlan_ether.h>
 104
 105 #include <netinet/ip.h>
 106 #include <netinet/tcp.h>
 107 #include <netinet/udp.h>
 108
 109 #include <bus/pci/pcivar.h>
 110 #include <bus/pci/pcireg.h>
 111
 112 #include <dev/netif/ig_hal/e1000_api.h>
 113 #include <dev/netif/ig_hal/e1000_82571.h>
 114 #include <dev/netif/em/if_em.h>
 115
 116 #define DEBUG_HW 0
 117
 118 #define EM_NAME "Intel(R) PRO/1000 Network Connection "
 119 #define EM_VER  " 7.3.8"
 120
 121 #define _EM_DEVICE(id, ret)     \
 122         { EM_VENDOR_ID, E1000_DEV_ID_##id, ret, EM_NAME #id EM_VER }
 123 #define EM_EMX_DEVICE(id)       _EM_DEVICE(id, -100)
 124 #define EM_DEVICE(id)           _EM_DEVICE(id, 0)
 125 #define EM_DEVICE_NULL  { 0, 0, 0, NULL }
 126
 127 static const struct em_vendor_info em_vendor_info_array[] = {
 128         EM_DEVICE(82540EM),
 129         EM_DEVICE(82540EM_LOM),
 130         EM_DEVICE(82540EP),
 131         EM_DEVICE(82540EP_LOM),
 132         EM_DEVICE(82540EP_LP),
 133
 134         EM_DEVICE(82541EI),
 135         EM_DEVICE(82541ER),
 136         EM_DEVICE(82541ER_LOM),
 137         EM_DEVICE(82541EI_MOBILE),
 138         EM_DEVICE(82541GI),
 139         EM_DEVICE(82541GI_LF),
 140         EM_DEVICE(82541GI_MOBILE),
 141
 142         EM_DEVICE(82542),
 143
 144         EM_DEVICE(82543GC_FIBER),
 145         EM_DEVICE(82543GC_COPPER),
 146
 147         EM_DEVICE(82544EI_COPPER),
 148         EM_DEVICE(82544EI_FIBER),
 149         EM_DEVICE(82544GC_COPPER),
 150         EM_DEVICE(82544GC_LOM),
 151
 152         EM_DEVICE(82545EM_COPPER),
 153         EM_DEVICE(82545EM_FIBER),
 154         EM_DEVICE(82545GM_COPPER),
 155         EM_DEVICE(82545GM_FIBER),
 156         EM_DEVICE(82545GM_SERDES),
 157
 158         EM_DEVICE(82546EB_COPPER),
 159         EM_DEVICE(82546EB_FIBER),
 160         EM_DEVICE(82546EB_QUAD_COPPER),
 161         EM_DEVICE(82546GB_COPPER),
 162         EM_DEVICE(82546GB_FIBER),
 163         EM_DEVICE(82546GB_SERDES),
 164         EM_DEVICE(82546GB_PCIE),
 165         EM_DEVICE(82546GB_QUAD_COPPER),
 166         EM_DEVICE(82546GB_QUAD_COPPER_KSP3),
 167
 168         EM_DEVICE(82547EI),
 169         EM_DEVICE(82547EI_MOBILE),
 170         EM_DEVICE(82547GI),
 171
 172         EM_EMX_DEVICE(82571EB_COPPER),
 173         EM_EMX_DEVICE(82571EB_FIBER),
 174         EM_EMX_DEVICE(82571EB_SERDES),
 175         EM_EMX_DEVICE(82571EB_SERDES_DUAL),
 176         EM_EMX_DEVICE(82571EB_SERDES_QUAD),
 177         EM_EMX_DEVICE(82571EB_QUAD_COPPER),
 178         EM_EMX_DEVICE(82571EB_QUAD_COPPER_BP),
 179         EM_EMX_DEVICE(82571EB_QUAD_COPPER_LP),
 180         EM_EMX_DEVICE(82571EB_QUAD_FIBER),
 181         EM_EMX_DEVICE(82571PT_QUAD_COPPER),
 182
 183         EM_EMX_DEVICE(82572EI_COPPER),
 184         EM_EMX_DEVICE(82572EI_FIBER),
 185         EM_EMX_DEVICE(82572EI_SERDES),
 186         EM_EMX_DEVICE(82572EI),
 187
 188         EM_EMX_DEVICE(82573E),
 189         EM_EMX_DEVICE(82573E_IAMT),
 190         EM_EMX_DEVICE(82573L),
 191
 192         EM_DEVICE(82583V),
 193
 194         EM_EMX_DEVICE(80003ES2LAN_COPPER_SPT),
 195         EM_EMX_DEVICE(80003ES2LAN_SERDES_SPT),
 196         EM_EMX_DEVICE(80003ES2LAN_COPPER_DPT),
 197         EM_EMX_DEVICE(80003ES2LAN_SERDES_DPT),
 198
 199         EM_DEVICE(ICH8_IGP_M_AMT),
 200         EM_DEVICE(ICH8_IGP_AMT),
 201         EM_DEVICE(ICH8_IGP_C),
 202         EM_DEVICE(ICH8_IFE),
 203         EM_DEVICE(ICH8_IFE_GT),
 204         EM_DEVICE(ICH8_IFE_G),
 205         EM_DEVICE(ICH8_IGP_M),
 206         EM_DEVICE(ICH8_82567V_3),
 207
 208         EM_DEVICE(ICH9_IGP_M_AMT),
 209         EM_DEVICE(ICH9_IGP_AMT),
 210         EM_DEVICE(ICH9_IGP_C),
 211         EM_DEVICE(ICH9_IGP_M),
 212         EM_DEVICE(ICH9_IGP_M_V),
 213         EM_DEVICE(ICH9_IFE),
 214         EM_DEVICE(ICH9_IFE_GT),
 215         EM_DEVICE(ICH9_IFE_G),
 216         EM_DEVICE(ICH9_BM),
 217
 218         EM_EMX_DEVICE(82574L),
 219         EM_EMX_DEVICE(82574LA),
 220
 221         EM_DEVICE(ICH10_R_BM_LM),
 222         EM_DEVICE(ICH10_R_BM_LF),
 223         EM_DEVICE(ICH10_R_BM_V),
 224         EM_DEVICE(ICH10_D_BM_LM),
 225         EM_DEVICE(ICH10_D_BM_LF),
 226         EM_DEVICE(ICH10_D_BM_V),
 227
 228         EM_DEVICE(PCH_M_HV_LM),
 229         EM_DEVICE(PCH_M_HV_LC),
 230         EM_DEVICE(PCH_D_HV_DM),
 231         EM_DEVICE(PCH_D_HV_DC),
 232
 233         EM_DEVICE(PCH2_LV_LM),
 234         EM_DEVICE(PCH2_LV_V),
 235
 236         EM_EMX_DEVICE(PCH_LPT_I217_LM),
 237         EM_EMX_DEVICE(PCH_LPT_I217_V),
 238         EM_EMX_DEVICE(PCH_LPTLP_I218_LM),
 239         EM_EMX_DEVICE(PCH_LPTLP_I218_V),
 240
 241         /* required last entry */
 242         EM_DEVICE_NULL
 243 };
 244
 245 static int      em_probe(device_t);
 246 static int      em_attach(device_t);
 247 static int      em_detach(device_t);
 248 static int      em_shutdown(device_t);
 249 static int      em_suspend(device_t);
 250 static int      em_resume(device_t);
 251
 252 static void     em_init(void *);
 253 static void     em_stop(struct adapter *);
 254 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 255 static void     em_start(struct ifnet *, struct ifaltq_subque *);
 256 #ifdef IFPOLL_ENABLE
 257 static void     em_npoll(struct ifnet *, struct ifpoll_info *);
 258 static void     em_npoll_compat(struct ifnet *, void *, int);
 259 #endif
 260 static void     em_watchdog(struct ifnet *);
 261 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 262 static int      em_media_change(struct ifnet *);
 263 static void     em_timer(void *);
 264
 265 static void     em_intr(void *);
 266 static void     em_intr_mask(void *);
 267 static void     em_intr_body(struct adapter *, boolean_t);
 268 static void     em_rxeof(struct adapter *, int);
 269 static void     em_txeof(struct adapter *);
 270 static void     em_tx_collect(struct adapter *);
 271 static void     em_tx_purge(struct adapter *);
 272 static void     em_enable_intr(struct adapter *);
 273 static void     em_disable_intr(struct adapter *);
 274
 275 static int      em_dma_malloc(struct adapter *, bus_size_t,
 276                     struct em_dma_alloc *);
 277 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 278 static void     em_init_tx_ring(struct adapter *);
 279 static int      em_init_rx_ring(struct adapter *);
 280 static int      em_create_tx_ring(struct adapter *);
 281 static int      em_create_rx_ring(struct adapter *);
 282 static void     em_destroy_tx_ring(struct adapter *, int);
 283 static void     em_destroy_rx_ring(struct adapter *, int);
 284 static int      em_newbuf(struct adapter *, int, int);
 285 static int      em_encap(struct adapter *, struct mbuf **, int *, int *);
 286 static void     em_rxcsum(struct adapter *, struct e1000_rx_desc *,
 287                     struct mbuf *);
 288 static int      em_txcsum(struct adapter *, struct mbuf *,
 289                     uint32_t *, uint32_t *);
 290 static int      em_tso_pullup(struct adapter *, struct mbuf **);
 291 static int      em_tso_setup(struct adapter *, struct mbuf *,
 292                     uint32_t *, uint32_t *);
 293
 294 static int      em_get_hw_info(struct adapter *);
 295 static int      em_is_valid_eaddr(const uint8_t *);
 296 static int      em_alloc_pci_res(struct adapter *);
 297 static void     em_free_pci_res(struct adapter *);
 298 static int      em_reset(struct adapter *);
 299 static void     em_setup_ifp(struct adapter *);
 300 static void     em_init_tx_unit(struct adapter *);
 301 static void     em_init_rx_unit(struct adapter *);
 302 static void     em_update_stats(struct adapter *);
 303 static void     em_set_promisc(struct adapter *);
 304 static void     em_disable_promisc(struct adapter *);
 305 static void     em_set_multi(struct adapter *);
 306 static void     em_update_link_status(struct adapter *);
 307 static void     em_smartspeed(struct adapter *);
 308 static void     em_set_itr(struct adapter *, uint32_t);
 309 static void     em_disable_aspm(struct adapter *);
 310
 311 /* Hardware workarounds */
 312 static int      em_82547_fifo_workaround(struct adapter *, int);
 313 static void     em_82547_update_fifo_head(struct adapter *, int);
 314 static int      em_82547_tx_fifo_reset(struct adapter *);
 315 static void     em_82547_move_tail(void *);
 316 static void     em_82547_move_tail_serialized(struct adapter *);
 317 static uint32_t em_82544_fill_desc(bus_addr_t, uint32_t, PDESC_ARRAY);
 318
 319 static void     em_print_debug_info(struct adapter *);
 320 static void     em_print_nvm_info(struct adapter *);
 321 static void     em_print_hw_stats(struct adapter *);
 322
 323 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 324 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 325 static int      em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 326 static int      em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS);
 327 static void     em_add_sysctl(struct adapter *adapter);
 328
 329 /* Management and WOL Support */
 330 static void     em_get_mgmt(struct adapter *);
 331 static void     em_rel_mgmt(struct adapter *);
 332 static void     em_get_hw_control(struct adapter *);
 333 static void     em_rel_hw_control(struct adapter *);
 334 static void     em_enable_wol(device_t);
 335
 336 static device_method_t em_methods[] = {
 337         /* Device interface */
 338         DEVMETHOD(device_probe,         em_probe),
 339         DEVMETHOD(device_attach,        em_attach),
 340         DEVMETHOD(device_detach,        em_detach),
 341         DEVMETHOD(device_shutdown,      em_shutdown),
 342         DEVMETHOD(device_suspend,       em_suspend),
 343         DEVMETHOD(device_resume,        em_resume),
 344         DEVMETHOD_END
 345 };
 346
 347 static driver_t em_driver = {
 348         "em",
 349         em_methods,
 350         sizeof(struct adapter),
 351 };
 352
 353 static devclass_t em_devclass;
 354
 355 DECLARE_DUMMY_MODULE(if_em);
 356 MODULE_DEPEND(em, ig_hal, 1, 1, 1);
 357 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, NULL, NULL);
 358
 359 /*
 360  * Tunables
 361  */
 362 static int      em_int_throttle_ceil = EM_DEFAULT_ITR;
 363 static int      em_rxd = EM_DEFAULT_RXD;
 364 static int      em_txd = EM_DEFAULT_TXD;
 365 static int      em_smart_pwr_down = 0;
 366
 367 /* Controls whether promiscuous also shows bad packets */
 368 static int      em_debug_sbp = FALSE;
 369
 370 static int      em_82573_workaround = 1;
 371 static int      em_msi_enable = 1;
 372
 373 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
 374 TUNABLE_INT("hw.em.rxd", &em_rxd);
 375 TUNABLE_INT("hw.em.txd", &em_txd);
 376 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 377 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 378 TUNABLE_INT("hw.em.82573_workaround", &em_82573_workaround);
 379 TUNABLE_INT("hw.em.msi.enable", &em_msi_enable);
 380
 381 /* Global used in WOL setup with multiport cards */
 382 static int      em_global_quad_port_a = 0;
 383
 384 /* Set this to one to display debug statistics */
 385 static int      em_display_debug_stats = 0;
 386
 387 #if !defined(KTR_IF_EM)
 388 #define KTR_IF_EM       KTR_ALL
 389 #endif
 390 KTR_INFO_MASTER(if_em);
 391 KTR_INFO(KTR_IF_EM, if_em, intr_beg, 0, "intr begin");
 392 KTR_INFO(KTR_IF_EM, if_em, intr_end, 1, "intr end");
 393 KTR_INFO(KTR_IF_EM, if_em, pkt_receive, 4, "rx packet");
 394 KTR_INFO(KTR_IF_EM, if_em, pkt_txqueue, 5, "tx packet");
 395 KTR_INFO(KTR_IF_EM, if_em, pkt_txclean, 6, "tx clean");
 396 #define logif(name)     KTR_LOG(if_em_ ## name)
 397
 398 static int
 399 em_probe(device_t dev)
 400 {
 401         const struct em_vendor_info *ent;
 402         uint16_t vid, did;
 403
 404         vid = pci_get_vendor(dev);
 405         did = pci_get_device(dev);
 406
 407         for (ent = em_vendor_info_array; ent->desc != NULL; ++ent) {
 408                 if (vid == ent->vendor_id && did == ent->device_id) {
 409                         device_set_desc(dev, ent->desc);
 410                         device_set_async_attach(dev, TRUE);
 411                         return (ent->ret);
 412                 }
 413         }
 414         return (ENXIO);
 415 }
 416
 417 static int
 418 em_attach(device_t dev)
 419 {
 420         struct adapter *adapter = device_get_softc(dev);
 421         struct ifnet *ifp = &adapter->arpcom.ac_if;
 422         int tsize, rsize;
 423         int error = 0;
 424         uint16_t eeprom_data, device_id, apme_mask;
 425         driver_intr_t *intr_func;
 426
 427         adapter->dev = adapter->osdep.dev = dev;
 428
 429         callout_init_mp(&adapter->timer);
 430         callout_init_mp(&adapter->tx_fifo_timer);
 431
 432         ifmedia_init(&adapter->media, IFM_IMASK,
 433             em_media_change, em_media_status);
 434
 435         /* Determine hardware and mac info */
 436         error = em_get_hw_info(adapter);
 437         if (error) {
 438                 device_printf(dev, "Identify hardware failed\n");
 439                 goto fail;
 440         }
 441
 442         /* Setup PCI resources */
 443         error = em_alloc_pci_res(adapter);
 444         if (error) {
 445                 device_printf(dev, "Allocation of PCI resources failed\n");
 446                 goto fail;
 447         }
 448
 449         /*
 450          * For ICH8 and family we need to map the flash memory,
 451          * and this must happen after the MAC is identified.
 452          */
 453         if (adapter->hw.mac.type == e1000_ich8lan ||
 454             adapter->hw.mac.type == e1000_ich9lan ||
 455             adapter->hw.mac.type == e1000_ich10lan ||
 456             adapter->hw.mac.type == e1000_pchlan ||
 457             adapter->hw.mac.type == e1000_pch2lan ||
 458             adapter->hw.mac.type == e1000_pch_lpt) {
 459                 adapter->flash_rid = EM_BAR_FLASH;
 460
 461                 adapter->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 462                                         &adapter->flash_rid, RF_ACTIVE);
 463                 if (adapter->flash == NULL) {
 464                         device_printf(dev, "Mapping of Flash failed\n");
 465                         error = ENXIO;
 466                         goto fail;
 467                 }
 468                 adapter->osdep.flash_bus_space_tag =
 469                     rman_get_bustag(adapter->flash);
 470                 adapter->osdep.flash_bus_space_handle =
 471                     rman_get_bushandle(adapter->flash);
 472
 473                 /*
 474                  * This is used in the shared code
 475                  * XXX this goof is actually not used.
 476                  */
 477                 adapter->hw.flash_address = (uint8_t *)adapter->flash;
 478         }
 479
 480         switch (adapter->hw.mac.type) {
 481         case e1000_82571:
 482         case e1000_82572:
 483                 /*
 484                  * Pullup extra 4bytes into the first data segment, see:
 485                  * 82571/82572 specification update errata #7
 486                  *
 487                  * NOTE:
 488                  * 4bytes instead of 2bytes, which are mentioned in the
 489                  * errata, are pulled; mainly to keep rest of the data
 490                  * properly aligned.
 491                  */
 492                 adapter->flags |= EM_FLAG_TSO_PULLEX;
 493                 /* FALL THROUGH */
 494
 495         default:
 496                 if (pci_is_pcie(dev))
 497                         adapter->flags |= EM_FLAG_TSO;
 498                 break;
 499         }
 500
 501         /* Do Shared Code initialization */
 502         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 503                 device_printf(dev, "Setup of Shared code failed\n");
 504                 error = ENXIO;
 505                 goto fail;
 506         }
 507
 508         e1000_get_bus_info(&adapter->hw);
 509
 510         /*
 511          * Validate number of transmit and receive descriptors.  It
 512          * must not exceed hardware maximum, and must be multiple
 513          * of E1000_DBA_ALIGN.
 514          */
 515         if ((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN != 0 ||
 516             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 517             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 518             em_txd < EM_MIN_TXD) {
 519                 if (adapter->hw.mac.type < e1000_82544)
 520                         adapter->num_tx_desc = EM_MAX_TXD_82543;
 521                 else
 522                         adapter->num_tx_desc = EM_DEFAULT_TXD;
 523                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 524                     adapter->num_tx_desc, em_txd);
 525         } else {
 526                 adapter->num_tx_desc = em_txd;
 527         }
 528         if ((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN != 0 ||
 529             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 530             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 531             em_rxd < EM_MIN_RXD) {
 532                 if (adapter->hw.mac.type < e1000_82544)
 533                         adapter->num_rx_desc = EM_MAX_RXD_82543;
 534                 else
 535                         adapter->num_rx_desc = EM_DEFAULT_RXD;
 536                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 537                     adapter->num_rx_desc, em_rxd);
 538         } else {
 539                 adapter->num_rx_desc = em_rxd;
 540         }
 541
 542         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 543         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 544         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 545         adapter->rx_buffer_len = MCLBYTES;
 546
 547         /*
 548          * Interrupt throttle rate
 549          */
 550         if (em_int_throttle_ceil == 0) {
 551                 adapter->int_throttle_ceil = 0;
 552         } else {
 553                 int throttle = em_int_throttle_ceil;
 554
 555                 if (throttle < 0)
 556                         throttle = EM_DEFAULT_ITR;
 557
 558                 /* Recalculate the tunable value to get the exact frequency. */
 559                 throttle = 1000000000 / 256 / throttle;
 560
 561                 /* Upper 16bits of ITR is reserved and should be zero */
 562                 if (throttle & 0xffff0000)
 563                         throttle = 1000000000 / 256 / EM_DEFAULT_ITR;
 564
 565                 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
 566         }
 567
 568         e1000_init_script_state_82541(&adapter->hw, TRUE);
 569         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 570
 571         /* Copper options */
 572         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 573                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 574                 adapter->hw.phy.disable_polarity_correction = FALSE;
 575                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 576         }
 577
 578         /* Set the frame limits assuming standard ethernet sized frames. */
 579         adapter->hw.mac.max_frame_size =
 580             ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 581         adapter->min_frame_size = ETH_ZLEN + ETHER_CRC_LEN;
 582
 583         /* This controls when hardware reports transmit completion status. */
 584         adapter->hw.mac.report_tx_early = 1;
 585
 586         /*
 587          * Create top level busdma tag
 588          */
 589         error = bus_dma_tag_create(NULL, 1, 0,
 590                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 591                         NULL, NULL,
 592                         BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
 593                         0, &adapter->parent_dtag);
 594         if (error) {
 595                 device_printf(dev, "could not create top level DMA tag\n");
 596                 goto fail;
 597         }
 598
 599         /*
 600          * Allocate Transmit Descriptor ring
 601          */
 602         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 603                          EM_DBA_ALIGN);
 604         error = em_dma_malloc(adapter, tsize, &adapter->txdma);
 605         if (error) {
 606                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 607                 goto fail;
 608         }
 609         adapter->tx_desc_base = adapter->txdma.dma_vaddr;
 610
 611         /*
 612          * Allocate Receive Descriptor ring
 613          */
 614         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 615                          EM_DBA_ALIGN);
 616         error = em_dma_malloc(adapter, rsize, &adapter->rxdma);
 617         if (error) {
 618                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 619                 goto fail;
 620         }
 621         adapter->rx_desc_base = adapter->rxdma.dma_vaddr;
 622
 623         /* Allocate multicast array memory. */
 624         adapter->mta = kmalloc(ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES,
 625             M_DEVBUF, M_WAITOK);
 626
 627         /* Indicate SOL/IDER usage */
 628         if (e1000_check_reset_block(&adapter->hw)) {
 629                 device_printf(dev,
 630                     "PHY reset is blocked due to SOL/IDER session.\n");
 631         }
 632
 633         /* Disable EEE */
 634         adapter->hw.dev_spec.ich8lan.eee_disable = 1;
 635
 636         /*
 637          * Start from a known state, this is important in reading the
 638          * nvm and mac from that.
 639          */
 640         e1000_reset_hw(&adapter->hw);
 641
 642         /* Make sure we have a good EEPROM before we read from it */
 643         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 644                 /*
 645                  * Some PCI-E parts fail the first check due to
 646                  * the link being in sleep state, call it again,
 647                  * if it fails a second time its a real issue.
 648                  */
 649                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 650                         device_printf(dev,
 651                             "The EEPROM Checksum Is Not Valid\n");
 652                         error = EIO;
 653                         goto fail;
 654                 }
 655         }
 656
 657         /* Copy the permanent MAC address out of the EEPROM */
 658         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 659                 device_printf(dev, "EEPROM read error while reading MAC"
 660                     " address\n");
 661                 error = EIO;
 662                 goto fail;
 663         }
 664         if (!em_is_valid_eaddr(adapter->hw.mac.addr)) {
 665                 device_printf(dev, "Invalid MAC address\n");
 666                 error = EIO;
 667                 goto fail;
 668         }
 669
 670         /* Allocate transmit descriptors and buffers */
 671         error = em_create_tx_ring(adapter);
 672         if (error) {
 673                 device_printf(dev, "Could not setup transmit structures\n");
 674                 goto fail;
 675         }
 676
 677         /* Allocate receive descriptors and buffers */
 678         error = em_create_rx_ring(adapter);
 679         if (error) {
 680                 device_printf(dev, "Could not setup receive structures\n");
 681                 goto fail;
 682         }
 683
 684         /* Manually turn off all interrupts */
 685         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 686
 687         /* Determine if we have to control management hardware */
 688         if (e1000_enable_mng_pass_thru(&adapter->hw))
 689                 adapter->flags |= EM_FLAG_HAS_MGMT;
 690
 691         /*
 692          * Setup Wake-on-Lan
 693          */
 694         apme_mask = EM_EEPROM_APME;
 695         eeprom_data = 0;
 696         switch (adapter->hw.mac.type) {
 697         case e1000_82542:
 698         case e1000_82543:
 699                 break;
 700
 701         case e1000_82573:
 702         case e1000_82583:
 703                 adapter->flags |= EM_FLAG_HAS_AMT;
 704                 /* FALL THROUGH */
 705
 706         case e1000_82546:
 707         case e1000_82546_rev_3:
 708         case e1000_82571:
 709         case e1000_82572:
 710         case e1000_80003es2lan:
 711                 if (adapter->hw.bus.func == 1) {
 712                         e1000_read_nvm(&adapter->hw,
 713                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 714                 } else {
 715                         e1000_read_nvm(&adapter->hw,
 716                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 717                 }
 718                 break;
 719
 720         case e1000_ich8lan:
 721         case e1000_ich9lan:
 722         case e1000_ich10lan:
 723         case e1000_pchlan:
 724         case e1000_pch2lan:
 725                 apme_mask = E1000_WUC_APME;
 726                 adapter->flags |= EM_FLAG_HAS_AMT;
 727                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
 728                 break;
 729
 730         default:
 731                 e1000_read_nvm(&adapter->hw,
 732                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 733                 break;
 734         }
 735         if (eeprom_data & apme_mask)
 736                 adapter->wol = E1000_WUFC_MAG | E1000_WUFC_MC;
 737
 738         /*
 739          * We have the eeprom settings, now apply the special cases
 740          * where the eeprom may be wrong or the board won't support
 741          * wake on lan on a particular port
 742          */
 743         device_id = pci_get_device(dev);
 744         switch (device_id) {
 745         case E1000_DEV_ID_82546GB_PCIE:
 746                 adapter->wol = 0;
 747                 break;
 748
 749         case E1000_DEV_ID_82546EB_FIBER:
 750         case E1000_DEV_ID_82546GB_FIBER:
 751         case E1000_DEV_ID_82571EB_FIBER:
 752                 /*
 753                  * Wake events only supported on port A for dual fiber
 754                  * regardless of eeprom setting
 755                  */
 756                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 757                     E1000_STATUS_FUNC_1)
 758                         adapter->wol = 0;
 759                 break;
 760
 761         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
 762         case E1000_DEV_ID_82571EB_QUAD_COPPER:
 763         case E1000_DEV_ID_82571EB_QUAD_FIBER:
 764         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
 765                 /* if quad port adapter, disable WoL on all but port A */
 766                 if (em_global_quad_port_a != 0)
 767                         adapter->wol = 0;
 768                 /* Reset for multiple quad port adapters */
 769                 if (++em_global_quad_port_a == 4)
 770                         em_global_quad_port_a = 0;
 771                 break;
 772         }
 773
 774         /* XXX disable wol */
 775         adapter->wol = 0;
 776
 777         /* Setup OS specific network interface */
 778         em_setup_ifp(adapter);
 779
 780         /* Add sysctl tree, must after em_setup_ifp() */
 781         em_add_sysctl(adapter);
 782
 783 #ifdef IFPOLL_ENABLE
 784         /* Polling setup */
 785         ifpoll_compat_setup(&adapter->npoll,
 786             &adapter->sysctl_ctx, adapter->sysctl_tree, device_get_unit(dev),
 787             ifp->if_serializer);
 788 #endif
 789
 790         /* Reset the hardware */
 791         error = em_reset(adapter);
 792         if (error) {
 793                 /*
 794                  * Some 82573 parts fail the first reset, call it again,
 795                  * if it fails a second time its a real issue.
 796                  */
 797                 error = em_reset(adapter);
 798                 if (error) {
 799                         device_printf(dev, "Unable to reset the hardware\n");
 800                         ether_ifdetach(ifp);
 801                         goto fail;
 802                 }
 803         }
 804
 805         /* Initialize statistics */
 806         em_update_stats(adapter);
 807
 808         adapter->hw.mac.get_link_status = 1;
 809         em_update_link_status(adapter);
 810
 811         /* Do we need workaround for 82544 PCI-X adapter? */
 812         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 813             adapter->hw.mac.type == e1000_82544)
 814                 adapter->pcix_82544 = TRUE;
 815         else
 816                 adapter->pcix_82544 = FALSE;
 817
 818         if (adapter->pcix_82544) {
 819                 /*
 820                  * 82544 on PCI-X may split one TX segment
 821                  * into two TX descs, so we double its number
 822                  * of spare TX desc here.
 823                  */
 824                 adapter->spare_tx_desc = 2 * EM_TX_SPARE;
 825         } else {
 826                 adapter->spare_tx_desc = EM_TX_SPARE;
 827         }
 828         if (adapter->flags & EM_FLAG_TSO)
 829                 adapter->spare_tx_desc = EM_TX_SPARE_TSO;
 830         adapter->tx_wreg_nsegs = EM_DEFAULT_TXWREG;
 831
 832         /*
 833          * Keep following relationship between spare_tx_desc, oact_tx_desc
 834          * and tx_int_nsegs:
 835          * (spare_tx_desc + EM_TX_RESERVED) <=
 836          * oact_tx_desc <= EM_TX_OACTIVE_MAX <= tx_int_nsegs
 837          */
 838         adapter->oact_tx_desc = adapter->num_tx_desc / 8;
 839         if (adapter->oact_tx_desc > EM_TX_OACTIVE_MAX)
 840                 adapter->oact_tx_desc = EM_TX_OACTIVE_MAX;
 841         if (adapter->oact_tx_desc < adapter->spare_tx_desc + EM_TX_RESERVED)
 842                 adapter->oact_tx_desc = adapter->spare_tx_desc + EM_TX_RESERVED;
 843
 844         adapter->tx_int_nsegs = adapter->num_tx_desc / 16;
 845         if (adapter->tx_int_nsegs < adapter->oact_tx_desc)
 846                 adapter->tx_int_nsegs = adapter->oact_tx_desc;
 847
 848         /* Non-AMT based hardware can now take control from firmware */
 849         if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
 850             EM_FLAG_HAS_MGMT && adapter->hw.mac.type >= e1000_82571)
 851                 em_get_hw_control(adapter);
 852
 853         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
 854
 855         /*
 856          * Missing Interrupt Following ICR read:
 857          *
 858          * 82571/82572 specification update errata #76
 859          * 82573 specification update errata #31
 860          * 82574 specification update errata #12
 861          * 82583 specification update errata #4
 862          */
 863         intr_func = em_intr;
 864         if ((adapter->flags & EM_FLAG_SHARED_INTR) &&
 865             (adapter->hw.mac.type == e1000_82571 ||
 866              adapter->hw.mac.type == e1000_82572 ||
 867              adapter->hw.mac.type == e1000_82573 ||
 868              adapter->hw.mac.type == e1000_82574 ||
 869              adapter->hw.mac.type == e1000_82583))
 870                 intr_func = em_intr_mask;
 871
 872         error = bus_setup_intr(dev, adapter->intr_res, INTR_MPSAFE,
 873                                intr_func, adapter, &adapter->intr_tag,
 874                                ifp->if_serializer);
 875         if (error) {
 876                 device_printf(dev, "Failed to register interrupt handler");
 877                 ether_ifdetach(ifp);
 878                 goto fail;
 879         }
 880         return (0);
 881 fail:
 882         em_detach(dev);
 883         return (error);
 884 }
 885
 886 static int
 887 em_detach(device_t dev)
 888 {
 889         struct adapter *adapter = device_get_softc(dev);
 890
 891         if (device_is_attached(dev)) {
 892                 struct ifnet *ifp = &adapter->arpcom.ac_if;
 893
 894                 lwkt_serialize_enter(ifp->if_serializer);
 895
 896                 em_stop(adapter);
 897
 898                 e1000_phy_hw_reset(&adapter->hw);
 899
 900                 em_rel_mgmt(adapter);
 901                 em_rel_hw_control(adapter);
 902
 903                 if (adapter->wol) {
 904                         E1000_WRITE_REG(&adapter->hw, E1000_WUC,
 905                                         E1000_WUC_PME_EN);
 906                         E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 907                         em_enable_wol(dev);
 908                 }
 909
 910                 bus_teardown_intr(dev, adapter->intr_res, adapter->intr_tag);
 911
 912                 lwkt_serialize_exit(ifp->if_serializer);
 913
 914                 ether_ifdetach(ifp);
 915         } else if (adapter->memory != NULL) {
 916                 em_rel_hw_control(adapter);
 917         }
 918
 919         ifmedia_removeall(&adapter->media);
 920         bus_generic_detach(dev);
 921
 922         em_free_pci_res(adapter);
 923
 924         em_destroy_tx_ring(adapter, adapter->num_tx_desc);
 925         em_destroy_rx_ring(adapter, adapter->num_rx_desc);
 926
 927         /* Free Transmit Descriptor ring */
 928         if (adapter->tx_desc_base)
 929                 em_dma_free(adapter, &adapter->txdma);
 930
 931         /* Free Receive Descriptor ring */
 932         if (adapter->rx_desc_base)
 933                 em_dma_free(adapter, &adapter->rxdma);
 934
 935         /* Free top level busdma tag */
 936         if (adapter->parent_dtag != NULL)
 937                 bus_dma_tag_destroy(adapter->parent_dtag);
 938
 939         /* Free sysctl tree */
 940         if (adapter->sysctl_tree != NULL)
 941                 sysctl_ctx_free(&adapter->sysctl_ctx);
 942
 943         if (adapter->mta != NULL)
 944                 kfree(adapter->mta, M_DEVBUF);
 945
 946         return (0);
 947 }
 948
 949 static int
 950 em_shutdown(device_t dev)
 951 {
 952         return em_suspend(dev);
 953 }
 954
 955 static int
 956 em_suspend(device_t dev)
 957 {
 958         struct adapter *adapter = device_get_softc(dev);
 959         struct ifnet *ifp = &adapter->arpcom.ac_if;
 960
 961         lwkt_serialize_enter(ifp->if_serializer);
 962
 963         em_stop(adapter);
 964
 965         em_rel_mgmt(adapter);
 966         em_rel_hw_control(adapter);
 967
 968         if (adapter->wol) {
 969                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 970                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 971                 em_enable_wol(dev);
 972         }
 973
 974         lwkt_serialize_exit(ifp->if_serializer);
 975
 976         return bus_generic_suspend(dev);
 977 }
 978
 979 static int
 980 em_resume(device_t dev)
 981 {
 982         struct adapter *adapter = device_get_softc(dev);
 983         struct ifnet *ifp = &adapter->arpcom.ac_if;
 984
 985         lwkt_serialize_enter(ifp->if_serializer);
 986
 987         if (adapter->hw.mac.type == e1000_pch2lan)
 988                 e1000_resume_workarounds_pchlan(&adapter->hw);
 989
 990         em_init(adapter);
 991         em_get_mgmt(adapter);
 992         if_devstart(ifp);
 993
 994         lwkt_serialize_exit(ifp->if_serializer);
 995
 996         return bus_generic_resume(dev);
 997 }
 998
 999 static void
1000 em_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1001 {
1002         struct adapter *adapter = ifp->if_softc;
1003         struct mbuf *m_head;
1004         int idx = -1, nsegs = 0;
1005
1006         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1007         ASSERT_SERIALIZED(ifp->if_serializer);
1008
1009         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
1010                 return;
1011
1012         if (!adapter->link_active) {
1013                 ifq_purge(&ifp->if_snd);
1014                 return;
1015         }
1016
1017         while (!ifq_is_empty(&ifp->if_snd)) {
1018                 /* Now do we at least have a minimal? */
1019                 if (EM_IS_OACTIVE(adapter)) {
1020                         em_tx_collect(adapter);
1021                         if (EM_IS_OACTIVE(adapter)) {
1022                                 ifq_set_oactive(&ifp->if_snd);
1023                                 adapter->no_tx_desc_avail1++;
1024                                 break;
1025                         }
1026                 }
1027
1028                 logif(pkt_txqueue);
1029                 m_head = ifq_dequeue(&ifp->if_snd);
1030                 if (m_head == NULL)
1031                         break;
1032
1033                 if (em_encap(adapter, &m_head, &nsegs, &idx)) {
1034                         IFNET_STAT_INC(ifp, oerrors, 1);
1035                         em_tx_collect(adapter);
1036                         continue;
1037                 }
1038
1039                 if (nsegs >= adapter->tx_wreg_nsegs && idx >= 0) {
1040                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1041                         nsegs = 0;
1042                         idx = -1;
1043                 }
1044
1045                 /* Send a copy of the frame to the BPF listener */
1046                 ETHER_BPF_MTAP(ifp, m_head);
1047
1048                 /* Set timeout in case hardware has problems transmitting. */
1049                 ifp->if_timer = EM_TX_TIMEOUT;
1050         }
1051         if (idx >= 0)
1052                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1053 }
1054
1055 static int
1056 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
1057 {
1058         struct adapter *adapter = ifp->if_softc;
1059         struct ifreq *ifr = (struct ifreq *)data;
1060         uint16_t eeprom_data = 0;
1061         int max_frame_size, mask, reinit;
1062         int error = 0;
1063
1064         ASSERT_SERIALIZED(ifp->if_serializer);
1065
1066         switch (command) {
1067         case SIOCSIFMTU:
1068                 switch (adapter->hw.mac.type) {
1069                 case e1000_82573:
1070                         /*
1071                          * 82573 only supports jumbo frames
1072                          * if ASPM is disabled.
1073                          */
1074                         e1000_read_nvm(&adapter->hw,
1075                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1076                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1077                                 max_frame_size = ETHER_MAX_LEN;
1078                                 break;
1079                         }
1080                         /* FALL THROUGH */
1081
1082                 /* Limit Jumbo Frame size */
1083                 case e1000_82571:
1084                 case e1000_82572:
1085                 case e1000_ich9lan:
1086                 case e1000_ich10lan:
1087                 case e1000_pch2lan:
1088                 case e1000_pch_lpt:
1089                 case e1000_82574:
1090                 case e1000_82583:
1091                 case e1000_80003es2lan:
1092                         max_frame_size = 9234;
1093                         break;
1094
1095                 case e1000_pchlan:
1096                         max_frame_size = 4096;
1097                         break;
1098
1099                 /* Adapters that do not support jumbo frames */
1100                 case e1000_82542:
1101                 case e1000_ich8lan:
1102                         max_frame_size = ETHER_MAX_LEN;
1103                         break;
1104
1105                 default:
1106                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1107                         break;
1108                 }
1109                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1110                     ETHER_CRC_LEN) {
1111                         error = EINVAL;
1112                         break;
1113                 }
1114
1115                 ifp->if_mtu = ifr->ifr_mtu;
1116                 adapter->hw.mac.max_frame_size =
1117                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1118
1119                 if (ifp->if_flags & IFF_RUNNING)
1120                         em_init(adapter);
1121                 break;
1122
1123         case SIOCSIFFLAGS:
1124                 if (ifp->if_flags & IFF_UP) {
1125                         if ((ifp->if_flags & IFF_RUNNING)) {
1126                                 if ((ifp->if_flags ^ adapter->if_flags) &
1127                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1128                                         em_disable_promisc(adapter);
1129                                         em_set_promisc(adapter);
1130                                 }
1131                         } else {
1132                                 em_init(adapter);
1133                         }
1134                 } else if (ifp->if_flags & IFF_RUNNING) {
1135                         em_stop(adapter);
1136                 }
1137                 adapter->if_flags = ifp->if_flags;
1138                 break;
1139
1140         case SIOCADDMULTI:
1141         case SIOCDELMULTI:
1142                 if (ifp->if_flags & IFF_RUNNING) {
1143                         em_disable_intr(adapter);
1144                         em_set_multi(adapter);
1145                         if (adapter->hw.mac.type == e1000_82542 &&
1146                             adapter->hw.revision_id == E1000_REVISION_2)
1147                                 em_init_rx_unit(adapter);
1148 #ifdef IFPOLL_ENABLE
1149                         if (!(ifp->if_flags & IFF_NPOLLING))
1150 #endif
1151                                 em_enable_intr(adapter);
1152                 }
1153                 break;
1154
1155         case SIOCSIFMEDIA:
1156                 /* Check SOL/IDER usage */
1157                 if (e1000_check_reset_block(&adapter->hw)) {
1158                         device_printf(adapter->dev, "Media change is"
1159                             " blocked due to SOL/IDER session.\n");
1160                         break;
1161                 }
1162                 /* FALL THROUGH */
1163
1164         case SIOCGIFMEDIA:
1165                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1166                 break;
1167
1168         case SIOCSIFCAP:
1169                 reinit = 0;
1170                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1171                 if (mask & IFCAP_RXCSUM) {
1172                         ifp->if_capenable ^= IFCAP_RXCSUM;
1173                         reinit = 1;
1174                 }
1175                 if (mask & IFCAP_TXCSUM) {
1176                         ifp->if_capenable ^= IFCAP_TXCSUM;
1177                         if (ifp->if_capenable & IFCAP_TXCSUM)
1178                                 ifp->if_hwassist |= EM_CSUM_FEATURES;
1179                         else
1180                                 ifp->if_hwassist &= ~EM_CSUM_FEATURES;
1181                 }
1182                 if (mask & IFCAP_TSO) {
1183                         ifp->if_capenable ^= IFCAP_TSO;
1184                         if (ifp->if_capenable & IFCAP_TSO)
1185                                 ifp->if_hwassist |= CSUM_TSO;
1186                         else
1187                                 ifp->if_hwassist &= ~CSUM_TSO;
1188                 }
1189                 if (mask & IFCAP_VLAN_HWTAGGING) {
1190                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1191                         reinit = 1;
1192                 }
1193                 if (reinit && (ifp->if_flags & IFF_RUNNING))
1194                         em_init(adapter);
1195                 break;
1196
1197         default:
1198                 error = ether_ioctl(ifp, command, data);
1199                 break;
1200         }
1201         return (error);
1202 }
1203
1204 static void
1205 em_watchdog(struct ifnet *ifp)
1206 {
1207         struct adapter *adapter = ifp->if_softc;
1208
1209         ASSERT_SERIALIZED(ifp->if_serializer);
1210
1211         /*
1212          * The timer is set to 5 every time start queues a packet.
1213          * Then txeof keeps resetting it as long as it cleans at
1214          * least one descriptor.
1215          * Finally, anytime all descriptors are clean the timer is
1216          * set to 0.
1217          */
1218
1219         if (E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1220             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) {
1221                 /*
1222                  * If we reach here, all TX jobs are completed and
1223                  * the TX engine should have been idled for some time.
1224                  * We don't need to call if_devstart() here.
1225                  */
1226                 ifq_clr_oactive(&ifp->if_snd);
1227                 ifp->if_timer = 0;
1228                 return;
1229         }
1230
1231         /*
1232          * If we are in this routine because of pause frames, then
1233          * don't reset the hardware.
1234          */
1235         if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1236             E1000_STATUS_TXOFF) {
1237                 ifp->if_timer = EM_TX_TIMEOUT;
1238                 return;
1239         }
1240
1241         if (e1000_check_for_link(&adapter->hw) == 0)
1242                 if_printf(ifp, "watchdog timeout -- resetting\n");
1243
1244         IFNET_STAT_INC(ifp, oerrors, 1);
1245         adapter->watchdog_events++;
1246
1247         em_init(adapter);
1248
1249         if (!ifq_is_empty(&ifp->if_snd))
1250                 if_devstart(ifp);
1251 }
1252
1253 static void
1254 em_init(void *xsc)
1255 {
1256         struct adapter *adapter = xsc;
1257         struct ifnet *ifp = &adapter->arpcom.ac_if;
1258         device_t dev = adapter->dev;
1259
1260         ASSERT_SERIALIZED(ifp->if_serializer);
1261
1262         em_stop(adapter);
1263
1264         /* Get the latest mac address, User can use a LAA */
1265         bcopy(IF_LLADDR(ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN);
1266
1267         /* Put the address into the Receive Address Array */
1268         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1269
1270         /*
1271          * With the 82571 adapter, RAR[0] may be overwritten
1272          * when the other port is reset, we make a duplicate
1273          * in RAR[14] for that eventuality, this assures
1274          * the interface continues to function.
1275          */
1276         if (adapter->hw.mac.type == e1000_82571) {
1277                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1278                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1279                     E1000_RAR_ENTRIES - 1);
1280         }
1281
1282         /* Reset the hardware */
1283         if (em_reset(adapter)) {
1284                 device_printf(dev, "Unable to reset the hardware\n");
1285                 /* XXX em_stop()? */
1286                 return;
1287         }
1288         em_update_link_status(adapter);
1289
1290         /* Setup VLAN support, basic and offload if available */
1291         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1292
1293         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1294                 uint32_t ctrl;
1295
1296                 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1297                 ctrl |= E1000_CTRL_VME;
1298                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1299         }
1300
1301         /* Configure for OS presence */
1302         em_get_mgmt(adapter);
1303
1304         /* Prepare transmit descriptors and buffers */
1305         em_init_tx_ring(adapter);
1306         em_init_tx_unit(adapter);
1307
1308         /* Setup Multicast table */
1309         em_set_multi(adapter);
1310
1311         /* Prepare receive descriptors and buffers */
1312         if (em_init_rx_ring(adapter)) {
1313                 device_printf(dev, "Could not setup receive structures\n");
1314                 em_stop(adapter);
1315                 return;
1316         }
1317         em_init_rx_unit(adapter);
1318
1319         /* Don't lose promiscuous settings */
1320         em_set_promisc(adapter);
1321
1322         ifp->if_flags |= IFF_RUNNING;
1323         ifq_clr_oactive(&ifp->if_snd);
1324
1325         callout_reset(&adapter->timer, hz, em_timer, adapter);
1326         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1327
1328         /* MSI/X configuration for 82574 */
1329         if (adapter->hw.mac.type == e1000_82574) {
1330                 int tmp;
1331
1332                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1333                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1334                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1335                 /*
1336                  * XXX MSIX
1337                  * Set the IVAR - interrupt vector routing.
1338                  * Each nibble represents a vector, high bit
1339                  * is enable, other 3 bits are the MSIX table
1340                  * entry, we map RXQ0 to 0, TXQ0 to 1, and
1341                  * Link (other) to 2, hence the magic number.
1342                  */
1343                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1344         }
1345
1346 #ifdef IFPOLL_ENABLE
1347         /*
1348          * Only enable interrupts if we are not polling, make sure
1349          * they are off otherwise.
1350          */
1351         if (ifp->if_flags & IFF_NPOLLING)
1352                 em_disable_intr(adapter);
1353         else
1354 #endif /* IFPOLL_ENABLE */
1355                 em_enable_intr(adapter);
1356
1357         /* AMT based hardware can now take control from firmware */
1358         if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
1359             (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT) &&
1360             adapter->hw.mac.type >= e1000_82571)
1361                 em_get_hw_control(adapter);
1362 }
1363
1364 #ifdef IFPOLL_ENABLE
1365
1366 static void
1367 em_npoll_compat(struct ifnet *ifp, void *arg __unused, int count)
1368 {
1369         struct adapter *adapter = ifp->if_softc;
1370
1371         ASSERT_SERIALIZED(ifp->if_serializer);
1372
1373         if (adapter->npoll.ifpc_stcount-- == 0) {
1374                 uint32_t reg_icr;
1375
1376                 adapter->npoll.ifpc_stcount = adapter->npoll.ifpc_stfrac;
1377
1378                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1379                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1380                         callout_stop(&adapter->timer);
1381                         adapter->hw.mac.get_link_status = 1;
1382                         em_update_link_status(adapter);
1383                         callout_reset(&adapter->timer, hz, em_timer, adapter);
1384                 }
1385         }
1386
1387         em_rxeof(adapter, count);
1388         em_txeof(adapter);
1389
1390         if (!ifq_is_empty(&ifp->if_snd))
1391                 if_devstart(ifp);
1392 }
1393
1394 static void
1395 em_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1396 {
1397         struct adapter *adapter = ifp->if_softc;
1398
1399         ASSERT_SERIALIZED(ifp->if_serializer);
1400
1401         if (info != NULL) {
1402                 int cpuid = adapter->npoll.ifpc_cpuid;
1403
1404                 info->ifpi_rx[cpuid].poll_func = em_npoll_compat;
1405                 info->ifpi_rx[cpuid].arg = NULL;
1406                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
1407
1408                 if (ifp->if_flags & IFF_RUNNING)
1409                         em_disable_intr(adapter);
1410                 ifq_set_cpuid(&ifp->if_snd, cpuid);
1411         } else {
1412                 if (ifp->if_flags & IFF_RUNNING)
1413                         em_enable_intr(adapter);
1414                 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
1415         }
1416 }
1417
1418 #endif /* IFPOLL_ENABLE */
1419
1420 static void
1421 em_intr(void *xsc)
1422 {
1423         em_intr_body(xsc, TRUE);
1424 }
1425
1426 static void
1427 em_intr_body(struct adapter *adapter, boolean_t chk_asserted)
1428 {
1429         struct ifnet *ifp = &adapter->arpcom.ac_if;
1430         uint32_t reg_icr;
1431
1432         logif(intr_beg);
1433         ASSERT_SERIALIZED(ifp->if_serializer);
1434
1435         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1436
1437         if (chk_asserted &&
1438             ((adapter->hw.mac.type >= e1000_82571 &&
1439               (reg_icr & E1000_ICR_INT_ASSERTED) == 0) ||
1440              reg_icr == 0)) {
1441                 logif(intr_end);
1442                 return;
1443         }
1444
1445         /*
1446          * XXX: some laptops trigger several spurious interrupts
1447          * on em(4) when in the resume cycle. The ICR register
1448          * reports all-ones value in this case. Processing such
1449          * interrupts would lead to a freeze. I don't know why.
1450          */
1451         if (reg_icr == 0xffffffff) {
1452                 logif(intr_end);
1453                 return;
1454         }
1455
1456         if (ifp->if_flags & IFF_RUNNING) {
1457                 if (reg_icr &
1458                     (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO))
1459                         em_rxeof(adapter, -1);
1460                 if (reg_icr & E1000_ICR_TXDW) {
1461                         em_txeof(adapter);
1462                         if (!ifq_is_empty(&ifp->if_snd))
1463                                 if_devstart(ifp);
1464                 }
1465         }
1466
1467         /* Link status change */
1468         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1469                 callout_stop(&adapter->timer);
1470                 adapter->hw.mac.get_link_status = 1;
1471                 em_update_link_status(adapter);
1472
1473                 /* Deal with TX cruft when link lost */
1474                 em_tx_purge(adapter);
1475
1476                 callout_reset(&adapter->timer, hz, em_timer, adapter);
1477         }
1478
1479         if (reg_icr & E1000_ICR_RXO)
1480                 adapter->rx_overruns++;
1481
1482         logif(intr_end);
1483 }
1484
1485 static void
1486 em_intr_mask(void *xsc)
1487 {
1488         struct adapter *adapter = xsc;
1489
1490         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
1491         /*
1492          * NOTE:
1493          * ICR.INT_ASSERTED bit will never be set if IMS is 0,
1494          * so don't check it.
1495          */
1496         em_intr_body(adapter, FALSE);
1497         E1000_WRITE_REG(&adapter->hw, E1000_IMS, IMS_ENABLE_MASK);
1498 }
1499
1500 static void
1501 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1502 {
1503         struct adapter *adapter = ifp->if_softc;
1504         u_char fiber_type = IFM_1000_SX;
1505
1506         ASSERT_SERIALIZED(ifp->if_serializer);
1507
1508         em_update_link_status(adapter);
1509
1510         ifmr->ifm_status = IFM_AVALID;
1511         ifmr->ifm_active = IFM_ETHER;
1512
1513         if (!adapter->link_active)
1514                 return;
1515
1516         ifmr->ifm_status |= IFM_ACTIVE;
1517
1518         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
1519             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
1520                 if (adapter->hw.mac.type == e1000_82545)
1521                         fiber_type = IFM_1000_LX;
1522                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1523         } else {
1524                 switch (adapter->link_speed) {
1525                 case 10:
1526                         ifmr->ifm_active |= IFM_10_T;
1527                         break;
1528                 case 100:
1529                         ifmr->ifm_active |= IFM_100_TX;
1530                         break;
1531
1532                 case 1000:
1533                         ifmr->ifm_active |= IFM_1000_T;
1534                         break;
1535                 }
1536                 if (adapter->link_duplex == FULL_DUPLEX)
1537                         ifmr->ifm_active |= IFM_FDX;
1538                 else
1539                         ifmr->ifm_active |= IFM_HDX;
1540         }
1541 }
1542
1543 static int
1544 em_media_change(struct ifnet *ifp)
1545 {
1546         struct adapter *adapter = ifp->if_softc;
1547         struct ifmedia *ifm = &adapter->media;
1548
1549         ASSERT_SERIALIZED(ifp->if_serializer);
1550
1551         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1552                 return (EINVAL);
1553
1554         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1555         case IFM_AUTO:
1556                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1557                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1558                 break;
1559
1560         case IFM_1000_LX:
1561         case IFM_1000_SX:
1562         case IFM_1000_T:
1563                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1564                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1565                 break;
1566
1567         case IFM_100_TX:
1568                 adapter->hw.mac.autoneg = FALSE;
1569                 adapter->hw.phy.autoneg_advertised = 0;
1570                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1571                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1572                 else
1573                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1574                 break;
1575
1576         case IFM_10_T:
1577                 adapter->hw.mac.autoneg = FALSE;
1578                 adapter->hw.phy.autoneg_advertised = 0;
1579                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1580                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1581                 else
1582                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1583                 break;
1584
1585         default:
1586                 if_printf(ifp, "Unsupported media type\n");
1587                 break;
1588         }
1589
1590         em_init(adapter);
1591
1592         return (0);
1593 }
1594
1595 static int
1596 em_encap(struct adapter *adapter, struct mbuf **m_headp,
1597     int *segs_used, int *idx)
1598 {
1599         bus_dma_segment_t segs[EM_MAX_SCATTER];
1600         bus_dmamap_t map;
1601         struct em_buffer *tx_buffer, *tx_buffer_mapped;
1602         struct e1000_tx_desc *ctxd = NULL;
1603         struct mbuf *m_head = *m_headp;
1604         uint32_t txd_upper, txd_lower, txd_used, cmd = 0;
1605         int maxsegs, nsegs, i, j, first, last = 0, error;
1606
1607         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1608                 error = em_tso_pullup(adapter, m_headp);
1609                 if (error)
1610                         return error;
1611                 m_head = *m_headp;
1612         }
1613
1614         txd_upper = txd_lower = 0;
1615         txd_used = 0;
1616
1617         /*
1618          * Capture the first descriptor index, this descriptor
1619          * will have the index of the EOP which is the only one
1620          * that now gets a DONE bit writeback.
1621          */
1622         first = adapter->next_avail_tx_desc;
1623         tx_buffer = &adapter->tx_buffer_area[first];
1624         tx_buffer_mapped = tx_buffer;
1625         map = tx_buffer->map;
1626
1627         maxsegs = adapter->num_tx_desc_avail - EM_TX_RESERVED;
1628         KASSERT(maxsegs >= adapter->spare_tx_desc,
1629                 ("not enough spare TX desc"));
1630         if (adapter->pcix_82544) {
1631                 /* Half it; see the comment in em_attach() */
1632                 maxsegs >>= 1;
1633         }
1634         if (maxsegs > EM_MAX_SCATTER)
1635                 maxsegs = EM_MAX_SCATTER;
1636
1637         error = bus_dmamap_load_mbuf_defrag(adapter->txtag, map, m_headp,
1638                         segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1639         if (error) {
1640                 if (error == ENOBUFS)
1641                         adapter->mbuf_alloc_failed++;
1642                 else
1643                         adapter->no_tx_dma_setup++;
1644
1645                 m_freem(*m_headp);
1646                 *m_headp = NULL;
1647                 return error;
1648         }
1649         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1650
1651         m_head = *m_headp;
1652         adapter->tx_nsegs += nsegs;
1653         *segs_used += nsegs;
1654
1655         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1656                 /* TSO will consume one TX desc */
1657                 i = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
1658                 adapter->tx_nsegs += i;
1659                 *segs_used += i;
1660         } else if (m_head->m_pkthdr.csum_flags & EM_CSUM_FEATURES) {
1661                 /* TX csum offloading will consume one TX desc */
1662                 i = em_txcsum(adapter, m_head, &txd_upper, &txd_lower);
1663                 adapter->tx_nsegs += i;
1664                 *segs_used += i;
1665         }
1666
1667         /* Handle VLAN tag */
1668         if (m_head->m_flags & M_VLANTAG) {
1669                 /* Set the vlan id. */
1670                 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16);
1671                 /* Tell hardware to add tag */
1672                 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1673         }
1674
1675         i = adapter->next_avail_tx_desc;
1676
1677         /* Set up our transmit descriptors */
1678         for (j = 0; j < nsegs; j++) {
1679                 /* If adapter is 82544 and on PCIX bus */
1680                 if(adapter->pcix_82544) {
1681                         DESC_ARRAY desc_array;
1682                         uint32_t array_elements, counter;
1683
1684                         /*
1685                          * Check the Address and Length combination and
1686                          * split the data accordingly
1687                          */
1688                         array_elements = em_82544_fill_desc(segs[j].ds_addr,
1689                                                 segs[j].ds_len, &desc_array);
1690                         for (counter = 0; counter < array_elements; counter++) {
1691                                 KKASSERT(txd_used < adapter->num_tx_desc_avail);
1692
1693                                 tx_buffer = &adapter->tx_buffer_area[i];
1694                                 ctxd = &adapter->tx_desc_base[i];
1695
1696                                 ctxd->buffer_addr = htole64(
1697                                     desc_array.descriptor[counter].address);
1698                                 ctxd->lower.data = htole32(
1699                                     E1000_TXD_CMD_IFCS | txd_lower |
1700                                     desc_array.descriptor[counter].length);
1701                                 ctxd->upper.data = htole32(txd_upper);
1702
1703                                 last = i;
1704                                 if (++i == adapter->num_tx_desc)
1705                                         i = 0;
1706
1707                                 txd_used++;
1708                         }
1709                 } else {
1710                         tx_buffer = &adapter->tx_buffer_area[i];
1711                         ctxd = &adapter->tx_desc_base[i];
1712
1713                         ctxd->buffer_addr = htole64(segs[j].ds_addr);
1714                         ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS |
1715                                                    txd_lower | segs[j].ds_len);
1716                         ctxd->upper.data = htole32(txd_upper);
1717
1718                         last = i;
1719                         if (++i == adapter->num_tx_desc)
1720                                 i = 0;
1721                 }
1722         }
1723
1724         adapter->next_avail_tx_desc = i;
1725         if (adapter->pcix_82544) {
1726                 KKASSERT(adapter->num_tx_desc_avail > txd_used);
1727                 adapter->num_tx_desc_avail -= txd_used;
1728         } else {
1729                 KKASSERT(adapter->num_tx_desc_avail > nsegs);
1730                 adapter->num_tx_desc_avail -= nsegs;
1731         }
1732
1733         tx_buffer->m_head = m_head;
1734         tx_buffer_mapped->map = tx_buffer->map;
1735         tx_buffer->map = map;
1736
1737         if (adapter->tx_nsegs >= adapter->tx_int_nsegs) {
1738                 adapter->tx_nsegs = 0;
1739
1740                 /*
1741                  * Report Status (RS) is turned on
1742                  * every tx_int_nsegs descriptors.
1743                  */
1744                 cmd = E1000_TXD_CMD_RS;
1745
1746                 /*
1747                  * Keep track of the descriptor, which will
1748                  * be written back by hardware.
1749                  */
1750                 adapter->tx_dd[adapter->tx_dd_tail] = last;
1751                 EM_INC_TXDD_IDX(adapter->tx_dd_tail);
1752                 KKASSERT(adapter->tx_dd_tail != adapter->tx_dd_head);
1753         }
1754
1755         /*
1756          * Last Descriptor of Packet needs End Of Packet (EOP)
1757          */
1758         ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd);
1759
1760         if (adapter->hw.mac.type == e1000_82547) {
1761                 /*
1762                  * Advance the Transmit Descriptor Tail (TDT), this tells the
1763                  * E1000 that this frame is available to transmit.
1764                  */
1765                 if (adapter->link_duplex == HALF_DUPLEX) {
1766                         em_82547_move_tail_serialized(adapter);
1767                 } else {
1768                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
1769                         em_82547_update_fifo_head(adapter,
1770                             m_head->m_pkthdr.len);
1771                 }
1772         } else {
1773                 /*
1774                  * Defer TDT updating, until enough descriptors are setup
1775                  */
1776                 *idx = i;
1777         }
1778         return (0);
1779 }
1780
1781 /*
1782  * 82547 workaround to avoid controller hang in half-duplex environment.
1783  * The workaround is to avoid queuing a large packet that would span
1784  * the internal Tx FIFO ring boundary.  We need to reset the FIFO pointers
1785  * in this case.  We do that only when FIFO is quiescent.
1786  */
1787 static void
1788 em_82547_move_tail_serialized(struct adapter *adapter)
1789 {
1790         struct e1000_tx_desc *tx_desc;
1791         uint16_t hw_tdt, sw_tdt, length = 0;
1792         bool eop = 0;
1793
1794         ASSERT_SERIALIZED(adapter->arpcom.ac_if.if_serializer);
1795
1796         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
1797         sw_tdt = adapter->next_avail_tx_desc;
1798
1799         while (hw_tdt != sw_tdt) {
1800                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1801                 length += tx_desc->lower.flags.length;
1802                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1803                 if (++hw_tdt == adapter->num_tx_desc)
1804                         hw_tdt = 0;
1805
1806                 if (eop) {
1807                         if (em_82547_fifo_workaround(adapter, length)) {
1808                                 adapter->tx_fifo_wrk_cnt++;
1809                                 callout_reset(&adapter->tx_fifo_timer, 1,
1810                                         em_82547_move_tail, adapter);
1811                                 break;
1812                         }
1813                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
1814                         em_82547_update_fifo_head(adapter, length);
1815                         length = 0;
1816                 }
1817         }
1818 }
1819
1820 static void
1821 em_82547_move_tail(void *xsc)
1822 {
1823         struct adapter *adapter = xsc;
1824         struct ifnet *ifp = &adapter->arpcom.ac_if;
1825
1826         lwkt_serialize_enter(ifp->if_serializer);
1827         em_82547_move_tail_serialized(adapter);
1828         lwkt_serialize_exit(ifp->if_serializer);
1829 }
1830
1831 static int
1832 em_82547_fifo_workaround(struct adapter *adapter, int len)
1833 {
1834         int fifo_space, fifo_pkt_len;
1835
1836         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1837
1838         if (adapter->link_duplex == HALF_DUPLEX) {
1839                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1840
1841                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1842                         if (em_82547_tx_fifo_reset(adapter))
1843                                 return (0);
1844                         else
1845                                 return (1);
1846                 }
1847         }
1848         return (0);
1849 }
1850
1851 static void
1852 em_82547_update_fifo_head(struct adapter *adapter, int len)
1853 {
1854         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1855
1856         /* tx_fifo_head is always 16 byte aligned */
1857         adapter->tx_fifo_head += fifo_pkt_len;
1858         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
1859                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1860 }
1861
1862 static int
1863 em_82547_tx_fifo_reset(struct adapter *adapter)
1864 {
1865         uint32_t tctl;
1866
1867         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1868              E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
1869             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
1870              E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
1871             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
1872              E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
1873             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
1874                 /* Disable TX unit */
1875                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
1876                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
1877                     tctl & ~E1000_TCTL_EN);
1878
1879                 /* Reset FIFO pointers */
1880                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
1881                     adapter->tx_head_addr);
1882                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
1883                     adapter->tx_head_addr);
1884                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
1885                     adapter->tx_head_addr);
1886                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
1887                     adapter->tx_head_addr);
1888
1889                 /* Re-enable TX unit */
1890                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
1891                 E1000_WRITE_FLUSH(&adapter->hw);
1892
1893                 adapter->tx_fifo_head = 0;
1894                 adapter->tx_fifo_reset_cnt++;
1895
1896                 return (TRUE);
1897         } else {
1898                 return (FALSE);
1899         }
1900 }
1901
1902 static void
1903 em_set_promisc(struct adapter *adapter)
1904 {
1905         struct ifnet *ifp = &adapter->arpcom.ac_if;
1906         uint32_t reg_rctl;
1907
1908         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1909
1910         if (ifp->if_flags & IFF_PROMISC) {
1911                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1912                 /* Turn this on if you want to see bad packets */
1913                 if (em_debug_sbp)
1914                         reg_rctl |= E1000_RCTL_SBP;
1915                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1916         } else if (ifp->if_flags & IFF_ALLMULTI) {
1917                 reg_rctl |= E1000_RCTL_MPE;
1918                 reg_rctl &= ~E1000_RCTL_UPE;
1919                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1920         }
1921 }
1922
1923 static void
1924 em_disable_promisc(struct adapter *adapter)
1925 {
1926         uint32_t reg_rctl;
1927
1928         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1929
1930         reg_rctl &= ~E1000_RCTL_UPE;
1931         reg_rctl &= ~E1000_RCTL_MPE;
1932         reg_rctl &= ~E1000_RCTL_SBP;
1933         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1934 }
1935
1936 static void
1937 em_set_multi(struct adapter *adapter)
1938 {
1939         struct ifnet *ifp = &adapter->arpcom.ac_if;
1940         struct ifmultiaddr *ifma;
1941         uint32_t reg_rctl = 0;
1942         uint8_t *mta;
1943         int mcnt = 0;
1944
1945         mta = adapter->mta;
1946         bzero(mta, ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
1947
1948         if (adapter->hw.mac.type == e1000_82542 &&
1949             adapter->hw.revision_id == E1000_REVISION_2) {
1950                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1951                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1952                         e1000_pci_clear_mwi(&adapter->hw);
1953                 reg_rctl |= E1000_RCTL_RST;
1954                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1955                 msec_delay(5);
1956         }
1957
1958         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1959                 if (ifma->ifma_addr->sa_family != AF_LINK)
1960                         continue;
1961
1962                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1963                         break;
1964
1965                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1966                     &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
1967                 mcnt++;
1968         }
1969
1970         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1971                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1972                 reg_rctl |= E1000_RCTL_MPE;
1973                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1974         } else {
1975                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1976         }
1977
1978         if (adapter->hw.mac.type == e1000_82542 &&
1979             adapter->hw.revision_id == E1000_REVISION_2) {
1980                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1981                 reg_rctl &= ~E1000_RCTL_RST;
1982                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1983                 msec_delay(5);
1984                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1985                         e1000_pci_set_mwi(&adapter->hw);
1986         }
1987 }
1988
1989 /*
1990  * This routine checks for link status and updates statistics.
1991  */
1992 static void
1993 em_timer(void *xsc)
1994 {
1995         struct adapter *adapter = xsc;
1996         struct ifnet *ifp = &adapter->arpcom.ac_if;
1997
1998         lwkt_serialize_enter(ifp->if_serializer);
1999
2000         em_update_link_status(adapter);
2001         em_update_stats(adapter);
2002
2003         /* Reset LAA into RAR[0] on 82571 */
2004         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2005                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2006
2007         if (em_display_debug_stats && (ifp->if_flags & IFF_RUNNING))
2008                 em_print_hw_stats(adapter);
2009
2010         em_smartspeed(adapter);
2011
2012         callout_reset(&adapter->timer, hz, em_timer, adapter);
2013
2014         lwkt_serialize_exit(ifp->if_serializer);
2015 }
2016
2017 static void
2018 em_update_link_status(struct adapter *adapter)
2019 {
2020         struct e1000_hw *hw = &adapter->hw;
2021         struct ifnet *ifp = &adapter->arpcom.ac_if;
2022         device_t dev = adapter->dev;
2023         uint32_t link_check = 0;
2024
2025         /* Get the cached link value or read phy for real */
2026         switch (hw->phy.media_type) {
2027         case e1000_media_type_copper:
2028                 if (hw->mac.get_link_status) {
2029                         /* Do the work to read phy */
2030                         e1000_check_for_link(hw);
2031                         link_check = !hw->mac.get_link_status;
2032                         if (link_check) /* ESB2 fix */
2033                                 e1000_cfg_on_link_up(hw);
2034                 } else {
2035                         link_check = TRUE;
2036                 }
2037                 break;
2038
2039         case e1000_media_type_fiber:
2040                 e1000_check_for_link(hw);
2041                 link_check =
2042                         E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU;
2043                 break;
2044
2045         case e1000_media_type_internal_serdes:
2046                 e1000_check_for_link(hw);
2047                 link_check = adapter->hw.mac.serdes_has_link;
2048                 break;
2049
2050         case e1000_media_type_unknown:
2051         default:
2052                 break;
2053         }
2054
2055         /* Now check for a transition */
2056         if (link_check && adapter->link_active == 0) {
2057                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2058                     &adapter->link_duplex);
2059
2060                 /*
2061                  * Check if we should enable/disable SPEED_MODE bit on
2062                  * 82571/82572
2063                  */
2064                 if (adapter->link_speed != SPEED_1000 &&
2065                     (hw->mac.type == e1000_82571 ||
2066                      hw->mac.type == e1000_82572)) {
2067                         int tarc0;
2068
2069                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2070                         tarc0 &= ~SPEED_MODE_BIT;
2071                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2072                 }
2073                 if (bootverbose) {
2074                         device_printf(dev, "Link is up %d Mbps %s\n",
2075                             adapter->link_speed,
2076                             ((adapter->link_duplex == FULL_DUPLEX) ?
2077                             "Full Duplex" : "Half Duplex"));
2078                 }
2079                 adapter->link_active = 1;
2080                 adapter->smartspeed = 0;
2081                 ifp->if_baudrate = adapter->link_speed * 1000000;
2082                 ifp->if_link_state = LINK_STATE_UP;
2083                 if_link_state_change(ifp);
2084         } else if (!link_check && adapter->link_active == 1) {
2085                 ifp->if_baudrate = adapter->link_speed = 0;
2086                 adapter->link_duplex = 0;
2087                 if (bootverbose)
2088                         device_printf(dev, "Link is Down\n");
2089                 adapter->link_active = 0;
2090 #if 0
2091                 /* Link down, disable watchdog */
2092                 if->if_timer = 0;
2093 #endif
2094                 ifp->if_link_state = LINK_STATE_DOWN;
2095                 if_link_state_change(ifp);
2096         }
2097 }
2098
2099 static void
2100 em_stop(struct adapter *adapter)
2101 {
2102         struct ifnet *ifp = &adapter->arpcom.ac_if;
2103         int i;
2104
2105         ASSERT_SERIALIZED(ifp->if_serializer);
2106
2107         em_disable_intr(adapter);
2108
2109         callout_stop(&adapter->timer);
2110         callout_stop(&adapter->tx_fifo_timer);
2111
2112         ifp->if_flags &= ~IFF_RUNNING;
2113         ifq_clr_oactive(&ifp->if_snd);
2114         ifp->if_timer = 0;
2115
2116         e1000_reset_hw(&adapter->hw);
2117         if (adapter->hw.mac.type >= e1000_82544)
2118                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2119
2120         for (i = 0; i < adapter->num_tx_desc; i++) {
2121                 struct em_buffer *tx_buffer = &adapter->tx_buffer_area[i];
2122
2123                 if (tx_buffer->m_head != NULL) {
2124                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2125                         m_freem(tx_buffer->m_head);
2126                         tx_buffer->m_head = NULL;
2127                 }
2128         }
2129
2130         for (i = 0; i < adapter->num_rx_desc; i++) {
2131                 struct em_buffer *rx_buffer = &adapter->rx_buffer_area[i];
2132
2133                 if (rx_buffer->m_head != NULL) {
2134                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2135                         m_freem(rx_buffer->m_head);
2136                         rx_buffer->m_head = NULL;
2137                 }
2138         }
2139
2140         if (adapter->fmp != NULL)
2141                 m_freem(adapter->fmp);
2142         adapter->fmp = NULL;
2143         adapter->lmp = NULL;
2144
2145         adapter->csum_flags = 0;
2146         adapter->csum_lhlen = 0;
2147         adapter->csum_iphlen = 0;
2148         adapter->csum_thlen = 0;
2149         adapter->csum_mss = 0;
2150         adapter->csum_pktlen = 0;
2151
2152         adapter->tx_dd_head = 0;
2153         adapter->tx_dd_tail = 0;
2154         adapter->tx_nsegs = 0;
2155 }
2156
2157 static int
2158 em_get_hw_info(struct adapter *adapter)
2159 {
2160         device_t dev = adapter->dev;
2161
2162         /* Save off the information about this board */
2163         adapter->hw.vendor_id = pci_get_vendor(dev);
2164         adapter->hw.device_id = pci_get_device(dev);
2165         adapter->hw.revision_id = pci_get_revid(dev);
2166         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
2167         adapter->hw.subsystem_device_id = pci_get_subdevice(dev);
2168
2169         /* Do Shared Code Init and Setup */
2170         if (e1000_set_mac_type(&adapter->hw))
2171                 return ENXIO;
2172         return 0;
2173 }
2174
2175 static int
2176 em_alloc_pci_res(struct adapter *adapter)
2177 {
2178         device_t dev = adapter->dev;
2179         u_int intr_flags;
2180         int val, rid, msi_enable;
2181
2182         /* Enable bus mastering */
2183         pci_enable_busmaster(dev);
2184
2185         adapter->memory_rid = EM_BAR_MEM;
2186         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2187                                 &adapter->memory_rid, RF_ACTIVE);
2188         if (adapter->memory == NULL) {
2189                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2190                 return (ENXIO);
2191         }
2192         adapter->osdep.mem_bus_space_tag =
2193             rman_get_bustag(adapter->memory);
2194         adapter->osdep.mem_bus_space_handle =
2195             rman_get_bushandle(adapter->memory);
2196
2197         /* XXX This is quite goofy, it is not actually used */
2198         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2199
2200         /* Only older adapters use IO mapping */
2201         if (adapter->hw.mac.type > e1000_82543 &&
2202             adapter->hw.mac.type < e1000_82571) {
2203                 /* Figure our where our IO BAR is ? */
2204                 for (rid = PCIR_BAR(0); rid < PCIR_CARDBUSCIS;) {
2205                         val = pci_read_config(dev, rid, 4);
2206                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2207                                 adapter->io_rid = rid;
2208                                 break;
2209                         }
2210                         rid += 4;
2211                         /* check for 64bit BAR */
2212                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2213                                 rid += 4;
2214                 }
2215                 if (rid >= PCIR_CARDBUSCIS) {
2216                         device_printf(dev, "Unable to locate IO BAR\n");
2217                         return (ENXIO);
2218                 }
2219                 adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
2220                                         &adapter->io_rid, RF_ACTIVE);
2221                 if (adapter->ioport == NULL) {
2222                         device_printf(dev, "Unable to allocate bus resource: "
2223                             "ioport\n");
2224                         return (ENXIO);
2225                 }
2226                 adapter->hw.io_base = 0;
2227                 adapter->osdep.io_bus_space_tag =
2228                     rman_get_bustag(adapter->ioport);
2229                 adapter->osdep.io_bus_space_handle =
2230                     rman_get_bushandle(adapter->ioport);
2231         }
2232
2233         /*
2234          * Don't enable MSI-X on 82574, see:
2235          * 82574 specification update errata #15
2236          *
2237          * Don't enable MSI on PCI/PCI-X chips, see:
2238          * 82540 specification update errata #6
2239          * 82545 specification update errata #4
2240          *
2241          * Don't enable MSI on 82571/82572, see:
2242          * 82571/82572 specification update errata #63
2243          */
2244         msi_enable = em_msi_enable;
2245         if (msi_enable &&
2246             (!pci_is_pcie(dev) ||
2247              adapter->hw.mac.type == e1000_82571 ||
2248              adapter->hw.mac.type == e1000_82572))
2249                 msi_enable = 0;
2250
2251         adapter->intr_type = pci_alloc_1intr(dev, msi_enable,
2252             &adapter->intr_rid, &intr_flags);
2253
2254         if (adapter->intr_type == PCI_INTR_TYPE_LEGACY) {
2255                 int unshared;
2256
2257                 unshared = device_getenv_int(dev, "irq.unshared", 0);
2258                 if (!unshared) {
2259                         adapter->flags |= EM_FLAG_SHARED_INTR;
2260                         if (bootverbose)
2261                                 device_printf(dev, "IRQ shared\n");
2262                 } else {
2263                         intr_flags &= ~RF_SHAREABLE;
2264                         if (bootverbose)
2265                                 device_printf(dev, "IRQ unshared\n");
2266                 }
2267         }
2268
2269         adapter->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2270             &adapter->intr_rid, intr_flags);
2271         if (adapter->intr_res == NULL) {
2272                 device_printf(dev, "Unable to allocate bus resource: "
2273                     "interrupt\n");
2274                 return (ENXIO);
2275         }
2276
2277         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2278         adapter->hw.back = &adapter->osdep;
2279         return (0);
2280 }
2281
2282 static void
2283 em_free_pci_res(struct adapter *adapter)
2284 {
2285         device_t dev = adapter->dev;
2286
2287         if (adapter->intr_res != NULL) {
2288                 bus_release_resource(dev, SYS_RES_IRQ,
2289                     adapter->intr_rid, adapter->intr_res);
2290         }
2291
2292         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2293                 pci_release_msi(dev);
2294
2295         if (adapter->memory != NULL) {
2296                 bus_release_resource(dev, SYS_RES_MEMORY,
2297                     adapter->memory_rid, adapter->memory);
2298         }
2299
2300         if (adapter->flash != NULL) {
2301                 bus_release_resource(dev, SYS_RES_MEMORY,
2302                     adapter->flash_rid, adapter->flash);
2303         }
2304
2305         if (adapter->ioport != NULL) {
2306                 bus_release_resource(dev, SYS_RES_IOPORT,
2307                     adapter->io_rid, adapter->ioport);
2308         }
2309 }
2310
2311 static int
2312 em_reset(struct adapter *adapter)
2313 {
2314         device_t dev = adapter->dev;
2315         uint16_t rx_buffer_size;
2316         uint32_t pba;
2317
2318         /* When hardware is reset, fifo_head is also reset */
2319         adapter->tx_fifo_head = 0;
2320
2321         /* Set up smart power down as default off on newer adapters. */
2322         if (!em_smart_pwr_down &&
2323             (adapter->hw.mac.type == e1000_82571 ||
2324              adapter->hw.mac.type == e1000_82572)) {
2325                 uint16_t phy_tmp = 0;
2326
2327                 /* Speed up time to link by disabling smart power down. */
2328                 e1000_read_phy_reg(&adapter->hw,
2329                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2330                 phy_tmp &= ~IGP02E1000_PM_SPD;
2331                 e1000_write_phy_reg(&adapter->hw,
2332                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2333         }
2334
2335         /*
2336          * Packet Buffer Allocation (PBA)
2337          * Writing PBA sets the receive portion of the buffer
2338          * the remainder is used for the transmit buffer.
2339          *
2340          * Devices before the 82547 had a Packet Buffer of 64K.
2341          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
2342          * After the 82547 the buffer was reduced to 40K.
2343          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
2344          *   Note: default does not leave enough room for Jumbo Frame >10k.
2345          */
2346         switch (adapter->hw.mac.type) {
2347         case e1000_82547:
2348         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
2349                 if (adapter->hw.mac.max_frame_size > 8192)
2350                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
2351                 else
2352                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
2353                 adapter->tx_fifo_head = 0;
2354                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
2355                 adapter->tx_fifo_size =
2356                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
2357                 break;
2358
2359         /* Total Packet Buffer on these is 48K */
2360         case e1000_82571:
2361         case e1000_82572:
2362         case e1000_80003es2lan:
2363                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2364                 break;
2365
2366         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2367                 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2368                 break;
2369
2370         case e1000_82574:
2371         case e1000_82583:
2372                 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2373                 break;
2374
2375         case e1000_ich8lan:
2376                 pba = E1000_PBA_8K;
2377                 break;
2378
2379         case e1000_ich9lan:
2380         case e1000_ich10lan:
2381 #define E1000_PBA_10K   0x000A
2382                 pba = E1000_PBA_10K;
2383                 break;
2384
2385         case e1000_pchlan:
2386         case e1000_pch2lan:
2387         case e1000_pch_lpt:
2388                 pba = E1000_PBA_26K;
2389                 break;
2390
2391         default:
2392                 /* Devices before 82547 had a Packet Buffer of 64K.   */
2393                 if (adapter->hw.mac.max_frame_size > 8192)
2394                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2395                 else
2396                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2397         }
2398         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2399
2400         /*
2401          * These parameters control the automatic generation (Tx) and
2402          * response (Rx) to Ethernet PAUSE frames.
2403          * - High water mark should allow for at least two frames to be
2404          *   received after sending an XOFF.
2405          * - Low water mark works best when it is very near the high water mark.
2406          *   This allows the receiver to restart by sending XON when it has
2407          *   drained a bit. Here we use an arbitary value of 1500 which will
2408          *   restart after one full frame is pulled from the buffer. There
2409          *   could be several smaller frames in the buffer and if so they will
2410          *   not trigger the XON until their total number reduces the buffer
2411          *   by 1500.
2412          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2413          */
2414         rx_buffer_size =
2415                 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) << 10;
2416
2417         adapter->hw.fc.high_water = rx_buffer_size -
2418             roundup2(adapter->hw.mac.max_frame_size, 1024);
2419         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
2420
2421         if (adapter->hw.mac.type == e1000_80003es2lan)
2422                 adapter->hw.fc.pause_time = 0xFFFF;
2423         else
2424                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
2425
2426         adapter->hw.fc.send_xon = TRUE;
2427
2428         adapter->hw.fc.requested_mode = e1000_fc_full;
2429
2430         /*
2431          * Device specific overrides/settings
2432          */
2433         switch (adapter->hw.mac.type) {
2434         case e1000_pchlan:
2435                 /* Workaround: no TX flow ctrl for PCH */
2436                 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
2437                 adapter->hw.fc.pause_time = 0xFFFF; /* override */
2438                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2439                         adapter->hw.fc.high_water = 0x3500;
2440                         adapter->hw.fc.low_water = 0x1500;
2441                 } else {
2442                         adapter->hw.fc.high_water = 0x5000;
2443                         adapter->hw.fc.low_water = 0x3000;
2444                 }
2445                 adapter->hw.fc.refresh_time = 0x1000;
2446                 break;
2447
2448         case e1000_pch2lan:
2449         case e1000_pch_lpt:
2450                 adapter->hw.fc.high_water = 0x5C20;
2451                 adapter->hw.fc.low_water = 0x5048;
2452                 adapter->hw.fc.pause_time = 0x0650;
2453                 adapter->hw.fc.refresh_time = 0x0400;
2454                 /* Jumbos need adjusted PBA */
2455                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU)
2456                         E1000_WRITE_REG(&adapter->hw, E1000_PBA, 12);
2457                 else
2458                         E1000_WRITE_REG(&adapter->hw, E1000_PBA, 26);
2459                 break;
2460
2461         case e1000_ich9lan:
2462         case e1000_ich10lan:
2463                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2464                         adapter->hw.fc.high_water = 0x2800;
2465                         adapter->hw.fc.low_water =
2466                             adapter->hw.fc.high_water - 8;
2467                         break;
2468                 }
2469                 /* FALL THROUGH */
2470         default:
2471                 if (adapter->hw.mac.type == e1000_80003es2lan)
2472                         adapter->hw.fc.pause_time = 0xFFFF;
2473                 break;
2474         }
2475
2476         /* Issue a global reset */
2477         e1000_reset_hw(&adapter->hw);
2478         if (adapter->hw.mac.type >= e1000_82544)
2479                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2480         em_disable_aspm(adapter);
2481
2482         if (e1000_init_hw(&adapter->hw) < 0) {
2483                 device_printf(dev, "Hardware Initialization Failed\n");
2484                 return (EIO);
2485         }
2486
2487         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2488         e1000_get_phy_info(&adapter->hw);
2489         e1000_check_for_link(&adapter->hw);
2490
2491         return (0);
2492 }
2493
2494 static void
2495 em_setup_ifp(struct adapter *adapter)
2496 {
2497         struct ifnet *ifp = &adapter->arpcom.ac_if;
2498
2499         if_initname(ifp, device_get_name(adapter->dev),
2500                     device_get_unit(adapter->dev));
2501         ifp->if_softc = adapter;
2502         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2503         ifp->if_init =  em_init;
2504         ifp->if_ioctl = em_ioctl;
2505         ifp->if_start = em_start;
2506 #ifdef IFPOLL_ENABLE
2507         ifp->if_npoll = em_npoll;
2508 #endif
2509         ifp->if_watchdog = em_watchdog;
2510         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
2511         ifq_set_ready(&ifp->if_snd);
2512
2513         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2514
2515         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2516         if (adapter->hw.mac.type >= e1000_82543)
2517                 ifp->if_capabilities |= IFCAP_HWCSUM;
2518         if (adapter->flags & EM_FLAG_TSO)
2519                 ifp->if_capabilities |= IFCAP_TSO;
2520         ifp->if_capenable = ifp->if_capabilities;
2521
2522         if (ifp->if_capenable & IFCAP_TXCSUM)
2523                 ifp->if_hwassist |= EM_CSUM_FEATURES;
2524         if (ifp->if_capenable & IFCAP_TSO)
2525                 ifp->if_hwassist |= CSUM_TSO;
2526
2527         /*
2528          * Tell the upper layer(s) we support long frames.
2529          */
2530         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2531
2532         /*
2533          * Specify the media types supported by this adapter and register
2534          * callbacks to update media and link information
2535          */
2536         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
2537             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
2538                 u_char fiber_type = IFM_1000_SX; /* default type */
2539
2540                 if (adapter->hw.mac.type == e1000_82545)
2541                         fiber_type = IFM_1000_LX;
2542                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2543                             0, NULL);
2544                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2545         } else {
2546                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2547                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2548                             0, NULL);
2549                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2550                             0, NULL);
2551                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2552                             0, NULL);
2553                 if (adapter->hw.phy.type != e1000_phy_ife) {
2554                         ifmedia_add(&adapter->media,
2555                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2556                         ifmedia_add(&adapter->media,
2557                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2558                 }
2559         }
2560         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2561         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2562 }
2563
2564
2565 /*
2566  * Workaround for SmartSpeed on 82541 and 82547 controllers
2567  */
2568 static void
2569 em_smartspeed(struct adapter *adapter)
2570 {
2571         uint16_t phy_tmp;
2572
2573         if (adapter->link_active || adapter->hw.phy.type != e1000_phy_igp ||
2574             adapter->hw.mac.autoneg == 0 ||
2575             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2576                 return;
2577
2578         if (adapter->smartspeed == 0) {
2579                 /*
2580                  * If Master/Slave config fault is asserted twice,
2581                  * we assume back-to-back
2582                  */
2583                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2584                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2585                         return;
2586                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2587                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2588                         e1000_read_phy_reg(&adapter->hw,
2589                             PHY_1000T_CTRL, &phy_tmp);
2590                         if (phy_tmp & CR_1000T_MS_ENABLE) {
2591                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
2592                                 e1000_write_phy_reg(&adapter->hw,
2593                                     PHY_1000T_CTRL, phy_tmp);
2594                                 adapter->smartspeed++;
2595                                 if (adapter->hw.mac.autoneg &&
2596                                     !e1000_phy_setup_autoneg(&adapter->hw) &&
2597                                     !e1000_read_phy_reg(&adapter->hw,
2598                                      PHY_CONTROL, &phy_tmp)) {
2599                                         phy_tmp |= MII_CR_AUTO_NEG_EN |
2600                                                    MII_CR_RESTART_AUTO_NEG;
2601                                         e1000_write_phy_reg(&adapter->hw,
2602                                             PHY_CONTROL, phy_tmp);
2603                                 }
2604                         }
2605                 }
2606                 return;
2607         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2608                 /* If still no link, perhaps using 2/3 pair cable */
2609                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2610                 phy_tmp |= CR_1000T_MS_ENABLE;
2611                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2612                 if (adapter->hw.mac.autoneg &&
2613                     !e1000_phy_setup_autoneg(&adapter->hw) &&
2614                     !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
2615                         phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG;
2616                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
2617                 }
2618         }
2619
2620         /* Restart process after EM_SMARTSPEED_MAX iterations */
2621         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2622                 adapter->smartspeed = 0;
2623 }
2624
2625 static int
2626 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2627               struct em_dma_alloc *dma)
2628 {
2629         dma->dma_vaddr = bus_dmamem_coherent_any(adapter->parent_dtag,
2630                                 EM_DBA_ALIGN, size, BUS_DMA_WAITOK,
2631                                 &dma->dma_tag, &dma->dma_map,
2632                                 &dma->dma_paddr);
2633         if (dma->dma_vaddr == NULL)
2634                 return ENOMEM;
2635         else
2636                 return 0;
2637 }
2638
2639 static void
2640 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2641 {
2642         if (dma->dma_tag == NULL)
2643                 return;
2644         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2645         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2646         bus_dma_tag_destroy(dma->dma_tag);
2647 }
2648
2649 static int
2650 em_create_tx_ring(struct adapter *adapter)
2651 {
2652         device_t dev = adapter->dev;
2653         struct em_buffer *tx_buffer;
2654         int error, i;
2655
2656         adapter->tx_buffer_area =
2657                 kmalloc(sizeof(struct em_buffer) * adapter->num_tx_desc,
2658                         M_DEVBUF, M_WAITOK | M_ZERO);
2659
2660         /*
2661          * Create DMA tags for tx buffers
2662          */
2663         error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
2664                         1, 0,                   /* alignment, bounds */
2665                         BUS_SPACE_MAXADDR,      /* lowaddr */
2666                         BUS_SPACE_MAXADDR,      /* highaddr */
2667                         NULL, NULL,             /* filter, filterarg */
2668                         EM_TSO_SIZE,            /* maxsize */
2669                         EM_MAX_SCATTER,         /* nsegments */
2670                         PAGE_SIZE,              /* maxsegsize */
2671                         BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
2672                         BUS_DMA_ONEBPAGE,       /* flags */
2673                         &adapter->txtag);
2674         if (error) {
2675                 device_printf(dev, "Unable to allocate TX DMA tag\n");
2676                 kfree(adapter->tx_buffer_area, M_DEVBUF);
2677                 adapter->tx_buffer_area = NULL;
2678                 return error;
2679         }
2680
2681         /*
2682          * Create DMA maps for tx buffers
2683          */
2684         for (i = 0; i < adapter->num_tx_desc; i++) {
2685                 tx_buffer = &adapter->tx_buffer_area[i];
2686
2687                 error = bus_dmamap_create(adapter->txtag,
2688                                           BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
2689                                           &tx_buffer->map);
2690                 if (error) {
2691                         device_printf(dev, "Unable to create TX DMA map\n");
2692                         em_destroy_tx_ring(adapter, i);
2693                         return error;
2694                 }
2695         }
2696         return (0);
2697 }
2698
2699 static void
2700 em_init_tx_ring(struct adapter *adapter)
2701 {
2702         /* Clear the old ring contents */
2703         bzero(adapter->tx_desc_base,
2704             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
2705
2706         /* Reset state */
2707         adapter->next_avail_tx_desc = 0;
2708         adapter->next_tx_to_clean = 0;
2709         adapter->num_tx_desc_avail = adapter->num_tx_desc;
2710 }
2711
2712 static void
2713 em_init_tx_unit(struct adapter *adapter)
2714 {
2715         uint32_t tctl, tarc, tipg = 0;
2716         uint64_t bus_addr;
2717
2718         /* Setup the Base and Length of the Tx Descriptor Ring */
2719         bus_addr = adapter->txdma.dma_paddr;
2720         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
2721             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2722         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
2723             (uint32_t)(bus_addr >> 32));
2724         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
2725             (uint32_t)bus_addr);
2726         /* Setup the HW Tx Head and Tail descriptor pointers */
2727         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
2728         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
2729
2730         /* Set the default values for the Tx Inter Packet Gap timer */
2731         switch (adapter->hw.mac.type) {
2732         case e1000_82542:
2733                 tipg = DEFAULT_82542_TIPG_IPGT;
2734                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2735                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2736                 break;
2737
2738         case e1000_80003es2lan:
2739                 tipg = DEFAULT_82543_TIPG_IPGR1;
2740                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
2741                     E1000_TIPG_IPGR2_SHIFT;
2742                 break;
2743
2744         default:
2745                 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
2746                     adapter->hw.phy.media_type ==
2747                     e1000_media_type_internal_serdes)
2748                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2749                 else
2750                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2751                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2752                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2753                 break;
2754         }
2755
2756         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
2757
2758         /* NOTE: 0 is not allowed for TIDV */
2759         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, 1);
2760         if(adapter->hw.mac.type >= e1000_82540)
2761                 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 0);
2762
2763         if (adapter->hw.mac.type == e1000_82571 ||
2764             adapter->hw.mac.type == e1000_82572) {
2765                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
2766                 tarc |= SPEED_MODE_BIT;
2767                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
2768         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
2769                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
2770                 tarc |= 1;
2771                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
2772                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
2773                 tarc |= 1;
2774                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
2775         }
2776
2777         /* Program the Transmit Control Register */
2778         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2779         tctl &= ~E1000_TCTL_CT;
2780         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
2781                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2782
2783         if (adapter->hw.mac.type >= e1000_82571)
2784                 tctl |= E1000_TCTL_MULR;
2785
2786         /* This write will effectively turn on the transmit unit. */
2787         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2788
2789         if (adapter->hw.mac.type == e1000_82571 ||
2790             adapter->hw.mac.type == e1000_82572 ||
2791             adapter->hw.mac.type == e1000_80003es2lan) {
2792                 /* Bit 28 of TARC1 must be cleared when MULR is enabled */
2793                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
2794                 tarc &= ~(1 << 28);
2795                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
2796         }
2797 }
2798
2799 static void
2800 em_destroy_tx_ring(struct adapter *adapter, int ndesc)
2801 {
2802         struct em_buffer *tx_buffer;
2803         int i;
2804
2805         if (adapter->tx_buffer_area == NULL)
2806                 return;
2807
2808         for (i = 0; i < ndesc; i++) {
2809                 tx_buffer = &adapter->tx_buffer_area[i];
2810
2811                 KKASSERT(tx_buffer->m_head == NULL);
2812                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2813         }
2814         bus_dma_tag_destroy(adapter->txtag);
2815
2816         kfree(adapter->tx_buffer_area, M_DEVBUF);
2817         adapter->tx_buffer_area = NULL;
2818 }
2819
2820 /*
2821  * The offload context needs to be set when we transfer the first
2822  * packet of a particular protocol (TCP/UDP).  This routine has been
2823  * enhanced to deal with inserted VLAN headers.
2824  *
2825  * If the new packet's ether header length, ip header length and
2826  * csum offloading type are same as the previous packet, we should
2827  * avoid allocating a new csum context descriptor; mainly to take
2828  * advantage of the pipeline effect of the TX data read request.
2829  *
2830  * This function returns number of TX descrptors allocated for
2831  * csum context.
2832  */
2833 static int
2834 em_txcsum(struct adapter *adapter, struct mbuf *mp,
2835           uint32_t *txd_upper, uint32_t *txd_lower)
2836 {
2837         struct e1000_context_desc *TXD;
2838         int curr_txd, ehdrlen, csum_flags;
2839         uint32_t cmd, hdr_len, ip_hlen;
2840
2841         csum_flags = mp->m_pkthdr.csum_flags & EM_CSUM_FEATURES;
2842         ip_hlen = mp->m_pkthdr.csum_iphlen;
2843         ehdrlen = mp->m_pkthdr.csum_lhlen;
2844
2845         if (adapter->csum_lhlen == ehdrlen &&
2846             adapter->csum_iphlen == ip_hlen &&
2847             adapter->csum_flags == csum_flags) {
2848                 /*
2849                  * Same csum offload context as the previous packets;
2850                  * just return.
2851                  */
2852                 *txd_upper = adapter->csum_txd_upper;
2853                 *txd_lower = adapter->csum_txd_lower;
2854                 return 0;
2855         }
2856
2857         /*
2858          * Setup a new csum offload context.
2859          */
2860
2861         curr_txd = adapter->next_avail_tx_desc;
2862         TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
2863
2864         cmd = 0;
2865
2866         /* Setup of IP header checksum. */
2867         if (csum_flags & CSUM_IP) {
2868                 /*
2869                  * Start offset for header checksum calculation.
2870                  * End offset for header checksum calculation.
2871                  * Offset of place to put the checksum.
2872                  */
2873                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
2874                 TXD->lower_setup.ip_fields.ipcse =
2875                     htole16(ehdrlen + ip_hlen - 1);
2876                 TXD->lower_setup.ip_fields.ipcso =
2877                     ehdrlen + offsetof(struct ip, ip_sum);
2878                 cmd |= E1000_TXD_CMD_IP;
2879                 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2880         }
2881         hdr_len = ehdrlen + ip_hlen;
2882
2883         if (csum_flags & CSUM_TCP) {
2884                 /*
2885                  * Start offset for payload checksum calculation.
2886                  * End offset for payload checksum calculation.
2887                  * Offset of place to put the checksum.
2888                  */
2889                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
2890                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2891                 TXD->upper_setup.tcp_fields.tucso =
2892                     hdr_len + offsetof(struct tcphdr, th_sum);
2893                 cmd |= E1000_TXD_CMD_TCP;
2894                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2895         } else if (csum_flags & CSUM_UDP) {
2896                 /*
2897                  * Start offset for header checksum calculation.
2898                  * End offset for header checksum calculation.
2899                  * Offset of place to put the checksum.
2900                  */
2901                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
2902                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2903                 TXD->upper_setup.tcp_fields.tucso =
2904                     hdr_len + offsetof(struct udphdr, uh_sum);
2905                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2906         }
2907
2908         *txd_lower = E1000_TXD_CMD_DEXT |       /* Extended descr type */
2909                      E1000_TXD_DTYP_D;          /* Data descr */
2910
2911         /* Save the information for this csum offloading context */
2912         adapter->csum_lhlen = ehdrlen;
2913         adapter->csum_iphlen = ip_hlen;
2914         adapter->csum_flags = csum_flags;
2915         adapter->csum_txd_upper = *txd_upper;
2916         adapter->csum_txd_lower = *txd_lower;
2917
2918         TXD->tcp_seg_setup.data = htole32(0);
2919         TXD->cmd_and_length =
2920             htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
2921
2922         if (++curr_txd == adapter->num_tx_desc)
2923                 curr_txd = 0;
2924
2925         KKASSERT(adapter->num_tx_desc_avail > 0);
2926         adapter->num_tx_desc_avail--;
2927
2928         adapter->next_avail_tx_desc = curr_txd;
2929         return 1;
2930 }
2931
2932 static void
2933 em_txeof(struct adapter *adapter)
2934 {
2935         struct ifnet *ifp = &adapter->arpcom.ac_if;
2936         struct em_buffer *tx_buffer;
2937         int first, num_avail;
2938
2939         if (adapter->tx_dd_head == adapter->tx_dd_tail)
2940                 return;
2941
2942         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2943                 return;
2944
2945         num_avail = adapter->num_tx_desc_avail;
2946         first = adapter->next_tx_to_clean;
2947
2948         while (adapter->tx_dd_head != adapter->tx_dd_tail) {
2949                 struct e1000_tx_desc *tx_desc;
2950                 int dd_idx = adapter->tx_dd[adapter->tx_dd_head];
2951
2952                 tx_desc = &adapter->tx_desc_base[dd_idx];
2953                 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2954                         EM_INC_TXDD_IDX(adapter->tx_dd_head);
2955
2956                         if (++dd_idx == adapter->num_tx_desc)
2957                                 dd_idx = 0;
2958
2959                         while (first != dd_idx) {
2960                                 logif(pkt_txclean);
2961
2962                                 num_avail++;
2963
2964                                 tx_buffer = &adapter->tx_buffer_area[first];
2965                                 if (tx_buffer->m_head) {
2966                                         IFNET_STAT_INC(ifp, opackets, 1);
2967                                         bus_dmamap_unload(adapter->txtag,
2968                                                           tx_buffer->map);
2969                                         m_freem(tx_buffer->m_head);
2970                                         tx_buffer->m_head = NULL;
2971                                 }
2972
2973                                 if (++first == adapter->num_tx_desc)
2974                                         first = 0;
2975                         }
2976                 } else {
2977                         break;
2978                 }
2979         }
2980         adapter->next_tx_to_clean = first;
2981         adapter->num_tx_desc_avail = num_avail;
2982
2983         if (adapter->tx_dd_head == adapter->tx_dd_tail) {
2984                 adapter->tx_dd_head = 0;
2985                 adapter->tx_dd_tail = 0;
2986         }
2987
2988         if (!EM_IS_OACTIVE(adapter)) {
2989                 ifq_clr_oactive(&ifp->if_snd);
2990
2991                 /* All clean, turn off the timer */
2992                 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2993                         ifp->if_timer = 0;
2994         }
2995 }
2996
2997 static void
2998 em_tx_collect(struct adapter *adapter)
2999 {
3000         struct ifnet *ifp = &adapter->arpcom.ac_if;
3001         struct em_buffer *tx_buffer;
3002         int tdh, first, num_avail, dd_idx = -1;
3003
3004         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3005                 return;
3006
3007         tdh = E1000_READ_REG(&adapter->hw, E1000_TDH(0));
3008         if (tdh == adapter->next_tx_to_clean)
3009                 return;
3010
3011         if (adapter->tx_dd_head != adapter->tx_dd_tail)
3012                 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3013
3014         num_avail = adapter->num_tx_desc_avail;
3015         first = adapter->next_tx_to_clean;
3016
3017         while (first != tdh) {
3018                 logif(pkt_txclean);
3019
3020                 num_avail++;
3021
3022                 tx_buffer = &adapter->tx_buffer_area[first];
3023                 if (tx_buffer->m_head) {
3024                         IFNET_STAT_INC(ifp, opackets, 1);
3025                         bus_dmamap_unload(adapter->txtag,
3026                                           tx_buffer->map);
3027                         m_freem(tx_buffer->m_head);
3028                         tx_buffer->m_head = NULL;
3029                 }
3030
3031                 if (first == dd_idx) {
3032                         EM_INC_TXDD_IDX(adapter->tx_dd_head);
3033                         if (adapter->tx_dd_head == adapter->tx_dd_tail) {
3034                                 adapter->tx_dd_head = 0;
3035                                 adapter->tx_dd_tail = 0;
3036                                 dd_idx = -1;
3037                         } else {
3038                                 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3039                         }
3040                 }
3041
3042                 if (++first == adapter->num_tx_desc)
3043                         first = 0;
3044         }
3045         adapter->next_tx_to_clean = first;
3046         adapter->num_tx_desc_avail = num_avail;
3047
3048         if (!EM_IS_OACTIVE(adapter)) {
3049                 ifq_clr_oactive(&ifp->if_snd);
3050
3051                 /* All clean, turn off the timer */
3052                 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3053                         ifp->if_timer = 0;
3054         }
3055 }
3056
3057 /*
3058  * When Link is lost sometimes there is work still in the TX ring
3059  * which will result in a watchdog, rather than allow that do an
3060  * attempted cleanup and then reinit here.  Note that this has been
3061  * seens mostly with fiber adapters.
3062  */
3063 static void
3064 em_tx_purge(struct adapter *adapter)
3065 {
3066         struct ifnet *ifp = &adapter->arpcom.ac_if;
3067
3068         if (!adapter->link_active && ifp->if_timer) {
3069                 em_tx_collect(adapter);
3070                 if (ifp->if_timer) {
3071                         if_printf(ifp, "Link lost, TX pending, reinit\n");
3072                         ifp->if_timer = 0;
3073                         em_init(adapter);
3074                 }
3075         }
3076 }
3077
3078 static int
3079 em_newbuf(struct adapter *adapter, int i, int init)
3080 {
3081         struct mbuf *m;
3082         bus_dma_segment_t seg;
3083         bus_dmamap_t map;
3084         struct em_buffer *rx_buffer;
3085         int error, nseg;
3086
3087         m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
3088         if (m == NULL) {
3089                 adapter->mbuf_cluster_failed++;
3090                 if (init) {
3091                         if_printf(&adapter->arpcom.ac_if,
3092                                   "Unable to allocate RX mbuf\n");
3093                 }
3094                 return (ENOBUFS);
3095         }
3096         m->m_len = m->m_pkthdr.len = MCLBYTES;
3097
3098         if (adapter->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN)
3099                 m_adj(m, ETHER_ALIGN);
3100
3101         error = bus_dmamap_load_mbuf_segment(adapter->rxtag,
3102                         adapter->rx_sparemap, m,
3103                         &seg, 1, &nseg, BUS_DMA_NOWAIT);
3104         if (error) {
3105                 m_freem(m);
3106                 if (init) {
3107                         if_printf(&adapter->arpcom.ac_if,
3108                                   "Unable to load RX mbuf\n");
3109                 }
3110                 return (error);
3111         }
3112
3113         rx_buffer = &adapter->rx_buffer_area[i];
3114         if (rx_buffer->m_head != NULL)
3115                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3116
3117         map = rx_buffer->map;
3118         rx_buffer->map = adapter->rx_sparemap;
3119         adapter->rx_sparemap = map;
3120
3121         rx_buffer->m_head = m;
3122
3123         adapter->rx_desc_base[i].buffer_addr = htole64(seg.ds_addr);
3124         return (0);
3125 }
3126
3127 static int
3128 em_create_rx_ring(struct adapter *adapter)
3129 {
3130         device_t dev = adapter->dev;
3131         struct em_buffer *rx_buffer;
3132         int i, error;
3133
3134         adapter->rx_buffer_area =
3135                 kmalloc(sizeof(struct em_buffer) * adapter->num_rx_desc,
3136                         M_DEVBUF, M_WAITOK | M_ZERO);
3137
3138         /*
3139          * Create DMA tag for rx buffers
3140          */
3141         error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
3142                         1, 0,                   /* alignment, bounds */
3143                         BUS_SPACE_MAXADDR,      /* lowaddr */
3144                         BUS_SPACE_MAXADDR,      /* highaddr */
3145                         NULL, NULL,             /* filter, filterarg */
3146                         MCLBYTES,               /* maxsize */
3147                         1,                      /* nsegments */
3148                         MCLBYTES,               /* maxsegsize */
3149                         BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */
3150                         &adapter->rxtag);
3151         if (error) {
3152                 device_printf(dev, "Unable to allocate RX DMA tag\n");
3153                 kfree(adapter->rx_buffer_area, M_DEVBUF);
3154                 adapter->rx_buffer_area = NULL;
3155                 return error;
3156         }
3157
3158         /*
3159          * Create spare DMA map for rx buffers
3160          */
3161         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3162                                   &adapter->rx_sparemap);
3163         if (error) {
3164                 device_printf(dev, "Unable to create spare RX DMA map\n");
3165                 bus_dma_tag_destroy(adapter->rxtag);
3166                 kfree(adapter->rx_buffer_area, M_DEVBUF);
3167                 adapter->rx_buffer_area = NULL;
3168                 return error;
3169         }
3170
3171         /*
3172          * Create DMA maps for rx buffers
3173          */
3174         for (i = 0; i < adapter->num_rx_desc; i++) {
3175                 rx_buffer = &adapter->rx_buffer_area[i];
3176
3177                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3178                                           &rx_buffer->map);
3179                 if (error) {
3180                         device_printf(dev, "Unable to create RX DMA map\n");
3181                         em_destroy_rx_ring(adapter, i);
3182                         return error;
3183                 }
3184         }
3185         return (0);
3186 }
3187
3188 static int
3189 em_init_rx_ring(struct adapter *adapter)
3190 {
3191         int i, error;
3192
3193         /* Reset descriptor ring */
3194         bzero(adapter->rx_desc_base,
3195             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
3196
3197         /* Allocate new ones. */
3198         for (i = 0; i < adapter->num_rx_desc; i++) {
3199                 error = em_newbuf(adapter, i, 1);
3200                 if (error)
3201                         return (error);
3202         }
3203
3204         /* Setup our descriptor pointers */
3205         adapter->next_rx_desc_to_check = 0;
3206
3207         return (0);
3208 }
3209
3210 static void
3211 em_init_rx_unit(struct adapter *adapter)
3212 {
3213         struct ifnet *ifp = &adapter->arpcom.ac_if;
3214         uint64_t bus_addr;
3215         uint32_t rctl;
3216
3217         /*
3218          * Make sure receives are disabled while setting
3219          * up the descriptor ring
3220          */
3221         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3222         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3223
3224         if (adapter->hw.mac.type >= e1000_82540) {
3225                 uint32_t itr;
3226
3227                 /*
3228                  * Set the interrupt throttling rate. Value is calculated
3229                  * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns)
3230                  */
3231                 if (adapter->int_throttle_ceil)
3232                         itr = 1000000000 / 256 / adapter->int_throttle_ceil;
3233                 else
3234                         itr = 0;
3235                 em_set_itr(adapter, itr);
3236         }
3237
3238         /* Disable accelerated ackknowledge */
3239         if (adapter->hw.mac.type == e1000_82574) {
3240                 E1000_WRITE_REG(&adapter->hw,
3241                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
3242         }
3243
3244         /* Receive Checksum Offload for TCP and UDP */
3245         if (ifp->if_capenable & IFCAP_RXCSUM) {
3246                 uint32_t rxcsum;
3247
3248                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3249                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3250                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
3251         }
3252
3253         /*
3254          * XXX TEMPORARY WORKAROUND: on some systems with 82573
3255          * long latencies are observed, like Lenovo X60. This
3256          * change eliminates the problem, but since having positive
3257          * values in RDTR is a known source of problems on other
3258          * platforms another solution is being sought.
3259          */
3260         if (em_82573_workaround && adapter->hw.mac.type == e1000_82573) {
3261                 E1000_WRITE_REG(&adapter->hw, E1000_RADV, EM_RADV_82573);
3262                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, EM_RDTR_82573);
3263         }
3264
3265         /*
3266          * Setup the Base and Length of the Rx Descriptor Ring
3267          */
3268         bus_addr = adapter->rxdma.dma_paddr;
3269         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
3270             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3271         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
3272             (uint32_t)(bus_addr >> 32));
3273         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
3274             (uint32_t)bus_addr);
3275
3276         /*
3277          * Setup the HW Rx Head and Tail Descriptor Pointers
3278          */
3279         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
3280         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
3281
3282         /* Set PTHRESH for improved jumbo performance */
3283         if (((adapter->hw.mac.type == e1000_ich9lan) ||
3284             (adapter->hw.mac.type == e1000_pch2lan) ||
3285             (adapter->hw.mac.type == e1000_ich10lan)) &&
3286             (ifp->if_mtu > ETHERMTU)) {
3287                 uint32_t rxdctl;
3288
3289                 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(0));
3290                 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(0), rxdctl | 3);
3291         }
3292
3293         if (adapter->hw.mac.type >= e1000_pch2lan) {
3294                 if (ifp->if_mtu > ETHERMTU)
3295                         e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, TRUE);
3296                 else
3297                         e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, FALSE);
3298         }
3299
3300         /* Setup the Receive Control Register */
3301         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3302         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3303                 E1000_RCTL_RDMTS_HALF |
3304                 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3305
3306         /* Make sure VLAN Filters are off */
3307         rctl &= ~E1000_RCTL_VFE;
3308
3309         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
3310                 rctl |= E1000_RCTL_SBP;
3311         else
3312                 rctl &= ~E1000_RCTL_SBP;
3313
3314         switch (adapter->rx_buffer_len) {
3315         default:
3316         case 2048:
3317                 rctl |= E1000_RCTL_SZ_2048;
3318                 break;
3319
3320         case 4096:
3321                 rctl |= E1000_RCTL_SZ_4096 |
3322                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3323                 break;
3324
3325         case 8192:
3326                 rctl |= E1000_RCTL_SZ_8192 |
3327                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3328                 break;
3329
3330         case 16384:
3331                 rctl |= E1000_RCTL_SZ_16384 |
3332                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3333                 break;
3334         }
3335
3336         if (ifp->if_mtu > ETHERMTU)
3337                 rctl |= E1000_RCTL_LPE;
3338         else
3339                 rctl &= ~E1000_RCTL_LPE;
3340
3341         /* Enable Receives */
3342         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3343 }
3344
3345 static void
3346 em_destroy_rx_ring(struct adapter *adapter, int ndesc)
3347 {
3348         struct em_buffer *rx_buffer;
3349         int i;
3350
3351         if (adapter->rx_buffer_area == NULL)
3352                 return;
3353
3354         for (i = 0; i < ndesc; i++) {
3355                 rx_buffer = &adapter->rx_buffer_area[i];
3356
3357                 KKASSERT(rx_buffer->m_head == NULL);
3358                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
3359         }
3360         bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3361         bus_dma_tag_destroy(adapter->rxtag);
3362
3363         kfree(adapter->rx_buffer_area, M_DEVBUF);
3364         adapter->rx_buffer_area = NULL;
3365 }
3366
3367 static void
3368 em_rxeof(struct adapter *adapter, int count)
3369 {
3370         struct ifnet *ifp = &adapter->arpcom.ac_if;
3371         uint8_t status, accept_frame = 0, eop = 0;
3372         uint16_t len, desc_len, prev_len_adj;
3373         struct e1000_rx_desc *current_desc;
3374         struct mbuf *mp;
3375         int i;
3376
3377         i = adapter->next_rx_desc_to_check;
3378         current_desc = &adapter->rx_desc_base[i];
3379
3380         if (!(current_desc->status & E1000_RXD_STAT_DD))
3381                 return;
3382
3383         while ((current_desc->status & E1000_RXD_STAT_DD) && count != 0) {
3384                 struct mbuf *m = NULL;
3385
3386                 logif(pkt_receive);
3387
3388                 mp = adapter->rx_buffer_area[i].m_head;
3389
3390                 /*
3391                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3392                  * needs to access the last received byte in the mbuf.
3393                  */
3394                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3395                                 BUS_DMASYNC_POSTREAD);
3396
3397                 accept_frame = 1;
3398                 prev_len_adj = 0;
3399                 desc_len = le16toh(current_desc->length);
3400                 status = current_desc->status;
3401                 if (status & E1000_RXD_STAT_EOP) {
3402                         count--;
3403                         eop = 1;
3404                         if (desc_len < ETHER_CRC_LEN) {
3405                                 len = 0;
3406                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
3407                         } else {
3408                                 len = desc_len - ETHER_CRC_LEN;
3409                         }
3410                 } else {
3411                         eop = 0;
3412                         len = desc_len;
3413                 }
3414
3415                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3416                         uint8_t last_byte;
3417                         uint32_t pkt_len = desc_len;
3418
3419                         if (adapter->fmp != NULL)
3420                                 pkt_len += adapter->fmp->m_pkthdr.len;
3421
3422                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3423                         if (TBI_ACCEPT(&adapter->hw, status,
3424                             current_desc->errors, pkt_len, last_byte,
3425                             adapter->min_frame_size,
3426                             adapter->hw.mac.max_frame_size)) {
3427                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
3428                                     &adapter->stats, pkt_len,
3429                                     adapter->hw.mac.addr,
3430                                     adapter->hw.mac.max_frame_size);
3431                                 if (len > 0)
3432                                         len--;
3433                         } else {
3434                                 accept_frame = 0;
3435                         }
3436                 }
3437
3438                 if (accept_frame) {
3439                         if (em_newbuf(adapter, i, 0) != 0) {
3440                                 IFNET_STAT_INC(ifp, iqdrops, 1);
3441                                 goto discard;
3442                         }
3443
3444                         /* Assign correct length to the current fragment */
3445                         mp->m_len = len;
3446
3447                         if (adapter->fmp == NULL) {
3448                                 mp->m_pkthdr.len = len;
3449                                 adapter->fmp = mp; /* Store the first mbuf */
3450                                 adapter->lmp = mp;
3451                         } else {
3452                                 /*
3453                                  * Chain mbuf's together
3454                                  */
3455
3456                                 /*
3457                                  * Adjust length of previous mbuf in chain if
3458                                  * we received less than 4 bytes in the last
3459                                  * descriptor.
3460                                  */
3461                                 if (prev_len_adj > 0) {
3462                                         adapter->lmp->m_len -= prev_len_adj;
3463                                         adapter->fmp->m_pkthdr.len -=
3464                                             prev_len_adj;
3465                                 }
3466                                 adapter->lmp->m_next = mp;
3467                                 adapter->lmp = adapter->lmp->m_next;
3468                                 adapter->fmp->m_pkthdr.len += len;
3469                         }
3470
3471                         if (eop) {
3472                                 adapter->fmp->m_pkthdr.rcvif = ifp;
3473                                 IFNET_STAT_INC(ifp, ipackets, 1);
3474
3475                                 if (ifp->if_capenable & IFCAP_RXCSUM) {
3476                                         em_rxcsum(adapter, current_desc,
3477                                                   adapter->fmp);
3478                                 }
3479
3480                                 if (status & E1000_RXD_STAT_VP) {
3481                                         adapter->fmp->m_pkthdr.ether_vlantag =
3482                                             (le16toh(current_desc->special) &
3483                                             E1000_RXD_SPC_VLAN_MASK);
3484                                         adapter->fmp->m_flags |= M_VLANTAG;
3485                                 }
3486                                 m = adapter->fmp;
3487                                 adapter->fmp = NULL;
3488                                 adapter->lmp = NULL;
3489                         }
3490                 } else {
3491                         IFNET_STAT_INC(ifp, ierrors, 1);
3492 discard:
3493 #ifdef foo
3494                         /* Reuse loaded DMA map and just update mbuf chain */
3495                         mp = adapter->rx_buffer_area[i].m_head;
3496                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3497                         mp->m_data = mp->m_ext.ext_buf;
3498                         mp->m_next = NULL;
3499                         if (adapter->hw.mac.max_frame_size <=
3500                             (MCLBYTES - ETHER_ALIGN))
3501                                 m_adj(mp, ETHER_ALIGN);
3502 #endif
3503                         if (adapter->fmp != NULL) {
3504                                 m_freem(adapter->fmp);
3505                                 adapter->fmp = NULL;
3506                                 adapter->lmp = NULL;
3507                         }
3508                         m = NULL;
3509                 }
3510
3511                 /* Zero out the receive descriptors status. */
3512                 current_desc->status = 0;
3513
3514                 if (m != NULL)
3515                         ifp->if_input(ifp, m, NULL, -1);
3516
3517                 /* Advance our pointers to the next descriptor. */
3518                 if (++i == adapter->num_rx_desc)
3519                         i = 0;
3520                 current_desc = &adapter->rx_desc_base[i];
3521         }
3522         adapter->next_rx_desc_to_check = i;
3523
3524         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3525         if (--i < 0)
3526                 i = adapter->num_rx_desc - 1;
3527         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3528 }
3529
3530 static void
3531 em_rxcsum(struct adapter *adapter, struct e1000_rx_desc *rx_desc,
3532           struct mbuf *mp)
3533 {
3534         /* 82543 or newer only */
3535         if (adapter->hw.mac.type < e1000_82543 ||
3536             /* Ignore Checksum bit is set */
3537             (rx_desc->status & E1000_RXD_STAT_IXSM))
3538                 return;
3539
3540         if ((rx_desc->status & E1000_RXD_STAT_IPCS) &&
3541             !(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3542                 /* IP Checksum Good */
3543                 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
3544         }
3545
3546         if ((rx_desc->status & E1000_RXD_STAT_TCPCS) &&
3547             !(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3548                 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
3549                                            CSUM_PSEUDO_HDR |
3550                                            CSUM_FRAG_NOT_CHECKED;
3551                 mp->m_pkthdr.csum_data = htons(0xffff);
3552         }
3553 }
3554
3555 static void
3556 em_enable_intr(struct adapter *adapter)
3557 {
3558         uint32_t ims_mask = IMS_ENABLE_MASK;
3559
3560         lwkt_serialize_handler_enable(adapter->arpcom.ac_if.if_serializer);
3561
3562 #if 0
3563         /* XXX MSIX */
3564         if (adapter->hw.mac.type == e1000_82574) {
3565                 E1000_WRITE_REG(&adapter->hw, EM_EIAC, EM_MSIX_MASK);
3566                 ims_mask |= EM_MSIX_MASK;
3567         }
3568 #endif
3569         E1000_WRITE_REG(&adapter->hw, E1000_IMS, ims_mask);
3570 }
3571
3572 static void
3573 em_disable_intr(struct adapter *adapter)
3574 {
3575         uint32_t clear = 0xffffffff;
3576
3577         /*
3578          * The first version of 82542 had an errata where when link was forced
3579          * it would stay up even up even if the cable was disconnected.
3580          * Sequence errors were used to detect the disconnect and then the
3581          * driver would unforce the link.  This code in the in the ISR.  For
3582          * this to work correctly the Sequence error interrupt had to be
3583          * enabled all the time.
3584          */
3585         if (adapter->hw.mac.type == e1000_82542 &&
3586             adapter->hw.revision_id == E1000_REVISION_2)
3587                 clear &= ~E1000_ICR_RXSEQ;
3588         else if (adapter->hw.mac.type == e1000_82574)
3589                 E1000_WRITE_REG(&adapter->hw, EM_EIAC, 0);
3590
3591         E1000_WRITE_REG(&adapter->hw, E1000_IMC, clear);
3592
3593         adapter->npoll.ifpc_stcount = 0;
3594
3595         lwkt_serialize_handler_disable(adapter->arpcom.ac_if.if_serializer);
3596 }
3597
3598 /*
3599  * Bit of a misnomer, what this really means is
3600  * to enable OS management of the system... aka
3601  * to disable special hardware management features
3602  */
3603 static void
3604 em_get_mgmt(struct adapter *adapter)
3605 {
3606         /* A shared code workaround */
3607 #define E1000_82542_MANC2H E1000_MANC2H
3608         if (adapter->flags & EM_FLAG_HAS_MGMT) {
3609                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
3610                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3611
3612                 /* disable hardware interception of ARP */
3613                 manc &= ~(E1000_MANC_ARP_EN);
3614
3615                 /* enable receiving management packets to the host */
3616                 if (adapter->hw.mac.type >= e1000_82571) {
3617                         manc |= E1000_MANC_EN_MNG2HOST;
3618 #define E1000_MNG2HOST_PORT_623 (1 << 5)
3619 #define E1000_MNG2HOST_PORT_664 (1 << 6)
3620                         manc2h |= E1000_MNG2HOST_PORT_623;
3621                         manc2h |= E1000_MNG2HOST_PORT_664;
3622                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
3623                 }
3624
3625                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3626         }
3627 }
3628
3629 /*
3630  * Give control back to hardware management
3631  * controller if there is one.
3632  */
3633 static void
3634 em_rel_mgmt(struct adapter *adapter)
3635 {
3636         if (adapter->flags & EM_FLAG_HAS_MGMT) {
3637                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3638
3639                 /* re-enable hardware interception of ARP */
3640                 manc |= E1000_MANC_ARP_EN;
3641
3642                 if (adapter->hw.mac.type >= e1000_82571)
3643                         manc &= ~E1000_MANC_EN_MNG2HOST;
3644
3645                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3646         }
3647 }
3648
3649 /*
3650  * em_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3651  * For ASF and Pass Through versions of f/w this means that
3652  * the driver is loaded.  For AMT version (only with 82573)
3653  * of the f/w this means that the network i/f is open.
3654  */
3655 static void
3656 em_get_hw_control(struct adapter *adapter)
3657 {
3658         /* Let firmware know the driver has taken over */
3659         if (adapter->hw.mac.type == e1000_82573) {
3660                 uint32_t swsm;
3661
3662                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3663                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3664                     swsm | E1000_SWSM_DRV_LOAD);
3665         } else {
3666                 uint32_t ctrl_ext;
3667
3668                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3669                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3670                     ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
3671         }
3672         adapter->flags |= EM_FLAG_HW_CTRL;
3673 }
3674
3675 /*
3676  * em_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3677  * For ASF and Pass Through versions of f/w this means that the
3678  * driver is no longer loaded.  For AMT version (only with 82573)
3679  * of the f/w this means that the network i/f is closed.
3680  */
3681 static void
3682 em_rel_hw_control(struct adapter *adapter)
3683 {
3684         if ((adapter->flags & EM_FLAG_HW_CTRL) == 0)
3685                 return;
3686         adapter->flags &= ~EM_FLAG_HW_CTRL;
3687
3688         /* Let firmware taken over control of h/w */
3689         if (adapter->hw.mac.type == e1000_82573) {
3690                 uint32_t swsm;
3691
3692                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3693                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3694                     swsm & ~E1000_SWSM_DRV_LOAD);
3695         } else {
3696                 uint32_t ctrl_ext;
3697
3698                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3699                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3700                     ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
3701         }
3702 }
3703
3704 static int
3705 em_is_valid_eaddr(const uint8_t *addr)
3706 {
3707         char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
3708
3709         if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN))
3710                 return (FALSE);
3711
3712         return (TRUE);
3713 }
3714
3715 /*
3716  * Enable PCI Wake On Lan capability
3717  */
3718 void
3719 em_enable_wol(device_t dev)
3720 {
3721         uint16_t cap, status;
3722         uint8_t id;
3723
3724         /* First find the capabilities pointer*/
3725         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
3726
3727         /* Read the PM Capabilities */
3728         id = pci_read_config(dev, cap, 1);
3729         if (id != PCIY_PMG)     /* Something wrong */
3730                 return;
3731
3732         /*
3733          * OK, we have the power capabilities,
3734          * so now get the status register
3735          */
3736         cap += PCIR_POWER_STATUS;
3737         status = pci_read_config(dev, cap, 2);
3738         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3739         pci_write_config(dev, cap, status, 2);
3740 }
3741
3742
3743 /*
3744  * 82544 Coexistence issue workaround.
3745  *    There are 2 issues.
3746  *       1. Transmit Hang issue.
3747  *    To detect this issue, following equation can be used...
3748  *        SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3749  *        If SUM[3:0] is in between 1 to 4, we will have this issue.
3750  *
3751  *       2. DAC issue.
3752  *    To detect this issue, following equation can be used...
3753  *        SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3754  *        If SUM[3:0] is in between 9 to c, we will have this issue.
3755  *
3756  *    WORKAROUND:
3757  *        Make sure we do not have ending address
3758  *        as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3759  */
3760 static uint32_t
3761 em_82544_fill_desc(bus_addr_t address, uint32_t length, PDESC_ARRAY desc_array)
3762 {
3763         uint32_t safe_terminator;
3764
3765         /*
3766          * Since issue is sensitive to length and address.
3767          * Let us first check the address...
3768          */
3769         if (length <= 4) {
3770                 desc_array->descriptor[0].address = address;
3771                 desc_array->descriptor[0].length = length;
3772                 desc_array->elements = 1;
3773                 return (desc_array->elements);
3774         }
3775
3776         safe_terminator =
3777         (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
3778
3779         /* If it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3780         if (safe_terminator == 0 ||
3781             (safe_terminator > 4 && safe_terminator < 9) ||
3782             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
3783                 desc_array->descriptor[0].address = address;
3784                 desc_array->descriptor[0].length = length;
3785                 desc_array->elements = 1;
3786                 return (desc_array->elements);
3787         }
3788
3789         desc_array->descriptor[0].address = address;
3790         desc_array->descriptor[0].length = length - 4;
3791         desc_array->descriptor[1].address = address + (length - 4);
3792         desc_array->descriptor[1].length = 4;
3793         desc_array->elements = 2;
3794         return (desc_array->elements);
3795 }
3796
3797 static void
3798 em_update_stats(struct adapter *adapter)
3799 {
3800         struct ifnet *ifp = &adapter->arpcom.ac_if;
3801
3802         if (adapter->hw.phy.media_type == e1000_media_type_copper ||
3803             (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
3804                 adapter->stats.symerrs +=
3805                         E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
3806                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
3807         }
3808         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
3809         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
3810         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
3811         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
3812
3813         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
3814         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
3815         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
3816         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
3817         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
3818         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
3819         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
3820         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
3821         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
3822         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
3823         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
3824         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
3825         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
3826         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
3827         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
3828         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
3829         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
3830         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
3831         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
3832         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
3833
3834         /* For the 64-bit byte counters the low dword must be read first. */
3835         /* Both registers clear on the read of the high dword */
3836
3837         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
3838         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
3839
3840         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
3841         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
3842         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
3843         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
3844         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
3845
3846         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
3847         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
3848
3849         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
3850         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
3851         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
3852         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
3853         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
3854         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
3855         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
3856         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
3857         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
3858         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
3859
3860         if (adapter->hw.mac.type >= e1000_82543) {
3861                 adapter->stats.algnerrc +=
3862                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
3863                 adapter->stats.rxerrc +=
3864                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
3865                 adapter->stats.tncrs +=
3866                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
3867                 adapter->stats.cexterr +=
3868                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
3869                 adapter->stats.tsctc +=
3870                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
3871                 adapter->stats.tsctfc +=
3872                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
3873         }
3874
3875         IFNET_STAT_SET(ifp, collisions, adapter->stats.colc);
3876
3877         /* Rx Errors */
3878         IFNET_STAT_SET(ifp, ierrors,
3879             adapter->dropped_pkts + adapter->stats.rxerrc +
3880             adapter->stats.crcerrs + adapter->stats.algnerrc +
3881             adapter->stats.ruc + adapter->stats.roc +
3882             adapter->stats.mpc + adapter->stats.cexterr);
3883
3884         /* Tx Errors */
3885         IFNET_STAT_SET(ifp, oerrors,
3886             adapter->stats.ecol + adapter->stats.latecol +
3887             adapter->watchdog_events);
3888 }
3889
3890 static void
3891 em_print_debug_info(struct adapter *adapter)
3892 {
3893         device_t dev = adapter->dev;
3894         uint8_t *hw_addr = adapter->hw.hw_addr;
3895
3896         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
3897         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
3898             E1000_READ_REG(&adapter->hw, E1000_CTRL),
3899             E1000_READ_REG(&adapter->hw, E1000_RCTL));
3900         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
3901             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
3902             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
3903         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
3904             adapter->hw.fc.high_water,
3905             adapter->hw.fc.low_water);
3906         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3907             E1000_READ_REG(&adapter->hw, E1000_TIDV),
3908             E1000_READ_REG(&adapter->hw, E1000_TADV));
3909         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3910             E1000_READ_REG(&adapter->hw, E1000_RDTR),
3911             E1000_READ_REG(&adapter->hw, E1000_RADV));
3912         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
3913             (long long)adapter->tx_fifo_wrk_cnt,
3914             (long long)adapter->tx_fifo_reset_cnt);
3915         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
3916             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
3917             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
3918         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
3919             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
3920             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
3921         device_printf(dev, "Num Tx descriptors avail = %d\n",
3922             adapter->num_tx_desc_avail);
3923         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
3924             adapter->no_tx_desc_avail1);
3925         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
3926             adapter->no_tx_desc_avail2);
3927         device_printf(dev, "Std mbuf failed = %ld\n",
3928             adapter->mbuf_alloc_failed);
3929         device_printf(dev, "Std mbuf cluster failed = %ld\n",
3930             adapter->mbuf_cluster_failed);
3931         device_printf(dev, "Driver dropped packets = %ld\n",
3932             adapter->dropped_pkts);
3933         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
3934             adapter->no_tx_dma_setup);
3935 }
3936
3937 static void
3938 em_print_hw_stats(struct adapter *adapter)
3939 {
3940         device_t dev = adapter->dev;
3941
3942         device_printf(dev, "Excessive collisions = %lld\n",
3943             (long long)adapter->stats.ecol);
3944 #if (DEBUG_HW > 0)  /* Dont output these errors normally */
3945         device_printf(dev, "Symbol errors = %lld\n",
3946             (long long)adapter->stats.symerrs);
3947 #endif
3948         device_printf(dev, "Sequence errors = %lld\n",
3949             (long long)adapter->stats.sec);
3950         device_printf(dev, "Defer count = %lld\n",
3951             (long long)adapter->stats.dc);
3952         device_printf(dev, "Missed Packets = %lld\n",
3953             (long long)adapter->stats.mpc);
3954         device_printf(dev, "Receive No Buffers = %lld\n",
3955             (long long)adapter->stats.rnbc);
3956         /* RLEC is inaccurate on some hardware, calculate our own. */
3957         device_printf(dev, "Receive Length Errors = %lld\n",
3958             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
3959         device_printf(dev, "Receive errors = %lld\n",
3960             (long long)adapter->stats.rxerrc);
3961         device_printf(dev, "Crc errors = %lld\n",
3962             (long long)adapter->stats.crcerrs);
3963         device_printf(dev, "Alignment errors = %lld\n",
3964             (long long)adapter->stats.algnerrc);
3965         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
3966             (long long)adapter->stats.cexterr);
3967         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
3968         device_printf(dev, "watchdog timeouts = %ld\n",
3969             adapter->watchdog_events);
3970         device_printf(dev, "XON Rcvd = %lld\n",
3971             (long long)adapter->stats.xonrxc);
3972         device_printf(dev, "XON Xmtd = %lld\n",
3973             (long long)adapter->stats.xontxc);
3974         device_printf(dev, "XOFF Rcvd = %lld\n",
3975             (long long)adapter->stats.xoffrxc);
3976         device_printf(dev, "XOFF Xmtd = %lld\n",
3977             (long long)adapter->stats.xofftxc);
3978         device_printf(dev, "Good Packets Rcvd = %lld\n",
3979             (long long)adapter->stats.gprc);
3980         device_printf(dev, "Good Packets Xmtd = %lld\n",
3981             (long long)adapter->stats.gptc);
3982 }
3983
3984 static void
3985 em_print_nvm_info(struct adapter *adapter)
3986 {
3987         uint16_t eeprom_data;
3988         int i, j, row = 0;
3989
3990         /* Its a bit crude, but it gets the job done */
3991         kprintf("\nInterface EEPROM Dump:\n");
3992         kprintf("Offset\n0x0000  ");
3993         for (i = 0, j = 0; i < 32; i++, j++) {
3994                 if (j == 8) { /* Make the offset block */
3995                         j = 0; ++row;
3996                         kprintf("\n0x00%x0  ",row);
3997                 }
3998                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
3999                 kprintf("%04x ", eeprom_data);
4000         }
4001         kprintf("\n");
4002 }
4003
4004 static int
4005 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
4006 {
4007         struct adapter *adapter;
4008         struct ifnet *ifp;
4009         int error, result;
4010
4011         result = -1;
4012         error = sysctl_handle_int(oidp, &result, 0, req);
4013         if (error || !req->newptr)
4014                 return (error);
4015
4016         adapter = (struct adapter *)arg1;
4017         ifp = &adapter->arpcom.ac_if;
4018
4019         lwkt_serialize_enter(ifp->if_serializer);
4020
4021         if (result == 1)
4022                 em_print_debug_info(adapter);
4023
4024         /*
4025          * This value will cause a hex dump of the
4026          * first 32 16-bit words of the EEPROM to
4027          * the screen.
4028          */
4029         if (result == 2)
4030                 em_print_nvm_info(adapter);
4031
4032         lwkt_serialize_exit(ifp->if_serializer);
4033
4034         return (error);
4035 }
4036
4037 static int
4038 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
4039 {
4040         int error, result;
4041
4042         result = -1;
4043         error = sysctl_handle_int(oidp, &result, 0, req);
4044         if (error || !req->newptr)
4045                 return (error);
4046
4047         if (result == 1) {
4048                 struct adapter *adapter = (struct adapter *)arg1;
4049                 struct ifnet *ifp = &adapter->arpcom.ac_if;
4050
4051                 lwkt_serialize_enter(ifp->if_serializer);
4052                 em_print_hw_stats(adapter);
4053                 lwkt_serialize_exit(ifp->if_serializer);
4054         }
4055         return (error);
4056 }
4057
4058 static void
4059 em_add_sysctl(struct adapter *adapter)
4060 {
4061         sysctl_ctx_init(&adapter->sysctl_ctx);
4062         adapter->sysctl_tree = SYSCTL_ADD_NODE(&adapter->sysctl_ctx,
4063                                         SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
4064                                         device_get_nameunit(adapter->dev),
4065                                         CTLFLAG_RD, 0, "");
4066         if (adapter->sysctl_tree == NULL) {
4067                 device_printf(adapter->dev, "can't add sysctl node\n");
4068         } else {
4069                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4070                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4071                     OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4072                     em_sysctl_debug_info, "I", "Debug Information");
4073
4074                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4075                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4076                     OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4077                     em_sysctl_stats, "I", "Statistics");
4078
4079                 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
4080                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4081                     OID_AUTO, "rxd", CTLFLAG_RD,
4082                     &adapter->num_rx_desc, 0, NULL);
4083                 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
4084                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4085                     OID_AUTO, "txd", CTLFLAG_RD,
4086                     &adapter->num_tx_desc, 0, NULL);
4087
4088                 if (adapter->hw.mac.type >= e1000_82540) {
4089                         SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4090                             SYSCTL_CHILDREN(adapter->sysctl_tree),
4091                             OID_AUTO, "int_throttle_ceil",
4092                             CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4093                             em_sysctl_int_throttle, "I",
4094                             "interrupt throttling rate");
4095                 }
4096                 SYSCTL_ADD_PROC(&adapter->sysctl_ctx,
4097                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4098                     OID_AUTO, "int_tx_nsegs",
4099                     CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4100                     em_sysctl_int_tx_nsegs, "I",
4101                     "# segments per TX interrupt");
4102                 SYSCTL_ADD_INT(&adapter->sysctl_ctx,
4103                     SYSCTL_CHILDREN(adapter->sysctl_tree),
4104                     OID_AUTO, "wreg_tx_nsegs", CTLFLAG_RW,
4105                     &adapter->tx_wreg_nsegs, 0,
4106                     "# segments before write to hardware register");
4107         }
4108 }
4109
4110 static int
4111 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
4112 {
4113         struct adapter *adapter = (void *)arg1;
4114         struct ifnet *ifp = &adapter->arpcom.ac_if;
4115         int error, throttle;
4116
4117         throttle = adapter->int_throttle_ceil;
4118         error = sysctl_handle_int(oidp, &throttle, 0, req);
4119         if (error || req->newptr == NULL)
4120                 return error;
4121         if (throttle < 0 || throttle > 1000000000 / 256)
4122                 return EINVAL;
4123
4124         if (throttle) {
4125                 /*
4126                  * Set the interrupt throttling rate in 256ns increments,
4127                  * recalculate sysctl value assignment to get exact frequency.
4128                  */
4129                 throttle = 1000000000 / 256 / throttle;
4130
4131                 /* Upper 16bits of ITR is reserved and should be zero */
4132                 if (throttle & 0xffff0000)
4133                         return EINVAL;
4134         }
4135
4136         lwkt_serialize_enter(ifp->if_serializer);
4137
4138         if (throttle)
4139                 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
4140         else
4141                 adapter->int_throttle_ceil = 0;
4142
4143         if (ifp->if_flags & IFF_RUNNING)
4144                 em_set_itr(adapter, throttle);
4145
4146         lwkt_serialize_exit(ifp->if_serializer);
4147
4148         if (bootverbose) {
4149                 if_printf(ifp, "Interrupt moderation set to %d/sec\n",
4150                           adapter->int_throttle_ceil);
4151         }
4152         return 0;
4153 }
4154
4155 static int
4156 em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
4157 {
4158         struct adapter *adapter = (void *)arg1;
4159         struct ifnet *ifp = &adapter->arpcom.ac_if;
4160         int error, segs;
4161
4162         segs = adapter->tx_int_nsegs;
4163         error = sysctl_handle_int(oidp, &segs, 0, req);
4164         if (error || req->newptr == NULL)
4165                 return error;
4166         if (segs <= 0)
4167                 return EINVAL;
4168
4169         lwkt_serialize_enter(ifp->if_serializer);
4170
4171         /*
4172          * Don't allow int_tx_nsegs to become:
4173          * o  Less the oact_tx_desc
4174          * o  Too large that no TX desc will cause TX interrupt to
4175          *    be generated (OACTIVE will never recover)
4176          * o  Too small that will cause tx_dd[] overflow
4177          */
4178         if (segs < adapter->oact_tx_desc ||
4179             segs >= adapter->num_tx_desc - adapter->oact_tx_desc ||
4180             segs < adapter->num_tx_desc / EM_TXDD_SAFE) {
4181                 error = EINVAL;
4182         } else {
4183                 error = 0;
4184                 adapter->tx_int_nsegs = segs;
4185         }
4186
4187         lwkt_serialize_exit(ifp->if_serializer);
4188
4189         return error;
4190 }
4191
4192 static void
4193 em_set_itr(struct adapter *adapter, uint32_t itr)
4194 {
4195         E1000_WRITE_REG(&adapter->hw, E1000_ITR, itr);
4196         if (adapter->hw.mac.type == e1000_82574) {
4197                 int i;
4198
4199                 /*
4200                  * When using MSIX interrupts we need to
4201                  * throttle using the EITR register
4202                  */
4203                 for (i = 0; i < 4; ++i) {
4204                         E1000_WRITE_REG(&adapter->hw,
4205                             E1000_EITR_82574(i), itr);
4206                 }
4207         }
4208 }
4209
4210 static void
4211 em_disable_aspm(struct adapter *adapter)
4212 {
4213         uint16_t link_cap, link_ctrl, disable;
4214         uint8_t pcie_ptr, reg;
4215         device_t dev = adapter->dev;
4216
4217         switch (adapter->hw.mac.type) {
4218         case e1000_82571:
4219         case e1000_82572:
4220         case e1000_82573:
4221                 /*
4222                  * 82573 specification update
4223                  * errata #8 disable L0s
4224                  * errata #41 disable L1
4225                  *
4226                  * 82571/82572 specification update
4227                  # errata #13 disable L1
4228                  * errata #68 disable L0s
4229                  */
4230                 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1;
4231                 break;
4232
4233         case e1000_82574:
4234         case e1000_82583:
4235                 /*
4236                  * 82574 specification update errata #20
4237                  * 82583 specification update errata #9
4238                  *
4239                  * There is no need to disable L1
4240                  */
4241                 disable = PCIEM_LNKCTL_ASPM_L0S;
4242                 break;
4243
4244         default:
4245                 return;
4246         }
4247
4248         pcie_ptr = pci_get_pciecap_ptr(dev);
4249         if (pcie_ptr == 0)
4250                 return;
4251
4252         link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2);
4253         if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0)
4254                 return;
4255
4256         if (bootverbose) {
4257                 if_printf(&adapter->arpcom.ac_if,
4258                     "disable ASPM %#02x\n", disable);
4259         }
4260
4261         reg = pcie_ptr + PCIER_LINKCTRL;
4262         link_ctrl = pci_read_config(dev, reg, 2);
4263         link_ctrl &= ~disable;
4264         pci_write_config(dev, reg, link_ctrl, 2);
4265 }
4266
4267 static int
4268 em_tso_pullup(struct adapter *adapter, struct mbuf **mp)
4269 {
4270         int iphlen, hoff, thoff, ex = 0;
4271         struct mbuf *m;
4272         struct ip *ip;
4273
4274         m = *mp;
4275         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
4276
4277         iphlen = m->m_pkthdr.csum_iphlen;
4278         thoff = m->m_pkthdr.csum_thlen;
4279         hoff = m->m_pkthdr.csum_lhlen;
4280
4281         KASSERT(iphlen > 0, ("invalid ip hlen"));
4282         KASSERT(thoff > 0, ("invalid tcp hlen"));
4283         KASSERT(hoff > 0, ("invalid ether hlen"));
4284
4285         if (adapter->flags & EM_FLAG_TSO_PULLEX)
4286                 ex = 4;
4287
4288         if (m->m_len < hoff + iphlen + thoff + ex) {
4289                 m = m_pullup(m, hoff + iphlen + thoff + ex);
4290                 if (m == NULL) {
4291                         *mp = NULL;
4292                         return ENOBUFS;
4293                 }
4294                 *mp = m;
4295         }
4296         ip = mtodoff(m, struct ip *, hoff);
4297         ip->ip_len = 0;
4298
4299         return 0;
4300 }
4301
4302 static int
4303 em_tso_setup(struct adapter *adapter, struct mbuf *mp,
4304     uint32_t *txd_upper, uint32_t *txd_lower)
4305 {
4306         struct e1000_context_desc *TXD;
4307         int hoff, iphlen, thoff, hlen;
4308         int mss, pktlen, curr_txd;
4309
4310         iphlen = mp->m_pkthdr.csum_iphlen;
4311         thoff = mp->m_pkthdr.csum_thlen;
4312         hoff = mp->m_pkthdr.csum_lhlen;
4313         mss = mp->m_pkthdr.tso_segsz;
4314         pktlen = mp->m_pkthdr.len;
4315
4316         if (adapter->csum_flags == CSUM_TSO &&
4317             adapter->csum_iphlen == iphlen &&
4318             adapter->csum_lhlen == hoff &&
4319             adapter->csum_thlen == thoff &&
4320             adapter->csum_mss == mss &&
4321             adapter->csum_pktlen == pktlen) {
4322                 *txd_upper = adapter->csum_txd_upper;
4323                 *txd_lower = adapter->csum_txd_lower;
4324                 return 0;
4325         }
4326         hlen = hoff + iphlen + thoff;
4327
4328         /*
4329          * Setup a new TSO context.
4330          */
4331
4332         curr_txd = adapter->next_avail_tx_desc;
4333         TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
4334
4335         *txd_lower = E1000_TXD_CMD_DEXT |       /* Extended descr type */
4336                      E1000_TXD_DTYP_D |         /* Data descr type */
4337                      E1000_TXD_CMD_TSE;         /* Do TSE on this packet */
4338
4339         /* IP and/or TCP header checksum calculation and insertion. */
4340         *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
4341
4342         /*
4343          * Start offset for header checksum calculation.
4344          * End offset for header checksum calculation.
4345          * Offset of place put the checksum.
4346          */
4347         TXD->lower_setup.ip_fields.ipcss = hoff;
4348         TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1);
4349         TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum);
4350
4351         /*
4352          * Start offset for payload checksum calculation.
4353          * End offset for payload checksum calculation.
4354          * Offset of place to put the checksum.
4355          */
4356         TXD->upper_setup.tcp_fields.tucss = hoff + iphlen;
4357         TXD->upper_setup.tcp_fields.tucse = 0;
4358         TXD->upper_setup.tcp_fields.tucso =
4359             hoff + iphlen + offsetof(struct tcphdr, th_sum);
4360
4361         /*
4362          * Payload size per packet w/o any headers.
4363          * Length of all headers up to payload.
4364          */
4365         TXD->tcp_seg_setup.fields.mss = htole16(mss);
4366         TXD->tcp_seg_setup.fields.hdr_len = hlen;
4367         TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS |
4368                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
4369                                 E1000_TXD_CMD_TSE |     /* TSE context */
4370                                 E1000_TXD_CMD_IP |      /* Do IP csum */
4371                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
4372                                 (pktlen - hlen));       /* Total len */
4373
4374         /* Save the information for this TSO context */
4375         adapter->csum_flags = CSUM_TSO;
4376         adapter->csum_lhlen = hoff;
4377         adapter->csum_iphlen = iphlen;
4378         adapter->csum_thlen = thoff;
4379         adapter->csum_mss = mss;
4380         adapter->csum_pktlen = pktlen;
4381         adapter->csum_txd_upper = *txd_upper;
4382         adapter->csum_txd_lower = *txd_lower;
4383
4384         if (++curr_txd == adapter->num_tx_desc)
4385                 curr_txd = 0;
4386
4387         KKASSERT(adapter->num_tx_desc_avail > 0);
4388         adapter->num_tx_desc_avail--;
4389
4390         adapter->next_avail_tx_desc = curr_txd;
4391         return 1;
4392 }