sys/dev/netif/em/if_em.c

   1 /*
   2  * Copyright (c) 2004 Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   3  *
   4  * Copyright (c) 2001-2008, Intel Corporation
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions are met:
   9  *
  10  *  1. Redistributions of source code must retain the above copyright notice,
  11  *     this list of conditions and the following disclaimer.
  12  *
  13  *  2. Redistributions in binary form must reproduce the above copyright
  14  *     notice, this list of conditions and the following disclaimer in the
  15  *     documentation and/or other materials provided with the distribution.
  16  *
  17  *  3. Neither the name of the Intel Corporation nor the names of its
  18  *     contributors may be used to endorse or promote products derived from
  19  *     this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31  * POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  *
  34  * Copyright (c) 2005 The DragonFly Project.  All rights reserved.
  35  *
  36  * This code is derived from software contributed to The DragonFly Project
  37  * by Matthew Dillon <dillon@backplane.com>
  38  *
  39  * Redistribution and use in source and binary forms, with or without
  40  * modification, are permitted provided that the following conditions
  41  * are met:
  42  *
  43  * 1. Redistributions of source code must retain the above copyright
  44  *    notice, this list of conditions and the following disclaimer.
  45  * 2. Redistributions in binary form must reproduce the above copyright
  46  *    notice, this list of conditions and the following disclaimer in
  47  *    the documentation and/or other materials provided with the
  48  *    distribution.
  49  * 3. Neither the name of The DragonFly Project nor the names of its
  50  *    contributors may be used to endorse or promote products derived
  51  *    from this software without specific, prior written permission.
  52  *
  53  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  54  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  55  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  56  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  57  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  58  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  59  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  60  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  61  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  62  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  63  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  64  * SUCH DAMAGE.
  65  *
  66  */
  67 /*
  68  * SERIALIZATION API RULES:
  69  *
  70  * - We must call lwkt_serialize_handler_enable() prior to enabling the
  71  *   hardware interrupt and lwkt_serialize_handler_disable() after disabling
  72  *   the hardware interrupt in order to avoid handler execution races from
  73  *   scheduled interrupt threads.
  74  */
  75
  76 #include "opt_ifpoll.h"
  77
  78 #include <sys/param.h>
  79 #include <sys/bus.h>
  80 #include <sys/endian.h>
  81 #include <sys/interrupt.h>
  82 #include <sys/kernel.h>
  83 #include <sys/ktr.h>
  84 #include <sys/malloc.h>
  85 #include <sys/mbuf.h>
  86 #include <sys/proc.h>
  87 #include <sys/rman.h>
  88 #include <sys/serialize.h>
  89 #include <sys/socket.h>
  90 #include <sys/sockio.h>
  91 #include <sys/sysctl.h>
  92 #include <sys/systm.h>
  93
  94 #include <net/bpf.h>
  95 #include <net/ethernet.h>
  96 #include <net/if.h>
  97 #include <net/if_arp.h>
  98 #include <net/if_dl.h>
  99 #include <net/if_media.h>
 100 #include <net/if_poll.h>
 101 #include <net/ifq_var.h>
 102 #include <net/vlan/if_vlan_var.h>
 103 #include <net/vlan/if_vlan_ether.h>
 104
 105 #include <netinet/ip.h>
 106 #include <netinet/tcp.h>
 107 #include <netinet/udp.h>
 108
 109 #include <bus/pci/pcivar.h>
 110 #include <bus/pci/pcireg.h>
 111
 112 #include <dev/netif/ig_hal/e1000_api.h>
 113 #include <dev/netif/ig_hal/e1000_82571.h>
 114 #include <dev/netif/em/if_em.h>
 115
 116 #define DEBUG_HW 0
 117
 118 #define EM_NAME "Intel(R) PRO/1000 Network Connection "
 119 #define EM_VER  " 7.3.8"
 120
 121 #define _EM_DEVICE(id, ret)     \
 122         { EM_VENDOR_ID, E1000_DEV_ID_##id, ret, EM_NAME #id EM_VER }
 123 #define EM_EMX_DEVICE(id)       _EM_DEVICE(id, -100)
 124 #define EM_DEVICE(id)           _EM_DEVICE(id, 0)
 125 #define EM_DEVICE_NULL  { 0, 0, 0, NULL }
 126
 127 static const struct em_vendor_info em_vendor_info_array[] = {
 128         EM_DEVICE(82540EM),
 129         EM_DEVICE(82540EM_LOM),
 130         EM_DEVICE(82540EP),
 131         EM_DEVICE(82540EP_LOM),
 132         EM_DEVICE(82540EP_LP),
 133
 134         EM_DEVICE(82541EI),
 135         EM_DEVICE(82541ER),
 136         EM_DEVICE(82541ER_LOM),
 137         EM_DEVICE(82541EI_MOBILE),
 138         EM_DEVICE(82541GI),
 139         EM_DEVICE(82541GI_LF),
 140         EM_DEVICE(82541GI_MOBILE),
 141
 142         EM_DEVICE(82542),
 143
 144         EM_DEVICE(82543GC_FIBER),
 145         EM_DEVICE(82543GC_COPPER),
 146
 147         EM_DEVICE(82544EI_COPPER),
 148         EM_DEVICE(82544EI_FIBER),
 149         EM_DEVICE(82544GC_COPPER),
 150         EM_DEVICE(82544GC_LOM),
 151
 152         EM_DEVICE(82545EM_COPPER),
 153         EM_DEVICE(82545EM_FIBER),
 154         EM_DEVICE(82545GM_COPPER),
 155         EM_DEVICE(82545GM_FIBER),
 156         EM_DEVICE(82545GM_SERDES),
 157
 158         EM_DEVICE(82546EB_COPPER),
 159         EM_DEVICE(82546EB_FIBER),
 160         EM_DEVICE(82546EB_QUAD_COPPER),
 161         EM_DEVICE(82546GB_COPPER),
 162         EM_DEVICE(82546GB_FIBER),
 163         EM_DEVICE(82546GB_SERDES),
 164         EM_DEVICE(82546GB_PCIE),
 165         EM_DEVICE(82546GB_QUAD_COPPER),
 166         EM_DEVICE(82546GB_QUAD_COPPER_KSP3),
 167
 168         EM_DEVICE(82547EI),
 169         EM_DEVICE(82547EI_MOBILE),
 170         EM_DEVICE(82547GI),
 171
 172         EM_EMX_DEVICE(82571EB_COPPER),
 173         EM_EMX_DEVICE(82571EB_FIBER),
 174         EM_EMX_DEVICE(82571EB_SERDES),
 175         EM_EMX_DEVICE(82571EB_SERDES_DUAL),
 176         EM_EMX_DEVICE(82571EB_SERDES_QUAD),
 177         EM_EMX_DEVICE(82571EB_QUAD_COPPER),
 178         EM_EMX_DEVICE(82571EB_QUAD_COPPER_BP),
 179         EM_EMX_DEVICE(82571EB_QUAD_COPPER_LP),
 180         EM_EMX_DEVICE(82571EB_QUAD_FIBER),
 181         EM_EMX_DEVICE(82571PT_QUAD_COPPER),
 182
 183         EM_EMX_DEVICE(82572EI_COPPER),
 184         EM_EMX_DEVICE(82572EI_FIBER),
 185         EM_EMX_DEVICE(82572EI_SERDES),
 186         EM_EMX_DEVICE(82572EI),
 187
 188         EM_EMX_DEVICE(82573E),
 189         EM_EMX_DEVICE(82573E_IAMT),
 190         EM_EMX_DEVICE(82573L),
 191
 192         EM_DEVICE(82583V),
 193
 194         EM_EMX_DEVICE(80003ES2LAN_COPPER_SPT),
 195         EM_EMX_DEVICE(80003ES2LAN_SERDES_SPT),
 196         EM_EMX_DEVICE(80003ES2LAN_COPPER_DPT),
 197         EM_EMX_DEVICE(80003ES2LAN_SERDES_DPT),
 198
 199         EM_DEVICE(ICH8_IGP_M_AMT),
 200         EM_DEVICE(ICH8_IGP_AMT),
 201         EM_DEVICE(ICH8_IGP_C),
 202         EM_DEVICE(ICH8_IFE),
 203         EM_DEVICE(ICH8_IFE_GT),
 204         EM_DEVICE(ICH8_IFE_G),
 205         EM_DEVICE(ICH8_IGP_M),
 206         EM_DEVICE(ICH8_82567V_3),
 207
 208         EM_DEVICE(ICH9_IGP_M_AMT),
 209         EM_DEVICE(ICH9_IGP_AMT),
 210         EM_DEVICE(ICH9_IGP_C),
 211         EM_DEVICE(ICH9_IGP_M),
 212         EM_DEVICE(ICH9_IGP_M_V),
 213         EM_DEVICE(ICH9_IFE),
 214         EM_DEVICE(ICH9_IFE_GT),
 215         EM_DEVICE(ICH9_IFE_G),
 216         EM_DEVICE(ICH9_BM),
 217
 218         EM_EMX_DEVICE(82574L),
 219         EM_EMX_DEVICE(82574LA),
 220
 221         EM_DEVICE(ICH10_R_BM_LM),
 222         EM_DEVICE(ICH10_R_BM_LF),
 223         EM_DEVICE(ICH10_R_BM_V),
 224         EM_DEVICE(ICH10_D_BM_LM),
 225         EM_DEVICE(ICH10_D_BM_LF),
 226         EM_DEVICE(ICH10_D_BM_V),
 227
 228         EM_DEVICE(PCH_M_HV_LM),
 229         EM_DEVICE(PCH_M_HV_LC),
 230         EM_DEVICE(PCH_D_HV_DM),
 231         EM_DEVICE(PCH_D_HV_DC),
 232
 233         EM_DEVICE(PCH2_LV_LM),
 234         EM_DEVICE(PCH2_LV_V),
 235
 236         EM_EMX_DEVICE(PCH_LPT_I217_LM),
 237         EM_EMX_DEVICE(PCH_LPT_I217_V),
 238         EM_EMX_DEVICE(PCH_LPTLP_I218_LM),
 239         EM_EMX_DEVICE(PCH_LPTLP_I218_V),
 240
 241         /* required last entry */
 242         EM_DEVICE_NULL
 243 };
 244
 245 static int      em_probe(device_t);
 246 static int      em_attach(device_t);
 247 static int      em_detach(device_t);
 248 static int      em_shutdown(device_t);
 249 static int      em_suspend(device_t);
 250 static int      em_resume(device_t);
 251
 252 static void     em_init(void *);
 253 static void     em_stop(struct adapter *);
 254 static int      em_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 255 static void     em_start(struct ifnet *, struct ifaltq_subque *);
 256 #ifdef IFPOLL_ENABLE
 257 static void     em_npoll(struct ifnet *, struct ifpoll_info *);
 258 static void     em_npoll_compat(struct ifnet *, void *, int);
 259 #endif
 260 static void     em_watchdog(struct ifnet *);
 261 static void     em_media_status(struct ifnet *, struct ifmediareq *);
 262 static int      em_media_change(struct ifnet *);
 263 static void     em_timer(void *);
 264
 265 static void     em_intr(void *);
 266 static void     em_intr_mask(void *);
 267 static void     em_intr_body(struct adapter *, boolean_t);
 268 static void     em_rxeof(struct adapter *, int);
 269 static void     em_txeof(struct adapter *);
 270 static void     em_tx_collect(struct adapter *);
 271 static void     em_tx_purge(struct adapter *);
 272 static void     em_enable_intr(struct adapter *);
 273 static void     em_disable_intr(struct adapter *);
 274
 275 static int      em_dma_malloc(struct adapter *, bus_size_t,
 276                     struct em_dma_alloc *);
 277 static void     em_dma_free(struct adapter *, struct em_dma_alloc *);
 278 static void     em_init_tx_ring(struct adapter *);
 279 static int      em_init_rx_ring(struct adapter *);
 280 static int      em_create_tx_ring(struct adapter *);
 281 static int      em_create_rx_ring(struct adapter *);
 282 static void     em_destroy_tx_ring(struct adapter *, int);
 283 static void     em_destroy_rx_ring(struct adapter *, int);
 284 static int      em_newbuf(struct adapter *, int, int);
 285 static int      em_encap(struct adapter *, struct mbuf **, int *, int *);
 286 static void     em_rxcsum(struct adapter *, struct e1000_rx_desc *,
 287                     struct mbuf *);
 288 static int      em_txcsum(struct adapter *, struct mbuf *,
 289                     uint32_t *, uint32_t *);
 290 static int      em_tso_pullup(struct adapter *, struct mbuf **);
 291 static int      em_tso_setup(struct adapter *, struct mbuf *,
 292                     uint32_t *, uint32_t *);
 293
 294 static int      em_get_hw_info(struct adapter *);
 295 static int      em_is_valid_eaddr(const uint8_t *);
 296 static int      em_alloc_pci_res(struct adapter *);
 297 static void     em_free_pci_res(struct adapter *);
 298 static int      em_reset(struct adapter *);
 299 static void     em_setup_ifp(struct adapter *);
 300 static void     em_init_tx_unit(struct adapter *);
 301 static void     em_init_rx_unit(struct adapter *);
 302 static void     em_update_stats(struct adapter *);
 303 static void     em_set_promisc(struct adapter *);
 304 static void     em_disable_promisc(struct adapter *);
 305 static void     em_set_multi(struct adapter *);
 306 static void     em_update_link_status(struct adapter *);
 307 static void     em_smartspeed(struct adapter *);
 308 static void     em_set_itr(struct adapter *, uint32_t);
 309 static void     em_disable_aspm(struct adapter *);
 310
 311 /* Hardware workarounds */
 312 static int      em_82547_fifo_workaround(struct adapter *, int);
 313 static void     em_82547_update_fifo_head(struct adapter *, int);
 314 static int      em_82547_tx_fifo_reset(struct adapter *);
 315 static void     em_82547_move_tail(void *);
 316 static void     em_82547_move_tail_serialized(struct adapter *);
 317 static uint32_t em_82544_fill_desc(bus_addr_t, uint32_t, PDESC_ARRAY);
 318
 319 static void     em_print_debug_info(struct adapter *);
 320 static void     em_print_nvm_info(struct adapter *);
 321 static void     em_print_hw_stats(struct adapter *);
 322
 323 static int      em_sysctl_stats(SYSCTL_HANDLER_ARGS);
 324 static int      em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
 325 static int      em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS);
 326 static int      em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS);
 327 static void     em_add_sysctl(struct adapter *adapter);
 328
 329 /* Management and WOL Support */
 330 static void     em_get_mgmt(struct adapter *);
 331 static void     em_rel_mgmt(struct adapter *);
 332 static void     em_get_hw_control(struct adapter *);
 333 static void     em_rel_hw_control(struct adapter *);
 334 static void     em_enable_wol(device_t);
 335
 336 static device_method_t em_methods[] = {
 337         /* Device interface */
 338         DEVMETHOD(device_probe,         em_probe),
 339         DEVMETHOD(device_attach,        em_attach),
 340         DEVMETHOD(device_detach,        em_detach),
 341         DEVMETHOD(device_shutdown,      em_shutdown),
 342         DEVMETHOD(device_suspend,       em_suspend),
 343         DEVMETHOD(device_resume,        em_resume),
 344         DEVMETHOD_END
 345 };
 346
 347 static driver_t em_driver = {
 348         "em",
 349         em_methods,
 350         sizeof(struct adapter),
 351 };
 352
 353 static devclass_t em_devclass;
 354
 355 DECLARE_DUMMY_MODULE(if_em);
 356 MODULE_DEPEND(em, ig_hal, 1, 1, 1);
 357 DRIVER_MODULE(if_em, pci, em_driver, em_devclass, NULL, NULL);
 358
 359 /*
 360  * Tunables
 361  */
 362 static int      em_int_throttle_ceil = EM_DEFAULT_ITR;
 363 static int      em_rxd = EM_DEFAULT_RXD;
 364 static int      em_txd = EM_DEFAULT_TXD;
 365 static int      em_smart_pwr_down = 0;
 366
 367 /* Controls whether promiscuous also shows bad packets */
 368 static int      em_debug_sbp = FALSE;
 369
 370 static int      em_82573_workaround = 1;
 371 static int      em_msi_enable = 1;
 372
 373 TUNABLE_INT("hw.em.int_throttle_ceil", &em_int_throttle_ceil);
 374 TUNABLE_INT("hw.em.rxd", &em_rxd);
 375 TUNABLE_INT("hw.em.txd", &em_txd);
 376 TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
 377 TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
 378 TUNABLE_INT("hw.em.82573_workaround", &em_82573_workaround);
 379 TUNABLE_INT("hw.em.msi.enable", &em_msi_enable);
 380
 381 /* Global used in WOL setup with multiport cards */
 382 static int      em_global_quad_port_a = 0;
 383
 384 /* Set this to one to display debug statistics */
 385 static int      em_display_debug_stats = 0;
 386
 387 #if !defined(KTR_IF_EM)
 388 #define KTR_IF_EM       KTR_ALL
 389 #endif
 390 KTR_INFO_MASTER(if_em);
 391 KTR_INFO(KTR_IF_EM, if_em, intr_beg, 0, "intr begin");
 392 KTR_INFO(KTR_IF_EM, if_em, intr_end, 1, "intr end");
 393 KTR_INFO(KTR_IF_EM, if_em, pkt_receive, 4, "rx packet");
 394 KTR_INFO(KTR_IF_EM, if_em, pkt_txqueue, 5, "tx packet");
 395 KTR_INFO(KTR_IF_EM, if_em, pkt_txclean, 6, "tx clean");
 396 #define logif(name)     KTR_LOG(if_em_ ## name)
 397
 398 static int
 399 em_probe(device_t dev)
 400 {
 401         const struct em_vendor_info *ent;
 402         uint16_t vid, did;
 403
 404         vid = pci_get_vendor(dev);
 405         did = pci_get_device(dev);
 406
 407         for (ent = em_vendor_info_array; ent->desc != NULL; ++ent) {
 408                 if (vid == ent->vendor_id && did == ent->device_id) {
 409                         device_set_desc(dev, ent->desc);
 410                         device_set_async_attach(dev, TRUE);
 411                         return (ent->ret);
 412                 }
 413         }
 414         return (ENXIO);
 415 }
 416
 417 static int
 418 em_attach(device_t dev)
 419 {
 420         struct adapter *adapter = device_get_softc(dev);
 421         struct ifnet *ifp = &adapter->arpcom.ac_if;
 422         int tsize, rsize;
 423         int error = 0;
 424         uint16_t eeprom_data, device_id, apme_mask;
 425         driver_intr_t *intr_func;
 426
 427         adapter->dev = adapter->osdep.dev = dev;
 428
 429         callout_init_mp(&adapter->timer);
 430         callout_init_mp(&adapter->tx_fifo_timer);
 431
 432         ifmedia_init(&adapter->media, IFM_IMASK,
 433             em_media_change, em_media_status);
 434
 435         /* Determine hardware and mac info */
 436         error = em_get_hw_info(adapter);
 437         if (error) {
 438                 device_printf(dev, "Identify hardware failed\n");
 439                 goto fail;
 440         }
 441
 442         /* Setup PCI resources */
 443         error = em_alloc_pci_res(adapter);
 444         if (error) {
 445                 device_printf(dev, "Allocation of PCI resources failed\n");
 446                 goto fail;
 447         }
 448
 449         /*
 450          * For ICH8 and family we need to map the flash memory,
 451          * and this must happen after the MAC is identified.
 452          */
 453         if (adapter->hw.mac.type == e1000_ich8lan ||
 454             adapter->hw.mac.type == e1000_ich9lan ||
 455             adapter->hw.mac.type == e1000_ich10lan ||
 456             adapter->hw.mac.type == e1000_pchlan ||
 457             adapter->hw.mac.type == e1000_pch2lan ||
 458             adapter->hw.mac.type == e1000_pch_lpt) {
 459                 adapter->flash_rid = EM_BAR_FLASH;
 460
 461                 adapter->flash = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 462                                         &adapter->flash_rid, RF_ACTIVE);
 463                 if (adapter->flash == NULL) {
 464                         device_printf(dev, "Mapping of Flash failed\n");
 465                         error = ENXIO;
 466                         goto fail;
 467                 }
 468                 adapter->osdep.flash_bus_space_tag =
 469                     rman_get_bustag(adapter->flash);
 470                 adapter->osdep.flash_bus_space_handle =
 471                     rman_get_bushandle(adapter->flash);
 472
 473                 /*
 474                  * This is used in the shared code
 475                  * XXX this goof is actually not used.
 476                  */
 477                 adapter->hw.flash_address = (uint8_t *)adapter->flash;
 478         }
 479
 480         switch (adapter->hw.mac.type) {
 481         case e1000_82571:
 482         case e1000_82572:
 483                 /*
 484                  * Pullup extra 4bytes into the first data segment, see:
 485                  * 82571/82572 specification update errata #7
 486                  *
 487                  * NOTE:
 488                  * 4bytes instead of 2bytes, which are mentioned in the
 489                  * errata, are pulled; mainly to keep rest of the data
 490                  * properly aligned.
 491                  */
 492                 adapter->flags |= EM_FLAG_TSO_PULLEX;
 493                 /* FALL THROUGH */
 494
 495         default:
 496                 if (pci_is_pcie(dev))
 497                         adapter->flags |= EM_FLAG_TSO;
 498                 break;
 499         }
 500
 501         /* Do Shared Code initialization */
 502         if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
 503                 device_printf(dev, "Setup of Shared code failed\n");
 504                 error = ENXIO;
 505                 goto fail;
 506         }
 507
 508         e1000_get_bus_info(&adapter->hw);
 509
 510         /*
 511          * Validate number of transmit and receive descriptors.  It
 512          * must not exceed hardware maximum, and must be multiple
 513          * of E1000_DBA_ALIGN.
 514          */
 515         if ((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN != 0 ||
 516             (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
 517             (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
 518             em_txd < EM_MIN_TXD) {
 519                 if (adapter->hw.mac.type < e1000_82544)
 520                         adapter->num_tx_desc = EM_MAX_TXD_82543;
 521                 else
 522                         adapter->num_tx_desc = EM_DEFAULT_TXD;
 523                 device_printf(dev, "Using %d TX descriptors instead of %d!\n",
 524                     adapter->num_tx_desc, em_txd);
 525         } else {
 526                 adapter->num_tx_desc = em_txd;
 527         }
 528         if ((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN != 0 ||
 529             (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
 530             (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
 531             em_rxd < EM_MIN_RXD) {
 532                 if (adapter->hw.mac.type < e1000_82544)
 533                         adapter->num_rx_desc = EM_MAX_RXD_82543;
 534                 else
 535                         adapter->num_rx_desc = EM_DEFAULT_RXD;
 536                 device_printf(dev, "Using %d RX descriptors instead of %d!\n",
 537                     adapter->num_rx_desc, em_rxd);
 538         } else {
 539                 adapter->num_rx_desc = em_rxd;
 540         }
 541
 542         adapter->hw.mac.autoneg = DO_AUTO_NEG;
 543         adapter->hw.phy.autoneg_wait_to_complete = FALSE;
 544         adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
 545         adapter->rx_buffer_len = MCLBYTES;
 546
 547         /*
 548          * Interrupt throttle rate
 549          */
 550         if (em_int_throttle_ceil == 0) {
 551                 adapter->int_throttle_ceil = 0;
 552         } else {
 553                 int throttle = em_int_throttle_ceil;
 554
 555                 if (throttle < 0)
 556                         throttle = EM_DEFAULT_ITR;
 557
 558                 /* Recalculate the tunable value to get the exact frequency. */
 559                 throttle = 1000000000 / 256 / throttle;
 560
 561                 /* Upper 16bits of ITR is reserved and should be zero */
 562                 if (throttle & 0xffff0000)
 563                         throttle = 1000000000 / 256 / EM_DEFAULT_ITR;
 564
 565                 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
 566         }
 567
 568         e1000_init_script_state_82541(&adapter->hw, TRUE);
 569         e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
 570
 571         /* Copper options */
 572         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
 573                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
 574                 adapter->hw.phy.disable_polarity_correction = FALSE;
 575                 adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
 576         }
 577
 578         /* Set the frame limits assuming standard ethernet sized frames. */
 579         adapter->hw.mac.max_frame_size =
 580             ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
 581         adapter->min_frame_size = ETH_ZLEN + ETHER_CRC_LEN;
 582
 583         /* This controls when hardware reports transmit completion status. */
 584         adapter->hw.mac.report_tx_early = 1;
 585
 586         /*
 587          * Create top level busdma tag
 588          */
 589         error = bus_dma_tag_create(NULL, 1, 0,
 590                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 591                         NULL, NULL,
 592                         BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
 593                         0, &adapter->parent_dtag);
 594         if (error) {
 595                 device_printf(dev, "could not create top level DMA tag\n");
 596                 goto fail;
 597         }
 598
 599         /*
 600          * Allocate Transmit Descriptor ring
 601          */
 602         tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
 603                          EM_DBA_ALIGN);
 604         error = em_dma_malloc(adapter, tsize, &adapter->txdma);
 605         if (error) {
 606                 device_printf(dev, "Unable to allocate tx_desc memory\n");
 607                 goto fail;
 608         }
 609         adapter->tx_desc_base = adapter->txdma.dma_vaddr;
 610
 611         /*
 612          * Allocate Receive Descriptor ring
 613          */
 614         rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
 615                          EM_DBA_ALIGN);
 616         error = em_dma_malloc(adapter, rsize, &adapter->rxdma);
 617         if (error) {
 618                 device_printf(dev, "Unable to allocate rx_desc memory\n");
 619                 goto fail;
 620         }
 621         adapter->rx_desc_base = adapter->rxdma.dma_vaddr;
 622
 623         /* Allocate multicast array memory. */
 624         adapter->mta = kmalloc(ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES,
 625             M_DEVBUF, M_WAITOK);
 626
 627         /* Indicate SOL/IDER usage */
 628         if (e1000_check_reset_block(&adapter->hw)) {
 629                 device_printf(dev,
 630                     "PHY reset is blocked due to SOL/IDER session.\n");
 631         }
 632
 633         /* Disable EEE */
 634         adapter->hw.dev_spec.ich8lan.eee_disable = 1;
 635
 636         /*
 637          * Start from a known state, this is important in reading the
 638          * nvm and mac from that.
 639          */
 640         e1000_reset_hw(&adapter->hw);
 641
 642         /* Make sure we have a good EEPROM before we read from it */
 643         if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 644                 /*
 645                  * Some PCI-E parts fail the first check due to
 646                  * the link being in sleep state, call it again,
 647                  * if it fails a second time its a real issue.
 648                  */
 649                 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
 650                         device_printf(dev,
 651                             "The EEPROM Checksum Is Not Valid\n");
 652                         error = EIO;
 653                         goto fail;
 654                 }
 655         }
 656
 657         /* Copy the permanent MAC address out of the EEPROM */
 658         if (e1000_read_mac_addr(&adapter->hw) < 0) {
 659                 device_printf(dev, "EEPROM read error while reading MAC"
 660                     " address\n");
 661                 error = EIO;
 662                 goto fail;
 663         }
 664         if (!em_is_valid_eaddr(adapter->hw.mac.addr)) {
 665                 device_printf(dev, "Invalid MAC address\n");
 666                 error = EIO;
 667                 goto fail;
 668         }
 669
 670         /* Allocate transmit descriptors and buffers */
 671         error = em_create_tx_ring(adapter);
 672         if (error) {
 673                 device_printf(dev, "Could not setup transmit structures\n");
 674                 goto fail;
 675         }
 676
 677         /* Allocate receive descriptors and buffers */
 678         error = em_create_rx_ring(adapter);
 679         if (error) {
 680                 device_printf(dev, "Could not setup receive structures\n");
 681                 goto fail;
 682         }
 683
 684         /* Manually turn off all interrupts */
 685         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
 686
 687         /* Determine if we have to control management hardware */
 688         if (e1000_enable_mng_pass_thru(&adapter->hw))
 689                 adapter->flags |= EM_FLAG_HAS_MGMT;
 690
 691         /*
 692          * Setup Wake-on-Lan
 693          */
 694         apme_mask = EM_EEPROM_APME;
 695         eeprom_data = 0;
 696         switch (adapter->hw.mac.type) {
 697         case e1000_82542:
 698         case e1000_82543:
 699                 break;
 700
 701         case e1000_82573:
 702         case e1000_82583:
 703                 adapter->flags |= EM_FLAG_HAS_AMT;
 704                 /* FALL THROUGH */
 705
 706         case e1000_82546:
 707         case e1000_82546_rev_3:
 708         case e1000_82571:
 709         case e1000_82572:
 710         case e1000_80003es2lan:
 711                 if (adapter->hw.bus.func == 1) {
 712                         e1000_read_nvm(&adapter->hw,
 713                             NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
 714                 } else {
 715                         e1000_read_nvm(&adapter->hw,
 716                             NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 717                 }
 718                 break;
 719
 720         case e1000_ich8lan:
 721         case e1000_ich9lan:
 722         case e1000_ich10lan:
 723         case e1000_pchlan:
 724         case e1000_pch2lan:
 725                 apme_mask = E1000_WUC_APME;
 726                 adapter->flags |= EM_FLAG_HAS_AMT;
 727                 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC);
 728                 break;
 729
 730         default:
 731                 e1000_read_nvm(&adapter->hw,
 732                     NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
 733                 break;
 734         }
 735         if (eeprom_data & apme_mask)
 736                 adapter->wol = E1000_WUFC_MAG | E1000_WUFC_MC;
 737
 738         /*
 739          * We have the eeprom settings, now apply the special cases
 740          * where the eeprom may be wrong or the board won't support
 741          * wake on lan on a particular port
 742          */
 743         device_id = pci_get_device(dev);
 744         switch (device_id) {
 745         case E1000_DEV_ID_82546GB_PCIE:
 746                 adapter->wol = 0;
 747                 break;
 748
 749         case E1000_DEV_ID_82546EB_FIBER:
 750         case E1000_DEV_ID_82546GB_FIBER:
 751         case E1000_DEV_ID_82571EB_FIBER:
 752                 /*
 753                  * Wake events only supported on port A for dual fiber
 754                  * regardless of eeprom setting
 755                  */
 756                 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
 757                     E1000_STATUS_FUNC_1)
 758                         adapter->wol = 0;
 759                 break;
 760
 761         case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
 762         case E1000_DEV_ID_82571EB_QUAD_COPPER:
 763         case E1000_DEV_ID_82571EB_QUAD_FIBER:
 764         case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
 765                 /* if quad port adapter, disable WoL on all but port A */
 766                 if (em_global_quad_port_a != 0)
 767                         adapter->wol = 0;
 768                 /* Reset for multiple quad port adapters */
 769                 if (++em_global_quad_port_a == 4)
 770                         em_global_quad_port_a = 0;
 771                 break;
 772         }
 773
 774         /* XXX disable wol */
 775         adapter->wol = 0;
 776
 777         /* Setup OS specific network interface */
 778         em_setup_ifp(adapter);
 779
 780         /* Add sysctl tree, must after em_setup_ifp() */
 781         em_add_sysctl(adapter);
 782
 783 #ifdef IFPOLL_ENABLE
 784         /* Polling setup */
 785         ifpoll_compat_setup(&adapter->npoll,
 786             device_get_sysctl_ctx(dev), device_get_sysctl_tree(dev),
 787             device_get_unit(dev), ifp->if_serializer);
 788 #endif
 789
 790         /* Reset the hardware */
 791         error = em_reset(adapter);
 792         if (error) {
 793                 /*
 794                  * Some 82573 parts fail the first reset, call it again,
 795                  * if it fails a second time its a real issue.
 796                  */
 797                 error = em_reset(adapter);
 798                 if (error) {
 799                         device_printf(dev, "Unable to reset the hardware\n");
 800                         ether_ifdetach(ifp);
 801                         goto fail;
 802                 }
 803         }
 804
 805         /* Initialize statistics */
 806         em_update_stats(adapter);
 807
 808         adapter->hw.mac.get_link_status = 1;
 809         em_update_link_status(adapter);
 810
 811         /* Do we need workaround for 82544 PCI-X adapter? */
 812         if (adapter->hw.bus.type == e1000_bus_type_pcix &&
 813             adapter->hw.mac.type == e1000_82544)
 814                 adapter->pcix_82544 = TRUE;
 815         else
 816                 adapter->pcix_82544 = FALSE;
 817
 818         if (adapter->pcix_82544) {
 819                 /*
 820                  * 82544 on PCI-X may split one TX segment
 821                  * into two TX descs, so we double its number
 822                  * of spare TX desc here.
 823                  */
 824                 adapter->spare_tx_desc = 2 * EM_TX_SPARE;
 825         } else {
 826                 adapter->spare_tx_desc = EM_TX_SPARE;
 827         }
 828         if (adapter->flags & EM_FLAG_TSO)
 829                 adapter->spare_tx_desc = EM_TX_SPARE_TSO;
 830         adapter->tx_wreg_nsegs = EM_DEFAULT_TXWREG;
 831
 832         /*
 833          * Keep following relationship between spare_tx_desc, oact_tx_desc
 834          * and tx_int_nsegs:
 835          * (spare_tx_desc + EM_TX_RESERVED) <=
 836          * oact_tx_desc <= EM_TX_OACTIVE_MAX <= tx_int_nsegs
 837          */
 838         adapter->oact_tx_desc = adapter->num_tx_desc / 8;
 839         if (adapter->oact_tx_desc > EM_TX_OACTIVE_MAX)
 840                 adapter->oact_tx_desc = EM_TX_OACTIVE_MAX;
 841         if (adapter->oact_tx_desc < adapter->spare_tx_desc + EM_TX_RESERVED)
 842                 adapter->oact_tx_desc = adapter->spare_tx_desc + EM_TX_RESERVED;
 843
 844         adapter->tx_int_nsegs = adapter->num_tx_desc / 16;
 845         if (adapter->tx_int_nsegs < adapter->oact_tx_desc)
 846                 adapter->tx_int_nsegs = adapter->oact_tx_desc;
 847
 848         /* Non-AMT based hardware can now take control from firmware */
 849         if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
 850             EM_FLAG_HAS_MGMT && adapter->hw.mac.type >= e1000_82571)
 851                 em_get_hw_control(adapter);
 852
 853         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
 854
 855         /*
 856          * Missing Interrupt Following ICR read:
 857          *
 858          * 82571/82572 specification update errata #76
 859          * 82573 specification update errata #31
 860          * 82574 specification update errata #12
 861          * 82583 specification update errata #4
 862          */
 863         intr_func = em_intr;
 864         if ((adapter->flags & EM_FLAG_SHARED_INTR) &&
 865             (adapter->hw.mac.type == e1000_82571 ||
 866              adapter->hw.mac.type == e1000_82572 ||
 867              adapter->hw.mac.type == e1000_82573 ||
 868              adapter->hw.mac.type == e1000_82574 ||
 869              adapter->hw.mac.type == e1000_82583))
 870                 intr_func = em_intr_mask;
 871
 872         error = bus_setup_intr(dev, adapter->intr_res, INTR_MPSAFE,
 873                                intr_func, adapter, &adapter->intr_tag,
 874                                ifp->if_serializer);
 875         if (error) {
 876                 device_printf(dev, "Failed to register interrupt handler");
 877                 ether_ifdetach(ifp);
 878                 goto fail;
 879         }
 880         return (0);
 881 fail:
 882         em_detach(dev);
 883         return (error);
 884 }
 885
 886 static int
 887 em_detach(device_t dev)
 888 {
 889         struct adapter *adapter = device_get_softc(dev);
 890
 891         if (device_is_attached(dev)) {
 892                 struct ifnet *ifp = &adapter->arpcom.ac_if;
 893
 894                 lwkt_serialize_enter(ifp->if_serializer);
 895
 896                 em_stop(adapter);
 897
 898                 e1000_phy_hw_reset(&adapter->hw);
 899
 900                 em_rel_mgmt(adapter);
 901                 em_rel_hw_control(adapter);
 902
 903                 if (adapter->wol) {
 904                         E1000_WRITE_REG(&adapter->hw, E1000_WUC,
 905                                         E1000_WUC_PME_EN);
 906                         E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 907                         em_enable_wol(dev);
 908                 }
 909
 910                 bus_teardown_intr(dev, adapter->intr_res, adapter->intr_tag);
 911
 912                 lwkt_serialize_exit(ifp->if_serializer);
 913
 914                 ether_ifdetach(ifp);
 915         } else if (adapter->memory != NULL) {
 916                 em_rel_hw_control(adapter);
 917         }
 918
 919         ifmedia_removeall(&adapter->media);
 920         bus_generic_detach(dev);
 921
 922         em_free_pci_res(adapter);
 923
 924         em_destroy_tx_ring(adapter, adapter->num_tx_desc);
 925         em_destroy_rx_ring(adapter, adapter->num_rx_desc);
 926
 927         /* Free Transmit Descriptor ring */
 928         if (adapter->tx_desc_base)
 929                 em_dma_free(adapter, &adapter->txdma);
 930
 931         /* Free Receive Descriptor ring */
 932         if (adapter->rx_desc_base)
 933                 em_dma_free(adapter, &adapter->rxdma);
 934
 935         /* Free top level busdma tag */
 936         if (adapter->parent_dtag != NULL)
 937                 bus_dma_tag_destroy(adapter->parent_dtag);
 938
 939         if (adapter->mta != NULL)
 940                 kfree(adapter->mta, M_DEVBUF);
 941
 942         return (0);
 943 }
 944
 945 static int
 946 em_shutdown(device_t dev)
 947 {
 948         return em_suspend(dev);
 949 }
 950
 951 static int
 952 em_suspend(device_t dev)
 953 {
 954         struct adapter *adapter = device_get_softc(dev);
 955         struct ifnet *ifp = &adapter->arpcom.ac_if;
 956
 957         lwkt_serialize_enter(ifp->if_serializer);
 958
 959         em_stop(adapter);
 960
 961         em_rel_mgmt(adapter);
 962         em_rel_hw_control(adapter);
 963
 964         if (adapter->wol) {
 965                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
 966                 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
 967                 em_enable_wol(dev);
 968         }
 969
 970         lwkt_serialize_exit(ifp->if_serializer);
 971
 972         return bus_generic_suspend(dev);
 973 }
 974
 975 static int
 976 em_resume(device_t dev)
 977 {
 978         struct adapter *adapter = device_get_softc(dev);
 979         struct ifnet *ifp = &adapter->arpcom.ac_if;
 980
 981         lwkt_serialize_enter(ifp->if_serializer);
 982
 983         if (adapter->hw.mac.type == e1000_pch2lan)
 984                 e1000_resume_workarounds_pchlan(&adapter->hw);
 985
 986         em_init(adapter);
 987         em_get_mgmt(adapter);
 988         if_devstart(ifp);
 989
 990         lwkt_serialize_exit(ifp->if_serializer);
 991
 992         return bus_generic_resume(dev);
 993 }
 994
 995 static void
 996 em_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
 997 {
 998         struct adapter *adapter = ifp->if_softc;
 999         struct mbuf *m_head;
1000         int idx = -1, nsegs = 0;
1001
1002         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1003         ASSERT_SERIALIZED(ifp->if_serializer);
1004
1005         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
1006                 return;
1007
1008         if (!adapter->link_active) {
1009                 ifq_purge(&ifp->if_snd);
1010                 return;
1011         }
1012
1013         while (!ifq_is_empty(&ifp->if_snd)) {
1014                 /* Now do we at least have a minimal? */
1015                 if (EM_IS_OACTIVE(adapter)) {
1016                         em_tx_collect(adapter);
1017                         if (EM_IS_OACTIVE(adapter)) {
1018                                 ifq_set_oactive(&ifp->if_snd);
1019                                 adapter->no_tx_desc_avail1++;
1020                                 break;
1021                         }
1022                 }
1023
1024                 logif(pkt_txqueue);
1025                 m_head = ifq_dequeue(&ifp->if_snd);
1026                 if (m_head == NULL)
1027                         break;
1028
1029                 if (em_encap(adapter, &m_head, &nsegs, &idx)) {
1030                         IFNET_STAT_INC(ifp, oerrors, 1);
1031                         em_tx_collect(adapter);
1032                         continue;
1033                 }
1034
1035                 if (nsegs >= adapter->tx_wreg_nsegs && idx >= 0) {
1036                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1037                         nsegs = 0;
1038                         idx = -1;
1039                 }
1040
1041                 /* Send a copy of the frame to the BPF listener */
1042                 ETHER_BPF_MTAP(ifp, m_head);
1043
1044                 /* Set timeout in case hardware has problems transmitting. */
1045                 ifp->if_timer = EM_TX_TIMEOUT;
1046         }
1047         if (idx >= 0)
1048                 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), idx);
1049 }
1050
1051 static int
1052 em_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
1053 {
1054         struct adapter *adapter = ifp->if_softc;
1055         struct ifreq *ifr = (struct ifreq *)data;
1056         uint16_t eeprom_data = 0;
1057         int max_frame_size, mask, reinit;
1058         int error = 0;
1059
1060         ASSERT_SERIALIZED(ifp->if_serializer);
1061
1062         switch (command) {
1063         case SIOCSIFMTU:
1064                 switch (adapter->hw.mac.type) {
1065                 case e1000_82573:
1066                         /*
1067                          * 82573 only supports jumbo frames
1068                          * if ASPM is disabled.
1069                          */
1070                         e1000_read_nvm(&adapter->hw,
1071                             NVM_INIT_3GIO_3, 1, &eeprom_data);
1072                         if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1073                                 max_frame_size = ETHER_MAX_LEN;
1074                                 break;
1075                         }
1076                         /* FALL THROUGH */
1077
1078                 /* Limit Jumbo Frame size */
1079                 case e1000_82571:
1080                 case e1000_82572:
1081                 case e1000_ich9lan:
1082                 case e1000_ich10lan:
1083                 case e1000_pch2lan:
1084                 case e1000_pch_lpt:
1085                 case e1000_82574:
1086                 case e1000_82583:
1087                 case e1000_80003es2lan:
1088                         max_frame_size = 9234;
1089                         break;
1090
1091                 case e1000_pchlan:
1092                         max_frame_size = 4096;
1093                         break;
1094
1095                 /* Adapters that do not support jumbo frames */
1096                 case e1000_82542:
1097                 case e1000_ich8lan:
1098                         max_frame_size = ETHER_MAX_LEN;
1099                         break;
1100
1101                 default:
1102                         max_frame_size = MAX_JUMBO_FRAME_SIZE;
1103                         break;
1104                 }
1105                 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1106                     ETHER_CRC_LEN) {
1107                         error = EINVAL;
1108                         break;
1109                 }
1110
1111                 ifp->if_mtu = ifr->ifr_mtu;
1112                 adapter->hw.mac.max_frame_size =
1113                     ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1114
1115                 if (ifp->if_flags & IFF_RUNNING)
1116                         em_init(adapter);
1117                 break;
1118
1119         case SIOCSIFFLAGS:
1120                 if (ifp->if_flags & IFF_UP) {
1121                         if ((ifp->if_flags & IFF_RUNNING)) {
1122                                 if ((ifp->if_flags ^ adapter->if_flags) &
1123                                     (IFF_PROMISC | IFF_ALLMULTI)) {
1124                                         em_disable_promisc(adapter);
1125                                         em_set_promisc(adapter);
1126                                 }
1127                         } else {
1128                                 em_init(adapter);
1129                         }
1130                 } else if (ifp->if_flags & IFF_RUNNING) {
1131                         em_stop(adapter);
1132                 }
1133                 adapter->if_flags = ifp->if_flags;
1134                 break;
1135
1136         case SIOCADDMULTI:
1137         case SIOCDELMULTI:
1138                 if (ifp->if_flags & IFF_RUNNING) {
1139                         em_disable_intr(adapter);
1140                         em_set_multi(adapter);
1141                         if (adapter->hw.mac.type == e1000_82542 &&
1142                             adapter->hw.revision_id == E1000_REVISION_2)
1143                                 em_init_rx_unit(adapter);
1144 #ifdef IFPOLL_ENABLE
1145                         if (!(ifp->if_flags & IFF_NPOLLING))
1146 #endif
1147                                 em_enable_intr(adapter);
1148                 }
1149                 break;
1150
1151         case SIOCSIFMEDIA:
1152                 /* Check SOL/IDER usage */
1153                 if (e1000_check_reset_block(&adapter->hw)) {
1154                         device_printf(adapter->dev, "Media change is"
1155                             " blocked due to SOL/IDER session.\n");
1156                         break;
1157                 }
1158                 /* FALL THROUGH */
1159
1160         case SIOCGIFMEDIA:
1161                 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1162                 break;
1163
1164         case SIOCSIFCAP:
1165                 reinit = 0;
1166                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1167                 if (mask & IFCAP_RXCSUM) {
1168                         ifp->if_capenable ^= IFCAP_RXCSUM;
1169                         reinit = 1;
1170                 }
1171                 if (mask & IFCAP_TXCSUM) {
1172                         ifp->if_capenable ^= IFCAP_TXCSUM;
1173                         if (ifp->if_capenable & IFCAP_TXCSUM)
1174                                 ifp->if_hwassist |= EM_CSUM_FEATURES;
1175                         else
1176                                 ifp->if_hwassist &= ~EM_CSUM_FEATURES;
1177                 }
1178                 if (mask & IFCAP_TSO) {
1179                         ifp->if_capenable ^= IFCAP_TSO;
1180                         if (ifp->if_capenable & IFCAP_TSO)
1181                                 ifp->if_hwassist |= CSUM_TSO;
1182                         else
1183                                 ifp->if_hwassist &= ~CSUM_TSO;
1184                 }
1185                 if (mask & IFCAP_VLAN_HWTAGGING) {
1186                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1187                         reinit = 1;
1188                 }
1189                 if (reinit && (ifp->if_flags & IFF_RUNNING))
1190                         em_init(adapter);
1191                 break;
1192
1193         default:
1194                 error = ether_ioctl(ifp, command, data);
1195                 break;
1196         }
1197         return (error);
1198 }
1199
1200 static void
1201 em_watchdog(struct ifnet *ifp)
1202 {
1203         struct adapter *adapter = ifp->if_softc;
1204
1205         ASSERT_SERIALIZED(ifp->if_serializer);
1206
1207         /*
1208          * The timer is set to 5 every time start queues a packet.
1209          * Then txeof keeps resetting it as long as it cleans at
1210          * least one descriptor.
1211          * Finally, anytime all descriptors are clean the timer is
1212          * set to 0.
1213          */
1214
1215         if (E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1216             E1000_READ_REG(&adapter->hw, E1000_TDH(0))) {
1217                 /*
1218                  * If we reach here, all TX jobs are completed and
1219                  * the TX engine should have been idled for some time.
1220                  * We don't need to call if_devstart() here.
1221                  */
1222                 ifq_clr_oactive(&ifp->if_snd);
1223                 ifp->if_timer = 0;
1224                 return;
1225         }
1226
1227         /*
1228          * If we are in this routine because of pause frames, then
1229          * don't reset the hardware.
1230          */
1231         if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1232             E1000_STATUS_TXOFF) {
1233                 ifp->if_timer = EM_TX_TIMEOUT;
1234                 return;
1235         }
1236
1237         if (e1000_check_for_link(&adapter->hw) == 0)
1238                 if_printf(ifp, "watchdog timeout -- resetting\n");
1239
1240         IFNET_STAT_INC(ifp, oerrors, 1);
1241         adapter->watchdog_events++;
1242
1243         em_init(adapter);
1244
1245         if (!ifq_is_empty(&ifp->if_snd))
1246                 if_devstart(ifp);
1247 }
1248
1249 static void
1250 em_init(void *xsc)
1251 {
1252         struct adapter *adapter = xsc;
1253         struct ifnet *ifp = &adapter->arpcom.ac_if;
1254         device_t dev = adapter->dev;
1255
1256         ASSERT_SERIALIZED(ifp->if_serializer);
1257
1258         em_stop(adapter);
1259
1260         /* Get the latest mac address, User can use a LAA */
1261         bcopy(IF_LLADDR(ifp), adapter->hw.mac.addr, ETHER_ADDR_LEN);
1262
1263         /* Put the address into the Receive Address Array */
1264         e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1265
1266         /*
1267          * With the 82571 adapter, RAR[0] may be overwritten
1268          * when the other port is reset, we make a duplicate
1269          * in RAR[14] for that eventuality, this assures
1270          * the interface continues to function.
1271          */
1272         if (adapter->hw.mac.type == e1000_82571) {
1273                 e1000_set_laa_state_82571(&adapter->hw, TRUE);
1274                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1275                     E1000_RAR_ENTRIES - 1);
1276         }
1277
1278         /* Reset the hardware */
1279         if (em_reset(adapter)) {
1280                 device_printf(dev, "Unable to reset the hardware\n");
1281                 /* XXX em_stop()? */
1282                 return;
1283         }
1284         em_update_link_status(adapter);
1285
1286         /* Setup VLAN support, basic and offload if available */
1287         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1288
1289         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
1290                 uint32_t ctrl;
1291
1292                 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1293                 ctrl |= E1000_CTRL_VME;
1294                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1295         }
1296
1297         /* Configure for OS presence */
1298         em_get_mgmt(adapter);
1299
1300         /* Prepare transmit descriptors and buffers */
1301         em_init_tx_ring(adapter);
1302         em_init_tx_unit(adapter);
1303
1304         /* Setup Multicast table */
1305         em_set_multi(adapter);
1306
1307         /* Prepare receive descriptors and buffers */
1308         if (em_init_rx_ring(adapter)) {
1309                 device_printf(dev, "Could not setup receive structures\n");
1310                 em_stop(adapter);
1311                 return;
1312         }
1313         em_init_rx_unit(adapter);
1314
1315         /* Don't lose promiscuous settings */
1316         em_set_promisc(adapter);
1317
1318         ifp->if_flags |= IFF_RUNNING;
1319         ifq_clr_oactive(&ifp->if_snd);
1320
1321         callout_reset(&adapter->timer, hz, em_timer, adapter);
1322         e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1323
1324         /* MSI/X configuration for 82574 */
1325         if (adapter->hw.mac.type == e1000_82574) {
1326                 int tmp;
1327
1328                 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1329                 tmp |= E1000_CTRL_EXT_PBA_CLR;
1330                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1331                 /*
1332                  * XXX MSIX
1333                  * Set the IVAR - interrupt vector routing.
1334                  * Each nibble represents a vector, high bit
1335                  * is enable, other 3 bits are the MSIX table
1336                  * entry, we map RXQ0 to 0, TXQ0 to 1, and
1337                  * Link (other) to 2, hence the magic number.
1338                  */
1339                 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1340         }
1341
1342 #ifdef IFPOLL_ENABLE
1343         /*
1344          * Only enable interrupts if we are not polling, make sure
1345          * they are off otherwise.
1346          */
1347         if (ifp->if_flags & IFF_NPOLLING)
1348                 em_disable_intr(adapter);
1349         else
1350 #endif /* IFPOLL_ENABLE */
1351                 em_enable_intr(adapter);
1352
1353         /* AMT based hardware can now take control from firmware */
1354         if ((adapter->flags & (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT)) ==
1355             (EM_FLAG_HAS_MGMT | EM_FLAG_HAS_AMT) &&
1356             adapter->hw.mac.type >= e1000_82571)
1357                 em_get_hw_control(adapter);
1358 }
1359
1360 #ifdef IFPOLL_ENABLE
1361
1362 static void
1363 em_npoll_compat(struct ifnet *ifp, void *arg __unused, int count)
1364 {
1365         struct adapter *adapter = ifp->if_softc;
1366
1367         ASSERT_SERIALIZED(ifp->if_serializer);
1368
1369         if (adapter->npoll.ifpc_stcount-- == 0) {
1370                 uint32_t reg_icr;
1371
1372                 adapter->npoll.ifpc_stcount = adapter->npoll.ifpc_stfrac;
1373
1374                 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1375                 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1376                         callout_stop(&adapter->timer);
1377                         adapter->hw.mac.get_link_status = 1;
1378                         em_update_link_status(adapter);
1379                         callout_reset(&adapter->timer, hz, em_timer, adapter);
1380                 }
1381         }
1382
1383         em_rxeof(adapter, count);
1384         em_txeof(adapter);
1385
1386         if (!ifq_is_empty(&ifp->if_snd))
1387                 if_devstart(ifp);
1388 }
1389
1390 static void
1391 em_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1392 {
1393         struct adapter *adapter = ifp->if_softc;
1394
1395         ASSERT_SERIALIZED(ifp->if_serializer);
1396
1397         if (info != NULL) {
1398                 int cpuid = adapter->npoll.ifpc_cpuid;
1399
1400                 info->ifpi_rx[cpuid].poll_func = em_npoll_compat;
1401                 info->ifpi_rx[cpuid].arg = NULL;
1402                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
1403
1404                 if (ifp->if_flags & IFF_RUNNING)
1405                         em_disable_intr(adapter);
1406                 ifq_set_cpuid(&ifp->if_snd, cpuid);
1407         } else {
1408                 if (ifp->if_flags & IFF_RUNNING)
1409                         em_enable_intr(adapter);
1410                 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(adapter->intr_res));
1411         }
1412 }
1413
1414 #endif /* IFPOLL_ENABLE */
1415
1416 static void
1417 em_intr(void *xsc)
1418 {
1419         em_intr_body(xsc, TRUE);
1420 }
1421
1422 static void
1423 em_intr_body(struct adapter *adapter, boolean_t chk_asserted)
1424 {
1425         struct ifnet *ifp = &adapter->arpcom.ac_if;
1426         uint32_t reg_icr;
1427
1428         logif(intr_beg);
1429         ASSERT_SERIALIZED(ifp->if_serializer);
1430
1431         reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1432
1433         if (chk_asserted &&
1434             ((adapter->hw.mac.type >= e1000_82571 &&
1435               (reg_icr & E1000_ICR_INT_ASSERTED) == 0) ||
1436              reg_icr == 0)) {
1437                 logif(intr_end);
1438                 return;
1439         }
1440
1441         /*
1442          * XXX: some laptops trigger several spurious interrupts
1443          * on em(4) when in the resume cycle. The ICR register
1444          * reports all-ones value in this case. Processing such
1445          * interrupts would lead to a freeze. I don't know why.
1446          */
1447         if (reg_icr == 0xffffffff) {
1448                 logif(intr_end);
1449                 return;
1450         }
1451
1452         if (ifp->if_flags & IFF_RUNNING) {
1453                 if (reg_icr &
1454                     (E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_RXO))
1455                         em_rxeof(adapter, -1);
1456                 if (reg_icr & E1000_ICR_TXDW) {
1457                         em_txeof(adapter);
1458                         if (!ifq_is_empty(&ifp->if_snd))
1459                                 if_devstart(ifp);
1460                 }
1461         }
1462
1463         /* Link status change */
1464         if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1465                 callout_stop(&adapter->timer);
1466                 adapter->hw.mac.get_link_status = 1;
1467                 em_update_link_status(adapter);
1468
1469                 /* Deal with TX cruft when link lost */
1470                 em_tx_purge(adapter);
1471
1472                 callout_reset(&adapter->timer, hz, em_timer, adapter);
1473         }
1474
1475         if (reg_icr & E1000_ICR_RXO)
1476                 adapter->rx_overruns++;
1477
1478         logif(intr_end);
1479 }
1480
1481 static void
1482 em_intr_mask(void *xsc)
1483 {
1484         struct adapter *adapter = xsc;
1485
1486         E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
1487         /*
1488          * NOTE:
1489          * ICR.INT_ASSERTED bit will never be set if IMS is 0,
1490          * so don't check it.
1491          */
1492         em_intr_body(adapter, FALSE);
1493         E1000_WRITE_REG(&adapter->hw, E1000_IMS, IMS_ENABLE_MASK);
1494 }
1495
1496 static void
1497 em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1498 {
1499         struct adapter *adapter = ifp->if_softc;
1500         u_char fiber_type = IFM_1000_SX;
1501
1502         ASSERT_SERIALIZED(ifp->if_serializer);
1503
1504         em_update_link_status(adapter);
1505
1506         ifmr->ifm_status = IFM_AVALID;
1507         ifmr->ifm_active = IFM_ETHER;
1508
1509         if (!adapter->link_active)
1510                 return;
1511
1512         ifmr->ifm_status |= IFM_ACTIVE;
1513
1514         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
1515             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
1516                 if (adapter->hw.mac.type == e1000_82545)
1517                         fiber_type = IFM_1000_LX;
1518                 ifmr->ifm_active |= fiber_type | IFM_FDX;
1519         } else {
1520                 switch (adapter->link_speed) {
1521                 case 10:
1522                         ifmr->ifm_active |= IFM_10_T;
1523                         break;
1524                 case 100:
1525                         ifmr->ifm_active |= IFM_100_TX;
1526                         break;
1527
1528                 case 1000:
1529                         ifmr->ifm_active |= IFM_1000_T;
1530                         break;
1531                 }
1532                 if (adapter->link_duplex == FULL_DUPLEX)
1533                         ifmr->ifm_active |= IFM_FDX;
1534                 else
1535                         ifmr->ifm_active |= IFM_HDX;
1536         }
1537 }
1538
1539 static int
1540 em_media_change(struct ifnet *ifp)
1541 {
1542         struct adapter *adapter = ifp->if_softc;
1543         struct ifmedia *ifm = &adapter->media;
1544
1545         ASSERT_SERIALIZED(ifp->if_serializer);
1546
1547         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1548                 return (EINVAL);
1549
1550         switch (IFM_SUBTYPE(ifm->ifm_media)) {
1551         case IFM_AUTO:
1552                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1553                 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1554                 break;
1555
1556         case IFM_1000_LX:
1557         case IFM_1000_SX:
1558         case IFM_1000_T:
1559                 adapter->hw.mac.autoneg = DO_AUTO_NEG;
1560                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1561                 break;
1562
1563         case IFM_100_TX:
1564                 adapter->hw.mac.autoneg = FALSE;
1565                 adapter->hw.phy.autoneg_advertised = 0;
1566                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1567                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1568                 else
1569                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1570                 break;
1571
1572         case IFM_10_T:
1573                 adapter->hw.mac.autoneg = FALSE;
1574                 adapter->hw.phy.autoneg_advertised = 0;
1575                 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1576                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1577                 else
1578                         adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1579                 break;
1580
1581         default:
1582                 if_printf(ifp, "Unsupported media type\n");
1583                 break;
1584         }
1585
1586         em_init(adapter);
1587
1588         return (0);
1589 }
1590
1591 static int
1592 em_encap(struct adapter *adapter, struct mbuf **m_headp,
1593     int *segs_used, int *idx)
1594 {
1595         bus_dma_segment_t segs[EM_MAX_SCATTER];
1596         bus_dmamap_t map;
1597         struct em_buffer *tx_buffer, *tx_buffer_mapped;
1598         struct e1000_tx_desc *ctxd = NULL;
1599         struct mbuf *m_head = *m_headp;
1600         uint32_t txd_upper, txd_lower, txd_used, cmd = 0;
1601         int maxsegs, nsegs, i, j, first, last = 0, error;
1602
1603         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1604                 error = em_tso_pullup(adapter, m_headp);
1605                 if (error)
1606                         return error;
1607                 m_head = *m_headp;
1608         }
1609
1610         txd_upper = txd_lower = 0;
1611         txd_used = 0;
1612
1613         /*
1614          * Capture the first descriptor index, this descriptor
1615          * will have the index of the EOP which is the only one
1616          * that now gets a DONE bit writeback.
1617          */
1618         first = adapter->next_avail_tx_desc;
1619         tx_buffer = &adapter->tx_buffer_area[first];
1620         tx_buffer_mapped = tx_buffer;
1621         map = tx_buffer->map;
1622
1623         maxsegs = adapter->num_tx_desc_avail - EM_TX_RESERVED;
1624         KASSERT(maxsegs >= adapter->spare_tx_desc,
1625                 ("not enough spare TX desc"));
1626         if (adapter->pcix_82544) {
1627                 /* Half it; see the comment in em_attach() */
1628                 maxsegs >>= 1;
1629         }
1630         if (maxsegs > EM_MAX_SCATTER)
1631                 maxsegs = EM_MAX_SCATTER;
1632
1633         error = bus_dmamap_load_mbuf_defrag(adapter->txtag, map, m_headp,
1634                         segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1635         if (error) {
1636                 if (error == ENOBUFS)
1637                         adapter->mbuf_alloc_failed++;
1638                 else
1639                         adapter->no_tx_dma_setup++;
1640
1641                 m_freem(*m_headp);
1642                 *m_headp = NULL;
1643                 return error;
1644         }
1645         bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1646
1647         m_head = *m_headp;
1648         adapter->tx_nsegs += nsegs;
1649         *segs_used += nsegs;
1650
1651         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
1652                 /* TSO will consume one TX desc */
1653                 i = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
1654                 adapter->tx_nsegs += i;
1655                 *segs_used += i;
1656         } else if (m_head->m_pkthdr.csum_flags & EM_CSUM_FEATURES) {
1657                 /* TX csum offloading will consume one TX desc */
1658                 i = em_txcsum(adapter, m_head, &txd_upper, &txd_lower);
1659                 adapter->tx_nsegs += i;
1660                 *segs_used += i;
1661         }
1662
1663         /* Handle VLAN tag */
1664         if (m_head->m_flags & M_VLANTAG) {
1665                 /* Set the vlan id. */
1666                 txd_upper |= (htole16(m_head->m_pkthdr.ether_vlantag) << 16);
1667                 /* Tell hardware to add tag */
1668                 txd_lower |= htole32(E1000_TXD_CMD_VLE);
1669         }
1670
1671         i = adapter->next_avail_tx_desc;
1672
1673         /* Set up our transmit descriptors */
1674         for (j = 0; j < nsegs; j++) {
1675                 /* If adapter is 82544 and on PCIX bus */
1676                 if(adapter->pcix_82544) {
1677                         DESC_ARRAY desc_array;
1678                         uint32_t array_elements, counter;
1679
1680                         /*
1681                          * Check the Address and Length combination and
1682                          * split the data accordingly
1683                          */
1684                         array_elements = em_82544_fill_desc(segs[j].ds_addr,
1685                                                 segs[j].ds_len, &desc_array);
1686                         for (counter = 0; counter < array_elements; counter++) {
1687                                 KKASSERT(txd_used < adapter->num_tx_desc_avail);
1688
1689                                 tx_buffer = &adapter->tx_buffer_area[i];
1690                                 ctxd = &adapter->tx_desc_base[i];
1691
1692                                 ctxd->buffer_addr = htole64(
1693                                     desc_array.descriptor[counter].address);
1694                                 ctxd->lower.data = htole32(
1695                                     E1000_TXD_CMD_IFCS | txd_lower |
1696                                     desc_array.descriptor[counter].length);
1697                                 ctxd->upper.data = htole32(txd_upper);
1698
1699                                 last = i;
1700                                 if (++i == adapter->num_tx_desc)
1701                                         i = 0;
1702
1703                                 txd_used++;
1704                         }
1705                 } else {
1706                         tx_buffer = &adapter->tx_buffer_area[i];
1707                         ctxd = &adapter->tx_desc_base[i];
1708
1709                         ctxd->buffer_addr = htole64(segs[j].ds_addr);
1710                         ctxd->lower.data = htole32(E1000_TXD_CMD_IFCS |
1711                                                    txd_lower | segs[j].ds_len);
1712                         ctxd->upper.data = htole32(txd_upper);
1713
1714                         last = i;
1715                         if (++i == adapter->num_tx_desc)
1716                                 i = 0;
1717                 }
1718         }
1719
1720         adapter->next_avail_tx_desc = i;
1721         if (adapter->pcix_82544) {
1722                 KKASSERT(adapter->num_tx_desc_avail > txd_used);
1723                 adapter->num_tx_desc_avail -= txd_used;
1724         } else {
1725                 KKASSERT(adapter->num_tx_desc_avail > nsegs);
1726                 adapter->num_tx_desc_avail -= nsegs;
1727         }
1728
1729         tx_buffer->m_head = m_head;
1730         tx_buffer_mapped->map = tx_buffer->map;
1731         tx_buffer->map = map;
1732
1733         if (adapter->tx_nsegs >= adapter->tx_int_nsegs) {
1734                 adapter->tx_nsegs = 0;
1735
1736                 /*
1737                  * Report Status (RS) is turned on
1738                  * every tx_int_nsegs descriptors.
1739                  */
1740                 cmd = E1000_TXD_CMD_RS;
1741
1742                 /*
1743                  * Keep track of the descriptor, which will
1744                  * be written back by hardware.
1745                  */
1746                 adapter->tx_dd[adapter->tx_dd_tail] = last;
1747                 EM_INC_TXDD_IDX(adapter->tx_dd_tail);
1748                 KKASSERT(adapter->tx_dd_tail != adapter->tx_dd_head);
1749         }
1750
1751         /*
1752          * Last Descriptor of Packet needs End Of Packet (EOP)
1753          */
1754         ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | cmd);
1755
1756         if (adapter->hw.mac.type == e1000_82547) {
1757                 /*
1758                  * Advance the Transmit Descriptor Tail (TDT), this tells the
1759                  * E1000 that this frame is available to transmit.
1760                  */
1761                 if (adapter->link_duplex == HALF_DUPLEX) {
1762                         em_82547_move_tail_serialized(adapter);
1763                 } else {
1764                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
1765                         em_82547_update_fifo_head(adapter,
1766                             m_head->m_pkthdr.len);
1767                 }
1768         } else {
1769                 /*
1770                  * Defer TDT updating, until enough descriptors are setup
1771                  */
1772                 *idx = i;
1773         }
1774         return (0);
1775 }
1776
1777 /*
1778  * 82547 workaround to avoid controller hang in half-duplex environment.
1779  * The workaround is to avoid queuing a large packet that would span
1780  * the internal Tx FIFO ring boundary.  We need to reset the FIFO pointers
1781  * in this case.  We do that only when FIFO is quiescent.
1782  */
1783 static void
1784 em_82547_move_tail_serialized(struct adapter *adapter)
1785 {
1786         struct e1000_tx_desc *tx_desc;
1787         uint16_t hw_tdt, sw_tdt, length = 0;
1788         bool eop = 0;
1789
1790         ASSERT_SERIALIZED(adapter->arpcom.ac_if.if_serializer);
1791
1792         hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
1793         sw_tdt = adapter->next_avail_tx_desc;
1794
1795         while (hw_tdt != sw_tdt) {
1796                 tx_desc = &adapter->tx_desc_base[hw_tdt];
1797                 length += tx_desc->lower.flags.length;
1798                 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1799                 if (++hw_tdt == adapter->num_tx_desc)
1800                         hw_tdt = 0;
1801
1802                 if (eop) {
1803                         if (em_82547_fifo_workaround(adapter, length)) {
1804                                 adapter->tx_fifo_wrk_cnt++;
1805                                 callout_reset(&adapter->tx_fifo_timer, 1,
1806                                         em_82547_move_tail, adapter);
1807                                 break;
1808                         }
1809                         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
1810                         em_82547_update_fifo_head(adapter, length);
1811                         length = 0;
1812                 }
1813         }
1814 }
1815
1816 static void
1817 em_82547_move_tail(void *xsc)
1818 {
1819         struct adapter *adapter = xsc;
1820         struct ifnet *ifp = &adapter->arpcom.ac_if;
1821
1822         lwkt_serialize_enter(ifp->if_serializer);
1823         em_82547_move_tail_serialized(adapter);
1824         lwkt_serialize_exit(ifp->if_serializer);
1825 }
1826
1827 static int
1828 em_82547_fifo_workaround(struct adapter *adapter, int len)
1829 {
1830         int fifo_space, fifo_pkt_len;
1831
1832         fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1833
1834         if (adapter->link_duplex == HALF_DUPLEX) {
1835                 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1836
1837                 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1838                         if (em_82547_tx_fifo_reset(adapter))
1839                                 return (0);
1840                         else
1841                                 return (1);
1842                 }
1843         }
1844         return (0);
1845 }
1846
1847 static void
1848 em_82547_update_fifo_head(struct adapter *adapter, int len)
1849 {
1850         int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1851
1852         /* tx_fifo_head is always 16 byte aligned */
1853         adapter->tx_fifo_head += fifo_pkt_len;
1854         if (adapter->tx_fifo_head >= adapter->tx_fifo_size)
1855                 adapter->tx_fifo_head -= adapter->tx_fifo_size;
1856 }
1857
1858 static int
1859 em_82547_tx_fifo_reset(struct adapter *adapter)
1860 {
1861         uint32_t tctl;
1862
1863         if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
1864              E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
1865             (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
1866              E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
1867             (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
1868              E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
1869             (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
1870                 /* Disable TX unit */
1871                 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
1872                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
1873                     tctl & ~E1000_TCTL_EN);
1874
1875                 /* Reset FIFO pointers */
1876                 E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
1877                     adapter->tx_head_addr);
1878                 E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
1879                     adapter->tx_head_addr);
1880                 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
1881                     adapter->tx_head_addr);
1882                 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
1883                     adapter->tx_head_addr);
1884
1885                 /* Re-enable TX unit */
1886                 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
1887                 E1000_WRITE_FLUSH(&adapter->hw);
1888
1889                 adapter->tx_fifo_head = 0;
1890                 adapter->tx_fifo_reset_cnt++;
1891
1892                 return (TRUE);
1893         } else {
1894                 return (FALSE);
1895         }
1896 }
1897
1898 static void
1899 em_set_promisc(struct adapter *adapter)
1900 {
1901         struct ifnet *ifp = &adapter->arpcom.ac_if;
1902         uint32_t reg_rctl;
1903
1904         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1905
1906         if (ifp->if_flags & IFF_PROMISC) {
1907                 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1908                 /* Turn this on if you want to see bad packets */
1909                 if (em_debug_sbp)
1910                         reg_rctl |= E1000_RCTL_SBP;
1911                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1912         } else if (ifp->if_flags & IFF_ALLMULTI) {
1913                 reg_rctl |= E1000_RCTL_MPE;
1914                 reg_rctl &= ~E1000_RCTL_UPE;
1915                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1916         }
1917 }
1918
1919 static void
1920 em_disable_promisc(struct adapter *adapter)
1921 {
1922         uint32_t reg_rctl;
1923
1924         reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1925
1926         reg_rctl &= ~E1000_RCTL_UPE;
1927         reg_rctl &= ~E1000_RCTL_MPE;
1928         reg_rctl &= ~E1000_RCTL_SBP;
1929         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1930 }
1931
1932 static void
1933 em_set_multi(struct adapter *adapter)
1934 {
1935         struct ifnet *ifp = &adapter->arpcom.ac_if;
1936         struct ifmultiaddr *ifma;
1937         uint32_t reg_rctl = 0;
1938         uint8_t *mta;
1939         int mcnt = 0;
1940
1941         mta = adapter->mta;
1942         bzero(mta, ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES);
1943
1944         if (adapter->hw.mac.type == e1000_82542 &&
1945             adapter->hw.revision_id == E1000_REVISION_2) {
1946                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1947                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1948                         e1000_pci_clear_mwi(&adapter->hw);
1949                 reg_rctl |= E1000_RCTL_RST;
1950                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1951                 msec_delay(5);
1952         }
1953
1954         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1955                 if (ifma->ifma_addr->sa_family != AF_LINK)
1956                         continue;
1957
1958                 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1959                         break;
1960
1961                 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1962                     &mta[mcnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN);
1963                 mcnt++;
1964         }
1965
1966         if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1967                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1968                 reg_rctl |= E1000_RCTL_MPE;
1969                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1970         } else {
1971                 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt);
1972         }
1973
1974         if (adapter->hw.mac.type == e1000_82542 &&
1975             adapter->hw.revision_id == E1000_REVISION_2) {
1976                 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
1977                 reg_rctl &= ~E1000_RCTL_RST;
1978                 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
1979                 msec_delay(5);
1980                 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1981                         e1000_pci_set_mwi(&adapter->hw);
1982         }
1983 }
1984
1985 /*
1986  * This routine checks for link status and updates statistics.
1987  */
1988 static void
1989 em_timer(void *xsc)
1990 {
1991         struct adapter *adapter = xsc;
1992         struct ifnet *ifp = &adapter->arpcom.ac_if;
1993
1994         lwkt_serialize_enter(ifp->if_serializer);
1995
1996         em_update_link_status(adapter);
1997         em_update_stats(adapter);
1998
1999         /* Reset LAA into RAR[0] on 82571 */
2000         if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2001                 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2002
2003         if (em_display_debug_stats && (ifp->if_flags & IFF_RUNNING))
2004                 em_print_hw_stats(adapter);
2005
2006         em_smartspeed(adapter);
2007
2008         callout_reset(&adapter->timer, hz, em_timer, adapter);
2009
2010         lwkt_serialize_exit(ifp->if_serializer);
2011 }
2012
2013 static void
2014 em_update_link_status(struct adapter *adapter)
2015 {
2016         struct e1000_hw *hw = &adapter->hw;
2017         struct ifnet *ifp = &adapter->arpcom.ac_if;
2018         device_t dev = adapter->dev;
2019         uint32_t link_check = 0;
2020
2021         /* Get the cached link value or read phy for real */
2022         switch (hw->phy.media_type) {
2023         case e1000_media_type_copper:
2024                 if (hw->mac.get_link_status) {
2025                         /* Do the work to read phy */
2026                         e1000_check_for_link(hw);
2027                         link_check = !hw->mac.get_link_status;
2028                         if (link_check) /* ESB2 fix */
2029                                 e1000_cfg_on_link_up(hw);
2030                 } else {
2031                         link_check = TRUE;
2032                 }
2033                 break;
2034
2035         case e1000_media_type_fiber:
2036                 e1000_check_for_link(hw);
2037                 link_check =
2038                         E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU;
2039                 break;
2040
2041         case e1000_media_type_internal_serdes:
2042                 e1000_check_for_link(hw);
2043                 link_check = adapter->hw.mac.serdes_has_link;
2044                 break;
2045
2046         case e1000_media_type_unknown:
2047         default:
2048                 break;
2049         }
2050
2051         /* Now check for a transition */
2052         if (link_check && adapter->link_active == 0) {
2053                 e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2054                     &adapter->link_duplex);
2055
2056                 /*
2057                  * Check if we should enable/disable SPEED_MODE bit on
2058                  * 82571/82572
2059                  */
2060                 if (adapter->link_speed != SPEED_1000 &&
2061                     (hw->mac.type == e1000_82571 ||
2062                      hw->mac.type == e1000_82572)) {
2063                         int tarc0;
2064
2065                         tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2066                         tarc0 &= ~SPEED_MODE_BIT;
2067                         E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2068                 }
2069                 if (bootverbose) {
2070                         device_printf(dev, "Link is up %d Mbps %s\n",
2071                             adapter->link_speed,
2072                             ((adapter->link_duplex == FULL_DUPLEX) ?
2073                             "Full Duplex" : "Half Duplex"));
2074                 }
2075                 adapter->link_active = 1;
2076                 adapter->smartspeed = 0;
2077                 ifp->if_baudrate = adapter->link_speed * 1000000;
2078                 ifp->if_link_state = LINK_STATE_UP;
2079                 if_link_state_change(ifp);
2080         } else if (!link_check && adapter->link_active == 1) {
2081                 ifp->if_baudrate = adapter->link_speed = 0;
2082                 adapter->link_duplex = 0;
2083                 if (bootverbose)
2084                         device_printf(dev, "Link is Down\n");
2085                 adapter->link_active = 0;
2086 #if 0
2087                 /* Link down, disable watchdog */
2088                 if->if_timer = 0;
2089 #endif
2090                 ifp->if_link_state = LINK_STATE_DOWN;
2091                 if_link_state_change(ifp);
2092         }
2093 }
2094
2095 static void
2096 em_stop(struct adapter *adapter)
2097 {
2098         struct ifnet *ifp = &adapter->arpcom.ac_if;
2099         int i;
2100
2101         ASSERT_SERIALIZED(ifp->if_serializer);
2102
2103         em_disable_intr(adapter);
2104
2105         callout_stop(&adapter->timer);
2106         callout_stop(&adapter->tx_fifo_timer);
2107
2108         ifp->if_flags &= ~IFF_RUNNING;
2109         ifq_clr_oactive(&ifp->if_snd);
2110         ifp->if_timer = 0;
2111
2112         e1000_reset_hw(&adapter->hw);
2113         if (adapter->hw.mac.type >= e1000_82544)
2114                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2115
2116         for (i = 0; i < adapter->num_tx_desc; i++) {
2117                 struct em_buffer *tx_buffer = &adapter->tx_buffer_area[i];
2118
2119                 if (tx_buffer->m_head != NULL) {
2120                         bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2121                         m_freem(tx_buffer->m_head);
2122                         tx_buffer->m_head = NULL;
2123                 }
2124         }
2125
2126         for (i = 0; i < adapter->num_rx_desc; i++) {
2127                 struct em_buffer *rx_buffer = &adapter->rx_buffer_area[i];
2128
2129                 if (rx_buffer->m_head != NULL) {
2130                         bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2131                         m_freem(rx_buffer->m_head);
2132                         rx_buffer->m_head = NULL;
2133                 }
2134         }
2135
2136         if (adapter->fmp != NULL)
2137                 m_freem(adapter->fmp);
2138         adapter->fmp = NULL;
2139         adapter->lmp = NULL;
2140
2141         adapter->csum_flags = 0;
2142         adapter->csum_lhlen = 0;
2143         adapter->csum_iphlen = 0;
2144         adapter->csum_thlen = 0;
2145         adapter->csum_mss = 0;
2146         adapter->csum_pktlen = 0;
2147
2148         adapter->tx_dd_head = 0;
2149         adapter->tx_dd_tail = 0;
2150         adapter->tx_nsegs = 0;
2151 }
2152
2153 static int
2154 em_get_hw_info(struct adapter *adapter)
2155 {
2156         device_t dev = adapter->dev;
2157
2158         /* Save off the information about this board */
2159         adapter->hw.vendor_id = pci_get_vendor(dev);
2160         adapter->hw.device_id = pci_get_device(dev);
2161         adapter->hw.revision_id = pci_get_revid(dev);
2162         adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
2163         adapter->hw.subsystem_device_id = pci_get_subdevice(dev);
2164
2165         /* Do Shared Code Init and Setup */
2166         if (e1000_set_mac_type(&adapter->hw))
2167                 return ENXIO;
2168         return 0;
2169 }
2170
2171 static int
2172 em_alloc_pci_res(struct adapter *adapter)
2173 {
2174         device_t dev = adapter->dev;
2175         u_int intr_flags;
2176         int val, rid, msi_enable;
2177
2178         /* Enable bus mastering */
2179         pci_enable_busmaster(dev);
2180
2181         adapter->memory_rid = EM_BAR_MEM;
2182         adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2183                                 &adapter->memory_rid, RF_ACTIVE);
2184         if (adapter->memory == NULL) {
2185                 device_printf(dev, "Unable to allocate bus resource: memory\n");
2186                 return (ENXIO);
2187         }
2188         adapter->osdep.mem_bus_space_tag =
2189             rman_get_bustag(adapter->memory);
2190         adapter->osdep.mem_bus_space_handle =
2191             rman_get_bushandle(adapter->memory);
2192
2193         /* XXX This is quite goofy, it is not actually used */
2194         adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2195
2196         /* Only older adapters use IO mapping */
2197         if (adapter->hw.mac.type > e1000_82543 &&
2198             adapter->hw.mac.type < e1000_82571) {
2199                 /* Figure our where our IO BAR is ? */
2200                 for (rid = PCIR_BAR(0); rid < PCIR_CARDBUSCIS;) {
2201                         val = pci_read_config(dev, rid, 4);
2202                         if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2203                                 adapter->io_rid = rid;
2204                                 break;
2205                         }
2206                         rid += 4;
2207                         /* check for 64bit BAR */
2208                         if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2209                                 rid += 4;
2210                 }
2211                 if (rid >= PCIR_CARDBUSCIS) {
2212                         device_printf(dev, "Unable to locate IO BAR\n");
2213                         return (ENXIO);
2214                 }
2215                 adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
2216                                         &adapter->io_rid, RF_ACTIVE);
2217                 if (adapter->ioport == NULL) {
2218                         device_printf(dev, "Unable to allocate bus resource: "
2219                             "ioport\n");
2220                         return (ENXIO);
2221                 }
2222                 adapter->hw.io_base = 0;
2223                 adapter->osdep.io_bus_space_tag =
2224                     rman_get_bustag(adapter->ioport);
2225                 adapter->osdep.io_bus_space_handle =
2226                     rman_get_bushandle(adapter->ioport);
2227         }
2228
2229         /*
2230          * Don't enable MSI-X on 82574, see:
2231          * 82574 specification update errata #15
2232          *
2233          * Don't enable MSI on PCI/PCI-X chips, see:
2234          * 82540 specification update errata #6
2235          * 82545 specification update errata #4
2236          *
2237          * Don't enable MSI on 82571/82572, see:
2238          * 82571/82572 specification update errata #63
2239          */
2240         msi_enable = em_msi_enable;
2241         if (msi_enable &&
2242             (!pci_is_pcie(dev) ||
2243              adapter->hw.mac.type == e1000_82571 ||
2244              adapter->hw.mac.type == e1000_82572))
2245                 msi_enable = 0;
2246
2247         adapter->intr_type = pci_alloc_1intr(dev, msi_enable,
2248             &adapter->intr_rid, &intr_flags);
2249
2250         if (adapter->intr_type == PCI_INTR_TYPE_LEGACY) {
2251                 int unshared;
2252
2253                 unshared = device_getenv_int(dev, "irq.unshared", 0);
2254                 if (!unshared) {
2255                         adapter->flags |= EM_FLAG_SHARED_INTR;
2256                         if (bootverbose)
2257                                 device_printf(dev, "IRQ shared\n");
2258                 } else {
2259                         intr_flags &= ~RF_SHAREABLE;
2260                         if (bootverbose)
2261                                 device_printf(dev, "IRQ unshared\n");
2262                 }
2263         }
2264
2265         adapter->intr_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
2266             &adapter->intr_rid, intr_flags);
2267         if (adapter->intr_res == NULL) {
2268                 device_printf(dev, "Unable to allocate bus resource: "
2269                     "interrupt\n");
2270                 return (ENXIO);
2271         }
2272
2273         adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2274         adapter->hw.back = &adapter->osdep;
2275         return (0);
2276 }
2277
2278 static void
2279 em_free_pci_res(struct adapter *adapter)
2280 {
2281         device_t dev = adapter->dev;
2282
2283         if (adapter->intr_res != NULL) {
2284                 bus_release_resource(dev, SYS_RES_IRQ,
2285                     adapter->intr_rid, adapter->intr_res);
2286         }
2287
2288         if (adapter->intr_type == PCI_INTR_TYPE_MSI)
2289                 pci_release_msi(dev);
2290
2291         if (adapter->memory != NULL) {
2292                 bus_release_resource(dev, SYS_RES_MEMORY,
2293                     adapter->memory_rid, adapter->memory);
2294         }
2295
2296         if (adapter->flash != NULL) {
2297                 bus_release_resource(dev, SYS_RES_MEMORY,
2298                     adapter->flash_rid, adapter->flash);
2299         }
2300
2301         if (adapter->ioport != NULL) {
2302                 bus_release_resource(dev, SYS_RES_IOPORT,
2303                     adapter->io_rid, adapter->ioport);
2304         }
2305 }
2306
2307 static int
2308 em_reset(struct adapter *adapter)
2309 {
2310         device_t dev = adapter->dev;
2311         uint16_t rx_buffer_size;
2312         uint32_t pba;
2313
2314         /* When hardware is reset, fifo_head is also reset */
2315         adapter->tx_fifo_head = 0;
2316
2317         /* Set up smart power down as default off on newer adapters. */
2318         if (!em_smart_pwr_down &&
2319             (adapter->hw.mac.type == e1000_82571 ||
2320              adapter->hw.mac.type == e1000_82572)) {
2321                 uint16_t phy_tmp = 0;
2322
2323                 /* Speed up time to link by disabling smart power down. */
2324                 e1000_read_phy_reg(&adapter->hw,
2325                     IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2326                 phy_tmp &= ~IGP02E1000_PM_SPD;
2327                 e1000_write_phy_reg(&adapter->hw,
2328                     IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2329         }
2330
2331         /*
2332          * Packet Buffer Allocation (PBA)
2333          * Writing PBA sets the receive portion of the buffer
2334          * the remainder is used for the transmit buffer.
2335          *
2336          * Devices before the 82547 had a Packet Buffer of 64K.
2337          *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
2338          * After the 82547 the buffer was reduced to 40K.
2339          *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
2340          *   Note: default does not leave enough room for Jumbo Frame >10k.
2341          */
2342         switch (adapter->hw.mac.type) {
2343         case e1000_82547:
2344         case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
2345                 if (adapter->hw.mac.max_frame_size > 8192)
2346                         pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
2347                 else
2348                         pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
2349                 adapter->tx_fifo_head = 0;
2350                 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
2351                 adapter->tx_fifo_size =
2352                     (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
2353                 break;
2354
2355         /* Total Packet Buffer on these is 48K */
2356         case e1000_82571:
2357         case e1000_82572:
2358         case e1000_80003es2lan:
2359                 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
2360                 break;
2361
2362         case e1000_82573: /* 82573: Total Packet Buffer is 32K */
2363                 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
2364                 break;
2365
2366         case e1000_82574:
2367         case e1000_82583:
2368                 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
2369                 break;
2370
2371         case e1000_ich8lan:
2372                 pba = E1000_PBA_8K;
2373                 break;
2374
2375         case e1000_ich9lan:
2376         case e1000_ich10lan:
2377 #define E1000_PBA_10K   0x000A
2378                 pba = E1000_PBA_10K;
2379                 break;
2380
2381         case e1000_pchlan:
2382         case e1000_pch2lan:
2383         case e1000_pch_lpt:
2384                 pba = E1000_PBA_26K;
2385                 break;
2386
2387         default:
2388                 /* Devices before 82547 had a Packet Buffer of 64K.   */
2389                 if (adapter->hw.mac.max_frame_size > 8192)
2390                         pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
2391                 else
2392                         pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
2393         }
2394         E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
2395
2396         /*
2397          * These parameters control the automatic generation (Tx) and
2398          * response (Rx) to Ethernet PAUSE frames.
2399          * - High water mark should allow for at least two frames to be
2400          *   received after sending an XOFF.
2401          * - Low water mark works best when it is very near the high water mark.
2402          *   This allows the receiver to restart by sending XON when it has
2403          *   drained a bit. Here we use an arbitary value of 1500 which will
2404          *   restart after one full frame is pulled from the buffer. There
2405          *   could be several smaller frames in the buffer and if so they will
2406          *   not trigger the XON until their total number reduces the buffer
2407          *   by 1500.
2408          * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2409          */
2410         rx_buffer_size =
2411                 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) << 10;
2412
2413         adapter->hw.fc.high_water = rx_buffer_size -
2414             roundup2(adapter->hw.mac.max_frame_size, 1024);
2415         adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
2416
2417         if (adapter->hw.mac.type == e1000_80003es2lan)
2418                 adapter->hw.fc.pause_time = 0xFFFF;
2419         else
2420                 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
2421
2422         adapter->hw.fc.send_xon = TRUE;
2423
2424         adapter->hw.fc.requested_mode = e1000_fc_full;
2425
2426         /*
2427          * Device specific overrides/settings
2428          */
2429         switch (adapter->hw.mac.type) {
2430         case e1000_pchlan:
2431                 /* Workaround: no TX flow ctrl for PCH */
2432                 adapter->hw.fc.requested_mode = e1000_fc_rx_pause;
2433                 adapter->hw.fc.pause_time = 0xFFFF; /* override */
2434                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2435                         adapter->hw.fc.high_water = 0x3500;
2436                         adapter->hw.fc.low_water = 0x1500;
2437                 } else {
2438                         adapter->hw.fc.high_water = 0x5000;
2439                         adapter->hw.fc.low_water = 0x3000;
2440                 }
2441                 adapter->hw.fc.refresh_time = 0x1000;
2442                 break;
2443
2444         case e1000_pch2lan:
2445         case e1000_pch_lpt:
2446                 adapter->hw.fc.high_water = 0x5C20;
2447                 adapter->hw.fc.low_water = 0x5048;
2448                 adapter->hw.fc.pause_time = 0x0650;
2449                 adapter->hw.fc.refresh_time = 0x0400;
2450                 /* Jumbos need adjusted PBA */
2451                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU)
2452                         E1000_WRITE_REG(&adapter->hw, E1000_PBA, 12);
2453                 else
2454                         E1000_WRITE_REG(&adapter->hw, E1000_PBA, 26);
2455                 break;
2456
2457         case e1000_ich9lan:
2458         case e1000_ich10lan:
2459                 if (adapter->arpcom.ac_if.if_mtu > ETHERMTU) {
2460                         adapter->hw.fc.high_water = 0x2800;
2461                         adapter->hw.fc.low_water =
2462                             adapter->hw.fc.high_water - 8;
2463                         break;
2464                 }
2465                 /* FALL THROUGH */
2466         default:
2467                 if (adapter->hw.mac.type == e1000_80003es2lan)
2468                         adapter->hw.fc.pause_time = 0xFFFF;
2469                 break;
2470         }
2471
2472         /* Issue a global reset */
2473         e1000_reset_hw(&adapter->hw);
2474         if (adapter->hw.mac.type >= e1000_82544)
2475                 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2476         em_disable_aspm(adapter);
2477
2478         if (e1000_init_hw(&adapter->hw) < 0) {
2479                 device_printf(dev, "Hardware Initialization Failed\n");
2480                 return (EIO);
2481         }
2482
2483         E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
2484         e1000_get_phy_info(&adapter->hw);
2485         e1000_check_for_link(&adapter->hw);
2486
2487         return (0);
2488 }
2489
2490 static void
2491 em_setup_ifp(struct adapter *adapter)
2492 {
2493         struct ifnet *ifp = &adapter->arpcom.ac_if;
2494
2495         if_initname(ifp, device_get_name(adapter->dev),
2496                     device_get_unit(adapter->dev));
2497         ifp->if_softc = adapter;
2498         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2499         ifp->if_init =  em_init;
2500         ifp->if_ioctl = em_ioctl;
2501         ifp->if_start = em_start;
2502 #ifdef IFPOLL_ENABLE
2503         ifp->if_npoll = em_npoll;
2504 #endif
2505         ifp->if_watchdog = em_watchdog;
2506         ifq_set_maxlen(&ifp->if_snd, adapter->num_tx_desc - 1);
2507         ifq_set_ready(&ifp->if_snd);
2508
2509         ether_ifattach(ifp, adapter->hw.mac.addr, NULL);
2510
2511         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2512         if (adapter->hw.mac.type >= e1000_82543)
2513                 ifp->if_capabilities |= IFCAP_HWCSUM;
2514         if (adapter->flags & EM_FLAG_TSO)
2515                 ifp->if_capabilities |= IFCAP_TSO;
2516         ifp->if_capenable = ifp->if_capabilities;
2517
2518         if (ifp->if_capenable & IFCAP_TXCSUM)
2519                 ifp->if_hwassist |= EM_CSUM_FEATURES;
2520         if (ifp->if_capenable & IFCAP_TSO)
2521                 ifp->if_hwassist |= CSUM_TSO;
2522
2523         /*
2524          * Tell the upper layer(s) we support long frames.
2525          */
2526         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2527
2528         /*
2529          * Specify the media types supported by this adapter and register
2530          * callbacks to update media and link information
2531          */
2532         if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
2533             adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
2534                 u_char fiber_type = IFM_1000_SX; /* default type */
2535
2536                 if (adapter->hw.mac.type == e1000_82545)
2537                         fiber_type = IFM_1000_LX;
2538                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2539                             0, NULL);
2540                 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2541         } else {
2542                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2543                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2544                             0, NULL);
2545                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2546                             0, NULL);
2547                 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2548                             0, NULL);
2549                 if (adapter->hw.phy.type != e1000_phy_ife) {
2550                         ifmedia_add(&adapter->media,
2551                                 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2552                         ifmedia_add(&adapter->media,
2553                                 IFM_ETHER | IFM_1000_T, 0, NULL);
2554                 }
2555         }
2556         ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2557         ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2558 }
2559
2560
2561 /*
2562  * Workaround for SmartSpeed on 82541 and 82547 controllers
2563  */
2564 static void
2565 em_smartspeed(struct adapter *adapter)
2566 {
2567         uint16_t phy_tmp;
2568
2569         if (adapter->link_active || adapter->hw.phy.type != e1000_phy_igp ||
2570             adapter->hw.mac.autoneg == 0 ||
2571             (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2572                 return;
2573
2574         if (adapter->smartspeed == 0) {
2575                 /*
2576                  * If Master/Slave config fault is asserted twice,
2577                  * we assume back-to-back
2578                  */
2579                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2580                 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2581                         return;
2582                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2583                 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2584                         e1000_read_phy_reg(&adapter->hw,
2585                             PHY_1000T_CTRL, &phy_tmp);
2586                         if (phy_tmp & CR_1000T_MS_ENABLE) {
2587                                 phy_tmp &= ~CR_1000T_MS_ENABLE;
2588                                 e1000_write_phy_reg(&adapter->hw,
2589                                     PHY_1000T_CTRL, phy_tmp);
2590                                 adapter->smartspeed++;
2591                                 if (adapter->hw.mac.autoneg &&
2592                                     !e1000_phy_setup_autoneg(&adapter->hw) &&
2593                                     !e1000_read_phy_reg(&adapter->hw,
2594                                      PHY_CONTROL, &phy_tmp)) {
2595                                         phy_tmp |= MII_CR_AUTO_NEG_EN |
2596                                                    MII_CR_RESTART_AUTO_NEG;
2597                                         e1000_write_phy_reg(&adapter->hw,
2598                                             PHY_CONTROL, phy_tmp);
2599                                 }
2600                         }
2601                 }
2602                 return;
2603         } else if (adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2604                 /* If still no link, perhaps using 2/3 pair cable */
2605                 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2606                 phy_tmp |= CR_1000T_MS_ENABLE;
2607                 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2608                 if (adapter->hw.mac.autoneg &&
2609                     !e1000_phy_setup_autoneg(&adapter->hw) &&
2610                     !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
2611                         phy_tmp |= MII_CR_AUTO_NEG_EN | MII_CR_RESTART_AUTO_NEG;
2612                         e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
2613                 }
2614         }
2615
2616         /* Restart process after EM_SMARTSPEED_MAX iterations */
2617         if (adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2618                 adapter->smartspeed = 0;
2619 }
2620
2621 static int
2622 em_dma_malloc(struct adapter *adapter, bus_size_t size,
2623               struct em_dma_alloc *dma)
2624 {
2625         dma->dma_vaddr = bus_dmamem_coherent_any(adapter->parent_dtag,
2626                                 EM_DBA_ALIGN, size, BUS_DMA_WAITOK,
2627                                 &dma->dma_tag, &dma->dma_map,
2628                                 &dma->dma_paddr);
2629         if (dma->dma_vaddr == NULL)
2630                 return ENOMEM;
2631         else
2632                 return 0;
2633 }
2634
2635 static void
2636 em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2637 {
2638         if (dma->dma_tag == NULL)
2639                 return;
2640         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2641         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2642         bus_dma_tag_destroy(dma->dma_tag);
2643 }
2644
2645 static int
2646 em_create_tx_ring(struct adapter *adapter)
2647 {
2648         device_t dev = adapter->dev;
2649         struct em_buffer *tx_buffer;
2650         int error, i;
2651
2652         adapter->tx_buffer_area =
2653                 kmalloc(sizeof(struct em_buffer) * adapter->num_tx_desc,
2654                         M_DEVBUF, M_WAITOK | M_ZERO);
2655
2656         /*
2657          * Create DMA tags for tx buffers
2658          */
2659         error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
2660                         1, 0,                   /* alignment, bounds */
2661                         BUS_SPACE_MAXADDR,      /* lowaddr */
2662                         BUS_SPACE_MAXADDR,      /* highaddr */
2663                         NULL, NULL,             /* filter, filterarg */
2664                         EM_TSO_SIZE,            /* maxsize */
2665                         EM_MAX_SCATTER,         /* nsegments */
2666                         PAGE_SIZE,              /* maxsegsize */
2667                         BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW |
2668                         BUS_DMA_ONEBPAGE,       /* flags */
2669                         &adapter->txtag);
2670         if (error) {
2671                 device_printf(dev, "Unable to allocate TX DMA tag\n");
2672                 kfree(adapter->tx_buffer_area, M_DEVBUF);
2673                 adapter->tx_buffer_area = NULL;
2674                 return error;
2675         }
2676
2677         /*
2678          * Create DMA maps for tx buffers
2679          */
2680         for (i = 0; i < adapter->num_tx_desc; i++) {
2681                 tx_buffer = &adapter->tx_buffer_area[i];
2682
2683                 error = bus_dmamap_create(adapter->txtag,
2684                                           BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
2685                                           &tx_buffer->map);
2686                 if (error) {
2687                         device_printf(dev, "Unable to create TX DMA map\n");
2688                         em_destroy_tx_ring(adapter, i);
2689                         return error;
2690                 }
2691         }
2692         return (0);
2693 }
2694
2695 static void
2696 em_init_tx_ring(struct adapter *adapter)
2697 {
2698         /* Clear the old ring contents */
2699         bzero(adapter->tx_desc_base,
2700             (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
2701
2702         /* Reset state */
2703         adapter->next_avail_tx_desc = 0;
2704         adapter->next_tx_to_clean = 0;
2705         adapter->num_tx_desc_avail = adapter->num_tx_desc;
2706 }
2707
2708 static void
2709 em_init_tx_unit(struct adapter *adapter)
2710 {
2711         uint32_t tctl, tarc, tipg = 0;
2712         uint64_t bus_addr;
2713
2714         /* Setup the Base and Length of the Tx Descriptor Ring */
2715         bus_addr = adapter->txdma.dma_paddr;
2716         E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
2717             adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
2718         E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
2719             (uint32_t)(bus_addr >> 32));
2720         E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
2721             (uint32_t)bus_addr);
2722         /* Setup the HW Tx Head and Tail descriptor pointers */
2723         E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
2724         E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
2725
2726         /* Set the default values for the Tx Inter Packet Gap timer */
2727         switch (adapter->hw.mac.type) {
2728         case e1000_82542:
2729                 tipg = DEFAULT_82542_TIPG_IPGT;
2730                 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2731                 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2732                 break;
2733
2734         case e1000_80003es2lan:
2735                 tipg = DEFAULT_82543_TIPG_IPGR1;
2736                 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
2737                     E1000_TIPG_IPGR2_SHIFT;
2738                 break;
2739
2740         default:
2741                 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
2742                     adapter->hw.phy.media_type ==
2743                     e1000_media_type_internal_serdes)
2744                         tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2745                 else
2746                         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2747                 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2748                 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2749                 break;
2750         }
2751
2752         E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
2753
2754         /* NOTE: 0 is not allowed for TIDV */
2755         E1000_WRITE_REG(&adapter->hw, E1000_TIDV, 1);
2756         if(adapter->hw.mac.type >= e1000_82540)
2757                 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 0);
2758
2759         if (adapter->hw.mac.type == e1000_82571 ||
2760             adapter->hw.mac.type == e1000_82572) {
2761                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
2762                 tarc |= SPEED_MODE_BIT;
2763                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
2764         } else if (adapter->hw.mac.type == e1000_80003es2lan) {
2765                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
2766                 tarc |= 1;
2767                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
2768                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
2769                 tarc |= 1;
2770                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
2771         }
2772
2773         /* Program the Transmit Control Register */
2774         tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2775         tctl &= ~E1000_TCTL_CT;
2776         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
2777                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2778
2779         if (adapter->hw.mac.type >= e1000_82571)
2780                 tctl |= E1000_TCTL_MULR;
2781
2782         /* This write will effectively turn on the transmit unit. */
2783         E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2784
2785         if (adapter->hw.mac.type == e1000_82571 ||
2786             adapter->hw.mac.type == e1000_82572 ||
2787             adapter->hw.mac.type == e1000_80003es2lan) {
2788                 /* Bit 28 of TARC1 must be cleared when MULR is enabled */
2789                 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
2790                 tarc &= ~(1 << 28);
2791                 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
2792         }
2793 }
2794
2795 static void
2796 em_destroy_tx_ring(struct adapter *adapter, int ndesc)
2797 {
2798         struct em_buffer *tx_buffer;
2799         int i;
2800
2801         if (adapter->tx_buffer_area == NULL)
2802                 return;
2803
2804         for (i = 0; i < ndesc; i++) {
2805                 tx_buffer = &adapter->tx_buffer_area[i];
2806
2807                 KKASSERT(tx_buffer->m_head == NULL);
2808                 bus_dmamap_destroy(adapter->txtag, tx_buffer->map);
2809         }
2810         bus_dma_tag_destroy(adapter->txtag);
2811
2812         kfree(adapter->tx_buffer_area, M_DEVBUF);
2813         adapter->tx_buffer_area = NULL;
2814 }
2815
2816 /*
2817  * The offload context needs to be set when we transfer the first
2818  * packet of a particular protocol (TCP/UDP).  This routine has been
2819  * enhanced to deal with inserted VLAN headers.
2820  *
2821  * If the new packet's ether header length, ip header length and
2822  * csum offloading type are same as the previous packet, we should
2823  * avoid allocating a new csum context descriptor; mainly to take
2824  * advantage of the pipeline effect of the TX data read request.
2825  *
2826  * This function returns number of TX descrptors allocated for
2827  * csum context.
2828  */
2829 static int
2830 em_txcsum(struct adapter *adapter, struct mbuf *mp,
2831           uint32_t *txd_upper, uint32_t *txd_lower)
2832 {
2833         struct e1000_context_desc *TXD;
2834         int curr_txd, ehdrlen, csum_flags;
2835         uint32_t cmd, hdr_len, ip_hlen;
2836
2837         csum_flags = mp->m_pkthdr.csum_flags & EM_CSUM_FEATURES;
2838         ip_hlen = mp->m_pkthdr.csum_iphlen;
2839         ehdrlen = mp->m_pkthdr.csum_lhlen;
2840
2841         if (adapter->csum_lhlen == ehdrlen &&
2842             adapter->csum_iphlen == ip_hlen &&
2843             adapter->csum_flags == csum_flags) {
2844                 /*
2845                  * Same csum offload context as the previous packets;
2846                  * just return.
2847                  */
2848                 *txd_upper = adapter->csum_txd_upper;
2849                 *txd_lower = adapter->csum_txd_lower;
2850                 return 0;
2851         }
2852
2853         /*
2854          * Setup a new csum offload context.
2855          */
2856
2857         curr_txd = adapter->next_avail_tx_desc;
2858         TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
2859
2860         cmd = 0;
2861
2862         /* Setup of IP header checksum. */
2863         if (csum_flags & CSUM_IP) {
2864                 /*
2865                  * Start offset for header checksum calculation.
2866                  * End offset for header checksum calculation.
2867                  * Offset of place to put the checksum.
2868                  */
2869                 TXD->lower_setup.ip_fields.ipcss = ehdrlen;
2870                 TXD->lower_setup.ip_fields.ipcse =
2871                     htole16(ehdrlen + ip_hlen - 1);
2872                 TXD->lower_setup.ip_fields.ipcso =
2873                     ehdrlen + offsetof(struct ip, ip_sum);
2874                 cmd |= E1000_TXD_CMD_IP;
2875                 *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
2876         }
2877         hdr_len = ehdrlen + ip_hlen;
2878
2879         if (csum_flags & CSUM_TCP) {
2880                 /*
2881                  * Start offset for payload checksum calculation.
2882                  * End offset for payload checksum calculation.
2883                  * Offset of place to put the checksum.
2884                  */
2885                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
2886                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2887                 TXD->upper_setup.tcp_fields.tucso =
2888                     hdr_len + offsetof(struct tcphdr, th_sum);
2889                 cmd |= E1000_TXD_CMD_TCP;
2890                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2891         } else if (csum_flags & CSUM_UDP) {
2892                 /*
2893                  * Start offset for header checksum calculation.
2894                  * End offset for header checksum calculation.
2895                  * Offset of place to put the checksum.
2896                  */
2897                 TXD->upper_setup.tcp_fields.tucss = hdr_len;
2898                 TXD->upper_setup.tcp_fields.tucse = htole16(0);
2899                 TXD->upper_setup.tcp_fields.tucso =
2900                     hdr_len + offsetof(struct udphdr, uh_sum);
2901                 *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
2902         }
2903
2904         *txd_lower = E1000_TXD_CMD_DEXT |       /* Extended descr type */
2905                      E1000_TXD_DTYP_D;          /* Data descr */
2906
2907         /* Save the information for this csum offloading context */
2908         adapter->csum_lhlen = ehdrlen;
2909         adapter->csum_iphlen = ip_hlen;
2910         adapter->csum_flags = csum_flags;
2911         adapter->csum_txd_upper = *txd_upper;
2912         adapter->csum_txd_lower = *txd_lower;
2913
2914         TXD->tcp_seg_setup.data = htole32(0);
2915         TXD->cmd_and_length =
2916             htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
2917
2918         if (++curr_txd == adapter->num_tx_desc)
2919                 curr_txd = 0;
2920
2921         KKASSERT(adapter->num_tx_desc_avail > 0);
2922         adapter->num_tx_desc_avail--;
2923
2924         adapter->next_avail_tx_desc = curr_txd;
2925         return 1;
2926 }
2927
2928 static void
2929 em_txeof(struct adapter *adapter)
2930 {
2931         struct ifnet *ifp = &adapter->arpcom.ac_if;
2932         struct em_buffer *tx_buffer;
2933         int first, num_avail;
2934
2935         if (adapter->tx_dd_head == adapter->tx_dd_tail)
2936                 return;
2937
2938         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2939                 return;
2940
2941         num_avail = adapter->num_tx_desc_avail;
2942         first = adapter->next_tx_to_clean;
2943
2944         while (adapter->tx_dd_head != adapter->tx_dd_tail) {
2945                 struct e1000_tx_desc *tx_desc;
2946                 int dd_idx = adapter->tx_dd[adapter->tx_dd_head];
2947
2948                 tx_desc = &adapter->tx_desc_base[dd_idx];
2949                 if (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2950                         EM_INC_TXDD_IDX(adapter->tx_dd_head);
2951
2952                         if (++dd_idx == adapter->num_tx_desc)
2953                                 dd_idx = 0;
2954
2955                         while (first != dd_idx) {
2956                                 logif(pkt_txclean);
2957
2958                                 num_avail++;
2959
2960                                 tx_buffer = &adapter->tx_buffer_area[first];
2961                                 if (tx_buffer->m_head) {
2962                                         IFNET_STAT_INC(ifp, opackets, 1);
2963                                         bus_dmamap_unload(adapter->txtag,
2964                                                           tx_buffer->map);
2965                                         m_freem(tx_buffer->m_head);
2966                                         tx_buffer->m_head = NULL;
2967                                 }
2968
2969                                 if (++first == adapter->num_tx_desc)
2970                                         first = 0;
2971                         }
2972                 } else {
2973                         break;
2974                 }
2975         }
2976         adapter->next_tx_to_clean = first;
2977         adapter->num_tx_desc_avail = num_avail;
2978
2979         if (adapter->tx_dd_head == adapter->tx_dd_tail) {
2980                 adapter->tx_dd_head = 0;
2981                 adapter->tx_dd_tail = 0;
2982         }
2983
2984         if (!EM_IS_OACTIVE(adapter)) {
2985                 ifq_clr_oactive(&ifp->if_snd);
2986
2987                 /* All clean, turn off the timer */
2988                 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2989                         ifp->if_timer = 0;
2990         }
2991 }
2992
2993 static void
2994 em_tx_collect(struct adapter *adapter)
2995 {
2996         struct ifnet *ifp = &adapter->arpcom.ac_if;
2997         struct em_buffer *tx_buffer;
2998         int tdh, first, num_avail, dd_idx = -1;
2999
3000         if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3001                 return;
3002
3003         tdh = E1000_READ_REG(&adapter->hw, E1000_TDH(0));
3004         if (tdh == adapter->next_tx_to_clean)
3005                 return;
3006
3007         if (adapter->tx_dd_head != adapter->tx_dd_tail)
3008                 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3009
3010         num_avail = adapter->num_tx_desc_avail;
3011         first = adapter->next_tx_to_clean;
3012
3013         while (first != tdh) {
3014                 logif(pkt_txclean);
3015
3016                 num_avail++;
3017
3018                 tx_buffer = &adapter->tx_buffer_area[first];
3019                 if (tx_buffer->m_head) {
3020                         IFNET_STAT_INC(ifp, opackets, 1);
3021                         bus_dmamap_unload(adapter->txtag,
3022                                           tx_buffer->map);
3023                         m_freem(tx_buffer->m_head);
3024                         tx_buffer->m_head = NULL;
3025                 }
3026
3027                 if (first == dd_idx) {
3028                         EM_INC_TXDD_IDX(adapter->tx_dd_head);
3029                         if (adapter->tx_dd_head == adapter->tx_dd_tail) {
3030                                 adapter->tx_dd_head = 0;
3031                                 adapter->tx_dd_tail = 0;
3032                                 dd_idx = -1;
3033                         } else {
3034                                 dd_idx = adapter->tx_dd[adapter->tx_dd_head];
3035                         }
3036                 }
3037
3038                 if (++first == adapter->num_tx_desc)
3039                         first = 0;
3040         }
3041         adapter->next_tx_to_clean = first;
3042         adapter->num_tx_desc_avail = num_avail;
3043
3044         if (!EM_IS_OACTIVE(adapter)) {
3045                 ifq_clr_oactive(&ifp->if_snd);
3046
3047                 /* All clean, turn off the timer */
3048                 if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3049                         ifp->if_timer = 0;
3050         }
3051 }
3052
3053 /*
3054  * When Link is lost sometimes there is work still in the TX ring
3055  * which will result in a watchdog, rather than allow that do an
3056  * attempted cleanup and then reinit here.  Note that this has been
3057  * seens mostly with fiber adapters.
3058  */
3059 static void
3060 em_tx_purge(struct adapter *adapter)
3061 {
3062         struct ifnet *ifp = &adapter->arpcom.ac_if;
3063
3064         if (!adapter->link_active && ifp->if_timer) {
3065                 em_tx_collect(adapter);
3066                 if (ifp->if_timer) {
3067                         if_printf(ifp, "Link lost, TX pending, reinit\n");
3068                         ifp->if_timer = 0;
3069                         em_init(adapter);
3070                 }
3071         }
3072 }
3073
3074 static int
3075 em_newbuf(struct adapter *adapter, int i, int init)
3076 {
3077         struct mbuf *m;
3078         bus_dma_segment_t seg;
3079         bus_dmamap_t map;
3080         struct em_buffer *rx_buffer;
3081         int error, nseg;
3082
3083         m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
3084         if (m == NULL) {
3085                 adapter->mbuf_cluster_failed++;
3086                 if (init) {
3087                         if_printf(&adapter->arpcom.ac_if,
3088                                   "Unable to allocate RX mbuf\n");
3089                 }
3090                 return (ENOBUFS);
3091         }
3092         m->m_len = m->m_pkthdr.len = MCLBYTES;
3093
3094         if (adapter->hw.mac.max_frame_size <= MCLBYTES - ETHER_ALIGN)
3095                 m_adj(m, ETHER_ALIGN);
3096
3097         error = bus_dmamap_load_mbuf_segment(adapter->rxtag,
3098                         adapter->rx_sparemap, m,
3099                         &seg, 1, &nseg, BUS_DMA_NOWAIT);
3100         if (error) {
3101                 m_freem(m);
3102                 if (init) {
3103                         if_printf(&adapter->arpcom.ac_if,
3104                                   "Unable to load RX mbuf\n");
3105                 }
3106                 return (error);
3107         }
3108
3109         rx_buffer = &adapter->rx_buffer_area[i];
3110         if (rx_buffer->m_head != NULL)
3111                 bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3112
3113         map = rx_buffer->map;
3114         rx_buffer->map = adapter->rx_sparemap;
3115         adapter->rx_sparemap = map;
3116
3117         rx_buffer->m_head = m;
3118
3119         adapter->rx_desc_base[i].buffer_addr = htole64(seg.ds_addr);
3120         return (0);
3121 }
3122
3123 static int
3124 em_create_rx_ring(struct adapter *adapter)
3125 {
3126         device_t dev = adapter->dev;
3127         struct em_buffer *rx_buffer;
3128         int i, error;
3129
3130         adapter->rx_buffer_area =
3131                 kmalloc(sizeof(struct em_buffer) * adapter->num_rx_desc,
3132                         M_DEVBUF, M_WAITOK | M_ZERO);
3133
3134         /*
3135          * Create DMA tag for rx buffers
3136          */
3137         error = bus_dma_tag_create(adapter->parent_dtag, /* parent */
3138                         1, 0,                   /* alignment, bounds */
3139                         BUS_SPACE_MAXADDR,      /* lowaddr */
3140                         BUS_SPACE_MAXADDR,      /* highaddr */
3141                         NULL, NULL,             /* filter, filterarg */
3142                         MCLBYTES,               /* maxsize */
3143                         1,                      /* nsegments */
3144                         MCLBYTES,               /* maxsegsize */
3145                         BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, /* flags */
3146                         &adapter->rxtag);
3147         if (error) {
3148                 device_printf(dev, "Unable to allocate RX DMA tag\n");
3149                 kfree(adapter->rx_buffer_area, M_DEVBUF);
3150                 adapter->rx_buffer_area = NULL;
3151                 return error;
3152         }
3153
3154         /*
3155          * Create spare DMA map for rx buffers
3156          */
3157         error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3158                                   &adapter->rx_sparemap);
3159         if (error) {
3160                 device_printf(dev, "Unable to create spare RX DMA map\n");
3161                 bus_dma_tag_destroy(adapter->rxtag);
3162                 kfree(adapter->rx_buffer_area, M_DEVBUF);
3163                 adapter->rx_buffer_area = NULL;
3164                 return error;
3165         }
3166
3167         /*
3168          * Create DMA maps for rx buffers
3169          */
3170         for (i = 0; i < adapter->num_rx_desc; i++) {
3171                 rx_buffer = &adapter->rx_buffer_area[i];
3172
3173                 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_WAITOK,
3174                                           &rx_buffer->map);
3175                 if (error) {
3176                         device_printf(dev, "Unable to create RX DMA map\n");
3177                         em_destroy_rx_ring(adapter, i);
3178                         return error;
3179                 }
3180         }
3181         return (0);
3182 }
3183
3184 static int
3185 em_init_rx_ring(struct adapter *adapter)
3186 {
3187         int i, error;
3188
3189         /* Reset descriptor ring */
3190         bzero(adapter->rx_desc_base,
3191             (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
3192
3193         /* Allocate new ones. */
3194         for (i = 0; i < adapter->num_rx_desc; i++) {
3195                 error = em_newbuf(adapter, i, 1);
3196                 if (error)
3197                         return (error);
3198         }
3199
3200         /* Setup our descriptor pointers */
3201         adapter->next_rx_desc_to_check = 0;
3202
3203         return (0);
3204 }
3205
3206 static void
3207 em_init_rx_unit(struct adapter *adapter)
3208 {
3209         struct ifnet *ifp = &adapter->arpcom.ac_if;
3210         uint64_t bus_addr;
3211         uint32_t rctl;
3212
3213         /*
3214          * Make sure receives are disabled while setting
3215          * up the descriptor ring
3216          */
3217         rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
3218         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
3219
3220         if (adapter->hw.mac.type >= e1000_82540) {
3221                 uint32_t itr;
3222
3223                 /*
3224                  * Set the interrupt throttling rate. Value is calculated
3225                  * as ITR = 1 / (INT_THROTTLE_CEIL * 256ns)
3226                  */
3227                 if (adapter->int_throttle_ceil)
3228                         itr = 1000000000 / 256 / adapter->int_throttle_ceil;
3229                 else
3230                         itr = 0;
3231                 em_set_itr(adapter, itr);
3232         }
3233
3234         /* Disable accelerated ackknowledge */
3235         if (adapter->hw.mac.type == e1000_82574) {
3236                 E1000_WRITE_REG(&adapter->hw,
3237                     E1000_RFCTL, E1000_RFCTL_ACK_DIS);
3238         }
3239
3240         /* Receive Checksum Offload for TCP and UDP */
3241         if (ifp->if_capenable & IFCAP_RXCSUM) {
3242                 uint32_t rxcsum;
3243
3244                 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
3245                 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3246                 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
3247         }
3248
3249         /*
3250          * XXX TEMPORARY WORKAROUND: on some systems with 82573
3251          * long latencies are observed, like Lenovo X60. This
3252          * change eliminates the problem, but since having positive
3253          * values in RDTR is a known source of problems on other
3254          * platforms another solution is being sought.
3255          */
3256         if (em_82573_workaround && adapter->hw.mac.type == e1000_82573) {
3257                 E1000_WRITE_REG(&adapter->hw, E1000_RADV, EM_RADV_82573);
3258                 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, EM_RDTR_82573);
3259         }
3260
3261         /*
3262          * Setup the Base and Length of the Rx Descriptor Ring
3263          */
3264         bus_addr = adapter->rxdma.dma_paddr;
3265         E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
3266             adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
3267         E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
3268             (uint32_t)(bus_addr >> 32));
3269         E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
3270             (uint32_t)bus_addr);
3271
3272         /*
3273          * Setup the HW Rx Head and Tail Descriptor Pointers
3274          */
3275         E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
3276         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
3277
3278         /* Set PTHRESH for improved jumbo performance */
3279         if (((adapter->hw.mac.type == e1000_ich9lan) ||
3280             (adapter->hw.mac.type == e1000_pch2lan) ||
3281             (adapter->hw.mac.type == e1000_ich10lan)) &&
3282             (ifp->if_mtu > ETHERMTU)) {
3283                 uint32_t rxdctl;
3284
3285                 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(0));
3286                 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(0), rxdctl | 3);
3287         }
3288
3289         if (adapter->hw.mac.type >= e1000_pch2lan) {
3290                 if (ifp->if_mtu > ETHERMTU)
3291                         e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, TRUE);
3292                 else
3293                         e1000_lv_jumbo_workaround_ich8lan(&adapter->hw, FALSE);
3294         }
3295
3296         /* Setup the Receive Control Register */
3297         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3298         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3299                 E1000_RCTL_RDMTS_HALF |
3300                 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3301
3302         /* Make sure VLAN Filters are off */
3303         rctl &= ~E1000_RCTL_VFE;
3304
3305         if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
3306                 rctl |= E1000_RCTL_SBP;
3307         else
3308                 rctl &= ~E1000_RCTL_SBP;
3309
3310         switch (adapter->rx_buffer_len) {
3311         default:
3312         case 2048:
3313                 rctl |= E1000_RCTL_SZ_2048;
3314                 break;
3315
3316         case 4096:
3317                 rctl |= E1000_RCTL_SZ_4096 |
3318                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3319                 break;
3320
3321         case 8192:
3322                 rctl |= E1000_RCTL_SZ_8192 |
3323                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3324                 break;
3325
3326         case 16384:
3327                 rctl |= E1000_RCTL_SZ_16384 |
3328                     E1000_RCTL_BSEX | E1000_RCTL_LPE;
3329                 break;
3330         }
3331
3332         if (ifp->if_mtu > ETHERMTU)
3333                 rctl |= E1000_RCTL_LPE;
3334         else
3335                 rctl &= ~E1000_RCTL_LPE;
3336
3337         /* Enable Receives */
3338         E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
3339 }
3340
3341 static void
3342 em_destroy_rx_ring(struct adapter *adapter, int ndesc)
3343 {
3344         struct em_buffer *rx_buffer;
3345         int i;
3346
3347         if (adapter->rx_buffer_area == NULL)
3348                 return;
3349
3350         for (i = 0; i < ndesc; i++) {
3351                 rx_buffer = &adapter->rx_buffer_area[i];
3352
3353                 KKASSERT(rx_buffer->m_head == NULL);
3354                 bus_dmamap_destroy(adapter->rxtag, rx_buffer->map);
3355         }
3356         bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3357         bus_dma_tag_destroy(adapter->rxtag);
3358
3359         kfree(adapter->rx_buffer_area, M_DEVBUF);
3360         adapter->rx_buffer_area = NULL;
3361 }
3362
3363 static void
3364 em_rxeof(struct adapter *adapter, int count)
3365 {
3366         struct ifnet *ifp = &adapter->arpcom.ac_if;
3367         uint8_t status, accept_frame = 0, eop = 0;
3368         uint16_t len, desc_len, prev_len_adj;
3369         struct e1000_rx_desc *current_desc;
3370         struct mbuf *mp;
3371         int i;
3372
3373         i = adapter->next_rx_desc_to_check;
3374         current_desc = &adapter->rx_desc_base[i];
3375
3376         if (!(current_desc->status & E1000_RXD_STAT_DD))
3377                 return;
3378
3379         while ((current_desc->status & E1000_RXD_STAT_DD) && count != 0) {
3380                 struct mbuf *m = NULL;
3381
3382                 logif(pkt_receive);
3383
3384                 mp = adapter->rx_buffer_area[i].m_head;
3385
3386                 /*
3387                  * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3388                  * needs to access the last received byte in the mbuf.
3389                  */
3390                 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3391                                 BUS_DMASYNC_POSTREAD);
3392
3393                 accept_frame = 1;
3394                 prev_len_adj = 0;
3395                 desc_len = le16toh(current_desc->length);
3396                 status = current_desc->status;
3397                 if (status & E1000_RXD_STAT_EOP) {
3398                         count--;
3399                         eop = 1;
3400                         if (desc_len < ETHER_CRC_LEN) {
3401                                 len = 0;
3402                                 prev_len_adj = ETHER_CRC_LEN - desc_len;
3403                         } else {
3404                                 len = desc_len - ETHER_CRC_LEN;
3405                         }
3406                 } else {
3407                         eop = 0;
3408                         len = desc_len;
3409                 }
3410
3411                 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3412                         uint8_t last_byte;
3413                         uint32_t pkt_len = desc_len;
3414
3415                         if (adapter->fmp != NULL)
3416                                 pkt_len += adapter->fmp->m_pkthdr.len;
3417
3418                         last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3419                         if (TBI_ACCEPT(&adapter->hw, status,
3420                             current_desc->errors, pkt_len, last_byte,
3421                             adapter->min_frame_size,
3422                             adapter->hw.mac.max_frame_size)) {
3423                                 e1000_tbi_adjust_stats_82543(&adapter->hw,
3424                                     &adapter->stats, pkt_len,
3425                                     adapter->hw.mac.addr,
3426                                     adapter->hw.mac.max_frame_size);
3427                                 if (len > 0)
3428                                         len--;
3429                         } else {
3430                                 accept_frame = 0;
3431                         }
3432                 }
3433
3434                 if (accept_frame) {
3435                         if (em_newbuf(adapter, i, 0) != 0) {
3436                                 IFNET_STAT_INC(ifp, iqdrops, 1);
3437                                 goto discard;
3438                         }
3439
3440                         /* Assign correct length to the current fragment */
3441                         mp->m_len = len;
3442
3443                         if (adapter->fmp == NULL) {
3444                                 mp->m_pkthdr.len = len;
3445                                 adapter->fmp = mp; /* Store the first mbuf */
3446                                 adapter->lmp = mp;
3447                         } else {
3448                                 /*
3449                                  * Chain mbuf's together
3450                                  */
3451
3452                                 /*
3453                                  * Adjust length of previous mbuf in chain if
3454                                  * we received less than 4 bytes in the last
3455                                  * descriptor.
3456                                  */
3457                                 if (prev_len_adj > 0) {
3458                                         adapter->lmp->m_len -= prev_len_adj;
3459                                         adapter->fmp->m_pkthdr.len -=
3460                                             prev_len_adj;
3461                                 }
3462                                 adapter->lmp->m_next = mp;
3463                                 adapter->lmp = adapter->lmp->m_next;
3464                                 adapter->fmp->m_pkthdr.len += len;
3465                         }
3466
3467                         if (eop) {
3468                                 adapter->fmp->m_pkthdr.rcvif = ifp;
3469                                 IFNET_STAT_INC(ifp, ipackets, 1);
3470
3471                                 if (ifp->if_capenable & IFCAP_RXCSUM) {
3472                                         em_rxcsum(adapter, current_desc,
3473                                                   adapter->fmp);
3474                                 }
3475
3476                                 if (status & E1000_RXD_STAT_VP) {
3477                                         adapter->fmp->m_pkthdr.ether_vlantag =
3478                                             (le16toh(current_desc->special) &
3479                                             E1000_RXD_SPC_VLAN_MASK);
3480                                         adapter->fmp->m_flags |= M_VLANTAG;
3481                                 }
3482                                 m = adapter->fmp;
3483                                 adapter->fmp = NULL;
3484                                 adapter->lmp = NULL;
3485                         }
3486                 } else {
3487                         IFNET_STAT_INC(ifp, ierrors, 1);
3488 discard:
3489 #ifdef foo
3490                         /* Reuse loaded DMA map and just update mbuf chain */
3491                         mp = adapter->rx_buffer_area[i].m_head;
3492                         mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3493                         mp->m_data = mp->m_ext.ext_buf;
3494                         mp->m_next = NULL;
3495                         if (adapter->hw.mac.max_frame_size <=
3496                             (MCLBYTES - ETHER_ALIGN))
3497                                 m_adj(mp, ETHER_ALIGN);
3498 #endif
3499                         if (adapter->fmp != NULL) {
3500                                 m_freem(adapter->fmp);
3501                                 adapter->fmp = NULL;
3502                                 adapter->lmp = NULL;
3503                         }
3504                         m = NULL;
3505                 }
3506
3507                 /* Zero out the receive descriptors status. */
3508                 current_desc->status = 0;
3509
3510                 if (m != NULL)
3511                         ifp->if_input(ifp, m, NULL, -1);
3512
3513                 /* Advance our pointers to the next descriptor. */
3514                 if (++i == adapter->num_rx_desc)
3515                         i = 0;
3516                 current_desc = &adapter->rx_desc_base[i];
3517         }
3518         adapter->next_rx_desc_to_check = i;
3519
3520         /* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3521         if (--i < 0)
3522                 i = adapter->num_rx_desc - 1;
3523         E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
3524 }
3525
3526 static void
3527 em_rxcsum(struct adapter *adapter, struct e1000_rx_desc *rx_desc,
3528           struct mbuf *mp)
3529 {
3530         /* 82543 or newer only */
3531         if (adapter->hw.mac.type < e1000_82543 ||
3532             /* Ignore Checksum bit is set */
3533             (rx_desc->status & E1000_RXD_STAT_IXSM))
3534                 return;
3535
3536         if ((rx_desc->status & E1000_RXD_STAT_IPCS) &&
3537             !(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3538                 /* IP Checksum Good */
3539                 mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED | CSUM_IP_VALID;
3540         }
3541
3542         if ((rx_desc->status & E1000_RXD_STAT_TCPCS) &&
3543             !(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3544                 mp->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
3545                                            CSUM_PSEUDO_HDR |
3546                                            CSUM_FRAG_NOT_CHECKED;
3547                 mp->m_pkthdr.csum_data = htons(0xffff);
3548         }
3549 }
3550
3551 static void
3552 em_enable_intr(struct adapter *adapter)
3553 {
3554         uint32_t ims_mask = IMS_ENABLE_MASK;
3555
3556         lwkt_serialize_handler_enable(adapter->arpcom.ac_if.if_serializer);
3557
3558 #if 0
3559         /* XXX MSIX */
3560         if (adapter->hw.mac.type == e1000_82574) {
3561                 E1000_WRITE_REG(&adapter->hw, EM_EIAC, EM_MSIX_MASK);
3562                 ims_mask |= EM_MSIX_MASK;
3563         }
3564 #endif
3565         E1000_WRITE_REG(&adapter->hw, E1000_IMS, ims_mask);
3566 }
3567
3568 static void
3569 em_disable_intr(struct adapter *adapter)
3570 {
3571         uint32_t clear = 0xffffffff;
3572
3573         /*
3574          * The first version of 82542 had an errata where when link was forced
3575          * it would stay up even up even if the cable was disconnected.
3576          * Sequence errors were used to detect the disconnect and then the
3577          * driver would unforce the link.  This code in the in the ISR.  For
3578          * this to work correctly the Sequence error interrupt had to be
3579          * enabled all the time.
3580          */
3581         if (adapter->hw.mac.type == e1000_82542 &&
3582             adapter->hw.revision_id == E1000_REVISION_2)
3583                 clear &= ~E1000_ICR_RXSEQ;
3584         else if (adapter->hw.mac.type == e1000_82574)
3585                 E1000_WRITE_REG(&adapter->hw, EM_EIAC, 0);
3586
3587         E1000_WRITE_REG(&adapter->hw, E1000_IMC, clear);
3588
3589         adapter->npoll.ifpc_stcount = 0;
3590
3591         lwkt_serialize_handler_disable(adapter->arpcom.ac_if.if_serializer);
3592 }
3593
3594 /*
3595  * Bit of a misnomer, what this really means is
3596  * to enable OS management of the system... aka
3597  * to disable special hardware management features
3598  */
3599 static void
3600 em_get_mgmt(struct adapter *adapter)
3601 {
3602         /* A shared code workaround */
3603 #define E1000_82542_MANC2H E1000_MANC2H
3604         if (adapter->flags & EM_FLAG_HAS_MGMT) {
3605                 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
3606                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3607
3608                 /* disable hardware interception of ARP */
3609                 manc &= ~(E1000_MANC_ARP_EN);
3610
3611                 /* enable receiving management packets to the host */
3612                 if (adapter->hw.mac.type >= e1000_82571) {
3613                         manc |= E1000_MANC_EN_MNG2HOST;
3614 #define E1000_MNG2HOST_PORT_623 (1 << 5)
3615 #define E1000_MNG2HOST_PORT_664 (1 << 6)
3616                         manc2h |= E1000_MNG2HOST_PORT_623;
3617                         manc2h |= E1000_MNG2HOST_PORT_664;
3618                         E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
3619                 }
3620
3621                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3622         }
3623 }
3624
3625 /*
3626  * Give control back to hardware management
3627  * controller if there is one.
3628  */
3629 static void
3630 em_rel_mgmt(struct adapter *adapter)
3631 {
3632         if (adapter->flags & EM_FLAG_HAS_MGMT) {
3633                 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
3634
3635                 /* re-enable hardware interception of ARP */
3636                 manc |= E1000_MANC_ARP_EN;
3637
3638                 if (adapter->hw.mac.type >= e1000_82571)
3639                         manc &= ~E1000_MANC_EN_MNG2HOST;
3640
3641                 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
3642         }
3643 }
3644
3645 /*
3646  * em_get_hw_control() sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3647  * For ASF and Pass Through versions of f/w this means that
3648  * the driver is loaded.  For AMT version (only with 82573)
3649  * of the f/w this means that the network i/f is open.
3650  */
3651 static void
3652 em_get_hw_control(struct adapter *adapter)
3653 {
3654         /* Let firmware know the driver has taken over */
3655         if (adapter->hw.mac.type == e1000_82573) {
3656                 uint32_t swsm;
3657
3658                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3659                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3660                     swsm | E1000_SWSM_DRV_LOAD);
3661         } else {
3662                 uint32_t ctrl_ext;
3663
3664                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3665                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3666                     ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
3667         }
3668         adapter->flags |= EM_FLAG_HW_CTRL;
3669 }
3670
3671 /*
3672  * em_rel_hw_control() resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
3673  * For ASF and Pass Through versions of f/w this means that the
3674  * driver is no longer loaded.  For AMT version (only with 82573)
3675  * of the f/w this means that the network i/f is closed.
3676  */
3677 static void
3678 em_rel_hw_control(struct adapter *adapter)
3679 {
3680         if ((adapter->flags & EM_FLAG_HW_CTRL) == 0)
3681                 return;
3682         adapter->flags &= ~EM_FLAG_HW_CTRL;
3683
3684         /* Let firmware taken over control of h/w */
3685         if (adapter->hw.mac.type == e1000_82573) {
3686                 uint32_t swsm;
3687
3688                 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
3689                 E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
3690                     swsm & ~E1000_SWSM_DRV_LOAD);
3691         } else {
3692                 uint32_t ctrl_ext;
3693
3694                 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
3695                 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
3696                     ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
3697         }
3698 }
3699
3700 static int
3701 em_is_valid_eaddr(const uint8_t *addr)
3702 {
3703         char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
3704
3705         if ((addr[0] & 1) || !bcmp(addr, zero_addr, ETHER_ADDR_LEN))
3706                 return (FALSE);
3707
3708         return (TRUE);
3709 }
3710
3711 /*
3712  * Enable PCI Wake On Lan capability
3713  */
3714 void
3715 em_enable_wol(device_t dev)
3716 {
3717         uint16_t cap, status;
3718         uint8_t id;
3719
3720         /* First find the capabilities pointer*/
3721         cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
3722
3723         /* Read the PM Capabilities */
3724         id = pci_read_config(dev, cap, 1);
3725         if (id != PCIY_PMG)     /* Something wrong */
3726                 return;
3727
3728         /*
3729          * OK, we have the power capabilities,
3730          * so now get the status register
3731          */
3732         cap += PCIR_POWER_STATUS;
3733         status = pci_read_config(dev, cap, 2);
3734         status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
3735         pci_write_config(dev, cap, status, 2);
3736 }
3737
3738
3739 /*
3740  * 82544 Coexistence issue workaround.
3741  *    There are 2 issues.
3742  *       1. Transmit Hang issue.
3743  *    To detect this issue, following equation can be used...
3744  *        SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3745  *        If SUM[3:0] is in between 1 to 4, we will have this issue.
3746  *
3747  *       2. DAC issue.
3748  *    To detect this issue, following equation can be used...
3749  *        SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3750  *        If SUM[3:0] is in between 9 to c, we will have this issue.
3751  *
3752  *    WORKAROUND:
3753  *        Make sure we do not have ending address
3754  *        as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3755  */
3756 static uint32_t
3757 em_82544_fill_desc(bus_addr_t address, uint32_t length, PDESC_ARRAY desc_array)
3758 {
3759         uint32_t safe_terminator;
3760
3761         /*
3762          * Since issue is sensitive to length and address.
3763          * Let us first check the address...
3764          */
3765         if (length <= 4) {
3766                 desc_array->descriptor[0].address = address;
3767                 desc_array->descriptor[0].length = length;
3768                 desc_array->elements = 1;
3769                 return (desc_array->elements);
3770         }
3771
3772         safe_terminator =
3773         (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
3774
3775         /* If it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3776         if (safe_terminator == 0 ||
3777             (safe_terminator > 4 && safe_terminator < 9) ||
3778             (safe_terminator > 0xC && safe_terminator <= 0xF)) {
3779                 desc_array->descriptor[0].address = address;
3780                 desc_array->descriptor[0].length = length;
3781                 desc_array->elements = 1;
3782                 return (desc_array->elements);
3783         }
3784
3785         desc_array->descriptor[0].address = address;
3786         desc_array->descriptor[0].length = length - 4;
3787         desc_array->descriptor[1].address = address + (length - 4);
3788         desc_array->descriptor[1].length = 4;
3789         desc_array->elements = 2;
3790         return (desc_array->elements);
3791 }
3792
3793 static void
3794 em_update_stats(struct adapter *adapter)
3795 {
3796         struct ifnet *ifp = &adapter->arpcom.ac_if;
3797
3798         if (adapter->hw.phy.media_type == e1000_media_type_copper ||
3799             (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
3800                 adapter->stats.symerrs +=
3801                         E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
3802                 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
3803         }
3804         adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
3805         adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
3806         adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
3807         adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
3808
3809         adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
3810         adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
3811         adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
3812         adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
3813         adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
3814         adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
3815         adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
3816         adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
3817         adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
3818         adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
3819         adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
3820         adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
3821         adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
3822         adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
3823         adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
3824         adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
3825         adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
3826         adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
3827         adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
3828         adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
3829
3830         /* For the 64-bit byte counters the low dword must be read first. */
3831         /* Both registers clear on the read of the high dword */
3832
3833         adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
3834         adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
3835
3836         adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
3837         adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
3838         adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
3839         adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
3840         adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
3841
3842         adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
3843         adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
3844
3845         adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
3846         adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
3847         adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
3848         adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
3849         adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
3850         adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
3851         adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
3852         adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
3853         adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
3854         adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
3855
3856         if (adapter->hw.mac.type >= e1000_82543) {
3857                 adapter->stats.algnerrc +=
3858                 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
3859                 adapter->stats.rxerrc +=
3860                 E1000_READ_REG(&adapter->hw, E1000_RXERRC);
3861                 adapter->stats.tncrs +=
3862                 E1000_READ_REG(&adapter->hw, E1000_TNCRS);
3863                 adapter->stats.cexterr +=
3864                 E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
3865                 adapter->stats.tsctc +=
3866                 E1000_READ_REG(&adapter->hw, E1000_TSCTC);
3867                 adapter->stats.tsctfc +=
3868                 E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
3869         }
3870
3871         IFNET_STAT_SET(ifp, collisions, adapter->stats.colc);
3872
3873         /* Rx Errors */
3874         IFNET_STAT_SET(ifp, ierrors,
3875             adapter->dropped_pkts + adapter->stats.rxerrc +
3876             adapter->stats.crcerrs + adapter->stats.algnerrc +
3877             adapter->stats.ruc + adapter->stats.roc +
3878             adapter->stats.mpc + adapter->stats.cexterr);
3879
3880         /* Tx Errors */
3881         IFNET_STAT_SET(ifp, oerrors,
3882             adapter->stats.ecol + adapter->stats.latecol +
3883             adapter->watchdog_events);
3884 }
3885
3886 static void
3887 em_print_debug_info(struct adapter *adapter)
3888 {
3889         device_t dev = adapter->dev;
3890         uint8_t *hw_addr = adapter->hw.hw_addr;
3891
3892         device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
3893         device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
3894             E1000_READ_REG(&adapter->hw, E1000_CTRL),
3895             E1000_READ_REG(&adapter->hw, E1000_RCTL));
3896         device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
3897             ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
3898             (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
3899         device_printf(dev, "Flow control watermarks high = %d low = %d\n",
3900             adapter->hw.fc.high_water,
3901             adapter->hw.fc.low_water);
3902         device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3903             E1000_READ_REG(&adapter->hw, E1000_TIDV),
3904             E1000_READ_REG(&adapter->hw, E1000_TADV));
3905         device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3906             E1000_READ_REG(&adapter->hw, E1000_RDTR),
3907             E1000_READ_REG(&adapter->hw, E1000_RADV));
3908         device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
3909             (long long)adapter->tx_fifo_wrk_cnt,
3910             (long long)adapter->tx_fifo_reset_cnt);
3911         device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
3912             E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
3913             E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
3914         device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
3915             E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
3916             E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
3917         device_printf(dev, "Num Tx descriptors avail = %d\n",
3918             adapter->num_tx_desc_avail);
3919         device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
3920             adapter->no_tx_desc_avail1);
3921         device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
3922             adapter->no_tx_desc_avail2);
3923         device_printf(dev, "Std mbuf failed = %ld\n",
3924             adapter->mbuf_alloc_failed);
3925         device_printf(dev, "Std mbuf cluster failed = %ld\n",
3926             adapter->mbuf_cluster_failed);
3927         device_printf(dev, "Driver dropped packets = %ld\n",
3928             adapter->dropped_pkts);
3929         device_printf(dev, "Driver tx dma failure in encap = %ld\n",
3930             adapter->no_tx_dma_setup);
3931 }
3932
3933 static void
3934 em_print_hw_stats(struct adapter *adapter)
3935 {
3936         device_t dev = adapter->dev;
3937
3938         device_printf(dev, "Excessive collisions = %lld\n",
3939             (long long)adapter->stats.ecol);
3940 #if (DEBUG_HW > 0)  /* Dont output these errors normally */
3941         device_printf(dev, "Symbol errors = %lld\n",
3942             (long long)adapter->stats.symerrs);
3943 #endif
3944         device_printf(dev, "Sequence errors = %lld\n",
3945             (long long)adapter->stats.sec);
3946         device_printf(dev, "Defer count = %lld\n",
3947             (long long)adapter->stats.dc);
3948         device_printf(dev, "Missed Packets = %lld\n",
3949             (long long)adapter->stats.mpc);
3950         device_printf(dev, "Receive No Buffers = %lld\n",
3951             (long long)adapter->stats.rnbc);
3952         /* RLEC is inaccurate on some hardware, calculate our own. */
3953         device_printf(dev, "Receive Length Errors = %lld\n",
3954             ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
3955         device_printf(dev, "Receive errors = %lld\n",
3956             (long long)adapter->stats.rxerrc);
3957         device_printf(dev, "Crc errors = %lld\n",
3958             (long long)adapter->stats.crcerrs);
3959         device_printf(dev, "Alignment errors = %lld\n",
3960             (long long)adapter->stats.algnerrc);
3961         device_printf(dev, "Collision/Carrier extension errors = %lld\n",
3962             (long long)adapter->stats.cexterr);
3963         device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
3964         device_printf(dev, "watchdog timeouts = %ld\n",
3965             adapter->watchdog_events);
3966         device_printf(dev, "XON Rcvd = %lld\n",
3967             (long long)adapter->stats.xonrxc);
3968         device_printf(dev, "XON Xmtd = %lld\n",
3969             (long long)adapter->stats.xontxc);
3970         device_printf(dev, "XOFF Rcvd = %lld\n",
3971             (long long)adapter->stats.xoffrxc);
3972         device_printf(dev, "XOFF Xmtd = %lld\n",
3973             (long long)adapter->stats.xofftxc);
3974         device_printf(dev, "Good Packets Rcvd = %lld\n",
3975             (long long)adapter->stats.gprc);
3976         device_printf(dev, "Good Packets Xmtd = %lld\n",
3977             (long long)adapter->stats.gptc);
3978 }
3979
3980 static void
3981 em_print_nvm_info(struct adapter *adapter)
3982 {
3983         uint16_t eeprom_data;
3984         int i, j, row = 0;
3985
3986         /* Its a bit crude, but it gets the job done */
3987         kprintf("\nInterface EEPROM Dump:\n");
3988         kprintf("Offset\n0x0000  ");
3989         for (i = 0, j = 0; i < 32; i++, j++) {
3990                 if (j == 8) { /* Make the offset block */
3991                         j = 0; ++row;
3992                         kprintf("\n0x00%x0  ",row);
3993                 }
3994                 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
3995                 kprintf("%04x ", eeprom_data);
3996         }
3997         kprintf("\n");
3998 }
3999
4000 static int
4001 em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
4002 {
4003         struct adapter *adapter;
4004         struct ifnet *ifp;
4005         int error, result;
4006
4007         result = -1;
4008         error = sysctl_handle_int(oidp, &result, 0, req);
4009         if (error || !req->newptr)
4010                 return (error);
4011
4012         adapter = (struct adapter *)arg1;
4013         ifp = &adapter->arpcom.ac_if;
4014
4015         lwkt_serialize_enter(ifp->if_serializer);
4016
4017         if (result == 1)
4018                 em_print_debug_info(adapter);
4019
4020         /*
4021          * This value will cause a hex dump of the
4022          * first 32 16-bit words of the EEPROM to
4023          * the screen.
4024          */
4025         if (result == 2)
4026                 em_print_nvm_info(adapter);
4027
4028         lwkt_serialize_exit(ifp->if_serializer);
4029
4030         return (error);
4031 }
4032
4033 static int
4034 em_sysctl_stats(SYSCTL_HANDLER_ARGS)
4035 {
4036         int error, result;
4037
4038         result = -1;
4039         error = sysctl_handle_int(oidp, &result, 0, req);
4040         if (error || !req->newptr)
4041                 return (error);
4042
4043         if (result == 1) {
4044                 struct adapter *adapter = (struct adapter *)arg1;
4045                 struct ifnet *ifp = &adapter->arpcom.ac_if;
4046
4047                 lwkt_serialize_enter(ifp->if_serializer);
4048                 em_print_hw_stats(adapter);
4049                 lwkt_serialize_exit(ifp->if_serializer);
4050         }
4051         return (error);
4052 }
4053
4054 static void
4055 em_add_sysctl(struct adapter *adapter)
4056 {
4057         struct sysctl_ctx_list *ctx;
4058         struct sysctl_oid *tree;
4059
4060         ctx = device_get_sysctl_ctx(adapter->dev);
4061         tree = device_get_sysctl_tree(adapter->dev);
4062         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4063             OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4064             em_sysctl_debug_info, "I", "Debug Information");
4065
4066         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4067             OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4068             em_sysctl_stats, "I", "Statistics");
4069
4070         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4071             OID_AUTO, "rxd", CTLFLAG_RD,
4072             &adapter->num_rx_desc, 0, NULL);
4073         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4074             OID_AUTO, "txd", CTLFLAG_RD,
4075             &adapter->num_tx_desc, 0, NULL);
4076
4077         if (adapter->hw.mac.type >= e1000_82540) {
4078                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4079                     OID_AUTO, "int_throttle_ceil",
4080                     CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4081                     em_sysctl_int_throttle, "I",
4082                     "interrupt throttling rate");
4083         }
4084         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
4085             OID_AUTO, "int_tx_nsegs",
4086             CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
4087             em_sysctl_int_tx_nsegs, "I",
4088             "# segments per TX interrupt");
4089         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree),
4090             OID_AUTO, "wreg_tx_nsegs", CTLFLAG_RW,
4091             &adapter->tx_wreg_nsegs, 0,
4092             "# segments before write to hardware register");
4093 }
4094
4095 static int
4096 em_sysctl_int_throttle(SYSCTL_HANDLER_ARGS)
4097 {
4098         struct adapter *adapter = (void *)arg1;
4099         struct ifnet *ifp = &adapter->arpcom.ac_if;
4100         int error, throttle;
4101
4102         throttle = adapter->int_throttle_ceil;
4103         error = sysctl_handle_int(oidp, &throttle, 0, req);
4104         if (error || req->newptr == NULL)
4105                 return error;
4106         if (throttle < 0 || throttle > 1000000000 / 256)
4107                 return EINVAL;
4108
4109         if (throttle) {
4110                 /*
4111                  * Set the interrupt throttling rate in 256ns increments,
4112                  * recalculate sysctl value assignment to get exact frequency.
4113                  */
4114                 throttle = 1000000000 / 256 / throttle;
4115
4116                 /* Upper 16bits of ITR is reserved and should be zero */
4117                 if (throttle & 0xffff0000)
4118                         return EINVAL;
4119         }
4120
4121         lwkt_serialize_enter(ifp->if_serializer);
4122
4123         if (throttle)
4124                 adapter->int_throttle_ceil = 1000000000 / 256 / throttle;
4125         else
4126                 adapter->int_throttle_ceil = 0;
4127
4128         if (ifp->if_flags & IFF_RUNNING)
4129                 em_set_itr(adapter, throttle);
4130
4131         lwkt_serialize_exit(ifp->if_serializer);
4132
4133         if (bootverbose) {
4134                 if_printf(ifp, "Interrupt moderation set to %d/sec\n",
4135                           adapter->int_throttle_ceil);
4136         }
4137         return 0;
4138 }
4139
4140 static int
4141 em_sysctl_int_tx_nsegs(SYSCTL_HANDLER_ARGS)
4142 {
4143         struct adapter *adapter = (void *)arg1;
4144         struct ifnet *ifp = &adapter->arpcom.ac_if;
4145         int error, segs;
4146
4147         segs = adapter->tx_int_nsegs;
4148         error = sysctl_handle_int(oidp, &segs, 0, req);
4149         if (error || req->newptr == NULL)
4150                 return error;
4151         if (segs <= 0)
4152                 return EINVAL;
4153
4154         lwkt_serialize_enter(ifp->if_serializer);
4155
4156         /*
4157          * Don't allow int_tx_nsegs to become:
4158          * o  Less the oact_tx_desc
4159          * o  Too large that no TX desc will cause TX interrupt to
4160          *    be generated (OACTIVE will never recover)
4161          * o  Too small that will cause tx_dd[] overflow
4162          */
4163         if (segs < adapter->oact_tx_desc ||
4164             segs >= adapter->num_tx_desc - adapter->oact_tx_desc ||
4165             segs < adapter->num_tx_desc / EM_TXDD_SAFE) {
4166                 error = EINVAL;
4167         } else {
4168                 error = 0;
4169                 adapter->tx_int_nsegs = segs;
4170         }
4171
4172         lwkt_serialize_exit(ifp->if_serializer);
4173
4174         return error;
4175 }
4176
4177 static void
4178 em_set_itr(struct adapter *adapter, uint32_t itr)
4179 {
4180         E1000_WRITE_REG(&adapter->hw, E1000_ITR, itr);
4181         if (adapter->hw.mac.type == e1000_82574) {
4182                 int i;
4183
4184                 /*
4185                  * When using MSIX interrupts we need to
4186                  * throttle using the EITR register
4187                  */
4188                 for (i = 0; i < 4; ++i) {
4189                         E1000_WRITE_REG(&adapter->hw,
4190                             E1000_EITR_82574(i), itr);
4191                 }
4192         }
4193 }
4194
4195 static void
4196 em_disable_aspm(struct adapter *adapter)
4197 {
4198         uint16_t link_cap, link_ctrl, disable;
4199         uint8_t pcie_ptr, reg;
4200         device_t dev = adapter->dev;
4201
4202         switch (adapter->hw.mac.type) {
4203         case e1000_82571:
4204         case e1000_82572:
4205         case e1000_82573:
4206                 /*
4207                  * 82573 specification update
4208                  * errata #8 disable L0s
4209                  * errata #41 disable L1
4210                  *
4211                  * 82571/82572 specification update
4212                  # errata #13 disable L1
4213                  * errata #68 disable L0s
4214                  */
4215                 disable = PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1;
4216                 break;
4217
4218         case e1000_82574:
4219         case e1000_82583:
4220                 /*
4221                  * 82574 specification update errata #20
4222                  * 82583 specification update errata #9
4223                  *
4224                  * There is no need to disable L1
4225                  */
4226                 disable = PCIEM_LNKCTL_ASPM_L0S;
4227                 break;
4228
4229         default:
4230                 return;
4231         }
4232
4233         pcie_ptr = pci_get_pciecap_ptr(dev);
4234         if (pcie_ptr == 0)
4235                 return;
4236
4237         link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2);
4238         if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0)
4239                 return;
4240
4241         if (bootverbose) {
4242                 if_printf(&adapter->arpcom.ac_if,
4243                     "disable ASPM %#02x\n", disable);
4244         }
4245
4246         reg = pcie_ptr + PCIER_LINKCTRL;
4247         link_ctrl = pci_read_config(dev, reg, 2);
4248         link_ctrl &= ~disable;
4249         pci_write_config(dev, reg, link_ctrl, 2);
4250 }
4251
4252 static int
4253 em_tso_pullup(struct adapter *adapter, struct mbuf **mp)
4254 {
4255         int iphlen, hoff, thoff, ex = 0;
4256         struct mbuf *m;
4257         struct ip *ip;
4258
4259         m = *mp;
4260         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
4261
4262         iphlen = m->m_pkthdr.csum_iphlen;
4263         thoff = m->m_pkthdr.csum_thlen;
4264         hoff = m->m_pkthdr.csum_lhlen;
4265
4266         KASSERT(iphlen > 0, ("invalid ip hlen"));
4267         KASSERT(thoff > 0, ("invalid tcp hlen"));
4268         KASSERT(hoff > 0, ("invalid ether hlen"));
4269
4270         if (adapter->flags & EM_FLAG_TSO_PULLEX)
4271                 ex = 4;
4272
4273         if (m->m_len < hoff + iphlen + thoff + ex) {
4274                 m = m_pullup(m, hoff + iphlen + thoff + ex);
4275                 if (m == NULL) {
4276                         *mp = NULL;
4277                         return ENOBUFS;
4278                 }
4279                 *mp = m;
4280         }
4281         ip = mtodoff(m, struct ip *, hoff);
4282         ip->ip_len = 0;
4283
4284         return 0;
4285 }
4286
4287 static int
4288 em_tso_setup(struct adapter *adapter, struct mbuf *mp,
4289     uint32_t *txd_upper, uint32_t *txd_lower)
4290 {
4291         struct e1000_context_desc *TXD;
4292         int hoff, iphlen, thoff, hlen;
4293         int mss, pktlen, curr_txd;
4294
4295         iphlen = mp->m_pkthdr.csum_iphlen;
4296         thoff = mp->m_pkthdr.csum_thlen;
4297         hoff = mp->m_pkthdr.csum_lhlen;
4298         mss = mp->m_pkthdr.tso_segsz;
4299         pktlen = mp->m_pkthdr.len;
4300
4301         if (adapter->csum_flags == CSUM_TSO &&
4302             adapter->csum_iphlen == iphlen &&
4303             adapter->csum_lhlen == hoff &&
4304             adapter->csum_thlen == thoff &&
4305             adapter->csum_mss == mss &&
4306             adapter->csum_pktlen == pktlen) {
4307                 *txd_upper = adapter->csum_txd_upper;
4308                 *txd_lower = adapter->csum_txd_lower;
4309                 return 0;
4310         }
4311         hlen = hoff + iphlen + thoff;
4312
4313         /*
4314          * Setup a new TSO context.
4315          */
4316
4317         curr_txd = adapter->next_avail_tx_desc;
4318         TXD = (struct e1000_context_desc *)&adapter->tx_desc_base[curr_txd];
4319
4320         *txd_lower = E1000_TXD_CMD_DEXT |       /* Extended descr type */
4321                      E1000_TXD_DTYP_D |         /* Data descr type */
4322                      E1000_TXD_CMD_TSE;         /* Do TSE on this packet */
4323
4324         /* IP and/or TCP header checksum calculation and insertion. */
4325         *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
4326
4327         /*
4328          * Start offset for header checksum calculation.
4329          * End offset for header checksum calculation.
4330          * Offset of place put the checksum.
4331          */
4332         TXD->lower_setup.ip_fields.ipcss = hoff;
4333         TXD->lower_setup.ip_fields.ipcse = htole16(hoff + iphlen - 1);
4334         TXD->lower_setup.ip_fields.ipcso = hoff + offsetof(struct ip, ip_sum);
4335
4336         /*
4337          * Start offset for payload checksum calculation.
4338          * End offset for payload checksum calculation.
4339          * Offset of place to put the checksum.
4340          */
4341         TXD->upper_setup.tcp_fields.tucss = hoff + iphlen;
4342         TXD->upper_setup.tcp_fields.tucse = 0;
4343         TXD->upper_setup.tcp_fields.tucso =
4344             hoff + iphlen + offsetof(struct tcphdr, th_sum);
4345
4346         /*
4347          * Payload size per packet w/o any headers.
4348          * Length of all headers up to payload.
4349          */
4350         TXD->tcp_seg_setup.fields.mss = htole16(mss);
4351         TXD->tcp_seg_setup.fields.hdr_len = hlen;
4352         TXD->cmd_and_length = htole32(E1000_TXD_CMD_IFCS |
4353                                 E1000_TXD_CMD_DEXT |    /* Extended descr */
4354                                 E1000_TXD_CMD_TSE |     /* TSE context */
4355                                 E1000_TXD_CMD_IP |      /* Do IP csum */
4356                                 E1000_TXD_CMD_TCP |     /* Do TCP checksum */
4357                                 (pktlen - hlen));       /* Total len */
4358
4359         /* Save the information for this TSO context */
4360         adapter->csum_flags = CSUM_TSO;
4361         adapter->csum_lhlen = hoff;
4362         adapter->csum_iphlen = iphlen;
4363         adapter->csum_thlen = thoff;
4364         adapter->csum_mss = mss;
4365         adapter->csum_pktlen = pktlen;
4366         adapter->csum_txd_upper = *txd_upper;
4367         adapter->csum_txd_lower = *txd_lower;
4368
4369         if (++curr_txd == adapter->num_tx_desc)
4370                 curr_txd = 0;
4371
4372         KKASSERT(adapter->num_tx_desc_avail > 0);
4373         adapter->num_tx_desc_avail--;
4374
4375         adapter->next_avail_tx_desc = curr_txd;
4376         return 1;
4377 }