sys/dev/netif/bce/if_bce.c

   1 /*-
   2  * Copyright (c) 2006-2007 Broadcom Corporation
   3  *      David Christensen <davidch@broadcom.com>.  All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. Neither the name of Broadcom Corporation nor the name of its contributors
  15  *    may be used to endorse or promote products derived from this software
  16  *    without specific prior written consent.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS'
  19  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  20  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
  22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  28  * THE POSSIBILITY OF SUCH DAMAGE.
  29  *
  30  * $FreeBSD: src/sys/dev/bce/if_bce.c,v 1.31 2007/05/16 23:34:11 davidch Exp $
  31  */
  32
  33 /*
  34  * The following controllers are supported by this driver:
  35  *   BCM5706C A2, A3
  36  *   BCM5706S A2, A3
  37  *   BCM5708C B1, B2
  38  *   BCM5708S B1, B2
  39  *   BCM5709C A1, B2, C0
  40  *   BCM5716  C0
  41  *
  42  * The following controllers are not supported by this driver:
  43  *   BCM5706C A0, A1
  44  *   BCM5706S A0, A1
  45  *   BCM5708C A0, B0
  46  *   BCM5708S A0, B0
  47  *   BCM5709C A0, B0, B1
  48  *   BCM5709S A0, A1, B0, B1, B2, C0
  49  *
  50  *
  51  * Note about MSI-X on 5709/5716:
  52  * - 9 MSI-X vectors are supported.
  53  * - MSI-X vectors, RX/TX rings and status blocks' association
  54  *   are fixed:
  55  *   o  The first RX ring and the first TX ring use the first
  56  *      status block.
  57  *   o  The first MSI-X vector is associated with the first
  58  *      status block.
  59  *   o  The second RX ring and the second TX ring use the second
  60  *      status block.
  61  *   o  The second MSI-X vector is associated with the second
  62  *      status block.
  63  *   ...
  64  *   and so on so forth.
  65  * - Status blocks must reside in physically contiguous memory
  66  *   and each status block consumes 128bytes.  In addition to
  67  *   this, the memory for the status blocks is aligned on 128bytes
  68  *   in this driver.  (see bce_dma_alloc() and HC_CONFIG)
  69  * - Each status block has its own coalesce parameters, which also
  70  *   serve as the related MSI-X vector's interrupt moderation
  71  *   parameters.  (see bce_coal_change())
  72  */
  73
  74 #include "opt_bce.h"
  75 #include "opt_ifpoll.h"
  76
  77 #include <sys/param.h>
  78 #include <sys/bus.h>
  79 #include <sys/endian.h>
  80 #include <sys/kernel.h>
  81 #include <sys/interrupt.h>
  82 #include <sys/mbuf.h>
  83 #include <sys/malloc.h>
  84 #include <sys/queue.h>
  85 #include <sys/rman.h>
  86 #include <sys/serialize.h>
  87 #include <sys/socket.h>
  88 #include <sys/sockio.h>
  89 #include <sys/sysctl.h>
  90
  91 #include <netinet/ip.h>
  92 #include <netinet/tcp.h>
  93
  94 #include <net/bpf.h>
  95 #include <net/ethernet.h>
  96 #include <net/if.h>
  97 #include <net/if_arp.h>
  98 #include <net/if_dl.h>
  99 #include <net/if_media.h>
 100 #include <net/if_poll.h>
 101 #include <net/if_types.h>
 102 #include <net/ifq_var.h>
 103 #include <net/toeplitz.h>
 104 #include <net/toeplitz2.h>
 105 #include <net/vlan/if_vlan_var.h>
 106 #include <net/vlan/if_vlan_ether.h>
 107
 108 #include <dev/netif/mii_layer/mii.h>
 109 #include <dev/netif/mii_layer/miivar.h>
 110 #include <dev/netif/mii_layer/brgphyreg.h>
 111
 112 #include <bus/pci/pcireg.h>
 113 #include <bus/pci/pcivar.h>
 114
 115 #include "miibus_if.h"
 116
 117 #include <dev/netif/bce/if_bcereg.h>
 118 #include <dev/netif/bce/if_bcefw.h>
 119
 120 #define BCE_MSI_CKINTVL         ((10 * hz) / 1000)      /* 10ms */
 121
 122 #ifdef BCE_RSS_DEBUG
 123 #define BCE_RSS_DPRINTF(sc, lvl, fmt, ...) \
 124 do { \
 125         if (sc->rss_debug >= lvl) \
 126                 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
 127 } while (0)
 128 #else   /* !BCE_RSS_DEBUG */
 129 #define BCE_RSS_DPRINTF(sc, lvl, fmt, ...)      ((void)0)
 130 #endif  /* BCE_RSS_DEBUG */
 131
 132 /****************************************************************************/
 133 /* PCI Device ID Table                                                      */
 134 /*                                                                          */
 135 /* Used by bce_probe() to identify the devices supported by this driver.    */
 136 /****************************************************************************/
 137 #define BCE_DEVDESC_MAX         64
 138
 139 static struct bce_type bce_devs[] = {
 140         /* BCM5706C Controllers and OEM boards. */
 141         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3101,
 142                 "HP NC370T Multifunction Gigabit Server Adapter" },
 143         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3106,
 144                 "HP NC370i Multifunction Gigabit Server Adapter" },
 145         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x3070,
 146                 "HP NC380T PCIe DP Multifunc Gig Server Adapter" },
 147         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  HP_VENDORID, 0x1709,
 148                 "HP NC371i Multifunction Gigabit Server Adapter" },
 149         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706,  PCI_ANY_ID,  PCI_ANY_ID,
 150                 "Broadcom NetXtreme II BCM5706 1000Base-T" },
 151
 152         /* BCM5706S controllers and OEM boards. */
 153         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, HP_VENDORID, 0x3102,
 154                 "HP NC370F Multifunction Gigabit Server Adapter" },
 155         { BRCM_VENDORID, BRCM_DEVICEID_BCM5706S, PCI_ANY_ID,  PCI_ANY_ID,
 156                 "Broadcom NetXtreme II BCM5706 1000Base-SX" },
 157
 158         /* BCM5708C controllers and OEM boards. */
 159         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7037,
 160                 "HP NC373T PCIe Multifunction Gig Server Adapter" },
 161         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7038,
 162                 "HP NC373i Multifunction Gigabit Server Adapter" },
 163         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  HP_VENDORID, 0x7045,
 164                 "HP NC374m PCIe Multifunction Adapter" },
 165         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708,  PCI_ANY_ID,  PCI_ANY_ID,
 166                 "Broadcom NetXtreme II BCM5708 1000Base-T" },
 167
 168         /* BCM5708S controllers and OEM boards. */
 169         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x1706,
 170                 "HP NC373m Multifunction Gigabit Server Adapter" },
 171         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703b,
 172                 "HP NC373i Multifunction Gigabit Server Adapter" },
 173         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  HP_VENDORID, 0x703d,
 174                 "HP NC373F PCIe Multifunc Giga Server Adapter" },
 175         { BRCM_VENDORID, BRCM_DEVICEID_BCM5708S,  PCI_ANY_ID,  PCI_ANY_ID,
 176                 "Broadcom NetXtreme II BCM5708S 1000Base-T" },
 177
 178         /* BCM5709C controllers and OEM boards. */
 179         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7055,
 180                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 181         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  HP_VENDORID, 0x7059,
 182                 "HP NC382T PCIe DP Multifunction Gigabit Server Adapter" },
 183         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709,  PCI_ANY_ID,  PCI_ANY_ID,
 184                 "Broadcom NetXtreme II BCM5709 1000Base-T" },
 185
 186         /* BCM5709S controllers and OEM boards. */
 187         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x171d,
 188                 "HP NC382m DP 1GbE Multifunction BL-c Adapter" },
 189         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  HP_VENDORID, 0x7056,
 190                 "HP NC382i DP Multifunction Gigabit Server Adapter" },
 191         { BRCM_VENDORID, BRCM_DEVICEID_BCM5709S,  PCI_ANY_ID,  PCI_ANY_ID,
 192                 "Broadcom NetXtreme II BCM5709 1000Base-SX" },
 193
 194         /* BCM5716 controllers and OEM boards. */
 195         { BRCM_VENDORID, BRCM_DEVICEID_BCM5716,   PCI_ANY_ID,  PCI_ANY_ID,
 196                 "Broadcom NetXtreme II BCM5716 1000Base-T" },
 197
 198         { 0, 0, 0, 0, NULL }
 199 };
 200
 201
 202 /****************************************************************************/
 203 /* Supported Flash NVRAM device data.                                       */
 204 /****************************************************************************/
 205 static const struct flash_spec flash_table[] =
 206 {
 207 #define BUFFERED_FLAGS          (BCE_NV_BUFFERED | BCE_NV_TRANSLATE)
 208 #define NONBUFFERED_FLAGS       (BCE_NV_WREN)
 209
 210         /* Slow EEPROM */
 211         {0x00000000, 0x40830380, 0x009f0081, 0xa184a053, 0xaf000400,
 212          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 213          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 214          "EEPROM - slow"},
 215         /* Expansion entry 0001 */
 216         {0x08000002, 0x4b808201, 0x00050081, 0x03840253, 0xaf020406,
 217          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 218          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 219          "Entry 0001"},
 220         /* Saifun SA25F010 (non-buffered flash) */
 221         /* strap, cfg1, & write1 need updates */
 222         {0x04000001, 0x47808201, 0x00050081, 0x03840253, 0xaf020406,
 223          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 224          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*2,
 225          "Non-buffered flash (128kB)"},
 226         /* Saifun SA25F020 (non-buffered flash) */
 227         /* strap, cfg1, & write1 need updates */
 228         {0x0c000003, 0x4f808201, 0x00050081, 0x03840253, 0xaf020406,
 229          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 230          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE*4,
 231          "Non-buffered flash (256kB)"},
 232         /* Expansion entry 0100 */
 233         {0x11000000, 0x53808201, 0x00050081, 0x03840253, 0xaf020406,
 234          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 235          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 236          "Entry 0100"},
 237         /* Entry 0101: ST M45PE10 (non-buffered flash, TetonII B0) */
 238         {0x19000002, 0x5b808201, 0x000500db, 0x03840253, 0xaf020406,
 239          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 240          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*2,
 241          "Entry 0101: ST M45PE10 (128kB non-bufferred)"},
 242         /* Entry 0110: ST M45PE20 (non-buffered flash)*/
 243         {0x15000001, 0x57808201, 0x000500db, 0x03840253, 0xaf020406,
 244          NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 245          ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*4,
 246          "Entry 0110: ST M45PE20 (256kB non-bufferred)"},
 247         /* Saifun SA25F005 (non-buffered flash) */
 248         /* strap, cfg1, & write1 need updates */
 249         {0x1d000003, 0x5f808201, 0x00050081, 0x03840253, 0xaf020406,
 250          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 251          SAIFUN_FLASH_BYTE_ADDR_MASK, SAIFUN_FLASH_BASE_TOTAL_SIZE,
 252          "Non-buffered flash (64kB)"},
 253         /* Fast EEPROM */
 254         {0x22000000, 0x62808380, 0x009f0081, 0xa184a053, 0xaf000400,
 255          BUFFERED_FLAGS, SEEPROM_PAGE_BITS, SEEPROM_PAGE_SIZE,
 256          SEEPROM_BYTE_ADDR_MASK, SEEPROM_TOTAL_SIZE,
 257          "EEPROM - fast"},
 258         /* Expansion entry 1001 */
 259         {0x2a000002, 0x6b808201, 0x00050081, 0x03840253, 0xaf020406,
 260          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 261          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 262          "Entry 1001"},
 263         /* Expansion entry 1010 */
 264         {0x26000001, 0x67808201, 0x00050081, 0x03840253, 0xaf020406,
 265          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 266          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 267          "Entry 1010"},
 268         /* ATMEL AT45DB011B (buffered flash) */
 269         {0x2e000003, 0x6e808273, 0x00570081, 0x68848353, 0xaf000400,
 270          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 271          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE,
 272          "Buffered flash (128kB)"},
 273         /* Expansion entry 1100 */
 274         {0x33000000, 0x73808201, 0x00050081, 0x03840253, 0xaf020406,
 275          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 276          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 277          "Entry 1100"},
 278         /* Expansion entry 1101 */
 279         {0x3b000002, 0x7b808201, 0x00050081, 0x03840253, 0xaf020406,
 280          NONBUFFERED_FLAGS, SAIFUN_FLASH_PAGE_BITS, SAIFUN_FLASH_PAGE_SIZE,
 281          SAIFUN_FLASH_BYTE_ADDR_MASK, 0,
 282          "Entry 1101"},
 283         /* Ateml Expansion entry 1110 */
 284         {0x37000001, 0x76808273, 0x00570081, 0x68848353, 0xaf000400,
 285          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 286          BUFFERED_FLASH_BYTE_ADDR_MASK, 0,
 287          "Entry 1110 (Atmel)"},
 288         /* ATMEL AT45DB021B (buffered flash) */
 289         {0x3f000003, 0x7e808273, 0x00570081, 0x68848353, 0xaf000400,
 290          BUFFERED_FLAGS, BUFFERED_FLASH_PAGE_BITS, BUFFERED_FLASH_PAGE_SIZE,
 291          BUFFERED_FLASH_BYTE_ADDR_MASK, BUFFERED_FLASH_TOTAL_SIZE*2,
 292          "Buffered flash (256kB)"},
 293 };
 294
 295 /*
 296  * The BCM5709 controllers transparently handle the
 297  * differences between Atmel 264 byte pages and all
 298  * flash devices which use 256 byte pages, so no
 299  * logical-to-physical mapping is required in the
 300  * driver.
 301  */
 302 static struct flash_spec flash_5709 = {
 303         .flags          = BCE_NV_BUFFERED,
 304         .page_bits      = BCM5709_FLASH_PAGE_BITS,
 305         .page_size      = BCM5709_FLASH_PAGE_SIZE,
 306         .addr_mask      = BCM5709_FLASH_BYTE_ADDR_MASK,
 307         .total_size     = BUFFERED_FLASH_TOTAL_SIZE * 2,
 308         .name           = "5709/5716 buffered flash (256kB)",
 309 };
 310
 311
 312 /****************************************************************************/
 313 /* DragonFly device entry points.                                           */
 314 /****************************************************************************/
 315 static int      bce_probe(device_t);
 316 static int      bce_attach(device_t);
 317 static int      bce_detach(device_t);
 318 static void     bce_shutdown(device_t);
 319 static int      bce_miibus_read_reg(device_t, int, int);
 320 static int      bce_miibus_write_reg(device_t, int, int, int);
 321 static void     bce_miibus_statchg(device_t);
 322
 323 /****************************************************************************/
 324 /* BCE Register/Memory Access Routines                                      */
 325 /****************************************************************************/
 326 static uint32_t bce_reg_rd_ind(struct bce_softc *, uint32_t);
 327 static void     bce_reg_wr_ind(struct bce_softc *, uint32_t, uint32_t);
 328 static void     bce_shmem_wr(struct bce_softc *, uint32_t, uint32_t);
 329 static uint32_t bce_shmem_rd(struct bce_softc *, u32);
 330 static void     bce_ctx_wr(struct bce_softc *, uint32_t, uint32_t, uint32_t);
 331
 332 /****************************************************************************/
 333 /* BCE NVRAM Access Routines                                                */
 334 /****************************************************************************/
 335 static int      bce_acquire_nvram_lock(struct bce_softc *);
 336 static int      bce_release_nvram_lock(struct bce_softc *);
 337 static void     bce_enable_nvram_access(struct bce_softc *);
 338 static void     bce_disable_nvram_access(struct bce_softc *);
 339 static int      bce_nvram_read_dword(struct bce_softc *, uint32_t, uint8_t *,
 340                     uint32_t);
 341 static int      bce_init_nvram(struct bce_softc *);
 342 static int      bce_nvram_read(struct bce_softc *, uint32_t, uint8_t *, int);
 343 static int      bce_nvram_test(struct bce_softc *);
 344
 345 /****************************************************************************/
 346 /* BCE DMA Allocate/Free Routines                                           */
 347 /****************************************************************************/
 348 static int      bce_dma_alloc(struct bce_softc *);
 349 static void     bce_dma_free(struct bce_softc *);
 350 static void     bce_dma_map_addr(void *, bus_dma_segment_t *, int, int);
 351
 352 /****************************************************************************/
 353 /* BCE Firmware Synchronization and Load                                    */
 354 /****************************************************************************/
 355 static int      bce_fw_sync(struct bce_softc *, uint32_t);
 356 static void     bce_load_rv2p_fw(struct bce_softc *, uint32_t *,
 357                     uint32_t, uint32_t);
 358 static void     bce_load_cpu_fw(struct bce_softc *, struct cpu_reg *,
 359                     struct fw_info *);
 360 static void     bce_start_cpu(struct bce_softc *, struct cpu_reg *);
 361 static void     bce_halt_cpu(struct bce_softc *, struct cpu_reg *);
 362 static void     bce_start_rxp_cpu(struct bce_softc *);
 363 static void     bce_init_rxp_cpu(struct bce_softc *);
 364 static void     bce_init_txp_cpu(struct bce_softc *);
 365 static void     bce_init_tpat_cpu(struct bce_softc *);
 366 static void     bce_init_cp_cpu(struct bce_softc *);
 367 static void     bce_init_com_cpu(struct bce_softc *);
 368 static void     bce_init_cpus(struct bce_softc *);
 369 static void     bce_setup_msix_table(struct bce_softc *);
 370 static void     bce_init_rss(struct bce_softc *);
 371
 372 static void     bce_stop(struct bce_softc *);
 373 static int      bce_reset(struct bce_softc *, uint32_t);
 374 static int      bce_chipinit(struct bce_softc *);
 375 static int      bce_blockinit(struct bce_softc *);
 376 static void     bce_probe_pci_caps(struct bce_softc *);
 377 static void     bce_print_adapter_info(struct bce_softc *);
 378 static void     bce_get_media(struct bce_softc *);
 379 static void     bce_mgmt_init(struct bce_softc *);
 380 static int      bce_init_ctx(struct bce_softc *);
 381 static void     bce_get_mac_addr(struct bce_softc *);
 382 static void     bce_set_mac_addr(struct bce_softc *);
 383 static void     bce_set_rx_mode(struct bce_softc *);
 384 static void     bce_coal_change(struct bce_softc *);
 385 static void     bce_npoll_coal_change(struct bce_softc *);
 386 static void     bce_setup_serialize(struct bce_softc *);
 387 static void     bce_serialize_skipmain(struct bce_softc *);
 388 static void     bce_deserialize_skipmain(struct bce_softc *);
 389 static void     bce_set_timer_cpuid(struct bce_softc *, boolean_t);
 390 static int      bce_alloc_intr(struct bce_softc *);
 391 static void     bce_free_intr(struct bce_softc *);
 392 static void     bce_try_alloc_msix(struct bce_softc *);
 393 static void     bce_free_msix(struct bce_softc *, boolean_t);
 394 static void     bce_setup_ring_cnt(struct bce_softc *);
 395 static int      bce_setup_intr(struct bce_softc *);
 396 static void     bce_teardown_intr(struct bce_softc *);
 397 static int      bce_setup_msix(struct bce_softc *);
 398 static void     bce_teardown_msix(struct bce_softc *, int);
 399
 400 static int      bce_create_tx_ring(struct bce_tx_ring *);
 401 static void     bce_destroy_tx_ring(struct bce_tx_ring *);
 402 static void     bce_init_tx_context(struct bce_tx_ring *);
 403 static int      bce_init_tx_chain(struct bce_tx_ring *);
 404 static void     bce_free_tx_chain(struct bce_tx_ring *);
 405 static void     bce_xmit(struct bce_tx_ring *);
 406 static int      bce_encap(struct bce_tx_ring *, struct mbuf **, int *);
 407 static int      bce_tso_setup(struct bce_tx_ring *, struct mbuf **,
 408                     uint16_t *, uint16_t *);
 409
 410 static int      bce_create_rx_ring(struct bce_rx_ring *);
 411 static void     bce_destroy_rx_ring(struct bce_rx_ring *);
 412 static void     bce_init_rx_context(struct bce_rx_ring *);
 413 static int      bce_init_rx_chain(struct bce_rx_ring *);
 414 static void     bce_free_rx_chain(struct bce_rx_ring *);
 415 static int      bce_newbuf_std(struct bce_rx_ring *, uint16_t *, uint16_t *,
 416                     uint32_t *, int);
 417 static void     bce_setup_rxdesc_std(struct bce_rx_ring *, uint16_t,
 418                     uint32_t *);
 419
 420 static void     bce_start(struct ifnet *, struct ifaltq_subque *);
 421 static int      bce_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 422 static void     bce_watchdog(struct ifaltq_subque *);
 423 static int      bce_ifmedia_upd(struct ifnet *);
 424 static void     bce_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 425 static void     bce_init(void *);
 426 #ifdef IFPOLL_ENABLE
 427 static void     bce_npoll(struct ifnet *, struct ifpoll_info *);
 428 static void     bce_npoll_rx(struct ifnet *, void *, int);
 429 static void     bce_npoll_tx(struct ifnet *, void *, int);
 430 static void     bce_npoll_status(struct ifnet *);
 431 static void     bce_npoll_rx_pack(struct ifnet *, void *, int);
 432 #endif
 433 static void     bce_serialize(struct ifnet *, enum ifnet_serialize);
 434 static void     bce_deserialize(struct ifnet *, enum ifnet_serialize);
 435 static int      bce_tryserialize(struct ifnet *, enum ifnet_serialize);
 436 #ifdef INVARIANTS
 437 static void     bce_serialize_assert(struct ifnet *, enum ifnet_serialize,
 438                     boolean_t);
 439 #endif
 440
 441 static void     bce_intr(struct bce_softc *);
 442 static void     bce_intr_legacy(void *);
 443 static void     bce_intr_msi(void *);
 444 static void     bce_intr_msi_oneshot(void *);
 445 static void     bce_intr_msix_rxtx(void *);
 446 static void     bce_intr_msix_rx(void *);
 447 static void     bce_tx_intr(struct bce_tx_ring *, uint16_t);
 448 static void     bce_rx_intr(struct bce_rx_ring *, int, uint16_t);
 449 static void     bce_phy_intr(struct bce_softc *);
 450 static void     bce_disable_intr(struct bce_softc *);
 451 static void     bce_enable_intr(struct bce_softc *);
 452 static void     bce_reenable_intr(struct bce_rx_ring *);
 453 static void     bce_check_msi(void *);
 454
 455 static void     bce_stats_update(struct bce_softc *);
 456 static void     bce_tick(void *);
 457 static void     bce_tick_serialized(struct bce_softc *);
 458 static void     bce_pulse(void *);
 459
 460 static void     bce_add_sysctls(struct bce_softc *);
 461 static int      bce_sysctl_tx_bds_int(SYSCTL_HANDLER_ARGS);
 462 static int      bce_sysctl_tx_bds(SYSCTL_HANDLER_ARGS);
 463 static int      bce_sysctl_tx_ticks_int(SYSCTL_HANDLER_ARGS);
 464 static int      bce_sysctl_tx_ticks(SYSCTL_HANDLER_ARGS);
 465 static int      bce_sysctl_rx_bds_int(SYSCTL_HANDLER_ARGS);
 466 static int      bce_sysctl_rx_bds(SYSCTL_HANDLER_ARGS);
 467 static int      bce_sysctl_rx_ticks_int(SYSCTL_HANDLER_ARGS);
 468 static int      bce_sysctl_rx_ticks(SYSCTL_HANDLER_ARGS);
 469 #ifdef IFPOLL_ENABLE
 470 static int      bce_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS);
 471 #endif
 472 static int      bce_sysctl_coal_change(SYSCTL_HANDLER_ARGS,
 473                     uint32_t *, uint32_t);
 474
 475 /*
 476  * NOTE:
 477  * Don't set bce_tx_ticks_int/bce_tx_ticks to 1023.  Linux's bnx2
 478  * takes 1023 as the TX ticks limit.  However, using 1023 will
 479  * cause 5708(B2) to generate extra interrupts (~2000/s) even when
 480  * there is _no_ network activity on the NIC.
 481  */
 482 static uint32_t bce_tx_bds_int = 255;           /* bcm: 20 */
 483 static uint32_t bce_tx_bds = 255;               /* bcm: 20 */
 484 static uint32_t bce_tx_ticks_int = 1022;        /* bcm: 80 */
 485 static uint32_t bce_tx_ticks = 1022;            /* bcm: 80 */
 486 static uint32_t bce_rx_bds_int = 128;           /* bcm: 6 */
 487 static uint32_t bce_rx_bds = 0;                 /* bcm: 6 */
 488 static uint32_t bce_rx_ticks_int = 150;         /* bcm: 18 */
 489 static uint32_t bce_rx_ticks = 150;             /* bcm: 18 */
 490
 491 static int      bce_tx_wreg = 8;
 492
 493 static int      bce_msi_enable = 1;
 494 static int      bce_msix_enable = 1;
 495
 496 static int      bce_rx_pages = RX_PAGES_DEFAULT;
 497 static int      bce_tx_pages = TX_PAGES_DEFAULT;
 498
 499 static int      bce_rx_rings = 0;       /* auto */
 500 static int      bce_tx_rings = 0;       /* auto */
 501
 502 TUNABLE_INT("hw.bce.tx_bds_int", &bce_tx_bds_int);
 503 TUNABLE_INT("hw.bce.tx_bds", &bce_tx_bds);
 504 TUNABLE_INT("hw.bce.tx_ticks_int", &bce_tx_ticks_int);
 505 TUNABLE_INT("hw.bce.tx_ticks", &bce_tx_ticks);
 506 TUNABLE_INT("hw.bce.rx_bds_int", &bce_rx_bds_int);
 507 TUNABLE_INT("hw.bce.rx_bds", &bce_rx_bds);
 508 TUNABLE_INT("hw.bce.rx_ticks_int", &bce_rx_ticks_int);
 509 TUNABLE_INT("hw.bce.rx_ticks", &bce_rx_ticks);
 510 TUNABLE_INT("hw.bce.msi.enable", &bce_msi_enable);
 511 TUNABLE_INT("hw.bce.msix.enable", &bce_msix_enable);
 512 TUNABLE_INT("hw.bce.rx_pages", &bce_rx_pages);
 513 TUNABLE_INT("hw.bce.tx_pages", &bce_tx_pages);
 514 TUNABLE_INT("hw.bce.tx_wreg", &bce_tx_wreg);
 515 TUNABLE_INT("hw.bce.tx_rings", &bce_tx_rings);
 516 TUNABLE_INT("hw.bce.rx_rings", &bce_rx_rings);
 517
 518 /****************************************************************************/
 519 /* DragonFly device dispatch table.                                         */
 520 /****************************************************************************/
 521 static device_method_t bce_methods[] = {
 522         /* Device interface */
 523         DEVMETHOD(device_probe,         bce_probe),
 524         DEVMETHOD(device_attach,        bce_attach),
 525         DEVMETHOD(device_detach,        bce_detach),
 526         DEVMETHOD(device_shutdown,      bce_shutdown),
 527
 528         /* bus interface */
 529         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 530         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 531
 532         /* MII interface */
 533         DEVMETHOD(miibus_readreg,       bce_miibus_read_reg),
 534         DEVMETHOD(miibus_writereg,      bce_miibus_write_reg),
 535         DEVMETHOD(miibus_statchg,       bce_miibus_statchg),
 536
 537         DEVMETHOD_END
 538 };
 539
 540 static driver_t bce_driver = {
 541         "bce",
 542         bce_methods,
 543         sizeof(struct bce_softc)
 544 };
 545
 546 static devclass_t bce_devclass;
 547
 548
 549 DECLARE_DUMMY_MODULE(if_bce);
 550 MODULE_DEPEND(bce, miibus, 1, 1, 1);
 551 DRIVER_MODULE(if_bce, pci, bce_driver, bce_devclass, NULL, NULL);
 552 DRIVER_MODULE(miibus, bce, miibus_driver, miibus_devclass, NULL, NULL);
 553
 554
 555 /****************************************************************************/
 556 /* Device probe function.                                                   */
 557 /*                                                                          */
 558 /* Compares the device to the driver's list of supported devices and        */
 559 /* reports back to the OS whether this is the right driver for the device.  */
 560 /*                                                                          */
 561 /* Returns:                                                                 */
 562 /*   BUS_PROBE_DEFAULT on success, positive value on failure.               */
 563 /****************************************************************************/
 564 static int
 565 bce_probe(device_t dev)
 566 {
 567         struct bce_type *t;
 568         uint16_t vid, did, svid, sdid;
 569
 570         /* Get the data for the device to be probed. */
 571         vid  = pci_get_vendor(dev);
 572         did  = pci_get_device(dev);
 573         svid = pci_get_subvendor(dev);
 574         sdid = pci_get_subdevice(dev);
 575
 576         /* Look through the list of known devices for a match. */
 577         for (t = bce_devs; t->bce_name != NULL; ++t) {
 578                 if (vid == t->bce_vid && did == t->bce_did &&
 579                     (svid == t->bce_svid || t->bce_svid == PCI_ANY_ID) &&
 580                     (sdid == t->bce_sdid || t->bce_sdid == PCI_ANY_ID)) {
 581                         uint32_t revid = pci_read_config(dev, PCIR_REVID, 4);
 582                         char *descbuf;
 583
 584                         descbuf = kmalloc(BCE_DEVDESC_MAX, M_TEMP, M_WAITOK);
 585
 586                         /* Print out the device identity. */
 587                         ksnprintf(descbuf, BCE_DEVDESC_MAX, "%s (%c%d)",
 588                                   t->bce_name,
 589                                   ((revid & 0xf0) >> 4) + 'A', revid & 0xf);
 590
 591                         device_set_desc_copy(dev, descbuf);
 592                         kfree(descbuf, M_TEMP);
 593                         return 0;
 594                 }
 595         }
 596         return ENXIO;
 597 }
 598
 599
 600 /****************************************************************************/
 601 /* PCI Capabilities Probe Function.                                         */
 602 /*                                                                          */
 603 /* Walks the PCI capabiites list for the device to find what features are   */
 604 /* supported.                                                               */
 605 /*                                                                          */
 606 /* Returns:                                                                 */
 607 /*   None.                                                                  */
 608 /****************************************************************************/
 609 static void
 610 bce_print_adapter_info(struct bce_softc *sc)
 611 {
 612         device_printf(sc->bce_dev, "ASIC (0x%08X); ", sc->bce_chipid);
 613
 614         kprintf("Rev (%c%d); ", ((BCE_CHIP_ID(sc) & 0xf000) >> 12) + 'A',
 615                 ((BCE_CHIP_ID(sc) & 0x0ff0) >> 4));
 616
 617         /* Bus info. */
 618         if (sc->bce_flags & BCE_PCIE_FLAG) {
 619                 kprintf("Bus (PCIe x%d, ", sc->link_width);
 620                 switch (sc->link_speed) {
 621                 case 1:
 622                         kprintf("2.5Gbps); ");
 623                         break;
 624                 case 2:
 625                         kprintf("5Gbps); ");
 626                         break;
 627                 default:
 628                         kprintf("Unknown link speed); ");
 629                         break;
 630                 }
 631         } else {
 632                 kprintf("Bus (PCI%s, %s, %dMHz); ",
 633                     ((sc->bce_flags & BCE_PCIX_FLAG) ? "-X" : ""),
 634                     ((sc->bce_flags & BCE_PCI_32BIT_FLAG) ? "32-bit" : "64-bit"),
 635                     sc->bus_speed_mhz);
 636         }
 637
 638         /* Firmware version and device features. */
 639         kprintf("B/C (%s)", sc->bce_bc_ver);
 640
 641         if ((sc->bce_flags & BCE_MFW_ENABLE_FLAG) ||
 642             (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)) {
 643                 kprintf("; Flags(");
 644                 if (sc->bce_flags & BCE_MFW_ENABLE_FLAG)
 645                         kprintf("MFW[%s]", sc->bce_mfw_ver);
 646                 if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
 647                         kprintf(" 2.5G");
 648                 kprintf(")");
 649         }
 650         kprintf("\n");
 651 }
 652
 653
 654 /****************************************************************************/
 655 /* PCI Capabilities Probe Function.                                         */
 656 /*                                                                          */
 657 /* Walks the PCI capabiites list for the device to find what features are   */
 658 /* supported.                                                               */
 659 /*                                                                          */
 660 /* Returns:                                                                 */
 661 /*   None.                                                                  */
 662 /****************************************************************************/
 663 static void
 664 bce_probe_pci_caps(struct bce_softc *sc)
 665 {
 666         device_t dev = sc->bce_dev;
 667         uint8_t ptr;
 668
 669         if (pci_is_pcix(dev))
 670                 sc->bce_cap_flags |= BCE_PCIX_CAPABLE_FLAG;
 671
 672         ptr = pci_get_pciecap_ptr(dev);
 673         if (ptr) {
 674                 uint16_t link_status = pci_read_config(dev, ptr + 0x12, 2);
 675
 676                 sc->link_speed = link_status & 0xf;
 677                 sc->link_width = (link_status >> 4) & 0x3f;
 678                 sc->bce_cap_flags |= BCE_PCIE_CAPABLE_FLAG;
 679                 sc->bce_flags |= BCE_PCIE_FLAG;
 680         }
 681 }
 682
 683
 684 /****************************************************************************/
 685 /* Device attach function.                                                  */
 686 /*                                                                          */
 687 /* Allocates device resources, performs secondary chip identification,      */
 688 /* resets and initializes the hardware, and initializes driver instance     */
 689 /* variables.                                                               */
 690 /*                                                                          */
 691 /* Returns:                                                                 */
 692 /*   0 on success, positive value on failure.                               */
 693 /****************************************************************************/
 694 static int
 695 bce_attach(device_t dev)
 696 {
 697         struct bce_softc *sc = device_get_softc(dev);
 698         struct ifnet *ifp = &sc->arpcom.ac_if;
 699         uint32_t val;
 700         int rid, rc = 0;
 701         int i, j;
 702         struct mii_probe_args mii_args;
 703         uintptr_t mii_priv = 0;
 704 #ifdef IFPOLL_ENABLE
 705         int offset, offset_def;
 706 #endif
 707
 708         sc->bce_dev = dev;
 709         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 710
 711         lwkt_serialize_init(&sc->main_serialize);
 712         for (i = 0; i < BCE_MSIX_MAX; ++i) {
 713                 struct bce_msix_data *msix = &sc->bce_msix[i];
 714
 715                 msix->msix_cpuid = -1;
 716                 msix->msix_rid = -1;
 717         }
 718
 719         pci_enable_busmaster(dev);
 720
 721         bce_probe_pci_caps(sc);
 722
 723         /* Allocate PCI memory resources. */
 724         rid = PCIR_BAR(0);
 725         sc->bce_res_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 726                                                  RF_ACTIVE | PCI_RF_DENSE);
 727         if (sc->bce_res_mem == NULL) {
 728                 device_printf(dev, "PCI memory allocation failed\n");
 729                 return ENXIO;
 730         }
 731         sc->bce_btag = rman_get_bustag(sc->bce_res_mem);
 732         sc->bce_bhandle = rman_get_bushandle(sc->bce_res_mem);
 733
 734         /*
 735          * Configure byte swap and enable indirect register access.
 736          * Rely on CPU to do target byte swapping on big endian systems.
 737          * Access to registers outside of PCI configurtion space are not
 738          * valid until this is done.
 739          */
 740         pci_write_config(dev, BCE_PCICFG_MISC_CONFIG,
 741                          BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
 742                          BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP, 4);
 743
 744         /* Save ASIC revsion info. */
 745         sc->bce_chipid =  REG_RD(sc, BCE_MISC_ID);
 746
 747         /* Weed out any non-production controller revisions. */
 748         switch (BCE_CHIP_ID(sc)) {
 749         case BCE_CHIP_ID_5706_A0:
 750         case BCE_CHIP_ID_5706_A1:
 751         case BCE_CHIP_ID_5708_A0:
 752         case BCE_CHIP_ID_5708_B0:
 753         case BCE_CHIP_ID_5709_A0:
 754         case BCE_CHIP_ID_5709_B0:
 755         case BCE_CHIP_ID_5709_B1:
 756 #ifdef foo
 757         /* 5709C B2 seems to work fine */
 758         case BCE_CHIP_ID_5709_B2:
 759 #endif
 760                 device_printf(dev, "Unsupported chip id 0x%08x!\n",
 761                               BCE_CHIP_ID(sc));
 762                 rc = ENODEV;
 763                 goto fail;
 764         }
 765
 766         mii_priv |= BRGPHY_FLAG_WIRESPEED;
 767         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
 768                 if (BCE_CHIP_REV(sc) == BCE_CHIP_REV_Ax ||
 769                     BCE_CHIP_REV(sc) == BCE_CHIP_REV_Bx)
 770                         mii_priv |= BRGPHY_FLAG_NO_EARLYDAC;
 771         } else {
 772                 mii_priv |= BRGPHY_FLAG_BER_BUG;
 773         }
 774
 775         /*
 776          * Find the base address for shared memory access.
 777          * Newer versions of bootcode use a signature and offset
 778          * while older versions use a fixed address.
 779          */
 780         val = REG_RD_IND(sc, BCE_SHM_HDR_SIGNATURE);
 781         if ((val & BCE_SHM_HDR_SIGNATURE_SIG_MASK) ==
 782             BCE_SHM_HDR_SIGNATURE_SIG) {
 783                 /* Multi-port devices use different offsets in shared memory. */
 784                 sc->bce_shmem_base = REG_RD_IND(sc,
 785                     BCE_SHM_HDR_ADDR_0 + (pci_get_function(sc->bce_dev) << 2));
 786         } else {
 787                 sc->bce_shmem_base = HOST_VIEW_SHMEM_BASE;
 788         }
 789
 790         /* Fetch the bootcode revision. */
 791         val = bce_shmem_rd(sc, BCE_DEV_INFO_BC_REV);
 792         for (i = 0, j = 0; i < 3; i++) {
 793                 uint8_t num;
 794                 int k, skip0;
 795
 796                 num = (uint8_t)(val >> (24 - (i * 8)));
 797                 for (k = 100, skip0 = 1; k >= 1; num %= k, k /= 10) {
 798                         if (num >= k || !skip0 || k == 1) {
 799                                 sc->bce_bc_ver[j++] = (num / k) + '0';
 800                                 skip0 = 0;
 801                         }
 802                 }
 803                 if (i != 2)
 804                         sc->bce_bc_ver[j++] = '.';
 805         }
 806
 807         /* Check if any management firwmare is running. */
 808         val = bce_shmem_rd(sc, BCE_PORT_FEATURE);
 809         if (val & BCE_PORT_FEATURE_ASF_ENABLED) {
 810                 sc->bce_flags |= BCE_MFW_ENABLE_FLAG;
 811
 812                 /* Allow time for firmware to enter the running state. */
 813                 for (i = 0; i < 30; i++) {
 814                         val = bce_shmem_rd(sc, BCE_BC_STATE_CONDITION);
 815                         if (val & BCE_CONDITION_MFW_RUN_MASK)
 816                                 break;
 817                         DELAY(10000);
 818                 }
 819         }
 820
 821         /* Check the current bootcode state. */
 822         val = bce_shmem_rd(sc, BCE_BC_STATE_CONDITION) &
 823             BCE_CONDITION_MFW_RUN_MASK;
 824         if (val != BCE_CONDITION_MFW_RUN_UNKNOWN &&
 825             val != BCE_CONDITION_MFW_RUN_NONE) {
 826                 uint32_t addr = bce_shmem_rd(sc, BCE_MFW_VER_PTR);
 827
 828                 for (i = 0, j = 0; j < 3; j++) {
 829                         val = bce_reg_rd_ind(sc, addr + j * 4);
 830                         val = bswap32(val);
 831                         memcpy(&sc->bce_mfw_ver[i], &val, 4);
 832                         i += 4;
 833                 }
 834         }
 835
 836         /* Get PCI bus information (speed and type). */
 837         val = REG_RD(sc, BCE_PCICFG_MISC_STATUS);
 838         if (val & BCE_PCICFG_MISC_STATUS_PCIX_DET) {
 839                 uint32_t clkreg;
 840
 841                 sc->bce_flags |= BCE_PCIX_FLAG;
 842
 843                 clkreg = REG_RD(sc, BCE_PCICFG_PCI_CLOCK_CONTROL_BITS) &
 844                          BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET;
 845                 switch (clkreg) {
 846                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_133MHZ:
 847                         sc->bus_speed_mhz = 133;
 848                         break;
 849
 850                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_95MHZ:
 851                         sc->bus_speed_mhz = 100;
 852                         break;
 853
 854                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_66MHZ:
 855                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_80MHZ:
 856                         sc->bus_speed_mhz = 66;
 857                         break;
 858
 859                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_48MHZ:
 860                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_55MHZ:
 861                         sc->bus_speed_mhz = 50;
 862                         break;
 863
 864                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_LOW:
 865                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_32MHZ:
 866                 case BCE_PCICFG_PCI_CLOCK_CONTROL_BITS_PCI_CLK_SPD_DET_38MHZ:
 867                         sc->bus_speed_mhz = 33;
 868                         break;
 869                 }
 870         } else {
 871                 if (val & BCE_PCICFG_MISC_STATUS_M66EN)
 872                         sc->bus_speed_mhz = 66;
 873                 else
 874                         sc->bus_speed_mhz = 33;
 875         }
 876
 877         if (val & BCE_PCICFG_MISC_STATUS_32BIT_DET)
 878                 sc->bce_flags |= BCE_PCI_32BIT_FLAG;
 879
 880         /* Reset the controller. */
 881         rc = bce_reset(sc, BCE_DRV_MSG_CODE_RESET);
 882         if (rc != 0)
 883                 goto fail;
 884
 885         /* Initialize the controller. */
 886         rc = bce_chipinit(sc);
 887         if (rc != 0) {
 888                 device_printf(dev, "Controller initialization failed!\n");
 889                 goto fail;
 890         }
 891
 892         /* Perform NVRAM test. */
 893         rc = bce_nvram_test(sc);
 894         if (rc != 0) {
 895                 device_printf(dev, "NVRAM test failed!\n");
 896                 goto fail;
 897         }
 898
 899         /* Fetch the permanent Ethernet MAC address. */
 900         bce_get_mac_addr(sc);
 901
 902         /*
 903          * Trip points control how many BDs
 904          * should be ready before generating an
 905          * interrupt while ticks control how long
 906          * a BD can sit in the chain before
 907          * generating an interrupt.  Set the default
 908          * values for the RX and TX rings.
 909          */
 910
 911 #ifdef BCE_DRBUG
 912         /* Force more frequent interrupts. */
 913         sc->bce_tx_quick_cons_trip_int = 1;
 914         sc->bce_tx_quick_cons_trip     = 1;
 915         sc->bce_tx_ticks_int           = 0;
 916         sc->bce_tx_ticks               = 0;
 917
 918         sc->bce_rx_quick_cons_trip_int = 1;
 919         sc->bce_rx_quick_cons_trip     = 1;
 920         sc->bce_rx_ticks_int           = 0;
 921         sc->bce_rx_ticks               = 0;
 922 #else
 923         sc->bce_tx_quick_cons_trip_int = bce_tx_bds_int;
 924         sc->bce_tx_quick_cons_trip     = bce_tx_bds;
 925         sc->bce_tx_ticks_int           = bce_tx_ticks_int;
 926         sc->bce_tx_ticks               = bce_tx_ticks;
 927
 928         sc->bce_rx_quick_cons_trip_int = bce_rx_bds_int;
 929         sc->bce_rx_quick_cons_trip     = bce_rx_bds;
 930         sc->bce_rx_ticks_int           = bce_rx_ticks_int;
 931         sc->bce_rx_ticks               = bce_rx_ticks;
 932 #endif
 933
 934         /* Update statistics once every second. */
 935         sc->bce_stats_ticks = 1000000 & 0xffff00;
 936
 937         /* Find the media type for the adapter. */
 938         bce_get_media(sc);
 939
 940         /* Find out RX/TX ring count */
 941         bce_setup_ring_cnt(sc);
 942
 943         /* Allocate DMA memory resources. */
 944         rc = bce_dma_alloc(sc);
 945         if (rc != 0) {
 946                 device_printf(dev, "DMA resource allocation failed!\n");
 947                 goto fail;
 948         }
 949
 950 #ifdef IFPOLL_ENABLE
 951         /*
 952          * NPOLLING RX/TX CPU offset
 953          */
 954         if (sc->rx_ring_cnt2 == ncpus2) {
 955                 offset = 0;
 956         } else {
 957                 offset_def = (sc->rx_ring_cnt2 * device_get_unit(dev)) % ncpus2;
 958                 offset = device_getenv_int(dev, "npoll.offset", offset_def);
 959                 if (offset >= ncpus2 ||
 960                     offset % sc->rx_ring_cnt2 != 0) {
 961                         device_printf(dev, "invalid npoll.offset %d, use %d\n",
 962                             offset, offset_def);
 963                         offset = offset_def;
 964                 }
 965         }
 966         sc->npoll_ofs = offset;
 967 #endif
 968
 969         /* Allocate PCI IRQ resources. */
 970         rc = bce_alloc_intr(sc);
 971         if (rc != 0)
 972                 goto fail;
 973
 974         /* Setup serializer */
 975         bce_setup_serialize(sc);
 976
 977         /* Initialize the ifnet interface. */
 978         ifp->if_softc = sc;
 979         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 980         ifp->if_ioctl = bce_ioctl;
 981         ifp->if_start = bce_start;
 982         ifp->if_init = bce_init;
 983         ifp->if_serialize = bce_serialize;
 984         ifp->if_deserialize = bce_deserialize;
 985         ifp->if_tryserialize = bce_tryserialize;
 986 #ifdef INVARIANTS
 987         ifp->if_serialize_assert = bce_serialize_assert;
 988 #endif
 989 #ifdef IFPOLL_ENABLE
 990         ifp->if_npoll = bce_npoll;
 991 #endif
 992
 993         ifp->if_mtu = ETHERMTU;
 994         ifp->if_hwassist = BCE_CSUM_FEATURES | CSUM_TSO;
 995         ifp->if_capabilities = BCE_IF_CAPABILITIES;
 996         ifp->if_capenable = ifp->if_capabilities;
 997
 998         if (sc->bce_phy_flags & BCE_PHY_2_5G_CAPABLE_FLAG)
 999                 ifp->if_baudrate = IF_Gbps(2.5);
1000         else
1001                 ifp->if_baudrate = IF_Gbps(1);
1002
1003         ifq_set_maxlen(&ifp->if_snd, USABLE_TX_BD(&sc->tx_rings[0]));
1004         ifq_set_ready(&ifp->if_snd);
1005         ifq_set_subq_cnt(&ifp->if_snd, sc->tx_ring_cnt);
1006
1007         if (sc->tx_ring_cnt > 1) {
1008                 ifp->if_mapsubq = ifq_mapsubq_mask;
1009                 ifq_set_subq_mask(&ifp->if_snd, sc->tx_ring_cnt - 1);
1010         }
1011
1012         /*
1013          * Look for our PHY.
1014          */
1015         mii_probe_args_init(&mii_args, bce_ifmedia_upd, bce_ifmedia_sts);
1016         mii_args.mii_probemask = 1 << sc->bce_phy_addr;
1017         mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
1018         mii_args.mii_priv = mii_priv;
1019
1020         rc = mii_probe(dev, &sc->bce_miibus, &mii_args);
1021         if (rc != 0) {
1022                 device_printf(dev, "PHY probe failed!\n");
1023                 goto fail;
1024         }
1025
1026         /* Attach to the Ethernet interface list. */
1027         ether_ifattach(ifp, sc->eaddr, NULL);
1028
1029         callout_init_mp(&sc->bce_tick_callout);
1030         callout_init_mp(&sc->bce_pulse_callout);
1031         callout_init_mp(&sc->bce_ckmsi_callout);
1032
1033         rc = bce_setup_intr(sc);
1034         if (rc != 0) {
1035                 device_printf(dev, "Failed to setup IRQ!\n");
1036                 ether_ifdetach(ifp);
1037                 goto fail;
1038         }
1039
1040         for (i = 0; i < sc->tx_ring_cnt; ++i) {
1041                 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
1042                 struct bce_tx_ring *txr = &sc->tx_rings[i];
1043
1044                 ifsq_set_cpuid(ifsq, sc->bce_msix[i].msix_cpuid);
1045                 ifsq_set_priv(ifsq, txr);
1046                 txr->ifsq = ifsq;
1047
1048                 ifsq_watchdog_init(&txr->tx_watchdog, ifsq, bce_watchdog);
1049         }
1050
1051         /* Set timer CPUID */
1052         bce_set_timer_cpuid(sc, FALSE);
1053
1054         /* Add the supported sysctls to the kernel. */
1055         bce_add_sysctls(sc);
1056
1057         /*
1058          * The chip reset earlier notified the bootcode that
1059          * a driver is present.  We now need to start our pulse
1060          * routine so that the bootcode is reminded that we're
1061          * still running.
1062          */
1063         bce_pulse(sc);
1064
1065         /* Get the firmware running so IPMI still works */
1066         bce_mgmt_init(sc);
1067
1068         if (bootverbose)
1069                 bce_print_adapter_info(sc);
1070
1071         return 0;
1072 fail:
1073         bce_detach(dev);
1074         return(rc);
1075 }
1076
1077
1078 /****************************************************************************/
1079 /* Device detach function.                                                  */
1080 /*                                                                          */
1081 /* Stops the controller, resets the controller, and releases resources.     */
1082 /*                                                                          */
1083 /* Returns:                                                                 */
1084 /*   0 on success, positive value on failure.                               */
1085 /****************************************************************************/
1086 static int
1087 bce_detach(device_t dev)
1088 {
1089         struct bce_softc *sc = device_get_softc(dev);
1090
1091         if (device_is_attached(dev)) {
1092                 struct ifnet *ifp = &sc->arpcom.ac_if;
1093                 uint32_t msg;
1094
1095                 ifnet_serialize_all(ifp);
1096
1097                 /* Stop and reset the controller. */
1098                 callout_stop(&sc->bce_pulse_callout);
1099                 bce_stop(sc);
1100                 if (sc->bce_flags & BCE_NO_WOL_FLAG)
1101                         msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1102                 else
1103                         msg = BCE_DRV_MSG_CODE_UNLOAD;
1104                 bce_reset(sc, msg);
1105
1106                 bce_teardown_intr(sc);
1107
1108                 ifnet_deserialize_all(ifp);
1109
1110                 ether_ifdetach(ifp);
1111         }
1112
1113         /* If we have a child device on the MII bus remove it too. */
1114         if (sc->bce_miibus)
1115                 device_delete_child(dev, sc->bce_miibus);
1116         bus_generic_detach(dev);
1117
1118         bce_free_intr(sc);
1119
1120         if (sc->bce_res_mem != NULL) {
1121                 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0),
1122                                      sc->bce_res_mem);
1123         }
1124
1125         bce_dma_free(sc);
1126
1127         if (sc->bce_sysctl_tree != NULL)
1128                 sysctl_ctx_free(&sc->bce_sysctl_ctx);
1129
1130         if (sc->serializes != NULL)
1131                 kfree(sc->serializes, M_DEVBUF);
1132
1133         return 0;
1134 }
1135
1136
1137 /****************************************************************************/
1138 /* Device shutdown function.                                                */
1139 /*                                                                          */
1140 /* Stops and resets the controller.                                         */
1141 /*                                                                          */
1142 /* Returns:                                                                 */
1143 /*   Nothing                                                                */
1144 /****************************************************************************/
1145 static void
1146 bce_shutdown(device_t dev)
1147 {
1148         struct bce_softc *sc = device_get_softc(dev);
1149         struct ifnet *ifp = &sc->arpcom.ac_if;
1150         uint32_t msg;
1151
1152         ifnet_serialize_all(ifp);
1153
1154         bce_stop(sc);
1155         if (sc->bce_flags & BCE_NO_WOL_FLAG)
1156                 msg = BCE_DRV_MSG_CODE_UNLOAD_LNK_DN;
1157         else
1158                 msg = BCE_DRV_MSG_CODE_UNLOAD;
1159         bce_reset(sc, msg);
1160
1161         ifnet_deserialize_all(ifp);
1162 }
1163
1164
1165 /****************************************************************************/
1166 /* Indirect register read.                                                  */
1167 /*                                                                          */
1168 /* Reads NetXtreme II registers using an index/data register pair in PCI    */
1169 /* configuration space.  Using this mechanism avoids issues with posted     */
1170 /* reads but is much slower than memory-mapped I/O.                         */
1171 /*                                                                          */
1172 /* Returns:                                                                 */
1173 /*   The value of the register.                                             */
1174 /****************************************************************************/
1175 static uint32_t
1176 bce_reg_rd_ind(struct bce_softc *sc, uint32_t offset)
1177 {
1178         device_t dev = sc->bce_dev;
1179
1180         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1181         return pci_read_config(dev, BCE_PCICFG_REG_WINDOW, 4);
1182 }
1183
1184
1185 /****************************************************************************/
1186 /* Indirect register write.                                                 */
1187 /*                                                                          */
1188 /* Writes NetXtreme II registers using an index/data register pair in PCI   */
1189 /* configuration space.  Using this mechanism avoids issues with posted     */
1190 /* writes but is muchh slower than memory-mapped I/O.                       */
1191 /*                                                                          */
1192 /* Returns:                                                                 */
1193 /*   Nothing.                                                               */
1194 /****************************************************************************/
1195 static void
1196 bce_reg_wr_ind(struct bce_softc *sc, uint32_t offset, uint32_t val)
1197 {
1198         device_t dev = sc->bce_dev;
1199
1200         pci_write_config(dev, BCE_PCICFG_REG_WINDOW_ADDRESS, offset, 4);
1201         pci_write_config(dev, BCE_PCICFG_REG_WINDOW, val, 4);
1202 }
1203
1204
1205 /****************************************************************************/
1206 /* Shared memory write.                                                     */
1207 /*                                                                          */
1208 /* Writes NetXtreme II shared memory region.                                */
1209 /*                                                                          */
1210 /* Returns:                                                                 */
1211 /*   Nothing.                                                               */
1212 /****************************************************************************/
1213 static void
1214 bce_shmem_wr(struct bce_softc *sc, uint32_t offset, uint32_t val)
1215 {
1216         bce_reg_wr_ind(sc, sc->bce_shmem_base + offset, val);
1217 }
1218
1219
1220 /****************************************************************************/
1221 /* Shared memory read.                                                      */
1222 /*                                                                          */
1223 /* Reads NetXtreme II shared memory region.                                 */
1224 /*                                                                          */
1225 /* Returns:                                                                 */
1226 /*   The 32 bit value read.                                                 */
1227 /****************************************************************************/
1228 static u32
1229 bce_shmem_rd(struct bce_softc *sc, uint32_t offset)
1230 {
1231         return bce_reg_rd_ind(sc, sc->bce_shmem_base + offset);
1232 }
1233
1234
1235 /****************************************************************************/
1236 /* Context memory write.                                                    */
1237 /*                                                                          */
1238 /* The NetXtreme II controller uses context memory to track connection      */
1239 /* information for L2 and higher network protocols.                         */
1240 /*                                                                          */
1241 /* Returns:                                                                 */
1242 /*   Nothing.                                                               */
1243 /****************************************************************************/
1244 static void
1245 bce_ctx_wr(struct bce_softc *sc, uint32_t cid_addr, uint32_t ctx_offset,
1246     uint32_t ctx_val)
1247 {
1248         uint32_t idx, offset = ctx_offset + cid_addr;
1249         uint32_t val, retry_cnt = 5;
1250
1251         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
1252             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
1253                 REG_WR(sc, BCE_CTX_CTX_DATA, ctx_val);
1254                 REG_WR(sc, BCE_CTX_CTX_CTRL, (offset | BCE_CTX_CTX_CTRL_WRITE_REQ));
1255
1256                 for (idx = 0; idx < retry_cnt; idx++) {
1257                         val = REG_RD(sc, BCE_CTX_CTX_CTRL);
1258                         if ((val & BCE_CTX_CTX_CTRL_WRITE_REQ) == 0)
1259                                 break;
1260                         DELAY(5);
1261                 }
1262
1263                 if (val & BCE_CTX_CTX_CTRL_WRITE_REQ) {
1264                         device_printf(sc->bce_dev,
1265                             "Unable to write CTX memory: "
1266                             "cid_addr = 0x%08X, offset = 0x%08X!\n",
1267                             cid_addr, ctx_offset);
1268                 }
1269         } else {
1270                 REG_WR(sc, BCE_CTX_DATA_ADR, offset);
1271                 REG_WR(sc, BCE_CTX_DATA, ctx_val);
1272         }
1273 }
1274
1275
1276 /****************************************************************************/
1277 /* PHY register read.                                                       */
1278 /*                                                                          */
1279 /* Implements register reads on the MII bus.                                */
1280 /*                                                                          */
1281 /* Returns:                                                                 */
1282 /*   The value of the register.                                             */
1283 /****************************************************************************/
1284 static int
1285 bce_miibus_read_reg(device_t dev, int phy, int reg)
1286 {
1287         struct bce_softc *sc = device_get_softc(dev);
1288         uint32_t val;
1289         int i;
1290
1291         /* Make sure we are accessing the correct PHY address. */
1292         KASSERT(phy == sc->bce_phy_addr,
1293             ("invalid phyno %d, should be %d\n", phy, sc->bce_phy_addr));
1294
1295         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1296                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1297                 val &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1298
1299                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1300                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1301
1302                 DELAY(40);
1303         }
1304
1305         val = BCE_MIPHY(phy) | BCE_MIREG(reg) |
1306               BCE_EMAC_MDIO_COMM_COMMAND_READ | BCE_EMAC_MDIO_COMM_DISEXT |
1307               BCE_EMAC_MDIO_COMM_START_BUSY;
1308         REG_WR(sc, BCE_EMAC_MDIO_COMM, val);
1309
1310         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1311                 DELAY(10);
1312
1313                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1314                 if (!(val & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1315                         DELAY(5);
1316
1317                         val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1318                         val &= BCE_EMAC_MDIO_COMM_DATA;
1319                         break;
1320                 }
1321         }
1322
1323         if (val & BCE_EMAC_MDIO_COMM_START_BUSY) {
1324                 if_printf(&sc->arpcom.ac_if,
1325                           "Error: PHY read timeout! phy = %d, reg = 0x%04X\n",
1326                           phy, reg);
1327                 val = 0x0;
1328         } else {
1329                 val = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1330         }
1331
1332         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1333                 val = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1334                 val |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1335
1336                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val);
1337                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1338
1339                 DELAY(40);
1340         }
1341         return (val & 0xffff);
1342 }
1343
1344
1345 /****************************************************************************/
1346 /* PHY register write.                                                      */
1347 /*                                                                          */
1348 /* Implements register writes on the MII bus.                               */
1349 /*                                                                          */
1350 /* Returns:                                                                 */
1351 /*   The value of the register.                                             */
1352 /****************************************************************************/
1353 static int
1354 bce_miibus_write_reg(device_t dev, int phy, int reg, int val)
1355 {
1356         struct bce_softc *sc = device_get_softc(dev);
1357         uint32_t val1;
1358         int i;
1359
1360         /* Make sure we are accessing the correct PHY address. */
1361         KASSERT(phy == sc->bce_phy_addr,
1362             ("invalid phyno %d, should be %d\n", phy, sc->bce_phy_addr));
1363
1364         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1365                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1366                 val1 &= ~BCE_EMAC_MDIO_MODE_AUTO_POLL;
1367
1368                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1369                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1370
1371                 DELAY(40);
1372         }
1373
1374         val1 = BCE_MIPHY(phy) | BCE_MIREG(reg) | val |
1375                 BCE_EMAC_MDIO_COMM_COMMAND_WRITE |
1376                 BCE_EMAC_MDIO_COMM_START_BUSY | BCE_EMAC_MDIO_COMM_DISEXT;
1377         REG_WR(sc, BCE_EMAC_MDIO_COMM, val1);
1378
1379         for (i = 0; i < BCE_PHY_TIMEOUT; i++) {
1380                 DELAY(10);
1381
1382                 val1 = REG_RD(sc, BCE_EMAC_MDIO_COMM);
1383                 if (!(val1 & BCE_EMAC_MDIO_COMM_START_BUSY)) {
1384                         DELAY(5);
1385                         break;
1386                 }
1387         }
1388
1389         if (val1 & BCE_EMAC_MDIO_COMM_START_BUSY)
1390                 if_printf(&sc->arpcom.ac_if, "PHY write timeout!\n");
1391
1392         if (sc->bce_phy_flags & BCE_PHY_INT_MODE_AUTO_POLLING_FLAG) {
1393                 val1 = REG_RD(sc, BCE_EMAC_MDIO_MODE);
1394                 val1 |= BCE_EMAC_MDIO_MODE_AUTO_POLL;
1395
1396                 REG_WR(sc, BCE_EMAC_MDIO_MODE, val1);
1397                 REG_RD(sc, BCE_EMAC_MDIO_MODE);
1398
1399                 DELAY(40);
1400         }
1401         return 0;
1402 }
1403
1404
1405 /****************************************************************************/
1406 /* MII bus status change.                                                   */
1407 /*                                                                          */
1408 /* Called by the MII bus driver when the PHY establishes link to set the    */
1409 /* MAC interface registers.                                                 */
1410 /*                                                                          */
1411 /* Returns:                                                                 */
1412 /*   Nothing.                                                               */
1413 /****************************************************************************/
1414 static void
1415 bce_miibus_statchg(device_t dev)
1416 {
1417         struct bce_softc *sc = device_get_softc(dev);
1418         struct mii_data *mii = device_get_softc(sc->bce_miibus);
1419
1420         BCE_CLRBIT(sc, BCE_EMAC_MODE, BCE_EMAC_MODE_PORT);
1421
1422         /*
1423          * Set MII or GMII interface based on the speed negotiated
1424          * by the PHY.
1425          */
1426         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
1427             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
1428                 BCE_SETBIT(sc, BCE_EMAC_MODE, BCE_EMAC_MODE_PORT_GMII);
1429         } else {
1430                 BCE_SETBIT(sc, BCE_EMAC_MODE, BCE_EMAC_MODE_PORT_MII);
1431         }
1432
1433         /*
1434          * Set half or full duplex based on the duplicity negotiated
1435          * by the PHY.
1436          */
1437         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
1438                 BCE_CLRBIT(sc, BCE_EMAC_MODE, BCE_EMAC_MODE_HALF_DUPLEX);
1439         } else {
1440                 BCE_SETBIT(sc, BCE_EMAC_MODE, BCE_EMAC_MODE_HALF_DUPLEX);
1441         }
1442 }
1443
1444
1445 /****************************************************************************/
1446 /* Acquire NVRAM lock.                                                      */
1447 /*                                                                          */
1448 /* Before the NVRAM can be accessed the caller must acquire an NVRAM lock.  */
1449 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1450 /* for use by the driver.                                                   */
1451 /*                                                                          */
1452 /* Returns:                                                                 */
1453 /*   0 on success, positive value on failure.                               */
1454 /****************************************************************************/
1455 static int
1456 bce_acquire_nvram_lock(struct bce_softc *sc)
1457 {
1458         uint32_t val;
1459         int j;
1460
1461         /* Request access to the flash interface. */
1462         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_SET2);
1463         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1464                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1465                 if (val & BCE_NVM_SW_ARB_ARB_ARB2)
1466                         break;
1467
1468                 DELAY(5);
1469         }
1470
1471         if (j >= NVRAM_TIMEOUT_COUNT) {
1472                 return EBUSY;
1473         }
1474         return 0;
1475 }
1476
1477
1478 /****************************************************************************/
1479 /* Release NVRAM lock.                                                      */
1480 /*                                                                          */
1481 /* When the caller is finished accessing NVRAM the lock must be released.   */
1482 /* Locks 0 and 2 are reserved, lock 1 is used by firmware and lock 2 is     */
1483 /* for use by the driver.                                                   */
1484 /*                                                                          */
1485 /* Returns:                                                                 */
1486 /*   0 on success, positive value on failure.                               */
1487 /****************************************************************************/
1488 static int
1489 bce_release_nvram_lock(struct bce_softc *sc)
1490 {
1491         int j;
1492         uint32_t val;
1493
1494         /*
1495          * Relinquish nvram interface.
1496          */
1497         REG_WR(sc, BCE_NVM_SW_ARB, BCE_NVM_SW_ARB_ARB_REQ_CLR2);
1498
1499         for (j = 0; j < NVRAM_TIMEOUT_COUNT; j++) {
1500                 val = REG_RD(sc, BCE_NVM_SW_ARB);
1501                 if (!(val & BCE_NVM_SW_ARB_ARB_ARB2))
1502                         break;
1503
1504                 DELAY(5);
1505         }
1506
1507         if (j >= NVRAM_TIMEOUT_COUNT) {
1508                 return EBUSY;
1509         }
1510         return 0;
1511 }
1512
1513
1514 /****************************************************************************/
1515 /* Enable NVRAM access.                                                     */
1516 /*                                                                          */
1517 /* Before accessing NVRAM for read or write operations the caller must      */
1518 /* enabled NVRAM access.                                                    */
1519 /*                                                                          */
1520 /* Returns:                                                                 */
1521 /*   Nothing.                                                               */
1522 /****************************************************************************/
1523 static void
1524 bce_enable_nvram_access(struct bce_softc *sc)
1525 {
1526         uint32_t val;
1527
1528         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1529         /* Enable both bits, even on read. */
1530         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1531                val | BCE_NVM_ACCESS_ENABLE_EN | BCE_NVM_ACCESS_ENABLE_WR_EN);
1532 }
1533
1534
1535 /****************************************************************************/
1536 /* Disable NVRAM access.                                                    */
1537 /*                                                                          */
1538 /* When the caller is finished accessing NVRAM access must be disabled.     */
1539 /*                                                                          */
1540 /* Returns:                                                                 */
1541 /*   Nothing.                                                               */
1542 /****************************************************************************/
1543 static void
1544 bce_disable_nvram_access(struct bce_softc *sc)
1545 {
1546         uint32_t val;
1547
1548         val = REG_RD(sc, BCE_NVM_ACCESS_ENABLE);
1549
1550         /* Disable both bits, even after read. */
1551         REG_WR(sc, BCE_NVM_ACCESS_ENABLE,
1552                val & ~(BCE_NVM_ACCESS_ENABLE_EN | BCE_NVM_ACCESS_ENABLE_WR_EN));
1553 }
1554
1555
1556 /****************************************************************************/
1557 /* Read a dword (32 bits) from NVRAM.                                       */
1558 /*                                                                          */
1559 /* Read a 32 bit word from NVRAM.  The caller is assumed to have already    */
1560 /* obtained the NVRAM lock and enabled the controller for NVRAM access.     */
1561 /*                                                                          */
1562 /* Returns:                                                                 */
1563 /*   0 on success and the 32 bit value read, positive value on failure.     */
1564 /****************************************************************************/
1565 static int
1566 bce_nvram_read_dword(struct bce_softc *sc, uint32_t offset, uint8_t *ret_val,
1567                      uint32_t cmd_flags)
1568 {
1569         uint32_t cmd;
1570         int i, rc = 0;
1571
1572         /* Build the command word. */
1573         cmd = BCE_NVM_COMMAND_DOIT | cmd_flags;
1574
1575         /* Calculate the offset for buffered flash. */
1576         if (sc->bce_flash_info->flags & BCE_NV_TRANSLATE) {
1577                 offset = ((offset / sc->bce_flash_info->page_size) <<
1578                           sc->bce_flash_info->page_bits) +
1579                          (offset % sc->bce_flash_info->page_size);
1580         }
1581
1582         /*
1583          * Clear the DONE bit separately, set the address to read,
1584          * and issue the read.
1585          */
1586         REG_WR(sc, BCE_NVM_COMMAND, BCE_NVM_COMMAND_DONE);
1587         REG_WR(sc, BCE_NVM_ADDR, offset & BCE_NVM_ADDR_NVM_ADDR_VALUE);
1588         REG_WR(sc, BCE_NVM_COMMAND, cmd);
1589
1590         /* Wait for completion. */
1591         for (i = 0; i < NVRAM_TIMEOUT_COUNT; i++) {
1592                 uint32_t val;
1593
1594                 DELAY(5);
1595
1596                 val = REG_RD(sc, BCE_NVM_COMMAND);
1597                 if (val & BCE_NVM_COMMAND_DONE) {
1598                         val = REG_RD(sc, BCE_NVM_READ);
1599
1600                         val = be32toh(val);
1601                         memcpy(ret_val, &val, 4);
1602                         break;
1603                 }
1604         }
1605
1606         /* Check for errors. */
1607         if (i >= NVRAM_TIMEOUT_COUNT) {
1608                 if_printf(&sc->arpcom.ac_if,
1609                           "Timeout error reading NVRAM at offset 0x%08X!\n",
1610                           offset);
1611                 rc = EBUSY;
1612         }
1613         return rc;
1614 }
1615
1616
1617 /****************************************************************************/
1618 /* Initialize NVRAM access.                                                 */
1619 /*                                                                          */
1620 /* Identify the NVRAM device in use and prepare the NVRAM interface to      */
1621 /* access that device.                                                      */
1622 /*                                                                          */
1623 /* Returns:                                                                 */
1624 /*   0 on success, positive value on failure.                               */
1625 /****************************************************************************/
1626 static int
1627 bce_init_nvram(struct bce_softc *sc)
1628 {
1629         uint32_t val;
1630         int j, entry_count, rc = 0;
1631         const struct flash_spec *flash;
1632
1633         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
1634             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
1635                 sc->bce_flash_info = &flash_5709;
1636                 goto bce_init_nvram_get_flash_size;
1637         }
1638
1639         /* Determine the selected interface. */
1640         val = REG_RD(sc, BCE_NVM_CFG1);
1641
1642         entry_count = sizeof(flash_table) / sizeof(struct flash_spec);
1643
1644         /*
1645          * Flash reconfiguration is required to support additional
1646          * NVRAM devices not directly supported in hardware.
1647          * Check if the flash interface was reconfigured
1648          * by the bootcode.
1649          */
1650
1651         if (val & 0x40000000) {
1652                 /* Flash interface reconfigured by bootcode. */
1653                 for (j = 0, flash = flash_table; j < entry_count;
1654                      j++, flash++) {
1655                         if ((val & FLASH_BACKUP_STRAP_MASK) ==
1656                             (flash->config1 & FLASH_BACKUP_STRAP_MASK)) {
1657                                 sc->bce_flash_info = flash;
1658                                 break;
1659                         }
1660                 }
1661         } else {
1662                 /* Flash interface not yet reconfigured. */
1663                 uint32_t mask;
1664
1665                 if (val & (1 << 23))
1666                         mask = FLASH_BACKUP_STRAP_MASK;
1667                 else
1668                         mask = FLASH_STRAP_MASK;
1669
1670                 /* Look for the matching NVRAM device configuration data. */
1671                 for (j = 0, flash = flash_table; j < entry_count;
1672                      j++, flash++) {
1673                         /* Check if the device matches any of the known devices. */
1674                         if ((val & mask) == (flash->strapping & mask)) {
1675                                 /* Found a device match. */
1676                                 sc->bce_flash_info = flash;
1677
1678                                 /* Request access to the flash interface. */
1679                                 rc = bce_acquire_nvram_lock(sc);
1680                                 if (rc != 0)
1681                                         return rc;
1682
1683                                 /* Reconfigure the flash interface. */
1684                                 bce_enable_nvram_access(sc);
1685                                 REG_WR(sc, BCE_NVM_CFG1, flash->config1);
1686                                 REG_WR(sc, BCE_NVM_CFG2, flash->config2);
1687                                 REG_WR(sc, BCE_NVM_CFG3, flash->config3);
1688                                 REG_WR(sc, BCE_NVM_WRITE1, flash->write1);
1689                                 bce_disable_nvram_access(sc);
1690                                 bce_release_nvram_lock(sc);
1691                                 break;
1692                         }
1693                 }
1694         }
1695
1696         /* Check if a matching device was found. */
1697         if (j == entry_count) {
1698                 sc->bce_flash_info = NULL;
1699                 if_printf(&sc->arpcom.ac_if, "Unknown Flash NVRAM found!\n");
1700                 return ENODEV;
1701         }
1702
1703 bce_init_nvram_get_flash_size:
1704         /* Write the flash config data to the shared memory interface. */
1705         val = bce_shmem_rd(sc, BCE_SHARED_HW_CFG_CONFIG2) &
1706             BCE_SHARED_HW_CFG2_NVM_SIZE_MASK;
1707         if (val)
1708                 sc->bce_flash_size = val;
1709         else
1710                 sc->bce_flash_size = sc->bce_flash_info->total_size;
1711
1712         return rc;
1713 }
1714
1715
1716 /****************************************************************************/
1717 /* Read an arbitrary range of data from NVRAM.                              */
1718 /*                                                                          */
1719 /* Prepares the NVRAM interface for access and reads the requested data     */
1720 /* into the supplied buffer.                                                */
1721 /*                                                                          */
1722 /* Returns:                                                                 */
1723 /*   0 on success and the data read, positive value on failure.             */
1724 /****************************************************************************/
1725 static int
1726 bce_nvram_read(struct bce_softc *sc, uint32_t offset, uint8_t *ret_buf,
1727                int buf_size)
1728 {
1729         uint32_t cmd_flags, offset32, len32, extra;
1730         int rc = 0;
1731
1732         if (buf_size == 0)
1733                 return 0;
1734
1735         /* Request access to the flash interface. */
1736         rc = bce_acquire_nvram_lock(sc);
1737         if (rc != 0)
1738                 return rc;
1739
1740         /* Enable access to flash interface */
1741         bce_enable_nvram_access(sc);
1742
1743         len32 = buf_size;
1744         offset32 = offset;
1745         extra = 0;
1746
1747         cmd_flags = 0;
1748
1749         /* XXX should we release nvram lock if read_dword() fails? */
1750         if (offset32 & 3) {
1751                 uint8_t buf[4];
1752                 uint32_t pre_len;
1753
1754                 offset32 &= ~3;
1755                 pre_len = 4 - (offset & 3);
1756
1757                 if (pre_len >= len32) {
1758                         pre_len = len32;
1759                         cmd_flags = BCE_NVM_COMMAND_FIRST | BCE_NVM_COMMAND_LAST;
1760                 } else {
1761                         cmd_flags = BCE_NVM_COMMAND_FIRST;
1762                 }
1763
1764                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
1765                 if (rc)
1766                         return rc;
1767
1768                 memcpy(ret_buf, buf + (offset & 3), pre_len);
1769
1770                 offset32 += 4;
1771                 ret_buf += pre_len;
1772                 len32 -= pre_len;
1773         }
1774
1775         if (len32 & 3) {
1776                 extra = 4 - (len32 & 3);
1777                 len32 = (len32 + 4) & ~3;
1778         }
1779
1780         if (len32 == 4) {
1781                 uint8_t buf[4];
1782
1783                 if (cmd_flags)
1784                         cmd_flags = BCE_NVM_COMMAND_LAST;
1785                 else
1786                         cmd_flags = BCE_NVM_COMMAND_FIRST |
1787                                     BCE_NVM_COMMAND_LAST;
1788
1789                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
1790
1791                 memcpy(ret_buf, buf, 4 - extra);
1792         } else if (len32 > 0) {
1793                 uint8_t buf[4];
1794
1795                 /* Read the first word. */
1796                 if (cmd_flags)
1797                         cmd_flags = 0;
1798                 else
1799                         cmd_flags = BCE_NVM_COMMAND_FIRST;
1800
1801                 rc = bce_nvram_read_dword(sc, offset32, ret_buf, cmd_flags);
1802
1803                 /* Advance to the next dword. */
1804                 offset32 += 4;
1805                 ret_buf += 4;
1806                 len32 -= 4;
1807
1808                 while (len32 > 4 && rc == 0) {
1809                         rc = bce_nvram_read_dword(sc, offset32, ret_buf, 0);
1810
1811                         /* Advance to the next dword. */
1812                         offset32 += 4;
1813                         ret_buf += 4;
1814                         len32 -= 4;
1815                 }
1816
1817                 if (rc)
1818                         goto bce_nvram_read_locked_exit;
1819
1820                 cmd_flags = BCE_NVM_COMMAND_LAST;
1821                 rc = bce_nvram_read_dword(sc, offset32, buf, cmd_flags);
1822
1823                 memcpy(ret_buf, buf, 4 - extra);
1824         }
1825
1826 bce_nvram_read_locked_exit:
1827         /* Disable access to flash interface and release the lock. */
1828         bce_disable_nvram_access(sc);
1829         bce_release_nvram_lock(sc);
1830
1831         return rc;
1832 }
1833
1834
1835 /****************************************************************************/
1836 /* Verifies that NVRAM is accessible and contains valid data.               */
1837 /*                                                                          */
1838 /* Reads the configuration data from NVRAM and verifies that the CRC is     */
1839 /* correct.                                                                 */
1840 /*                                                                          */
1841 /* Returns:                                                                 */
1842 /*   0 on success, positive value on failure.                               */
1843 /****************************************************************************/
1844 static int
1845 bce_nvram_test(struct bce_softc *sc)
1846 {
1847         uint32_t buf[BCE_NVRAM_SIZE / 4];
1848         uint32_t magic, csum;
1849         uint8_t *data = (uint8_t *)buf;
1850         int rc = 0;
1851
1852         /*
1853          * Check that the device NVRAM is valid by reading
1854          * the magic value at offset 0.
1855          */
1856         rc = bce_nvram_read(sc, 0, data, 4);
1857         if (rc != 0)
1858                 return rc;
1859
1860         magic = be32toh(buf[0]);
1861         if (magic != BCE_NVRAM_MAGIC) {
1862                 if_printf(&sc->arpcom.ac_if,
1863                           "Invalid NVRAM magic value! Expected: 0x%08X, "
1864                           "Found: 0x%08X\n", BCE_NVRAM_MAGIC, magic);
1865                 return ENODEV;
1866         }
1867
1868         /*
1869          * Verify that the device NVRAM includes valid
1870          * configuration data.
1871          */
1872         rc = bce_nvram_read(sc, 0x100, data, BCE_NVRAM_SIZE);
1873         if (rc != 0)
1874                 return rc;
1875
1876         csum = ether_crc32_le(data, 0x100);
1877         if (csum != BCE_CRC32_RESIDUAL) {
1878                 if_printf(&sc->arpcom.ac_if,
1879                           "Invalid Manufacturing Information NVRAM CRC! "
1880                           "Expected: 0x%08X, Found: 0x%08X\n",
1881                           BCE_CRC32_RESIDUAL, csum);
1882                 return ENODEV;
1883         }
1884
1885         csum = ether_crc32_le(data + 0x100, 0x100);
1886         if (csum != BCE_CRC32_RESIDUAL) {
1887                 if_printf(&sc->arpcom.ac_if,
1888                           "Invalid Feature Configuration Information "
1889                           "NVRAM CRC! Expected: 0x%08X, Found: 08%08X\n",
1890                           BCE_CRC32_RESIDUAL, csum);
1891                 rc = ENODEV;
1892         }
1893         return rc;
1894 }
1895
1896
1897 /****************************************************************************/
1898 /* Identifies the current media type of the controller and sets the PHY     */
1899 /* address.                                                                 */
1900 /*                                                                          */
1901 /* Returns:                                                                 */
1902 /*   Nothing.                                                               */
1903 /****************************************************************************/
1904 static void
1905 bce_get_media(struct bce_softc *sc)
1906 {
1907         uint32_t val;
1908
1909         sc->bce_phy_addr = 1;
1910
1911         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
1912             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
1913                 uint32_t val = REG_RD(sc, BCE_MISC_DUAL_MEDIA_CTRL);
1914                 uint32_t bond_id = val & BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID;
1915                 uint32_t strap;
1916
1917                 /*
1918                  * The BCM5709S is software configurable
1919                  * for Copper or SerDes operation.
1920                  */
1921                 if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_C) {
1922                         return;
1923                 } else if (bond_id == BCE_MISC_DUAL_MEDIA_CTRL_BOND_ID_S) {
1924                         sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
1925                         return;
1926                 }
1927
1928                 if (val & BCE_MISC_DUAL_MEDIA_CTRL_STRAP_OVERRIDE) {
1929                         strap = (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL) >> 21;
1930                 } else {
1931                         strap =
1932                         (val & BCE_MISC_DUAL_MEDIA_CTRL_PHY_CTRL_STRAP) >> 8;
1933                 }
1934
1935                 if (pci_get_function(sc->bce_dev) == 0) {
1936                         switch (strap) {
1937                         case 0x4:
1938                         case 0x5:
1939                         case 0x6:
1940                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
1941                                 break;
1942                         }
1943                 } else {
1944                         switch (strap) {
1945                         case 0x1:
1946                         case 0x2:
1947                         case 0x4:
1948                                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
1949                                 break;
1950                         }
1951                 }
1952         } else if (BCE_CHIP_BOND_ID(sc) & BCE_CHIP_BOND_ID_SERDES_BIT) {
1953                 sc->bce_phy_flags |= BCE_PHY_SERDES_FLAG;
1954         }
1955
1956         if (sc->bce_phy_flags & BCE_PHY_SERDES_FLAG) {
1957                 sc->bce_flags |= BCE_NO_WOL_FLAG;
1958                 if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5706) {
1959                         sc->bce_phy_addr = 2;
1960                         val = bce_shmem_rd(sc, BCE_SHARED_HW_CFG_CONFIG);
1961                         if (val & BCE_SHARED_HW_CFG_PHY_2_5G)
1962                                 sc->bce_phy_flags |= BCE_PHY_2_5G_CAPABLE_FLAG;
1963                 }
1964         } else if ((BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) ||
1965             (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708)) {
1966                 sc->bce_phy_flags |= BCE_PHY_CRC_FIX_FLAG;
1967         }
1968 }
1969
1970
1971 static void
1972 bce_destroy_tx_ring(struct bce_tx_ring *txr)
1973 {
1974         int i;
1975
1976         /* Destroy the TX buffer descriptor DMA stuffs. */
1977         if (txr->tx_bd_chain_tag != NULL) {
1978                 for (i = 0; i < txr->tx_pages; i++) {
1979                         if (txr->tx_bd_chain[i] != NULL) {
1980                                 bus_dmamap_unload(txr->tx_bd_chain_tag,
1981                                     txr->tx_bd_chain_map[i]);
1982                                 bus_dmamem_free(txr->tx_bd_chain_tag,
1983                                     txr->tx_bd_chain[i],
1984                                     txr->tx_bd_chain_map[i]);
1985                         }
1986                 }
1987                 bus_dma_tag_destroy(txr->tx_bd_chain_tag);
1988         }
1989
1990         /* Destroy the TX mbuf DMA stuffs. */
1991         if (txr->tx_mbuf_tag != NULL) {
1992                 for (i = 0; i < TOTAL_TX_BD(txr); i++) {
1993                         /* Must have been unloaded in bce_stop() */
1994                         KKASSERT(txr->tx_mbuf_ptr[i] == NULL);
1995                         bus_dmamap_destroy(txr->tx_mbuf_tag,
1996                             txr->tx_mbuf_map[i]);
1997                 }
1998                 bus_dma_tag_destroy(txr->tx_mbuf_tag);
1999         }
2000
2001         if (txr->tx_bd_chain_map != NULL)
2002                 kfree(txr->tx_bd_chain_map, M_DEVBUF);
2003         if (txr->tx_bd_chain != NULL)
2004                 kfree(txr->tx_bd_chain, M_DEVBUF);
2005         if (txr->tx_bd_chain_paddr != NULL)
2006                 kfree(txr->tx_bd_chain_paddr, M_DEVBUF);
2007
2008         if (txr->tx_mbuf_map != NULL)
2009                 kfree(txr->tx_mbuf_map, M_DEVBUF);
2010         if (txr->tx_mbuf_ptr != NULL)
2011                 kfree(txr->tx_mbuf_ptr, M_DEVBUF);
2012 }
2013
2014
2015 static void
2016 bce_destroy_rx_ring(struct bce_rx_ring *rxr)
2017 {
2018         int i;
2019
2020         /* Destroy the RX buffer descriptor DMA stuffs. */
2021         if (rxr->rx_bd_chain_tag != NULL) {
2022                 for (i = 0; i < rxr->rx_pages; i++) {
2023                         if (rxr->rx_bd_chain[i] != NULL) {
2024                                 bus_dmamap_unload(rxr->rx_bd_chain_tag,
2025                                     rxr->rx_bd_chain_map[i]);
2026                                 bus_dmamem_free(rxr->rx_bd_chain_tag,
2027                                     rxr->rx_bd_chain[i],
2028                                     rxr->rx_bd_chain_map[i]);
2029                         }
2030                 }
2031                 bus_dma_tag_destroy(rxr->rx_bd_chain_tag);
2032         }
2033
2034         /* Destroy the RX mbuf DMA stuffs. */
2035         if (rxr->rx_mbuf_tag != NULL) {
2036                 for (i = 0; i < TOTAL_RX_BD(rxr); i++) {
2037                         /* Must have been unloaded in bce_stop() */
2038                         KKASSERT(rxr->rx_mbuf_ptr[i] == NULL);
2039                         bus_dmamap_destroy(rxr->rx_mbuf_tag,
2040                             rxr->rx_mbuf_map[i]);
2041                 }
2042                 bus_dmamap_destroy(rxr->rx_mbuf_tag, rxr->rx_mbuf_tmpmap);
2043                 bus_dma_tag_destroy(rxr->rx_mbuf_tag);
2044         }
2045
2046         if (rxr->rx_bd_chain_map != NULL)
2047                 kfree(rxr->rx_bd_chain_map, M_DEVBUF);
2048         if (rxr->rx_bd_chain != NULL)
2049                 kfree(rxr->rx_bd_chain, M_DEVBUF);
2050         if (rxr->rx_bd_chain_paddr != NULL)
2051                 kfree(rxr->rx_bd_chain_paddr, M_DEVBUF);
2052
2053         if (rxr->rx_mbuf_map != NULL)
2054                 kfree(rxr->rx_mbuf_map, M_DEVBUF);
2055         if (rxr->rx_mbuf_ptr != NULL)
2056                 kfree(rxr->rx_mbuf_ptr, M_DEVBUF);
2057         if (rxr->rx_mbuf_paddr != NULL)
2058                 kfree(rxr->rx_mbuf_paddr, M_DEVBUF);
2059 }
2060
2061
2062 /****************************************************************************/
2063 /* Free any DMA memory owned by the driver.                                 */
2064 /*                                                                          */
2065 /* Scans through each data structre that requires DMA memory and frees      */
2066 /* the memory if allocated.                                                 */
2067 /*                                                                          */
2068 /* Returns:                                                                 */
2069 /*   Nothing.                                                               */
2070 /****************************************************************************/
2071 static void
2072 bce_dma_free(struct bce_softc *sc)
2073 {
2074         int i;
2075
2076         /* Destroy the status block. */
2077         if (sc->status_tag != NULL) {
2078                 if (sc->status_block != NULL) {
2079                         bus_dmamap_unload(sc->status_tag, sc->status_map);
2080                         bus_dmamem_free(sc->status_tag, sc->status_block,
2081                                         sc->status_map);
2082                 }
2083                 bus_dma_tag_destroy(sc->status_tag);
2084         }
2085
2086         /* Destroy the statistics block. */
2087         if (sc->stats_tag != NULL) {
2088                 if (sc->stats_block != NULL) {
2089                         bus_dmamap_unload(sc->stats_tag, sc->stats_map);
2090                         bus_dmamem_free(sc->stats_tag, sc->stats_block,
2091                                         sc->stats_map);
2092                 }
2093                 bus_dma_tag_destroy(sc->stats_tag);
2094         }
2095
2096         /* Destroy the CTX DMA stuffs. */
2097         if (sc->ctx_tag != NULL) {
2098                 for (i = 0; i < sc->ctx_pages; i++) {
2099                         if (sc->ctx_block[i] != NULL) {
2100                                 bus_dmamap_unload(sc->ctx_tag, sc->ctx_map[i]);
2101                                 bus_dmamem_free(sc->ctx_tag, sc->ctx_block[i],
2102                                                 sc->ctx_map[i]);
2103                         }
2104                 }
2105                 bus_dma_tag_destroy(sc->ctx_tag);
2106         }
2107
2108         /* Free TX rings */
2109         if (sc->tx_rings != NULL) {
2110                 for (i = 0; i < sc->tx_ring_cnt; ++i)
2111                         bce_destroy_tx_ring(&sc->tx_rings[i]);
2112                 kfree(sc->tx_rings, M_DEVBUF);
2113         }
2114
2115         /* Free RX rings */
2116         if (sc->rx_rings != NULL) {
2117                 for (i = 0; i < sc->rx_ring_cnt; ++i)
2118                         bce_destroy_rx_ring(&sc->rx_rings[i]);
2119                 kfree(sc->rx_rings, M_DEVBUF);
2120         }
2121
2122         /* Destroy the parent tag */
2123         if (sc->parent_tag != NULL)
2124                 bus_dma_tag_destroy(sc->parent_tag);
2125 }
2126
2127
2128 /****************************************************************************/
2129 /* Get DMA memory from the OS.                                              */
2130 /*                                                                          */
2131 /* Validates that the OS has provided DMA buffers in response to a          */
2132 /* bus_dmamap_load() call and saves the physical address of those buffers.  */
2133 /* When the callback is used the OS will return 0 for the mapping function  */
2134 /* (bus_dmamap_load()) so we use the value of map_arg->maxsegs to pass any  */
2135 /* failures back to the caller.                                             */
2136 /*                                                                          */
2137 /* Returns:                                                                 */
2138 /*   Nothing.                                                               */
2139 /****************************************************************************/
2140 static void
2141 bce_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2142 {
2143         bus_addr_t *busaddr = arg;
2144
2145         /* Check for an error and signal the caller that an error occurred. */
2146         if (error)
2147                 return;
2148
2149         KASSERT(nseg == 1, ("only one segment is allowed"));
2150         *busaddr = segs->ds_addr;
2151 }
2152
2153
2154 static int
2155 bce_create_tx_ring(struct bce_tx_ring *txr)
2156 {
2157         int pages, rc, i;
2158
2159         lwkt_serialize_init(&txr->tx_serialize);
2160         txr->tx_wreg = bce_tx_wreg;
2161
2162         pages = device_getenv_int(txr->sc->bce_dev, "tx_pages", bce_tx_pages);
2163         if (pages <= 0 || pages > TX_PAGES_MAX || !powerof2(pages)) {
2164                 device_printf(txr->sc->bce_dev, "invalid # of TX pages\n");
2165                 pages = TX_PAGES_DEFAULT;
2166         }
2167         txr->tx_pages = pages;
2168
2169         txr->tx_bd_chain_map = kmalloc(sizeof(bus_dmamap_t) * txr->tx_pages,
2170             M_DEVBUF, M_WAITOK | M_ZERO);
2171         txr->tx_bd_chain = kmalloc(sizeof(struct tx_bd *) * txr->tx_pages,
2172             M_DEVBUF, M_WAITOK | M_ZERO);
2173         txr->tx_bd_chain_paddr = kmalloc(sizeof(bus_addr_t) * txr->tx_pages,
2174             M_DEVBUF, M_WAITOK | M_ZERO);
2175
2176         txr->tx_mbuf_map = kmalloc(sizeof(bus_dmamap_t) * TOTAL_TX_BD(txr),
2177             M_DEVBUF, M_WAITOK | M_ZERO);
2178         txr->tx_mbuf_ptr = kmalloc(sizeof(struct mbuf *) * TOTAL_TX_BD(txr),
2179             M_DEVBUF, M_WAITOK | M_ZERO);
2180
2181         /*
2182          * Create a DMA tag for the TX buffer descriptor chain,
2183          * allocate and clear the  memory, and fetch the
2184          * physical address of the block.
2185          */
2186         rc = bus_dma_tag_create(txr->sc->parent_tag, BCM_PAGE_SIZE, 0,
2187             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
2188             BCE_TX_CHAIN_PAGE_SZ, 1, BCE_TX_CHAIN_PAGE_SZ,
2189             0, &txr->tx_bd_chain_tag);
2190         if (rc != 0) {
2191                 device_printf(txr->sc->bce_dev, "Could not allocate "
2192                     "TX descriptor chain DMA tag!\n");
2193                 return rc;
2194         }
2195
2196         for (i = 0; i < txr->tx_pages; i++) {
2197                 bus_addr_t busaddr;
2198
2199                 rc = bus_dmamem_alloc(txr->tx_bd_chain_tag,
2200                     (void **)&txr->tx_bd_chain[i],
2201                     BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
2202                     &txr->tx_bd_chain_map[i]);
2203                 if (rc != 0) {
2204                         device_printf(txr->sc->bce_dev,
2205                             "Could not allocate %dth TX descriptor "
2206                             "chain DMA memory!\n", i);
2207                         return rc;
2208                 }
2209
2210                 rc = bus_dmamap_load(txr->tx_bd_chain_tag,
2211                     txr->tx_bd_chain_map[i],
2212                     txr->tx_bd_chain[i],
2213                     BCE_TX_CHAIN_PAGE_SZ,
2214                     bce_dma_map_addr, &busaddr,
2215                     BUS_DMA_WAITOK);
2216                 if (rc != 0) {
2217                         if (rc == EINPROGRESS) {
2218                                 panic("%s coherent memory loading "
2219                                     "is still in progress!",
2220                                     txr->sc->arpcom.ac_if.if_xname);
2221                         }
2222                         device_printf(txr->sc->bce_dev, "Could not map %dth "
2223                             "TX descriptor chain DMA memory!\n", i);
2224                         bus_dmamem_free(txr->tx_bd_chain_tag,
2225                             txr->tx_bd_chain[i],
2226                             txr->tx_bd_chain_map[i]);
2227                         txr->tx_bd_chain[i] = NULL;
2228                         return rc;
2229                 }
2230
2231                 txr->tx_bd_chain_paddr[i] = busaddr;
2232         }
2233
2234         /* Create a DMA tag for TX mbufs. */
2235         rc = bus_dma_tag_create(txr->sc->parent_tag, 1, 0,
2236             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
2237             IP_MAXPACKET + sizeof(struct ether_vlan_header),
2238             BCE_MAX_SEGMENTS, PAGE_SIZE,
2239             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
2240             &txr->tx_mbuf_tag);
2241         if (rc != 0) {
2242                 device_printf(txr->sc->bce_dev,
2243                     "Could not allocate TX mbuf DMA tag!\n");
2244                 return rc;
2245         }
2246
2247         /* Create DMA maps for the TX mbufs clusters. */
2248         for (i = 0; i < TOTAL_TX_BD(txr); i++) {
2249                 rc = bus_dmamap_create(txr->tx_mbuf_tag,
2250                     BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
2251                     &txr->tx_mbuf_map[i]);
2252                 if (rc != 0) {
2253                         int j;
2254
2255                         for (j = 0; j < i; ++j) {
2256                                 bus_dmamap_destroy(txr->tx_mbuf_tag,
2257                                     txr->tx_mbuf_map[i]);
2258                         }
2259                         bus_dma_tag_destroy(txr->tx_mbuf_tag);
2260                         txr->tx_mbuf_tag = NULL;
2261
2262                         device_printf(txr->sc->bce_dev, "Unable to create "
2263                             "%dth TX mbuf DMA map!\n", i);
2264                         return rc;
2265                 }
2266         }
2267         return 0;
2268 }
2269
2270
2271 static int
2272 bce_create_rx_ring(struct bce_rx_ring *rxr)
2273 {
2274         int pages, rc, i;
2275
2276         lwkt_serialize_init(&rxr->rx_serialize);
2277
2278         pages = device_getenv_int(rxr->sc->bce_dev, "rx_pages", bce_rx_pages);
2279         if (pages <= 0 || pages > RX_PAGES_MAX || !powerof2(pages)) {
2280                 device_printf(rxr->sc->bce_dev, "invalid # of RX pages\n");
2281                 pages = RX_PAGES_DEFAULT;
2282         }
2283         rxr->rx_pages = pages;
2284
2285         rxr->rx_bd_chain_map = kmalloc(sizeof(bus_dmamap_t) * rxr->rx_pages,
2286             M_DEVBUF, M_WAITOK | M_ZERO);
2287         rxr->rx_bd_chain = kmalloc(sizeof(struct rx_bd *) * rxr->rx_pages,
2288             M_DEVBUF, M_WAITOK | M_ZERO);
2289         rxr->rx_bd_chain_paddr = kmalloc(sizeof(bus_addr_t) * rxr->rx_pages,
2290             M_DEVBUF, M_WAITOK | M_ZERO);
2291
2292         rxr->rx_mbuf_map = kmalloc(sizeof(bus_dmamap_t) * TOTAL_RX_BD(rxr),
2293             M_DEVBUF, M_WAITOK | M_ZERO);
2294         rxr->rx_mbuf_ptr = kmalloc(sizeof(struct mbuf *) * TOTAL_RX_BD(rxr),
2295             M_DEVBUF, M_WAITOK | M_ZERO);
2296         rxr->rx_mbuf_paddr = kmalloc(sizeof(bus_addr_t) * TOTAL_RX_BD(rxr),
2297             M_DEVBUF, M_WAITOK | M_ZERO);
2298
2299         /*
2300          * Create a DMA tag for the RX buffer descriptor chain,
2301          * allocate and clear the  memory, and fetch the physical
2302          * address of the blocks.
2303          */
2304         rc = bus_dma_tag_create(rxr->sc->parent_tag, BCM_PAGE_SIZE, 0,
2305             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
2306             BCE_RX_CHAIN_PAGE_SZ, 1, BCE_RX_CHAIN_PAGE_SZ,
2307             0, &rxr->rx_bd_chain_tag);
2308         if (rc != 0) {
2309                 device_printf(rxr->sc->bce_dev, "Could not allocate "
2310                     "RX descriptor chain DMA tag!\n");
2311                 return rc;
2312         }
2313
2314         for (i = 0; i < rxr->rx_pages; i++) {
2315                 bus_addr_t busaddr;
2316
2317                 rc = bus_dmamem_alloc(rxr->rx_bd_chain_tag,
2318                     (void **)&rxr->rx_bd_chain[i],
2319                     BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
2320                     &rxr->rx_bd_chain_map[i]);
2321                 if (rc != 0) {
2322                         device_printf(rxr->sc->bce_dev,
2323                             "Could not allocate %dth RX descriptor "
2324                             "chain DMA memory!\n", i);
2325                         return rc;
2326                 }
2327
2328                 rc = bus_dmamap_load(rxr->rx_bd_chain_tag,
2329                     rxr->rx_bd_chain_map[i],
2330                     rxr->rx_bd_chain[i],
2331                     BCE_RX_CHAIN_PAGE_SZ,
2332                     bce_dma_map_addr, &busaddr,
2333                     BUS_DMA_WAITOK);
2334                 if (rc != 0) {
2335                         if (rc == EINPROGRESS) {
2336                                 panic("%s coherent memory loading "
2337                                     "is still in progress!",
2338                                     rxr->sc->arpcom.ac_if.if_xname);
2339                         }
2340                         device_printf(rxr->sc->bce_dev,
2341                             "Could not map %dth RX descriptor "
2342                             "chain DMA memory!\n", i);
2343                         bus_dmamem_free(rxr->rx_bd_chain_tag,
2344                             rxr->rx_bd_chain[i],
2345                             rxr->rx_bd_chain_map[i]);
2346                         rxr->rx_bd_chain[i] = NULL;
2347                         return rc;
2348                 }
2349
2350                 rxr->rx_bd_chain_paddr[i] = busaddr;
2351         }
2352
2353         /* Create a DMA tag for RX mbufs. */
2354         rc = bus_dma_tag_create(rxr->sc->parent_tag, BCE_DMA_RX_ALIGN, 0,
2355             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
2356             MCLBYTES, 1, MCLBYTES,
2357             BUS_DMA_ALLOCNOW | BUS_DMA_ALIGNED | BUS_DMA_WAITOK,
2358             &rxr->rx_mbuf_tag);
2359         if (rc != 0) {
2360                 device_printf(rxr->sc->bce_dev,
2361                     "Could not allocate RX mbuf DMA tag!\n");
2362                 return rc;
2363         }
2364
2365         /* Create tmp DMA map for RX mbuf clusters. */
2366         rc = bus_dmamap_create(rxr->rx_mbuf_tag, BUS_DMA_WAITOK,
2367             &rxr->rx_mbuf_tmpmap);
2368         if (rc != 0) {
2369                 bus_dma_tag_destroy(rxr->rx_mbuf_tag);
2370                 rxr->rx_mbuf_tag = NULL;
2371
2372                 device_printf(rxr->sc->bce_dev,
2373                     "Could not create RX mbuf tmp DMA map!\n");
2374                 return rc;
2375         }
2376
2377         /* Create DMA maps for the RX mbuf clusters. */
2378         for (i = 0; i < TOTAL_RX_BD(rxr); i++) {
2379                 rc = bus_dmamap_create(rxr->rx_mbuf_tag, BUS_DMA_WAITOK,
2380                     &rxr->rx_mbuf_map[i]);
2381                 if (rc != 0) {
2382                         int j;
2383
2384                         for (j = 0; j < i; ++j) {
2385                                 bus_dmamap_destroy(rxr->rx_mbuf_tag,
2386                                     rxr->rx_mbuf_map[j]);
2387                         }
2388                         bus_dma_tag_destroy(rxr->rx_mbuf_tag);
2389                         rxr->rx_mbuf_tag = NULL;
2390
2391                         device_printf(rxr->sc->bce_dev, "Unable to create "
2392                             "%dth RX mbuf DMA map!\n", i);
2393                         return rc;
2394                 }
2395         }
2396         return 0;
2397 }
2398
2399
2400 /****************************************************************************/
2401 /* Allocate any DMA memory needed by the driver.                            */
2402 /*                                                                          */
2403 /* Allocates DMA memory needed for the various global structures needed by  */
2404 /* hardware.                                                                */
2405 /*                                                                          */
2406 /* Memory alignment requirements:                                           */
2407 /* -----------------+----------+----------+----------+----------+           */
2408 /*  Data Structure  |   5706   |   5708   |   5709   |   5716   |           */
2409 /* -----------------+----------+----------+----------+----------+           */
2410 /* Status Block     | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |           */
2411 /* Statistics Block | 8 bytes  | 8 bytes  | 16 bytes | 16 bytes |           */
2412 /* RX Buffers       | 16 bytes | 16 bytes | 16 bytes | 16 bytes |           */
2413 /* PG Buffers       |   none   |   none   |   none   |   none   |           */
2414 /* TX Buffers       |   none   |   none   |   none   |   none   |           */
2415 /* Chain Pages(1)   |   4KiB   |   4KiB   |   4KiB   |   4KiB   |           */
2416 /* Context Pages(1) |   N/A    |   N/A    |   4KiB   |   4KiB   |           */
2417 /* -----------------+----------+----------+----------+----------+           */
2418 /*                                                                          */
2419 /* (1) Must align with CPU page size (BCM_PAGE_SZIE).                       */
2420 /*                                                                          */
2421 /* Returns:                                                                 */
2422 /*   0 for success, positive value for failure.                             */
2423 /****************************************************************************/
2424 static int
2425 bce_dma_alloc(struct bce_softc *sc)
2426 {
2427         struct ifnet *ifp = &sc->arpcom.ac_if;
2428         int i, rc = 0;
2429         bus_addr_t busaddr, max_busaddr;
2430         bus_size_t status_align, stats_align, status_size;
2431
2432         /*
2433          * The embedded PCIe to PCI-X bridge (EPB)
2434          * in the 5708 cannot address memory above
2435          * 40 bits (E7_5708CB1_23043 & E6_5708SB1_23043).
2436          */
2437         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5708)
2438                 max_busaddr = BCE_BUS_SPACE_MAXADDR;
2439         else
2440                 max_busaddr = BUS_SPACE_MAXADDR;
2441
2442         /*
2443          * BCM5709 and BCM5716 uses host memory as cache for context memory.
2444          */
2445         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
2446             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
2447                 sc->ctx_pages = BCE_CTX_BLK_SZ / BCM_PAGE_SIZE;
2448                 if (sc->ctx_pages == 0)
2449                         sc->ctx_pages = 1;
2450                 if (sc->ctx_pages > BCE_CTX_PAGES) {
2451                         device_printf(sc->bce_dev, "excessive ctx pages %d\n",
2452                             sc->ctx_pages);
2453                         return ENOMEM;
2454                 }
2455                 status_align = 16;
2456                 stats_align = 16;
2457         } else {
2458                 status_align = 8;
2459                 stats_align = 8;
2460         }
2461
2462         /*
2463          * Each MSI-X vector needs a status block; each status block
2464          * consumes 128bytes and is 128bytes aligned.
2465          */
2466         if (sc->rx_ring_cnt > 1) {
2467                 status_size = BCE_MSIX_MAX * BCE_STATUS_BLK_MSIX_ALIGN;
2468                 status_align = BCE_STATUS_BLK_MSIX_ALIGN;
2469         } else {
2470                 status_size = BCE_STATUS_BLK_SZ;
2471         }
2472
2473         /*
2474          * Allocate the parent bus DMA tag appropriate for PCI.
2475          */
2476         rc = bus_dma_tag_create(NULL, 1, BCE_DMA_BOUNDARY,
2477                                 max_busaddr, BUS_SPACE_MAXADDR,
2478                                 NULL, NULL,
2479                                 BUS_SPACE_MAXSIZE_32BIT, 0,
2480                                 BUS_SPACE_MAXSIZE_32BIT,
2481                                 0, &sc->parent_tag);
2482         if (rc != 0) {
2483                 if_printf(ifp, "Could not allocate parent DMA tag!\n");
2484                 return rc;
2485         }
2486
2487         /*
2488          * Allocate status block.
2489          */
2490         sc->status_block = bus_dmamem_coherent_any(sc->parent_tag,
2491                                 status_align, status_size,
2492                                 BUS_DMA_WAITOK | BUS_DMA_ZERO,
2493                                 &sc->status_tag, &sc->status_map,
2494                                 &sc->status_block_paddr);
2495         if (sc->status_block == NULL) {
2496                 if_printf(ifp, "Could not allocate status block!\n");
2497                 return ENOMEM;
2498         }
2499
2500         /*
2501          * Allocate statistics block.
2502          */
2503         sc->stats_block = bus_dmamem_coherent_any(sc->parent_tag,
2504                                 stats_align, BCE_STATS_BLK_SZ,
2505                                 BUS_DMA_WAITOK | BUS_DMA_ZERO,
2506                                 &sc->stats_tag, &sc->stats_map,
2507                                 &sc->stats_block_paddr);
2508         if (sc->stats_block == NULL) {
2509                 if_printf(ifp, "Could not allocate statistics block!\n");
2510                 return ENOMEM;
2511         }
2512
2513         /*
2514          * Allocate context block, if needed
2515          */
2516         if (sc->ctx_pages != 0) {
2517                 rc = bus_dma_tag_create(sc->parent_tag, BCM_PAGE_SIZE, 0,
2518                                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
2519                                         NULL, NULL,
2520                                         BCM_PAGE_SIZE, 1, BCM_PAGE_SIZE,
2521                                         0, &sc->ctx_tag);
2522                 if (rc != 0) {
2523                         if_printf(ifp, "Could not allocate "
2524                                   "context block DMA tag!\n");
2525                         return rc;
2526                 }
2527
2528                 for (i = 0; i < sc->ctx_pages; i++) {
2529                         rc = bus_dmamem_alloc(sc->ctx_tag,
2530                                               (void **)&sc->ctx_block[i],
2531                                               BUS_DMA_WAITOK | BUS_DMA_ZERO |
2532                                               BUS_DMA_COHERENT,
2533                                               &sc->ctx_map[i]);
2534                         if (rc != 0) {
2535                                 if_printf(ifp, "Could not allocate %dth context "
2536                                           "DMA memory!\n", i);
2537                                 return rc;
2538                         }
2539
2540                         rc = bus_dmamap_load(sc->ctx_tag, sc->ctx_map[i],
2541                                              sc->ctx_block[i], BCM_PAGE_SIZE,
2542                                              bce_dma_map_addr, &busaddr,
2543                                              BUS_DMA_WAITOK);
2544                         if (rc != 0) {
2545                                 if (rc == EINPROGRESS) {
2546                                         panic("%s coherent memory loading "
2547                                               "is still in progress!", ifp->if_xname);
2548                                 }
2549                                 if_printf(ifp, "Could not map %dth context "
2550                                           "DMA memory!\n", i);
2551                                 bus_dmamem_free(sc->ctx_tag, sc->ctx_block[i],
2552                                                 sc->ctx_map[i]);
2553                                 sc->ctx_block[i] = NULL;
2554                                 return rc;
2555                         }
2556                         sc->ctx_paddr[i] = busaddr;
2557                 }
2558         }
2559
2560         sc->tx_rings = kmalloc_cachealign(
2561             sizeof(struct bce_tx_ring) * sc->tx_ring_cnt, M_DEVBUF,
2562             M_WAITOK | M_ZERO);
2563         for (i = 0; i < sc->tx_ring_cnt; ++i) {
2564                 sc->tx_rings[i].sc = sc;
2565                 if (i == 0) {
2566                         sc->tx_rings[i].tx_cid = TX_CID;
2567                         sc->tx_rings[i].tx_hw_cons =
2568                             &sc->status_block->status_tx_quick_consumer_index0;
2569                 } else {
2570                         struct status_block_msix *sblk =
2571                             (struct status_block_msix *)
2572                             (((uint8_t *)(sc->status_block)) +
2573                              (i * BCE_STATUS_BLK_MSIX_ALIGN));
2574
2575                         sc->tx_rings[i].tx_cid = TX_TSS_CID + i - 1;
2576                         sc->tx_rings[i].tx_hw_cons =
2577                             &sblk->status_tx_quick_consumer_index;
2578                 }
2579
2580                 rc = bce_create_tx_ring(&sc->tx_rings[i]);
2581                 if (rc != 0) {
2582                         device_printf(sc->bce_dev,
2583                             "can't create %dth tx ring\n", i);
2584                         return rc;
2585                 }
2586         }
2587
2588         sc->rx_rings = kmalloc_cachealign(
2589             sizeof(struct bce_rx_ring) * sc->rx_ring_cnt, M_DEVBUF,
2590             M_WAITOK | M_ZERO);
2591         for (i = 0; i < sc->rx_ring_cnt; ++i) {
2592                 sc->rx_rings[i].sc = sc;
2593                 sc->rx_rings[i].idx = i;
2594                 if (i == 0) {
2595                         sc->rx_rings[i].rx_cid = RX_CID;
2596                         sc->rx_rings[i].rx_hw_cons =
2597                             &sc->status_block->status_rx_quick_consumer_index0;
2598                         sc->rx_rings[i].hw_status_idx =
2599                             &sc->status_block->status_idx;
2600                 } else {
2601                         struct status_block_msix *sblk =
2602                             (struct status_block_msix *)
2603                             (((uint8_t *)(sc->status_block)) +
2604                              (i * BCE_STATUS_BLK_MSIX_ALIGN));
2605
2606                         sc->rx_rings[i].rx_cid = RX_RSS_CID + i - 1;
2607                         sc->rx_rings[i].rx_hw_cons =
2608                             &sblk->status_rx_quick_consumer_index;
2609                         sc->rx_rings[i].hw_status_idx = &sblk->status_idx;
2610                 }
2611
2612                 rc = bce_create_rx_ring(&sc->rx_rings[i]);
2613                 if (rc != 0) {
2614                         device_printf(sc->bce_dev,
2615                             "can't create %dth rx ring\n", i);
2616                         return rc;
2617                 }
2618         }
2619
2620         return 0;
2621 }
2622
2623
2624 /****************************************************************************/
2625 /* Firmware synchronization.                                                */
2626 /*                                                                          */
2627 /* Before performing certain events such as a chip reset, synchronize with  */
2628 /* the firmware first.                                                      */
2629 /*                                                                          */
2630 /* Returns:                                                                 */
2631 /*   0 for success, positive value for failure.                             */
2632 /****************************************************************************/
2633 static int
2634 bce_fw_sync(struct bce_softc *sc, uint32_t msg_data)
2635 {
2636         int i, rc = 0;
2637         uint32_t val;
2638
2639         /* Don't waste any time if we've timed out before. */
2640         if (sc->bce_fw_timed_out)
2641                 return EBUSY;
2642
2643         /* Increment the message sequence number. */
2644         sc->bce_fw_wr_seq++;
2645         msg_data |= sc->bce_fw_wr_seq;
2646
2647         /* Send the message to the bootcode driver mailbox. */
2648         bce_shmem_wr(sc, BCE_DRV_MB, msg_data);
2649
2650         /* Wait for the bootcode to acknowledge the message. */
2651         for (i = 0; i < FW_ACK_TIME_OUT_MS; i++) {
2652                 /* Check for a response in the bootcode firmware mailbox. */
2653                 val = bce_shmem_rd(sc, BCE_FW_MB);
2654                 if ((val & BCE_FW_MSG_ACK) == (msg_data & BCE_DRV_MSG_SEQ))
2655                         break;
2656                 DELAY(1000);
2657         }
2658
2659         /* If we've timed out, tell the bootcode that we've stopped waiting. */
2660         if ((val & BCE_FW_MSG_ACK) != (msg_data & BCE_DRV_MSG_SEQ) &&
2661             (msg_data & BCE_DRV_MSG_DATA) != BCE_DRV_MSG_DATA_WAIT0) {
2662                 if_printf(&sc->arpcom.ac_if,
2663                           "Firmware synchronization timeout! "
2664                           "msg_data = 0x%08X\n", msg_data);
2665
2666                 msg_data &= ~BCE_DRV_MSG_CODE;
2667                 msg_data |= BCE_DRV_MSG_CODE_FW_TIMEOUT;
2668
2669                 bce_shmem_wr(sc, BCE_DRV_MB, msg_data);
2670
2671                 sc->bce_fw_timed_out = 1;
2672                 rc = EBUSY;
2673         }
2674         return rc;
2675 }
2676
2677
2678 /****************************************************************************/
2679 /* Load Receive Virtual 2 Physical (RV2P) processor firmware.               */
2680 /*                                                                          */
2681 /* Returns:                                                                 */
2682 /*   Nothing.                                                               */
2683 /****************************************************************************/
2684 static void
2685 bce_load_rv2p_fw(struct bce_softc *sc, uint32_t *rv2p_code,
2686                  uint32_t rv2p_code_len, uint32_t rv2p_proc)
2687 {
2688         int i;
2689         uint32_t val;
2690
2691         for (i = 0; i < rv2p_code_len; i += 8) {
2692                 REG_WR(sc, BCE_RV2P_INSTR_HIGH, *rv2p_code);
2693                 rv2p_code++;
2694                 REG_WR(sc, BCE_RV2P_INSTR_LOW, *rv2p_code);
2695                 rv2p_code++;
2696
2697                 if (rv2p_proc == RV2P_PROC1) {
2698                         val = (i / 8) | BCE_RV2P_PROC1_ADDR_CMD_RDWR;
2699                         REG_WR(sc, BCE_RV2P_PROC1_ADDR_CMD, val);
2700                 } else {
2701                         val = (i / 8) | BCE_RV2P_PROC2_ADDR_CMD_RDWR;
2702                         REG_WR(sc, BCE_RV2P_PROC2_ADDR_CMD, val);
2703                 }
2704         }
2705
2706         /* Reset the processor, un-stall is done later. */
2707         if (rv2p_proc == RV2P_PROC1)
2708                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC1_RESET);
2709         else
2710                 REG_WR(sc, BCE_RV2P_COMMAND, BCE_RV2P_COMMAND_PROC2_RESET);
2711 }
2712
2713
2714 /****************************************************************************/
2715 /* Load RISC processor firmware.                                            */
2716 /*                                                                          */
2717 /* Loads firmware from the file if_bcefw.h into the scratchpad memory       */
2718 /* associated with a particular processor.                                  */
2719 /*                                                                          */
2720 /* Returns:                                                                 */
2721 /*   Nothing.                                                               */
2722 /****************************************************************************/
2723 static void
2724 bce_load_cpu_fw(struct bce_softc *sc, struct cpu_reg *cpu_reg,
2725                 struct fw_info *fw)
2726 {
2727         uint32_t offset;
2728         int j;
2729
2730         bce_halt_cpu(sc, cpu_reg);
2731
2732         /* Load the Text area. */
2733         offset = cpu_reg->spad_base + (fw->text_addr - cpu_reg->mips_view_base);
2734         if (fw->text) {
2735                 for (j = 0; j < (fw->text_len / 4); j++, offset += 4)
2736                         REG_WR_IND(sc, offset, fw->text[j]);
2737         }
2738
2739         /* Load the Data area. */
2740         offset = cpu_reg->spad_base + (fw->data_addr - cpu_reg->mips_view_base);
2741         if (fw->data) {
2742                 for (j = 0; j < (fw->data_len / 4); j++, offset += 4)
2743                         REG_WR_IND(sc, offset, fw->data[j]);
2744         }
2745
2746         /* Load the SBSS area. */
2747         offset = cpu_reg->spad_base + (fw->sbss_addr - cpu_reg->mips_view_base);
2748         if (fw->sbss) {
2749                 for (j = 0; j < (fw->sbss_len / 4); j++, offset += 4)
2750                         REG_WR_IND(sc, offset, fw->sbss[j]);
2751         }
2752
2753         /* Load the BSS area. */
2754         offset = cpu_reg->spad_base + (fw->bss_addr - cpu_reg->mips_view_base);
2755         if (fw->bss) {
2756                 for (j = 0; j < (fw->bss_len/4); j++, offset += 4)
2757                         REG_WR_IND(sc, offset, fw->bss[j]);
2758         }
2759
2760         /* Load the Read-Only area. */
2761         offset = cpu_reg->spad_base +
2762                 (fw->rodata_addr - cpu_reg->mips_view_base);
2763         if (fw->rodata) {
2764                 for (j = 0; j < (fw->rodata_len / 4); j++, offset += 4)
2765                         REG_WR_IND(sc, offset, fw->rodata[j]);
2766         }
2767
2768         /* Clear the pre-fetch instruction and set the FW start address. */
2769         REG_WR_IND(sc, cpu_reg->inst, 0);
2770         REG_WR_IND(sc, cpu_reg->pc, fw->start_addr);
2771 }
2772
2773
2774 /****************************************************************************/
2775 /* Starts the RISC processor.                                               */
2776 /*                                                                          */
2777 /* Assumes the CPU starting address has already been set.                   */
2778 /*                                                                          */
2779 /* Returns:                                                                 */
2780 /*   Nothing.                                                               */
2781 /****************************************************************************/
2782 static void
2783 bce_start_cpu(struct bce_softc *sc, struct cpu_reg *cpu_reg)
2784 {
2785         uint32_t val;
2786
2787         /* Start the CPU. */
2788         val = REG_RD_IND(sc, cpu_reg->mode);
2789         val &= ~cpu_reg->mode_value_halt;
2790         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
2791         REG_WR_IND(sc, cpu_reg->mode, val);
2792 }
2793
2794
2795 /****************************************************************************/
2796 /* Halts the RISC processor.                                                */
2797 /*                                                                          */
2798 /* Returns:                                                                 */
2799 /*   Nothing.                                                               */
2800 /****************************************************************************/
2801 static void
2802 bce_halt_cpu(struct bce_softc *sc, struct cpu_reg *cpu_reg)
2803 {
2804         uint32_t val;
2805
2806         /* Halt the CPU. */
2807         val = REG_RD_IND(sc, cpu_reg->mode);
2808         val |= cpu_reg->mode_value_halt;
2809         REG_WR_IND(sc, cpu_reg->mode, val);
2810         REG_WR_IND(sc, cpu_reg->state, cpu_reg->state_value_clear);
2811 }
2812
2813
2814 /****************************************************************************/
2815 /* Start the RX CPU.                                                        */
2816 /*                                                                          */
2817 /* Returns:                                                                 */
2818 /*   Nothing.                                                               */
2819 /****************************************************************************/
2820 static void
2821 bce_start_rxp_cpu(struct bce_softc *sc)
2822 {
2823         struct cpu_reg cpu_reg;
2824
2825         cpu_reg.mode = BCE_RXP_CPU_MODE;
2826         cpu_reg.mode_value_halt = BCE_RXP_CPU_MODE_SOFT_HALT;
2827         cpu_reg.mode_value_sstep = BCE_RXP_CPU_MODE_STEP_ENA;
2828         cpu_reg.state = BCE_RXP_CPU_STATE;
2829         cpu_reg.state_value_clear = 0xffffff;
2830         cpu_reg.gpr0 = BCE_RXP_CPU_REG_FILE;
2831         cpu_reg.evmask = BCE_RXP_CPU_EVENT_MASK;
2832         cpu_reg.pc = BCE_RXP_CPU_PROGRAM_COUNTER;
2833         cpu_reg.inst = BCE_RXP_CPU_INSTRUCTION;
2834         cpu_reg.bp = BCE_RXP_CPU_HW_BREAKPOINT;
2835         cpu_reg.spad_base = BCE_RXP_SCRATCH;
2836         cpu_reg.mips_view_base = 0x8000000;
2837
2838         bce_start_cpu(sc, &cpu_reg);
2839 }
2840
2841
2842 /****************************************************************************/
2843 /* Initialize the RX CPU.                                                   */
2844 /*                                                                          */
2845 /* Returns:                                                                 */
2846 /*   Nothing.                                                               */
2847 /****************************************************************************/
2848 static void
2849 bce_init_rxp_cpu(struct bce_softc *sc)
2850 {
2851         struct cpu_reg cpu_reg;
2852         struct fw_info fw;
2853
2854         cpu_reg.mode = BCE_RXP_CPU_MODE;
2855         cpu_reg.mode_value_halt = BCE_RXP_CPU_MODE_SOFT_HALT;
2856         cpu_reg.mode_value_sstep = BCE_RXP_CPU_MODE_STEP_ENA;
2857         cpu_reg.state = BCE_RXP_CPU_STATE;
2858         cpu_reg.state_value_clear = 0xffffff;
2859         cpu_reg.gpr0 = BCE_RXP_CPU_REG_FILE;
2860         cpu_reg.evmask = BCE_RXP_CPU_EVENT_MASK;
2861         cpu_reg.pc = BCE_RXP_CPU_PROGRAM_COUNTER;
2862         cpu_reg.inst = BCE_RXP_CPU_INSTRUCTION;
2863         cpu_reg.bp = BCE_RXP_CPU_HW_BREAKPOINT;
2864         cpu_reg.spad_base = BCE_RXP_SCRATCH;
2865         cpu_reg.mips_view_base = 0x8000000;
2866
2867         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
2868             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
2869                 fw.ver_major = bce_RXP_b09FwReleaseMajor;
2870                 fw.ver_minor = bce_RXP_b09FwReleaseMinor;
2871                 fw.ver_fix = bce_RXP_b09FwReleaseFix;
2872                 fw.start_addr = bce_RXP_b09FwStartAddr;
2873
2874                 fw.text_addr = bce_RXP_b09FwTextAddr;
2875                 fw.text_len = bce_RXP_b09FwTextLen;
2876                 fw.text_index = 0;
2877                 fw.text = bce_RXP_b09FwText;
2878
2879                 fw.data_addr = bce_RXP_b09FwDataAddr;
2880                 fw.data_len = bce_RXP_b09FwDataLen;
2881                 fw.data_index = 0;
2882                 fw.data = bce_RXP_b09FwData;
2883
2884                 fw.sbss_addr = bce_RXP_b09FwSbssAddr;
2885                 fw.sbss_len = bce_RXP_b09FwSbssLen;
2886                 fw.sbss_index = 0;
2887                 fw.sbss = bce_RXP_b09FwSbss;
2888
2889                 fw.bss_addr = bce_RXP_b09FwBssAddr;
2890                 fw.bss_len = bce_RXP_b09FwBssLen;
2891                 fw.bss_index = 0;
2892                 fw.bss = bce_RXP_b09FwBss;
2893
2894                 fw.rodata_addr = bce_RXP_b09FwRodataAddr;
2895                 fw.rodata_len = bce_RXP_b09FwRodataLen;
2896                 fw.rodata_index = 0;
2897                 fw.rodata = bce_RXP_b09FwRodata;
2898         } else {
2899                 fw.ver_major = bce_RXP_b06FwReleaseMajor;
2900                 fw.ver_minor = bce_RXP_b06FwReleaseMinor;
2901                 fw.ver_fix = bce_RXP_b06FwReleaseFix;
2902                 fw.start_addr = bce_RXP_b06FwStartAddr;
2903
2904                 fw.text_addr = bce_RXP_b06FwTextAddr;
2905                 fw.text_len = bce_RXP_b06FwTextLen;
2906                 fw.text_index = 0;
2907                 fw.text = bce_RXP_b06FwText;
2908
2909                 fw.data_addr = bce_RXP_b06FwDataAddr;
2910                 fw.data_len = bce_RXP_b06FwDataLen;
2911                 fw.data_index = 0;
2912                 fw.data = bce_RXP_b06FwData;
2913
2914                 fw.sbss_addr = bce_RXP_b06FwSbssAddr;
2915                 fw.sbss_len = bce_RXP_b06FwSbssLen;
2916                 fw.sbss_index = 0;
2917                 fw.sbss = bce_RXP_b06FwSbss;
2918
2919                 fw.bss_addr = bce_RXP_b06FwBssAddr;
2920                 fw.bss_len = bce_RXP_b06FwBssLen;
2921                 fw.bss_index = 0;
2922                 fw.bss = bce_RXP_b06FwBss;
2923
2924                 fw.rodata_addr = bce_RXP_b06FwRodataAddr;
2925                 fw.rodata_len = bce_RXP_b06FwRodataLen;
2926                 fw.rodata_index = 0;
2927                 fw.rodata = bce_RXP_b06FwRodata;
2928         }
2929
2930         bce_load_cpu_fw(sc, &cpu_reg, &fw);
2931         /* Delay RXP start until initialization is complete. */
2932 }
2933
2934
2935 /****************************************************************************/
2936 /* Initialize the TX CPU.                                                   */
2937 /*                                                                          */
2938 /* Returns:                                                                 */
2939 /*   Nothing.                                                               */
2940 /****************************************************************************/
2941 static void
2942 bce_init_txp_cpu(struct bce_softc *sc)
2943 {
2944         struct cpu_reg cpu_reg;
2945         struct fw_info fw;
2946
2947         cpu_reg.mode = BCE_TXP_CPU_MODE;
2948         cpu_reg.mode_value_halt = BCE_TXP_CPU_MODE_SOFT_HALT;
2949         cpu_reg.mode_value_sstep = BCE_TXP_CPU_MODE_STEP_ENA;
2950         cpu_reg.state = BCE_TXP_CPU_STATE;
2951         cpu_reg.state_value_clear = 0xffffff;
2952         cpu_reg.gpr0 = BCE_TXP_CPU_REG_FILE;
2953         cpu_reg.evmask = BCE_TXP_CPU_EVENT_MASK;
2954         cpu_reg.pc = BCE_TXP_CPU_PROGRAM_COUNTER;
2955         cpu_reg.inst = BCE_TXP_CPU_INSTRUCTION;
2956         cpu_reg.bp = BCE_TXP_CPU_HW_BREAKPOINT;
2957         cpu_reg.spad_base = BCE_TXP_SCRATCH;
2958         cpu_reg.mips_view_base = 0x8000000;
2959
2960         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
2961             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
2962                 fw.ver_major = bce_TXP_b09FwReleaseMajor;
2963                 fw.ver_minor = bce_TXP_b09FwReleaseMinor;
2964                 fw.ver_fix = bce_TXP_b09FwReleaseFix;
2965                 fw.start_addr = bce_TXP_b09FwStartAddr;
2966
2967                 fw.text_addr = bce_TXP_b09FwTextAddr;
2968                 fw.text_len = bce_TXP_b09FwTextLen;
2969                 fw.text_index = 0;
2970                 fw.text = bce_TXP_b09FwText;
2971
2972                 fw.data_addr = bce_TXP_b09FwDataAddr;
2973                 fw.data_len = bce_TXP_b09FwDataLen;
2974                 fw.data_index = 0;
2975                 fw.data = bce_TXP_b09FwData;
2976
2977                 fw.sbss_addr = bce_TXP_b09FwSbssAddr;
2978                 fw.sbss_len = bce_TXP_b09FwSbssLen;
2979                 fw.sbss_index = 0;
2980                 fw.sbss = bce_TXP_b09FwSbss;
2981
2982                 fw.bss_addr = bce_TXP_b09FwBssAddr;
2983                 fw.bss_len = bce_TXP_b09FwBssLen;
2984                 fw.bss_index = 0;
2985                 fw.bss = bce_TXP_b09FwBss;
2986
2987                 fw.rodata_addr = bce_TXP_b09FwRodataAddr;
2988                 fw.rodata_len = bce_TXP_b09FwRodataLen;
2989                 fw.rodata_index = 0;
2990                 fw.rodata = bce_TXP_b09FwRodata;
2991         } else {
2992                 fw.ver_major = bce_TXP_b06FwReleaseMajor;
2993                 fw.ver_minor = bce_TXP_b06FwReleaseMinor;
2994                 fw.ver_fix = bce_TXP_b06FwReleaseFix;
2995                 fw.start_addr = bce_TXP_b06FwStartAddr;
2996
2997                 fw.text_addr = bce_TXP_b06FwTextAddr;
2998                 fw.text_len = bce_TXP_b06FwTextLen;
2999                 fw.text_index = 0;
3000                 fw.text = bce_TXP_b06FwText;
3001
3002                 fw.data_addr = bce_TXP_b06FwDataAddr;
3003                 fw.data_len = bce_TXP_b06FwDataLen;
3004                 fw.data_index = 0;
3005                 fw.data = bce_TXP_b06FwData;
3006
3007                 fw.sbss_addr = bce_TXP_b06FwSbssAddr;
3008                 fw.sbss_len = bce_TXP_b06FwSbssLen;
3009                 fw.sbss_index = 0;
3010                 fw.sbss = bce_TXP_b06FwSbss;
3011
3012                 fw.bss_addr = bce_TXP_b06FwBssAddr;
3013                 fw.bss_len = bce_TXP_b06FwBssLen;
3014                 fw.bss_index = 0;
3015                 fw.bss = bce_TXP_b06FwBss;
3016
3017                 fw.rodata_addr = bce_TXP_b06FwRodataAddr;
3018                 fw.rodata_len = bce_TXP_b06FwRodataLen;
3019                 fw.rodata_index = 0;
3020                 fw.rodata = bce_TXP_b06FwRodata;
3021         }
3022
3023         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3024         bce_start_cpu(sc, &cpu_reg);
3025 }
3026
3027
3028 /****************************************************************************/
3029 /* Initialize the TPAT CPU.                                                 */
3030 /*                                                                          */
3031 /* Returns:                                                                 */
3032 /*   Nothing.                                                               */
3033 /****************************************************************************/
3034 static void
3035 bce_init_tpat_cpu(struct bce_softc *sc)
3036 {
3037         struct cpu_reg cpu_reg;
3038         struct fw_info fw;
3039
3040         cpu_reg.mode = BCE_TPAT_CPU_MODE;
3041         cpu_reg.mode_value_halt = BCE_TPAT_CPU_MODE_SOFT_HALT;
3042         cpu_reg.mode_value_sstep = BCE_TPAT_CPU_MODE_STEP_ENA;
3043         cpu_reg.state = BCE_TPAT_CPU_STATE;
3044         cpu_reg.state_value_clear = 0xffffff;
3045         cpu_reg.gpr0 = BCE_TPAT_CPU_REG_FILE;
3046         cpu_reg.evmask = BCE_TPAT_CPU_EVENT_MASK;
3047         cpu_reg.pc = BCE_TPAT_CPU_PROGRAM_COUNTER;
3048         cpu_reg.inst = BCE_TPAT_CPU_INSTRUCTION;
3049         cpu_reg.bp = BCE_TPAT_CPU_HW_BREAKPOINT;
3050         cpu_reg.spad_base = BCE_TPAT_SCRATCH;
3051         cpu_reg.mips_view_base = 0x8000000;
3052
3053         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3054             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3055                 fw.ver_major = bce_TPAT_b09FwReleaseMajor;
3056                 fw.ver_minor = bce_TPAT_b09FwReleaseMinor;
3057                 fw.ver_fix = bce_TPAT_b09FwReleaseFix;
3058                 fw.start_addr = bce_TPAT_b09FwStartAddr;
3059
3060                 fw.text_addr = bce_TPAT_b09FwTextAddr;
3061                 fw.text_len = bce_TPAT_b09FwTextLen;
3062                 fw.text_index = 0;
3063                 fw.text = bce_TPAT_b09FwText;
3064
3065                 fw.data_addr = bce_TPAT_b09FwDataAddr;
3066                 fw.data_len = bce_TPAT_b09FwDataLen;
3067                 fw.data_index = 0;
3068                 fw.data = bce_TPAT_b09FwData;
3069
3070                 fw.sbss_addr = bce_TPAT_b09FwSbssAddr;
3071                 fw.sbss_len = bce_TPAT_b09FwSbssLen;
3072                 fw.sbss_index = 0;
3073                 fw.sbss = bce_TPAT_b09FwSbss;
3074
3075                 fw.bss_addr = bce_TPAT_b09FwBssAddr;
3076                 fw.bss_len = bce_TPAT_b09FwBssLen;
3077                 fw.bss_index = 0;
3078                 fw.bss = bce_TPAT_b09FwBss;
3079
3080                 fw.rodata_addr = bce_TPAT_b09FwRodataAddr;
3081                 fw.rodata_len = bce_TPAT_b09FwRodataLen;
3082                 fw.rodata_index = 0;
3083                 fw.rodata = bce_TPAT_b09FwRodata;
3084         } else {
3085                 fw.ver_major = bce_TPAT_b06FwReleaseMajor;
3086                 fw.ver_minor = bce_TPAT_b06FwReleaseMinor;
3087                 fw.ver_fix = bce_TPAT_b06FwReleaseFix;
3088                 fw.start_addr = bce_TPAT_b06FwStartAddr;
3089
3090                 fw.text_addr = bce_TPAT_b06FwTextAddr;
3091                 fw.text_len = bce_TPAT_b06FwTextLen;
3092                 fw.text_index = 0;
3093                 fw.text = bce_TPAT_b06FwText;
3094
3095                 fw.data_addr = bce_TPAT_b06FwDataAddr;
3096                 fw.data_len = bce_TPAT_b06FwDataLen;
3097                 fw.data_index = 0;
3098                 fw.data = bce_TPAT_b06FwData;
3099
3100                 fw.sbss_addr = bce_TPAT_b06FwSbssAddr;
3101                 fw.sbss_len = bce_TPAT_b06FwSbssLen;
3102                 fw.sbss_index = 0;
3103                 fw.sbss = bce_TPAT_b06FwSbss;
3104
3105                 fw.bss_addr = bce_TPAT_b06FwBssAddr;
3106                 fw.bss_len = bce_TPAT_b06FwBssLen;
3107                 fw.bss_index = 0;
3108                 fw.bss = bce_TPAT_b06FwBss;
3109
3110                 fw.rodata_addr = bce_TPAT_b06FwRodataAddr;
3111                 fw.rodata_len = bce_TPAT_b06FwRodataLen;
3112                 fw.rodata_index = 0;
3113                 fw.rodata = bce_TPAT_b06FwRodata;
3114         }
3115
3116         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3117         bce_start_cpu(sc, &cpu_reg);
3118 }
3119
3120
3121 /****************************************************************************/
3122 /* Initialize the CP CPU.                                                   */
3123 /*                                                                          */
3124 /* Returns:                                                                 */
3125 /*   Nothing.                                                               */
3126 /****************************************************************************/
3127 static void
3128 bce_init_cp_cpu(struct bce_softc *sc)
3129 {
3130         struct cpu_reg cpu_reg;
3131         struct fw_info fw;
3132
3133         cpu_reg.mode = BCE_CP_CPU_MODE;
3134         cpu_reg.mode_value_halt = BCE_CP_CPU_MODE_SOFT_HALT;
3135         cpu_reg.mode_value_sstep = BCE_CP_CPU_MODE_STEP_ENA;
3136         cpu_reg.state = BCE_CP_CPU_STATE;
3137         cpu_reg.state_value_clear = 0xffffff;
3138         cpu_reg.gpr0 = BCE_CP_CPU_REG_FILE;
3139         cpu_reg.evmask = BCE_CP_CPU_EVENT_MASK;
3140         cpu_reg.pc = BCE_CP_CPU_PROGRAM_COUNTER;
3141         cpu_reg.inst = BCE_CP_CPU_INSTRUCTION;
3142         cpu_reg.bp = BCE_CP_CPU_HW_BREAKPOINT;
3143         cpu_reg.spad_base = BCE_CP_SCRATCH;
3144         cpu_reg.mips_view_base = 0x8000000;
3145
3146         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3147             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3148                 fw.ver_major = bce_CP_b09FwReleaseMajor;
3149                 fw.ver_minor = bce_CP_b09FwReleaseMinor;
3150                 fw.ver_fix = bce_CP_b09FwReleaseFix;
3151                 fw.start_addr = bce_CP_b09FwStartAddr;
3152
3153                 fw.text_addr = bce_CP_b09FwTextAddr;
3154                 fw.text_len = bce_CP_b09FwTextLen;
3155                 fw.text_index = 0;
3156                 fw.text = bce_CP_b09FwText;
3157
3158                 fw.data_addr = bce_CP_b09FwDataAddr;
3159                 fw.data_len = bce_CP_b09FwDataLen;
3160                 fw.data_index = 0;
3161                 fw.data = bce_CP_b09FwData;
3162
3163                 fw.sbss_addr = bce_CP_b09FwSbssAddr;
3164                 fw.sbss_len = bce_CP_b09FwSbssLen;
3165                 fw.sbss_index = 0;
3166                 fw.sbss = bce_CP_b09FwSbss;
3167
3168                 fw.bss_addr = bce_CP_b09FwBssAddr;
3169                 fw.bss_len = bce_CP_b09FwBssLen;
3170                 fw.bss_index = 0;
3171                 fw.bss = bce_CP_b09FwBss;
3172
3173                 fw.rodata_addr = bce_CP_b09FwRodataAddr;
3174                 fw.rodata_len = bce_CP_b09FwRodataLen;
3175                 fw.rodata_index = 0;
3176                 fw.rodata = bce_CP_b09FwRodata;
3177         } else {
3178                 fw.ver_major = bce_CP_b06FwReleaseMajor;
3179                 fw.ver_minor = bce_CP_b06FwReleaseMinor;
3180                 fw.ver_fix = bce_CP_b06FwReleaseFix;
3181                 fw.start_addr = bce_CP_b06FwStartAddr;
3182
3183                 fw.text_addr = bce_CP_b06FwTextAddr;
3184                 fw.text_len = bce_CP_b06FwTextLen;
3185                 fw.text_index = 0;
3186                 fw.text = bce_CP_b06FwText;
3187
3188                 fw.data_addr = bce_CP_b06FwDataAddr;
3189                 fw.data_len = bce_CP_b06FwDataLen;
3190                 fw.data_index = 0;
3191                 fw.data = bce_CP_b06FwData;
3192
3193                 fw.sbss_addr = bce_CP_b06FwSbssAddr;
3194                 fw.sbss_len = bce_CP_b06FwSbssLen;
3195                 fw.sbss_index = 0;
3196                 fw.sbss = bce_CP_b06FwSbss;
3197
3198                 fw.bss_addr = bce_CP_b06FwBssAddr;
3199                 fw.bss_len = bce_CP_b06FwBssLen;
3200                 fw.bss_index = 0;
3201                 fw.bss = bce_CP_b06FwBss;
3202
3203                 fw.rodata_addr = bce_CP_b06FwRodataAddr;
3204                 fw.rodata_len = bce_CP_b06FwRodataLen;
3205                 fw.rodata_index = 0;
3206                 fw.rodata = bce_CP_b06FwRodata;
3207         }
3208
3209         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3210         bce_start_cpu(sc, &cpu_reg);
3211 }
3212
3213
3214 /****************************************************************************/
3215 /* Initialize the COM CPU.                                                 */
3216 /*                                                                          */
3217 /* Returns:                                                                 */
3218 /*   Nothing.                                                               */
3219 /****************************************************************************/
3220 static void
3221 bce_init_com_cpu(struct bce_softc *sc)
3222 {
3223         struct cpu_reg cpu_reg;
3224         struct fw_info fw;
3225
3226         cpu_reg.mode = BCE_COM_CPU_MODE;
3227         cpu_reg.mode_value_halt = BCE_COM_CPU_MODE_SOFT_HALT;
3228         cpu_reg.mode_value_sstep = BCE_COM_CPU_MODE_STEP_ENA;
3229         cpu_reg.state = BCE_COM_CPU_STATE;
3230         cpu_reg.state_value_clear = 0xffffff;
3231         cpu_reg.gpr0 = BCE_COM_CPU_REG_FILE;
3232         cpu_reg.evmask = BCE_COM_CPU_EVENT_MASK;
3233         cpu_reg.pc = BCE_COM_CPU_PROGRAM_COUNTER;
3234         cpu_reg.inst = BCE_COM_CPU_INSTRUCTION;
3235         cpu_reg.bp = BCE_COM_CPU_HW_BREAKPOINT;
3236         cpu_reg.spad_base = BCE_COM_SCRATCH;
3237         cpu_reg.mips_view_base = 0x8000000;
3238
3239         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3240             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3241                 fw.ver_major = bce_COM_b09FwReleaseMajor;
3242                 fw.ver_minor = bce_COM_b09FwReleaseMinor;
3243                 fw.ver_fix = bce_COM_b09FwReleaseFix;
3244                 fw.start_addr = bce_COM_b09FwStartAddr;
3245
3246                 fw.text_addr = bce_COM_b09FwTextAddr;
3247                 fw.text_len = bce_COM_b09FwTextLen;
3248                 fw.text_index = 0;
3249                 fw.text = bce_COM_b09FwText;
3250
3251                 fw.data_addr = bce_COM_b09FwDataAddr;
3252                 fw.data_len = bce_COM_b09FwDataLen;
3253                 fw.data_index = 0;
3254                 fw.data = bce_COM_b09FwData;
3255
3256                 fw.sbss_addr = bce_COM_b09FwSbssAddr;
3257                 fw.sbss_len = bce_COM_b09FwSbssLen;
3258                 fw.sbss_index = 0;
3259                 fw.sbss = bce_COM_b09FwSbss;
3260
3261                 fw.bss_addr = bce_COM_b09FwBssAddr;
3262                 fw.bss_len = bce_COM_b09FwBssLen;
3263                 fw.bss_index = 0;
3264                 fw.bss = bce_COM_b09FwBss;
3265
3266                 fw.rodata_addr = bce_COM_b09FwRodataAddr;
3267                 fw.rodata_len = bce_COM_b09FwRodataLen;
3268                 fw.rodata_index = 0;
3269                 fw.rodata = bce_COM_b09FwRodata;
3270         } else {
3271                 fw.ver_major = bce_COM_b06FwReleaseMajor;
3272                 fw.ver_minor = bce_COM_b06FwReleaseMinor;
3273                 fw.ver_fix = bce_COM_b06FwReleaseFix;
3274                 fw.start_addr = bce_COM_b06FwStartAddr;
3275
3276                 fw.text_addr = bce_COM_b06FwTextAddr;
3277                 fw.text_len = bce_COM_b06FwTextLen;
3278                 fw.text_index = 0;
3279                 fw.text = bce_COM_b06FwText;
3280
3281                 fw.data_addr = bce_COM_b06FwDataAddr;
3282                 fw.data_len = bce_COM_b06FwDataLen;
3283                 fw.data_index = 0;
3284                 fw.data = bce_COM_b06FwData;
3285
3286                 fw.sbss_addr = bce_COM_b06FwSbssAddr;
3287                 fw.sbss_len = bce_COM_b06FwSbssLen;
3288                 fw.sbss_index = 0;
3289                 fw.sbss = bce_COM_b06FwSbss;
3290
3291                 fw.bss_addr = bce_COM_b06FwBssAddr;
3292                 fw.bss_len = bce_COM_b06FwBssLen;
3293                 fw.bss_index = 0;
3294                 fw.bss = bce_COM_b06FwBss;
3295
3296                 fw.rodata_addr = bce_COM_b06FwRodataAddr;
3297                 fw.rodata_len = bce_COM_b06FwRodataLen;
3298                 fw.rodata_index = 0;
3299                 fw.rodata = bce_COM_b06FwRodata;
3300         }
3301
3302         bce_load_cpu_fw(sc, &cpu_reg, &fw);
3303         bce_start_cpu(sc, &cpu_reg);
3304 }
3305
3306
3307 /****************************************************************************/
3308 /* Initialize the RV2P, RX, TX, TPAT, COM, and CP CPUs.                     */
3309 /*                                                                          */
3310 /* Loads the firmware for each CPU and starts the CPU.                      */
3311 /*                                                                          */
3312 /* Returns:                                                                 */
3313 /*   Nothing.                                                               */
3314 /****************************************************************************/
3315 static void
3316 bce_init_cpus(struct bce_softc *sc)
3317 {
3318         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3319             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3320                 if (BCE_CHIP_REV(sc) == BCE_CHIP_REV_Ax) {
3321                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc1,
3322                             sizeof(bce_xi90_rv2p_proc1), RV2P_PROC1);
3323                         bce_load_rv2p_fw(sc, bce_xi90_rv2p_proc2,
3324                             sizeof(bce_xi90_rv2p_proc2), RV2P_PROC2);
3325                 } else {
3326                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc1,
3327                             sizeof(bce_xi_rv2p_proc1), RV2P_PROC1);
3328                         bce_load_rv2p_fw(sc, bce_xi_rv2p_proc2,
3329                             sizeof(bce_xi_rv2p_proc2), RV2P_PROC2);
3330                 }
3331         } else {
3332                 bce_load_rv2p_fw(sc, bce_rv2p_proc1,
3333                     sizeof(bce_rv2p_proc1), RV2P_PROC1);
3334                 bce_load_rv2p_fw(sc, bce_rv2p_proc2,
3335                     sizeof(bce_rv2p_proc2), RV2P_PROC2);
3336         }
3337
3338         bce_init_rxp_cpu(sc);
3339         bce_init_txp_cpu(sc);
3340         bce_init_tpat_cpu(sc);
3341         bce_init_com_cpu(sc);
3342         bce_init_cp_cpu(sc);
3343 }
3344
3345
3346 /****************************************************************************/
3347 /* Initialize context memory.                                               */
3348 /*                                                                          */
3349 /* Clears the memory associated with each Context ID (CID).                 */
3350 /*                                                                          */
3351 /* Returns:                                                                 */
3352 /*   Nothing.                                                               */
3353 /****************************************************************************/
3354 static int
3355 bce_init_ctx(struct bce_softc *sc)
3356 {
3357         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3358             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3359                 /* DRC: Replace this constant value with a #define. */
3360                 int i, retry_cnt = 10;
3361                 uint32_t val;
3362
3363                 /*
3364                  * BCM5709 context memory may be cached
3365                  * in host memory so prepare the host memory
3366                  * for access.
3367                  */
3368                 val = BCE_CTX_COMMAND_ENABLED | BCE_CTX_COMMAND_MEM_INIT |
3369                     (1 << 12);
3370                 val |= (BCM_PAGE_BITS - 8) << 16;
3371                 REG_WR(sc, BCE_CTX_COMMAND, val);
3372
3373                 /* Wait for mem init command to complete. */
3374                 for (i = 0; i < retry_cnt; i++) {
3375                         val = REG_RD(sc, BCE_CTX_COMMAND);
3376                         if (!(val & BCE_CTX_COMMAND_MEM_INIT))
3377                                 break;
3378                         DELAY(2);
3379                 }
3380                 if (i == retry_cnt) {
3381                         device_printf(sc->bce_dev,
3382                             "Context memory initialization failed!\n");
3383                         return ETIMEDOUT;
3384                 }
3385
3386                 for (i = 0; i < sc->ctx_pages; i++) {
3387                         int j;
3388
3389                         /*
3390                          * Set the physical address of the context
3391                          * memory cache.
3392                          */
3393                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA0,
3394                             BCE_ADDR_LO(sc->ctx_paddr[i] & 0xfffffff0) |
3395                             BCE_CTX_HOST_PAGE_TBL_DATA0_VALID);
3396                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_DATA1,
3397                             BCE_ADDR_HI(sc->ctx_paddr[i]));
3398                         REG_WR(sc, BCE_CTX_HOST_PAGE_TBL_CTRL,
3399                             i | BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ);
3400
3401                         /*
3402                          * Verify that the context memory write was successful.
3403                          */
3404                         for (j = 0; j < retry_cnt; j++) {
3405                                 val = REG_RD(sc, BCE_CTX_HOST_PAGE_TBL_CTRL);
3406                                 if ((val &
3407                                     BCE_CTX_HOST_PAGE_TBL_CTRL_WRITE_REQ) == 0)
3408                                         break;
3409                                 DELAY(5);
3410                         }
3411                         if (j == retry_cnt) {
3412                                 device_printf(sc->bce_dev,
3413                                     "Failed to initialize context page!\n");
3414                                 return ETIMEDOUT;
3415                         }
3416                 }
3417         } else {
3418                 uint32_t vcid_addr, offset;
3419
3420                 /*
3421                  * For the 5706/5708, context memory is local to
3422                  * the controller, so initialize the controller
3423                  * context memory.
3424                  */
3425
3426                 vcid_addr = GET_CID_ADDR(96);
3427                 while (vcid_addr) {
3428                         vcid_addr -= PHY_CTX_SIZE;
3429
3430                         REG_WR(sc, BCE_CTX_VIRT_ADDR, 0);
3431                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
3432
3433                         for (offset = 0; offset < PHY_CTX_SIZE; offset += 4)
3434                                 CTX_WR(sc, 0x00, offset, 0);
3435
3436                         REG_WR(sc, BCE_CTX_VIRT_ADDR, vcid_addr);
3437                         REG_WR(sc, BCE_CTX_PAGE_TBL, vcid_addr);
3438                 }
3439         }
3440         return 0;
3441 }
3442
3443
3444 /****************************************************************************/
3445 /* Fetch the permanent MAC address of the controller.                       */
3446 /*                                                                          */
3447 /* Returns:                                                                 */
3448 /*   Nothing.                                                               */
3449 /****************************************************************************/
3450 static void
3451 bce_get_mac_addr(struct bce_softc *sc)
3452 {
3453         uint32_t mac_lo = 0, mac_hi = 0;
3454
3455         /*
3456          * The NetXtreme II bootcode populates various NIC
3457          * power-on and runtime configuration items in a
3458          * shared memory area.  The factory configured MAC
3459          * address is available from both NVRAM and the
3460          * shared memory area so we'll read the value from
3461          * shared memory for speed.
3462          */
3463
3464         mac_hi = bce_shmem_rd(sc,  BCE_PORT_HW_CFG_MAC_UPPER);
3465         mac_lo = bce_shmem_rd(sc, BCE_PORT_HW_CFG_MAC_LOWER);
3466
3467         if (mac_lo == 0 && mac_hi == 0) {
3468                 if_printf(&sc->arpcom.ac_if, "Invalid Ethernet address!\n");
3469         } else {
3470                 sc->eaddr[0] = (u_char)(mac_hi >> 8);
3471                 sc->eaddr[1] = (u_char)(mac_hi >> 0);
3472                 sc->eaddr[2] = (u_char)(mac_lo >> 24);
3473                 sc->eaddr[3] = (u_char)(mac_lo >> 16);
3474                 sc->eaddr[4] = (u_char)(mac_lo >> 8);
3475                 sc->eaddr[5] = (u_char)(mac_lo >> 0);
3476         }
3477 }
3478
3479
3480 /****************************************************************************/
3481 /* Program the MAC address.                                                 */
3482 /*                                                                          */
3483 /* Returns:                                                                 */
3484 /*   Nothing.                                                               */
3485 /****************************************************************************/
3486 static void
3487 bce_set_mac_addr(struct bce_softc *sc)
3488 {
3489         const uint8_t *mac_addr = sc->eaddr;
3490         uint32_t val;
3491
3492         val = (mac_addr[0] << 8) | mac_addr[1];
3493         REG_WR(sc, BCE_EMAC_MAC_MATCH0, val);
3494
3495         val = (mac_addr[2] << 24) |
3496               (mac_addr[3] << 16) |
3497               (mac_addr[4] << 8) |
3498               mac_addr[5];
3499         REG_WR(sc, BCE_EMAC_MAC_MATCH1, val);
3500 }
3501
3502
3503 /****************************************************************************/
3504 /* Stop the controller.                                                     */
3505 /*                                                                          */
3506 /* Returns:                                                                 */
3507 /*   Nothing.                                                               */
3508 /****************************************************************************/
3509 static void
3510 bce_stop(struct bce_softc *sc)
3511 {
3512         struct ifnet *ifp = &sc->arpcom.ac_if;
3513         int i;
3514
3515         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3516
3517         callout_stop(&sc->bce_tick_callout);
3518
3519         /* Disable the transmit/receive blocks. */
3520         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS, BCE_MISC_ENABLE_CLR_DEFAULT);
3521         REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
3522         DELAY(20);
3523
3524         bce_disable_intr(sc);
3525
3526         ifp->if_flags &= ~IFF_RUNNING;
3527         for (i = 0; i < sc->tx_ring_cnt; ++i) {
3528                 ifsq_clr_oactive(sc->tx_rings[i].ifsq);
3529                 ifsq_watchdog_stop(&sc->tx_rings[i].tx_watchdog);
3530         }
3531
3532         /* Free the RX lists. */
3533         for (i = 0; i < sc->rx_ring_cnt; ++i)
3534                 bce_free_rx_chain(&sc->rx_rings[i]);
3535
3536         /* Free TX buffers. */
3537         for (i = 0; i < sc->tx_ring_cnt; ++i)
3538                 bce_free_tx_chain(&sc->tx_rings[i]);
3539
3540         sc->bce_link = 0;
3541         sc->bce_coalchg_mask = 0;
3542 }
3543
3544
3545 static int
3546 bce_reset(struct bce_softc *sc, uint32_t reset_code)
3547 {
3548         uint32_t val;
3549         int i, rc = 0;
3550
3551         /* Wait for pending PCI transactions to complete. */
3552         REG_WR(sc, BCE_MISC_ENABLE_CLR_BITS,
3553                BCE_MISC_ENABLE_CLR_BITS_TX_DMA_ENABLE |
3554                BCE_MISC_ENABLE_CLR_BITS_DMA_ENGINE_ENABLE |
3555                BCE_MISC_ENABLE_CLR_BITS_RX_DMA_ENABLE |
3556                BCE_MISC_ENABLE_CLR_BITS_HOST_COALESCE_ENABLE);
3557         val = REG_RD(sc, BCE_MISC_ENABLE_CLR_BITS);
3558         DELAY(5);
3559
3560         /* Disable DMA */
3561         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3562             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3563                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
3564                 val &= ~BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
3565                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
3566         }
3567
3568         /* Assume bootcode is running. */
3569         sc->bce_fw_timed_out = 0;
3570         sc->bce_drv_cardiac_arrest = 0;
3571
3572         /* Give the firmware a chance to prepare for the reset. */
3573         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT0 | reset_code);
3574         if (rc) {
3575                 if_printf(&sc->arpcom.ac_if,
3576                           "Firmware is not ready for reset\n");
3577                 return rc;
3578         }
3579
3580         /* Set a firmware reminder that this is a soft reset. */
3581         bce_shmem_wr(sc, BCE_DRV_RESET_SIGNATURE,
3582             BCE_DRV_RESET_SIGNATURE_MAGIC);
3583
3584         /* Dummy read to force the chip to complete all current transactions. */
3585         val = REG_RD(sc, BCE_MISC_ID);
3586
3587         /* Chip reset. */
3588         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3589             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3590                 REG_WR(sc, BCE_MISC_COMMAND, BCE_MISC_COMMAND_SW_RESET);
3591                 REG_RD(sc, BCE_MISC_COMMAND);
3592                 DELAY(5);
3593
3594                 val = BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
3595                     BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
3596
3597                 pci_write_config(sc->bce_dev, BCE_PCICFG_MISC_CONFIG, val, 4);
3598         } else {
3599                 val = BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
3600                     BCE_PCICFG_MISC_CONFIG_REG_WINDOW_ENA |
3601                     BCE_PCICFG_MISC_CONFIG_TARGET_MB_WORD_SWAP;
3602                 REG_WR(sc, BCE_PCICFG_MISC_CONFIG, val);
3603
3604                 /* Allow up to 30us for reset to complete. */
3605                 for (i = 0; i < 10; i++) {
3606                         val = REG_RD(sc, BCE_PCICFG_MISC_CONFIG);
3607                         if ((val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
3608                             BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) == 0)
3609                                 break;
3610                         DELAY(10);
3611                 }
3612
3613                 /* Check that reset completed successfully. */
3614                 if (val & (BCE_PCICFG_MISC_CONFIG_CORE_RST_REQ |
3615                     BCE_PCICFG_MISC_CONFIG_CORE_RST_BSY)) {
3616                         if_printf(&sc->arpcom.ac_if, "Reset failed!\n");
3617                         return EBUSY;
3618                 }
3619         }
3620
3621         /* Make sure byte swapping is properly configured. */
3622         val = REG_RD(sc, BCE_PCI_SWAP_DIAG0);
3623         if (val != 0x01020304) {
3624                 if_printf(&sc->arpcom.ac_if, "Byte swap is incorrect!\n");
3625                 return ENODEV;
3626         }
3627
3628         /* Just completed a reset, assume that firmware is running again. */
3629         sc->bce_fw_timed_out = 0;
3630         sc->bce_drv_cardiac_arrest = 0;
3631
3632         /* Wait for the firmware to finish its initialization. */
3633         rc = bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT1 | reset_code);
3634         if (rc) {
3635                 if_printf(&sc->arpcom.ac_if,
3636                           "Firmware did not complete initialization!\n");
3637         }
3638
3639         if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX) {
3640                 bce_setup_msix_table(sc);
3641                 /* Prevent MSIX table reads and write from timing out */
3642                 REG_WR(sc, BCE_MISC_ECO_HW_CTL,
3643                     BCE_MISC_ECO_HW_CTL_LARGE_GRC_TMOUT_EN);
3644
3645         }
3646         return rc;
3647 }
3648
3649
3650 static int
3651 bce_chipinit(struct bce_softc *sc)
3652 {
3653         uint32_t val;
3654         int rc = 0;
3655
3656         /* Make sure the interrupt is not active. */
3657         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD, BCE_PCICFG_INT_ACK_CMD_MASK_INT);
3658         REG_RD(sc, BCE_PCICFG_INT_ACK_CMD);
3659
3660         /*
3661          * Initialize DMA byte/word swapping, configure the number of DMA
3662          * channels and PCI clock compensation delay.
3663          */
3664         val = BCE_DMA_CONFIG_DATA_BYTE_SWAP |
3665               BCE_DMA_CONFIG_DATA_WORD_SWAP |
3666 #if BYTE_ORDER == BIG_ENDIAN
3667               BCE_DMA_CONFIG_CNTL_BYTE_SWAP |
3668 #endif
3669               BCE_DMA_CONFIG_CNTL_WORD_SWAP |
3670               DMA_READ_CHANS << 12 |
3671               DMA_WRITE_CHANS << 16;
3672
3673         val |= (0x2 << 20) | BCE_DMA_CONFIG_CNTL_PCI_COMP_DLY;
3674
3675         if ((sc->bce_flags & BCE_PCIX_FLAG) && sc->bus_speed_mhz == 133)
3676                 val |= BCE_DMA_CONFIG_PCI_FAST_CLK_CMP;
3677
3678         /*
3679          * This setting resolves a problem observed on certain Intel PCI
3680          * chipsets that cannot handle multiple outstanding DMA operations.
3681          * See errata E9_5706A1_65.
3682          */
3683         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706 &&
3684             BCE_CHIP_ID(sc) != BCE_CHIP_ID_5706_A0 &&
3685             !(sc->bce_flags & BCE_PCIX_FLAG))
3686                 val |= BCE_DMA_CONFIG_CNTL_PING_PONG_DMA;
3687
3688         REG_WR(sc, BCE_DMA_CONFIG, val);
3689
3690         /* Enable the RX_V2P and Context state machines before access. */
3691         REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
3692                BCE_MISC_ENABLE_SET_BITS_HOST_COALESCE_ENABLE |
3693                BCE_MISC_ENABLE_STATUS_BITS_RX_V2P_ENABLE |
3694                BCE_MISC_ENABLE_STATUS_BITS_CONTEXT_ENABLE);
3695
3696         /* Initialize context mapping and zero out the quick contexts. */
3697         rc = bce_init_ctx(sc);
3698         if (rc != 0)
3699                 return rc;
3700
3701         /* Initialize the on-boards CPUs */
3702         bce_init_cpus(sc);
3703
3704         /* Enable management frames (NC-SI) to flow to the MCP. */
3705         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG) {
3706                 val = REG_RD(sc, BCE_RPM_MGMT_PKT_CTRL) |
3707                     BCE_RPM_MGMT_PKT_CTRL_MGMT_EN;
3708                 REG_WR(sc, BCE_RPM_MGMT_PKT_CTRL, val);
3709         }
3710
3711         /* Prepare NVRAM for access. */
3712         rc = bce_init_nvram(sc);
3713         if (rc != 0)
3714                 return rc;
3715
3716         /* Set the kernel bypass block size */
3717         val = REG_RD(sc, BCE_MQ_CONFIG);
3718         val &= ~BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE;
3719         val |= BCE_MQ_CONFIG_KNL_BYP_BLK_SIZE_256;
3720
3721         /* Enable bins used on the 5709/5716. */
3722         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3723             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3724                 val |= BCE_MQ_CONFIG_BIN_MQ_MODE;
3725                 if (BCE_CHIP_ID(sc) == BCE_CHIP_ID_5709_A1)
3726                         val |= BCE_MQ_CONFIG_HALT_DIS;
3727         }
3728
3729         REG_WR(sc, BCE_MQ_CONFIG, val);
3730
3731         val = 0x10000 + (MAX_CID_CNT * MB_KERNEL_CTX_SIZE);
3732         REG_WR(sc, BCE_MQ_KNL_BYP_WIND_START, val);
3733         REG_WR(sc, BCE_MQ_KNL_WIND_END, val);
3734
3735         /* Set the page size and clear the RV2P processor stall bits. */
3736         val = (BCM_PAGE_BITS - 8) << 24;
3737         REG_WR(sc, BCE_RV2P_CONFIG, val);
3738
3739         /* Configure page size. */
3740         val = REG_RD(sc, BCE_TBDR_CONFIG);
3741         val &= ~BCE_TBDR_CONFIG_PAGE_SIZE;
3742         val |= (BCM_PAGE_BITS - 8) << 24 | 0x40;
3743         REG_WR(sc, BCE_TBDR_CONFIG, val);
3744
3745         /* Set the perfect match control register to default. */
3746         REG_WR_IND(sc, BCE_RXP_PM_CTRL, 0);
3747
3748         return 0;
3749 }
3750
3751
3752 /****************************************************************************/
3753 /* Initialize the controller in preparation to send/receive traffic.        */
3754 /*                                                                          */
3755 /* Returns:                                                                 */
3756 /*   0 for success, positive value for failure.                             */
3757 /****************************************************************************/
3758 static int
3759 bce_blockinit(struct bce_softc *sc)
3760 {
3761         uint32_t reg, val;
3762         int i;
3763
3764         /* Load the hardware default MAC address. */
3765         bce_set_mac_addr(sc);
3766
3767         /* Set the Ethernet backoff seed value */
3768         val = sc->eaddr[0] + (sc->eaddr[1] << 8) + (sc->eaddr[2] << 16) +
3769               sc->eaddr[3] + (sc->eaddr[4] << 8) + (sc->eaddr[5] << 16);
3770         REG_WR(sc, BCE_EMAC_BACKOFF_SEED, val);
3771
3772         sc->rx_mode = BCE_EMAC_RX_MODE_SORT_MODE;
3773
3774         /* Set up link change interrupt generation. */
3775         REG_WR(sc, BCE_EMAC_ATTENTION_ENA, BCE_EMAC_ATTENTION_ENA_LINK);
3776
3777         /* Program the physical address of the status block. */
3778         REG_WR(sc, BCE_HC_STATUS_ADDR_L, BCE_ADDR_LO(sc->status_block_paddr));
3779         REG_WR(sc, BCE_HC_STATUS_ADDR_H, BCE_ADDR_HI(sc->status_block_paddr));
3780
3781         /* Program the physical address of the statistics block. */
3782         REG_WR(sc, BCE_HC_STATISTICS_ADDR_L,
3783                BCE_ADDR_LO(sc->stats_block_paddr));
3784         REG_WR(sc, BCE_HC_STATISTICS_ADDR_H,
3785                BCE_ADDR_HI(sc->stats_block_paddr));
3786
3787         /* Program various host coalescing parameters. */
3788         REG_WR(sc, BCE_HC_TX_QUICK_CONS_TRIP,
3789                (sc->bce_tx_quick_cons_trip_int << 16) |
3790                sc->bce_tx_quick_cons_trip);
3791         REG_WR(sc, BCE_HC_RX_QUICK_CONS_TRIP,
3792                (sc->bce_rx_quick_cons_trip_int << 16) |
3793                sc->bce_rx_quick_cons_trip);
3794         REG_WR(sc, BCE_HC_COMP_PROD_TRIP,
3795                (sc->bce_comp_prod_trip_int << 16) | sc->bce_comp_prod_trip);
3796         REG_WR(sc, BCE_HC_TX_TICKS,
3797                (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
3798         REG_WR(sc, BCE_HC_RX_TICKS,
3799                (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
3800         REG_WR(sc, BCE_HC_COM_TICKS,
3801                (sc->bce_com_ticks_int << 16) | sc->bce_com_ticks);
3802         REG_WR(sc, BCE_HC_CMD_TICKS,
3803                (sc->bce_cmd_ticks_int << 16) | sc->bce_cmd_ticks);
3804         REG_WR(sc, BCE_HC_STATS_TICKS, (sc->bce_stats_ticks & 0xffff00));
3805         REG_WR(sc, BCE_HC_STAT_COLLECT_TICKS, 0xbb8);   /* 3ms */
3806
3807         if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX)
3808                 REG_WR(sc, BCE_HC_MSIX_BIT_VECTOR, BCE_HC_MSIX_BIT_VECTOR_VAL);
3809
3810         val = BCE_HC_CONFIG_TX_TMR_MODE | BCE_HC_CONFIG_COLLECT_STATS;
3811         if ((sc->bce_flags & BCE_ONESHOT_MSI_FLAG) ||
3812             sc->bce_irq_type == PCI_INTR_TYPE_MSIX) {
3813                 if (bootverbose) {
3814                         if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX) {
3815                                 if_printf(&sc->arpcom.ac_if,
3816                                     "using MSI-X\n");
3817                         } else {
3818                                 if_printf(&sc->arpcom.ac_if,
3819                                     "using oneshot MSI\n");
3820                         }
3821                 }
3822                 val |= BCE_HC_CONFIG_ONE_SHOT | BCE_HC_CONFIG_USE_INT_PARAM;
3823                 if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX)
3824                         val |= BCE_HC_CONFIG_SB_ADDR_INC_128B;
3825         }
3826         REG_WR(sc, BCE_HC_CONFIG, val);
3827
3828         for (i = 1; i < sc->rx_ring_cnt; ++i) {
3829                 uint32_t base;
3830
3831                 base = ((i - 1) * BCE_HC_SB_CONFIG_SIZE) + BCE_HC_SB_CONFIG_1;
3832                 KKASSERT(base <= BCE_HC_SB_CONFIG_8);
3833
3834                 REG_WR(sc, base,
3835                     BCE_HC_SB_CONFIG_1_TX_TMR_MODE |
3836                     /* BCE_HC_SB_CONFIG_1_RX_TMR_MODE | */
3837                     BCE_HC_SB_CONFIG_1_ONE_SHOT);
3838
3839                 REG_WR(sc, base + BCE_HC_TX_QUICK_CONS_TRIP_OFF,
3840                     (sc->bce_tx_quick_cons_trip_int << 16) |
3841                     sc->bce_tx_quick_cons_trip);
3842                 REG_WR(sc, base + BCE_HC_RX_QUICK_CONS_TRIP_OFF,
3843                     (sc->bce_rx_quick_cons_trip_int << 16) |
3844                     sc->bce_rx_quick_cons_trip);
3845                 REG_WR(sc, base + BCE_HC_TX_TICKS_OFF,
3846                     (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
3847                 REG_WR(sc, base + BCE_HC_RX_TICKS_OFF,
3848                     (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
3849         }
3850
3851         /* Clear the internal statistics counters. */
3852         REG_WR(sc, BCE_HC_COMMAND, BCE_HC_COMMAND_CLR_STAT_NOW);
3853
3854         /* Verify that bootcode is running. */
3855         reg = bce_shmem_rd(sc, BCE_DEV_INFO_SIGNATURE);
3856
3857         if ((reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK) !=
3858             BCE_DEV_INFO_SIGNATURE_MAGIC) {
3859                 if_printf(&sc->arpcom.ac_if,
3860                           "Bootcode not running! Found: 0x%08X, "
3861                           "Expected: 08%08X\n",
3862                           reg & BCE_DEV_INFO_SIGNATURE_MAGIC_MASK,
3863                           BCE_DEV_INFO_SIGNATURE_MAGIC);
3864                 return ENODEV;
3865         }
3866
3867         /* Enable DMA */
3868         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3869             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3870                 val = REG_RD(sc, BCE_MISC_NEW_CORE_CTL);
3871                 val |= BCE_MISC_NEW_CORE_CTL_DMA_ENABLE;
3872                 REG_WR(sc, BCE_MISC_NEW_CORE_CTL, val);
3873         }
3874
3875         /* Allow bootcode to apply any additional fixes before enabling MAC. */
3876         bce_fw_sync(sc, BCE_DRV_MSG_DATA_WAIT2 | BCE_DRV_MSG_CODE_RESET);
3877
3878         /* Enable link state change interrupt generation. */
3879         REG_WR(sc, BCE_HC_ATTN_BITS_ENABLE, STATUS_ATTN_BITS_LINK_STATE);
3880
3881         /* Enable the RXP. */
3882         bce_start_rxp_cpu(sc);
3883
3884         /* Disable management frames (NC-SI) from flowing to the MCP. */
3885         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG) {
3886                 val = REG_RD(sc, BCE_RPM_MGMT_PKT_CTRL) &
3887                     ~BCE_RPM_MGMT_PKT_CTRL_MGMT_EN;
3888                 REG_WR(sc, BCE_RPM_MGMT_PKT_CTRL, val);
3889         }
3890
3891         /* Enable all remaining blocks in the MAC. */
3892         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
3893             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
3894                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
3895                     BCE_MISC_ENABLE_DEFAULT_XI);
3896         } else {
3897                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
3898         }
3899         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
3900         DELAY(20);
3901
3902         /* Save the current host coalescing block settings. */
3903         sc->hc_command = REG_RD(sc, BCE_HC_COMMAND);
3904
3905         return 0;
3906 }
3907
3908
3909 /****************************************************************************/
3910 /* Encapsulate an mbuf cluster into the rx_bd chain.                        */
3911 /*                                                                          */
3912 /* The NetXtreme II can support Jumbo frames by using multiple rx_bd's.     */
3913 /* This routine will map an mbuf cluster into 1 or more rx_bd's as          */
3914 /* necessary.                                                               */
3915 /*                                                                          */
3916 /* Returns:                                                                 */
3917 /*   0 for success, positive value for failure.                             */
3918 /****************************************************************************/
3919 static int
3920 bce_newbuf_std(struct bce_rx_ring *rxr, uint16_t *prod, uint16_t *chain_prod,
3921     uint32_t *prod_bseq, int init)
3922 {
3923         bus_dmamap_t map;
3924         bus_dma_segment_t seg;
3925         struct mbuf *m_new;
3926         int error, nseg;
3927
3928         /* This is a new mbuf allocation. */
3929         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
3930         if (m_new == NULL)
3931                 return ENOBUFS;
3932
3933         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
3934
3935         /* Map the mbuf cluster into device memory. */
3936         error = bus_dmamap_load_mbuf_segment(rxr->rx_mbuf_tag,
3937             rxr->rx_mbuf_tmpmap, m_new, &seg, 1, &nseg, BUS_DMA_NOWAIT);
3938         if (error) {
3939                 m_freem(m_new);
3940                 if (init) {
3941                         if_printf(&rxr->sc->arpcom.ac_if,
3942                             "Error mapping mbuf into RX chain!\n");
3943                 }
3944                 return error;
3945         }
3946
3947         if (rxr->rx_mbuf_ptr[*chain_prod] != NULL) {
3948                 bus_dmamap_unload(rxr->rx_mbuf_tag,
3949                     rxr->rx_mbuf_map[*chain_prod]);
3950         }
3951
3952         map = rxr->rx_mbuf_map[*chain_prod];
3953         rxr->rx_mbuf_map[*chain_prod] = rxr->rx_mbuf_tmpmap;
3954         rxr->rx_mbuf_tmpmap = map;
3955
3956         /* Save the mbuf and update our counter. */
3957         rxr->rx_mbuf_ptr[*chain_prod] = m_new;
3958         rxr->rx_mbuf_paddr[*chain_prod] = seg.ds_addr;
3959         rxr->free_rx_bd--;
3960
3961         bce_setup_rxdesc_std(rxr, *chain_prod, prod_bseq);
3962
3963         return 0;
3964 }
3965
3966
3967 static void
3968 bce_setup_rxdesc_std(struct bce_rx_ring *rxr, uint16_t chain_prod,
3969     uint32_t *prod_bseq)
3970 {
3971         struct rx_bd *rxbd;
3972         bus_addr_t paddr;
3973         int len;
3974
3975         paddr = rxr->rx_mbuf_paddr[chain_prod];
3976         len = rxr->rx_mbuf_ptr[chain_prod]->m_len;
3977
3978         /* Setup the rx_bd for the first segment. */
3979         rxbd = &rxr->rx_bd_chain[RX_PAGE(chain_prod)][RX_IDX(chain_prod)];
3980
3981         rxbd->rx_bd_haddr_lo = htole32(BCE_ADDR_LO(paddr));
3982         rxbd->rx_bd_haddr_hi = htole32(BCE_ADDR_HI(paddr));
3983         rxbd->rx_bd_len = htole32(len);
3984         rxbd->rx_bd_flags = htole32(RX_BD_FLAGS_START);
3985         *prod_bseq += len;
3986
3987         rxbd->rx_bd_flags |= htole32(RX_BD_FLAGS_END);
3988 }
3989
3990
3991 /****************************************************************************/
3992 /* Initialize the TX context memory.                                        */
3993 /*                                                                          */
3994 /* Returns:                                                                 */
3995 /*   Nothing                                                                */
3996 /****************************************************************************/
3997 static void
3998 bce_init_tx_context(struct bce_tx_ring *txr)
3999 {
4000         uint32_t val;
4001
4002         /* Initialize the context ID for an L2 TX chain. */
4003         if (BCE_CHIP_NUM(txr->sc) == BCE_CHIP_NUM_5709 ||
4004             BCE_CHIP_NUM(txr->sc) == BCE_CHIP_NUM_5716) {
4005                 /* Set the CID type to support an L2 connection. */
4006                 val = BCE_L2CTX_TX_TYPE_TYPE_L2 | BCE_L2CTX_TX_TYPE_SIZE_L2;
4007                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4008                     BCE_L2CTX_TX_TYPE_XI, val);
4009                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2 | (8 << 16);
4010                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4011                     BCE_L2CTX_TX_CMD_TYPE_XI, val);
4012
4013                 /* Point the hardware to the first page in the chain. */
4014                 val = BCE_ADDR_HI(txr->tx_bd_chain_paddr[0]);
4015                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4016                     BCE_L2CTX_TX_TBDR_BHADDR_HI_XI, val);
4017                 val = BCE_ADDR_LO(txr->tx_bd_chain_paddr[0]);
4018                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4019                     BCE_L2CTX_TX_TBDR_BHADDR_LO_XI, val);
4020         } else {
4021                 /* Set the CID type to support an L2 connection. */
4022                 val = BCE_L2CTX_TX_TYPE_TYPE_L2 | BCE_L2CTX_TX_TYPE_SIZE_L2;
4023                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4024                     BCE_L2CTX_TX_TYPE, val);
4025                 val = BCE_L2CTX_TX_CMD_TYPE_TYPE_L2 | (8 << 16);
4026                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4027                     BCE_L2CTX_TX_CMD_TYPE, val);
4028
4029                 /* Point the hardware to the first page in the chain. */
4030                 val = BCE_ADDR_HI(txr->tx_bd_chain_paddr[0]);
4031                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4032                     BCE_L2CTX_TX_TBDR_BHADDR_HI, val);
4033                 val = BCE_ADDR_LO(txr->tx_bd_chain_paddr[0]);
4034                 CTX_WR(txr->sc, GET_CID_ADDR(txr->tx_cid),
4035                     BCE_L2CTX_TX_TBDR_BHADDR_LO, val);
4036         }
4037 }
4038
4039
4040 /****************************************************************************/
4041 /* Allocate memory and initialize the TX data structures.                   */
4042 /*                                                                          */
4043 /* Returns:                                                                 */
4044 /*   0 for success, positive value for failure.                             */
4045 /****************************************************************************/
4046 static int
4047 bce_init_tx_chain(struct bce_tx_ring *txr)
4048 {
4049         struct tx_bd *txbd;
4050         int i, rc = 0;
4051
4052         /* Set the initial TX producer/consumer indices. */
4053         txr->tx_prod = 0;
4054         txr->tx_cons = 0;
4055         txr->tx_prod_bseq = 0;
4056         txr->used_tx_bd = 0;
4057         txr->max_tx_bd = USABLE_TX_BD(txr);
4058
4059         /*
4060          * The NetXtreme II supports a linked-list structre called
4061          * a Buffer Descriptor Chain (or BD chain).  A BD chain
4062          * consists of a series of 1 or more chain pages, each of which
4063          * consists of a fixed number of BD entries.
4064          * The last BD entry on each page is a pointer to the next page
4065          * in the chain, and the last pointer in the BD chain
4066          * points back to the beginning of the chain.
4067          */
4068
4069         /* Set the TX next pointer chain entries. */
4070         for (i = 0; i < txr->tx_pages; i++) {
4071                 int j;
4072
4073                 txbd = &txr->tx_bd_chain[i][USABLE_TX_BD_PER_PAGE];
4074
4075                 /* Check if we've reached the last page. */
4076                 if (i == (txr->tx_pages - 1))
4077                         j = 0;
4078                 else
4079                         j = i + 1;
4080
4081                 txbd->tx_bd_haddr_hi =
4082                     htole32(BCE_ADDR_HI(txr->tx_bd_chain_paddr[j]));
4083                 txbd->tx_bd_haddr_lo =
4084                     htole32(BCE_ADDR_LO(txr->tx_bd_chain_paddr[j]));
4085         }
4086         bce_init_tx_context(txr);
4087
4088         return(rc);
4089 }
4090
4091
4092 /****************************************************************************/
4093 /* Free memory and clear the TX data structures.                            */
4094 /*                                                                          */
4095 /* Returns:                                                                 */
4096 /*   Nothing.                                                               */
4097 /****************************************************************************/
4098 static void
4099 bce_free_tx_chain(struct bce_tx_ring *txr)
4100 {
4101         int i;
4102
4103         /* Unmap, unload, and free any mbufs still in the TX mbuf chain. */
4104         for (i = 0; i < TOTAL_TX_BD(txr); i++) {
4105                 if (txr->tx_mbuf_ptr[i] != NULL) {
4106                         bus_dmamap_unload(txr->tx_mbuf_tag,
4107                             txr->tx_mbuf_map[i]);
4108                         m_freem(txr->tx_mbuf_ptr[i]);
4109                         txr->tx_mbuf_ptr[i] = NULL;
4110                 }
4111         }
4112
4113         /* Clear each TX chain page. */
4114         for (i = 0; i < txr->tx_pages; i++)
4115                 bzero(txr->tx_bd_chain[i], BCE_TX_CHAIN_PAGE_SZ);
4116         txr->used_tx_bd = 0;
4117 }
4118
4119
4120 /****************************************************************************/
4121 /* Initialize the RX context memory.                                        */
4122 /*                                                                          */
4123 /* Returns:                                                                 */
4124 /*   Nothing                                                                */
4125 /****************************************************************************/
4126 static void
4127 bce_init_rx_context(struct bce_rx_ring *rxr)
4128 {
4129         uint32_t val;
4130
4131         /* Initialize the context ID for an L2 RX chain. */
4132         val = BCE_L2CTX_RX_CTX_TYPE_CTX_BD_CHN_TYPE_VALUE |
4133             BCE_L2CTX_RX_CTX_TYPE_SIZE_L2 | (0x02 << 8);
4134
4135         /*
4136          * Set the level for generating pause frames
4137          * when the number of available rx_bd's gets
4138          * too low (the low watermark) and the level
4139          * when pause frames can be stopped (the high
4140          * watermark).
4141          */
4142         if (BCE_CHIP_NUM(rxr->sc) == BCE_CHIP_NUM_5709 ||
4143             BCE_CHIP_NUM(rxr->sc) == BCE_CHIP_NUM_5716) {
4144                 uint32_t lo_water, hi_water;
4145
4146                 lo_water = BCE_L2CTX_RX_LO_WATER_MARK_DEFAULT;
4147                 hi_water = USABLE_RX_BD(rxr) / 4;
4148
4149                 lo_water /= BCE_L2CTX_RX_LO_WATER_MARK_SCALE;
4150                 hi_water /= BCE_L2CTX_RX_HI_WATER_MARK_SCALE;
4151
4152                 if (hi_water > 0xf)
4153                         hi_water = 0xf;
4154                 else if (hi_water == 0)
4155                         lo_water = 0;
4156                 val |= lo_water |
4157                     (hi_water << BCE_L2CTX_RX_HI_WATER_MARK_SHIFT);
4158         }
4159
4160         CTX_WR(rxr->sc, GET_CID_ADDR(rxr->rx_cid),
4161             BCE_L2CTX_RX_CTX_TYPE, val);
4162
4163         /* Setup the MQ BIN mapping for l2_ctx_host_bseq. */
4164         if (BCE_CHIP_NUM(rxr->sc) == BCE_CHIP_NUM_5709 ||
4165             BCE_CHIP_NUM(rxr->sc) == BCE_CHIP_NUM_5716) {
4166                 val = REG_RD(rxr->sc, BCE_MQ_MAP_L2_5);
4167                 REG_WR(rxr->sc, BCE_MQ_MAP_L2_5, val | BCE_MQ_MAP_L2_5_ARM);
4168         }
4169
4170         /* Point the hardware to the first page in the chain. */
4171         val = BCE_ADDR_HI(rxr->rx_bd_chain_paddr[0]);
4172         CTX_WR(rxr->sc, GET_CID_ADDR(rxr->rx_cid),
4173             BCE_L2CTX_RX_NX_BDHADDR_HI, val);
4174         val = BCE_ADDR_LO(rxr->rx_bd_chain_paddr[0]);
4175         CTX_WR(rxr->sc, GET_CID_ADDR(rxr->rx_cid),
4176             BCE_L2CTX_RX_NX_BDHADDR_LO, val);
4177 }
4178
4179
4180 /****************************************************************************/
4181 /* Allocate memory and initialize the RX data structures.                   */
4182 /*                                                                          */
4183 /* Returns:                                                                 */
4184 /*   0 for success, positive value for failure.                             */
4185 /****************************************************************************/
4186 static int
4187 bce_init_rx_chain(struct bce_rx_ring *rxr)
4188 {
4189         struct rx_bd *rxbd;
4190         int i, rc = 0;
4191         uint16_t prod, chain_prod;
4192         uint32_t prod_bseq;
4193
4194         /* Initialize the RX producer and consumer indices. */
4195         rxr->rx_prod = 0;
4196         rxr->rx_cons = 0;
4197         rxr->rx_prod_bseq = 0;
4198         rxr->free_rx_bd = USABLE_RX_BD(rxr);
4199         rxr->max_rx_bd = USABLE_RX_BD(rxr);
4200
4201         /* Clear cache status index */
4202         rxr->last_status_idx = 0;
4203
4204         /* Initialize the RX next pointer chain entries. */
4205         for (i = 0; i < rxr->rx_pages; i++) {
4206                 int j;
4207
4208                 rxbd = &rxr->rx_bd_chain[i][USABLE_RX_BD_PER_PAGE];
4209
4210                 /* Check if we've reached the last page. */
4211                 if (i == (rxr->rx_pages - 1))
4212                         j = 0;
4213                 else
4214                         j = i + 1;
4215
4216                 /* Setup the chain page pointers. */
4217                 rxbd->rx_bd_haddr_hi =
4218                     htole32(BCE_ADDR_HI(rxr->rx_bd_chain_paddr[j]));
4219                 rxbd->rx_bd_haddr_lo =
4220                     htole32(BCE_ADDR_LO(rxr->rx_bd_chain_paddr[j]));
4221         }
4222
4223         /* Allocate mbuf clusters for the rx_bd chain. */
4224         prod = prod_bseq = 0;
4225         while (prod < TOTAL_RX_BD(rxr)) {
4226                 chain_prod = RX_CHAIN_IDX(rxr, prod);
4227                 if (bce_newbuf_std(rxr, &prod, &chain_prod, &prod_bseq, 1)) {
4228                         if_printf(&rxr->sc->arpcom.ac_if,
4229                             "Error filling RX chain: rx_bd[0x%04X]!\n",
4230                             chain_prod);
4231                         rc = ENOBUFS;
4232                         break;
4233                 }
4234                 prod = NEXT_RX_BD(prod);
4235         }
4236
4237         /* Save the RX chain producer index. */
4238         rxr->rx_prod = prod;
4239         rxr->rx_prod_bseq = prod_bseq;
4240
4241         /* Tell the chip about the waiting rx_bd's. */
4242         REG_WR16(rxr->sc, MB_GET_CID_ADDR(rxr->rx_cid) + BCE_L2MQ_RX_HOST_BDIDX,
4243             rxr->rx_prod);
4244         REG_WR(rxr->sc, MB_GET_CID_ADDR(rxr->rx_cid) + BCE_L2MQ_RX_HOST_BSEQ,
4245             rxr->rx_prod_bseq);
4246
4247         bce_init_rx_context(rxr);
4248
4249         return(rc);
4250 }
4251
4252
4253 /****************************************************************************/
4254 /* Free memory and clear the RX data structures.                            */
4255 /*                                                                          */
4256 /* Returns:                                                                 */
4257 /*   Nothing.                                                               */
4258 /****************************************************************************/
4259 static void
4260 bce_free_rx_chain(struct bce_rx_ring *rxr)
4261 {
4262         int i;
4263
4264         /* Free any mbufs still in the RX mbuf chain. */
4265         for (i = 0; i < TOTAL_RX_BD(rxr); i++) {
4266                 if (rxr->rx_mbuf_ptr[i] != NULL) {
4267                         bus_dmamap_unload(rxr->rx_mbuf_tag,
4268                             rxr->rx_mbuf_map[i]);
4269                         m_freem(rxr->rx_mbuf_ptr[i]);
4270                         rxr->rx_mbuf_ptr[i] = NULL;
4271                 }
4272         }
4273
4274         /* Clear each RX chain page. */
4275         for (i = 0; i < rxr->rx_pages; i++)
4276                 bzero(rxr->rx_bd_chain[i], BCE_RX_CHAIN_PAGE_SZ);
4277 }
4278
4279
4280 /****************************************************************************/
4281 /* Set media options.                                                       */
4282 /*                                                                          */
4283 /* Returns:                                                                 */
4284 /*   0 for success, positive value for failure.                             */
4285 /****************************************************************************/
4286 static int
4287 bce_ifmedia_upd(struct ifnet *ifp)
4288 {
4289         struct bce_softc *sc = ifp->if_softc;
4290         struct mii_data *mii = device_get_softc(sc->bce_miibus);
4291         int error = 0;
4292
4293         /*
4294          * 'mii' will be NULL, when this function is called on following
4295          * code path: bce_attach() -> bce_mgmt_init()
4296          */
4297         if (mii != NULL) {
4298                 /* Make sure the MII bus has been enumerated. */
4299                 sc->bce_link = 0;
4300                 if (mii->mii_instance) {
4301                         struct mii_softc *miisc;
4302
4303                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
4304                                 mii_phy_reset(miisc);
4305                 }
4306                 error = mii_mediachg(mii);
4307         }
4308         return error;
4309 }
4310
4311
4312 /****************************************************************************/
4313 /* Reports current media status.                                            */
4314 /*                                                                          */
4315 /* Returns:                                                                 */
4316 /*   Nothing.                                                               */
4317 /****************************************************************************/
4318 static void
4319 bce_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
4320 {
4321         struct bce_softc *sc = ifp->if_softc;
4322         struct mii_data *mii = device_get_softc(sc->bce_miibus);
4323
4324         mii_pollstat(mii);
4325         ifmr->ifm_active = mii->mii_media_active;
4326         ifmr->ifm_status = mii->mii_media_status;
4327 }
4328
4329
4330 /****************************************************************************/
4331 /* Handles PHY generated interrupt events.                                  */
4332 /*                                                                          */
4333 /* Returns:                                                                 */
4334 /*   Nothing.                                                               */
4335 /****************************************************************************/
4336 static void
4337 bce_phy_intr(struct bce_softc *sc)
4338 {
4339         uint32_t new_link_state, old_link_state;
4340         struct ifnet *ifp = &sc->arpcom.ac_if;
4341
4342         ASSERT_SERIALIZED(&sc->main_serialize);
4343
4344         new_link_state = sc->status_block->status_attn_bits &
4345                          STATUS_ATTN_BITS_LINK_STATE;
4346         old_link_state = sc->status_block->status_attn_bits_ack &
4347                          STATUS_ATTN_BITS_LINK_STATE;
4348
4349         /* Handle any changes if the link state has changed. */
4350         if (new_link_state != old_link_state) { /* XXX redundant? */
4351                 /* Update the status_attn_bits_ack field in the status block. */
4352                 if (new_link_state) {
4353                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_SET_CMD,
4354                                STATUS_ATTN_BITS_LINK_STATE);
4355                         if (bootverbose)
4356                                 if_printf(ifp, "Link is now UP.\n");
4357                 } else {
4358                         REG_WR(sc, BCE_PCICFG_STATUS_BIT_CLEAR_CMD,
4359                                STATUS_ATTN_BITS_LINK_STATE);
4360                         if (bootverbose)
4361                                 if_printf(ifp, "Link is now DOWN.\n");
4362                 }
4363
4364                 /*
4365                  * Assume link is down and allow tick routine to
4366                  * update the state based on the actual media state.
4367                  */
4368                 sc->bce_link = 0;
4369                 callout_stop(&sc->bce_tick_callout);
4370                 bce_tick_serialized(sc);
4371         }
4372
4373         /* Acknowledge the link change interrupt. */
4374         REG_WR(sc, BCE_EMAC_STATUS, BCE_EMAC_STATUS_LINK_CHANGE);
4375 }
4376
4377
4378 /****************************************************************************/
4379 /* Reads the receive consumer value from the status block (skipping over    */
4380 /* chain page pointer if necessary).                                        */
4381 /*                                                                          */
4382 /* Returns:                                                                 */
4383 /*   hw_cons                                                                */
4384 /****************************************************************************/
4385 static __inline uint16_t
4386 bce_get_hw_rx_cons(struct bce_rx_ring *rxr)
4387 {
4388         uint16_t hw_cons = *rxr->rx_hw_cons;
4389
4390         if ((hw_cons & USABLE_RX_BD_PER_PAGE) == USABLE_RX_BD_PER_PAGE)
4391                 hw_cons++;
4392         return hw_cons;
4393 }
4394
4395
4396 /****************************************************************************/
4397 /* Handles received frame interrupt events.                                 */
4398 /*                                                                          */
4399 /* Returns:                                                                 */
4400 /*   Nothing.                                                               */
4401 /****************************************************************************/
4402 static void
4403 bce_rx_intr(struct bce_rx_ring *rxr, int count, uint16_t hw_cons)
4404 {
4405         struct ifnet *ifp = &rxr->sc->arpcom.ac_if;
4406         uint16_t sw_cons, sw_chain_cons, sw_prod, sw_chain_prod;
4407         uint32_t sw_prod_bseq;
4408
4409         ASSERT_SERIALIZED(&rxr->rx_serialize);
4410
4411         /* Get working copies of the driver's view of the RX indices. */
4412         sw_cons = rxr->rx_cons;
4413         sw_prod = rxr->rx_prod;
4414         sw_prod_bseq = rxr->rx_prod_bseq;
4415
4416         /* Scan through the receive chain as long as there is work to do. */
4417         while (sw_cons != hw_cons) {
4418                 struct mbuf *m = NULL;
4419                 struct l2_fhdr *l2fhdr = NULL;
4420                 unsigned int len;
4421                 uint32_t status = 0;
4422
4423 #ifdef IFPOLL_ENABLE
4424                 if (count >= 0 && count-- == 0)
4425                         break;
4426 #endif
4427
4428                 /*
4429                  * Convert the producer/consumer indices
4430                  * to an actual rx_bd index.
4431                  */
4432                 sw_chain_cons = RX_CHAIN_IDX(rxr, sw_cons);
4433                 sw_chain_prod = RX_CHAIN_IDX(rxr, sw_prod);
4434
4435                 rxr->free_rx_bd++;
4436
4437                 /* The mbuf is stored with the last rx_bd entry of a packet. */
4438                 if (rxr->rx_mbuf_ptr[sw_chain_cons] != NULL) {
4439                         if (sw_chain_cons != sw_chain_prod) {
4440                                 if_printf(ifp, "RX cons(%d) != prod(%d), "
4441                                     "drop!\n", sw_chain_cons, sw_chain_prod);
4442                                 IFNET_STAT_INC(ifp, ierrors, 1);
4443
4444                                 bce_setup_rxdesc_std(rxr, sw_chain_cons,
4445                                     &sw_prod_bseq);
4446                                 m = NULL;
4447                                 goto bce_rx_int_next_rx;
4448                         }
4449
4450                         /* Unmap the mbuf from DMA space. */
4451                         bus_dmamap_sync(rxr->rx_mbuf_tag,
4452                             rxr->rx_mbuf_map[sw_chain_cons],
4453                             BUS_DMASYNC_POSTREAD);
4454
4455                         /* Save the mbuf from the driver's chain. */
4456                         m = rxr->rx_mbuf_ptr[sw_chain_cons];
4457
4458                         /*
4459                          * Frames received on the NetXteme II are prepended
4460                          * with an l2_fhdr structure which provides status
4461                          * information about the received frame (including
4462                          * VLAN tags and checksum info).  The frames are also
4463                          * automatically adjusted to align the IP header
4464                          * (i.e. two null bytes are inserted before the
4465                          * Ethernet header).  As a result the data DMA'd by
4466                          * the controller into the mbuf is as follows:
4467                          *
4468                          * +---------+-----+---------------------+-----+
4469                          * | l2_fhdr | pad | packet data         | FCS |
4470                          * +---------+-----+---------------------+-----+
4471                          *
4472                          * The l2_fhdr needs to be checked and skipped and the
4473                          * FCS needs to be stripped before sending the packet
4474                          * up the stack.
4475                          */
4476                         l2fhdr = mtod(m, struct l2_fhdr *);
4477
4478                         len = l2fhdr->l2_fhdr_pkt_len;
4479                         status = l2fhdr->l2_fhdr_status;
4480
4481                         len -= ETHER_CRC_LEN;
4482
4483                         /* Check the received frame for errors. */
4484                         if (status & (L2_FHDR_ERRORS_BAD_CRC |
4485                                       L2_FHDR_ERRORS_PHY_DECODE |
4486                                       L2_FHDR_ERRORS_ALIGNMENT |
4487                                       L2_FHDR_ERRORS_TOO_SHORT |
4488                                       L2_FHDR_ERRORS_GIANT_FRAME)) {
4489                                 IFNET_STAT_INC(ifp, ierrors, 1);
4490
4491                                 /* Reuse the mbuf for a new frame. */
4492                                 bce_setup_rxdesc_std(rxr, sw_chain_prod,
4493                                     &sw_prod_bseq);
4494                                 m = NULL;
4495                                 goto bce_rx_int_next_rx;
4496                         }
4497
4498                         /*
4499                          * Get a new mbuf for the rx_bd.   If no new
4500                          * mbufs are available then reuse the current mbuf,
4501                          * log an ierror on the interface, and generate
4502                          * an error in the system log.
4503                          */
4504                         if (bce_newbuf_std(rxr, &sw_prod, &sw_chain_prod,
4505                             &sw_prod_bseq, 0)) {
4506                                 IFNET_STAT_INC(ifp, ierrors, 1);
4507
4508                                 /* Try and reuse the exisitng mbuf. */
4509                                 bce_setup_rxdesc_std(rxr, sw_chain_prod,
4510                                     &sw_prod_bseq);
4511                                 m = NULL;
4512                                 goto bce_rx_int_next_rx;
4513                         }
4514
4515                         /*
4516                          * Skip over the l2_fhdr when passing
4517                          * the data up the stack.
4518                          */
4519                         m_adj(m, sizeof(struct l2_fhdr) + ETHER_ALIGN);
4520
4521                         m->m_pkthdr.len = m->m_len = len;
4522                         m->m_pkthdr.rcvif = ifp;
4523
4524                         /* Validate the checksum if offload enabled. */
4525                         if (ifp->if_capenable & IFCAP_RXCSUM) {
4526                                 /* Check for an IP datagram. */
4527                                 if (status & L2_FHDR_STATUS_IP_DATAGRAM) {
4528                                         m->m_pkthdr.csum_flags |=
4529                                                 CSUM_IP_CHECKED;
4530
4531                                         /* Check if the IP checksum is valid. */
4532                                         if ((l2fhdr->l2_fhdr_ip_xsum ^
4533                                              0xffff) == 0) {
4534                                                 m->m_pkthdr.csum_flags |=
4535                                                         CSUM_IP_VALID;
4536                                         }
4537                                 }
4538
4539                                 /* Check for a valid TCP/UDP frame. */
4540                                 if (status & (L2_FHDR_STATUS_TCP_SEGMENT |
4541                                               L2_FHDR_STATUS_UDP_DATAGRAM)) {
4542
4543                                         /* Check for a good TCP/UDP checksum. */
4544                                         if ((status &
4545                                              (L2_FHDR_ERRORS_TCP_XSUM |
4546                                               L2_FHDR_ERRORS_UDP_XSUM)) == 0) {
4547                                                 m->m_pkthdr.csum_data =
4548                                                 l2fhdr->l2_fhdr_tcp_udp_xsum;
4549                                                 m->m_pkthdr.csum_flags |=
4550                                                         CSUM_DATA_VALID |
4551                                                         CSUM_PSEUDO_HDR;
4552                                         }
4553                                 }
4554                         }
4555
4556                         IFNET_STAT_INC(ifp, ipackets, 1);
4557 bce_rx_int_next_rx:
4558                         sw_prod = NEXT_RX_BD(sw_prod);
4559                 }
4560
4561                 sw_cons = NEXT_RX_BD(sw_cons);
4562
4563                 /* If we have a packet, pass it up the stack */
4564                 if (m) {
4565                         if (status & L2_FHDR_STATUS_L2_VLAN_TAG) {
4566                                 m->m_flags |= M_VLANTAG;
4567                                 m->m_pkthdr.ether_vlantag =
4568                                         l2fhdr->l2_fhdr_vlan_tag;
4569                         }
4570                         ifp->if_input(ifp, m);
4571                         rxr->rx_pkts++;
4572                 }
4573         }
4574
4575         rxr->rx_cons = sw_cons;
4576         rxr->rx_prod = sw_prod;
4577         rxr->rx_prod_bseq = sw_prod_bseq;
4578
4579         REG_WR16(rxr->sc, MB_GET_CID_ADDR(rxr->rx_cid) + BCE_L2MQ_RX_HOST_BDIDX,
4580             rxr->rx_prod);
4581         REG_WR(rxr->sc, MB_GET_CID_ADDR(rxr->rx_cid) + BCE_L2MQ_RX_HOST_BSEQ,
4582             rxr->rx_prod_bseq);
4583 }
4584
4585
4586 /****************************************************************************/
4587 /* Reads the transmit consumer value from the status block (skipping over   */
4588 /* chain page pointer if necessary).                                        */
4589 /*                                                                          */
4590 /* Returns:                                                                 */
4591 /*   hw_cons                                                                */
4592 /****************************************************************************/
4593 static __inline uint16_t
4594 bce_get_hw_tx_cons(struct bce_tx_ring *txr)
4595 {
4596         uint16_t hw_cons = *txr->tx_hw_cons;
4597
4598         if ((hw_cons & USABLE_TX_BD_PER_PAGE) == USABLE_TX_BD_PER_PAGE)
4599                 hw_cons++;
4600         return hw_cons;
4601 }
4602
4603
4604 /****************************************************************************/
4605 /* Handles transmit completion interrupt events.                            */
4606 /*                                                                          */
4607 /* Returns:                                                                 */
4608 /*   Nothing.                                                               */
4609 /****************************************************************************/
4610 static void
4611 bce_tx_intr(struct bce_tx_ring *txr, uint16_t hw_tx_cons)
4612 {
4613         struct ifnet *ifp = &txr->sc->arpcom.ac_if;
4614         uint16_t sw_tx_cons, sw_tx_chain_cons;
4615
4616         ASSERT_SERIALIZED(&txr->tx_serialize);
4617
4618         /* Get the hardware's view of the TX consumer index. */
4619         sw_tx_cons = txr->tx_cons;
4620
4621         /* Cycle through any completed TX chain page entries. */
4622         while (sw_tx_cons != hw_tx_cons) {
4623                 sw_tx_chain_cons = TX_CHAIN_IDX(txr, sw_tx_cons);
4624
4625                 /*
4626                  * Free the associated mbuf. Remember
4627                  * that only the last tx_bd of a packet
4628                  * has an mbuf pointer and DMA map.
4629                  */
4630                 if (txr->tx_mbuf_ptr[sw_tx_chain_cons] != NULL) {
4631                         /* Unmap the mbuf. */
4632                         bus_dmamap_unload(txr->tx_mbuf_tag,
4633                             txr->tx_mbuf_map[sw_tx_chain_cons]);
4634
4635                         /* Free the mbuf. */
4636                         m_freem(txr->tx_mbuf_ptr[sw_tx_chain_cons]);
4637                         txr->tx_mbuf_ptr[sw_tx_chain_cons] = NULL;
4638
4639                         IFNET_STAT_INC(ifp, opackets, 1);
4640 #ifdef BCE_TSS_DEBUG
4641                         txr->tx_pkts++;
4642 #endif
4643                 }
4644
4645                 txr->used_tx_bd--;
4646                 sw_tx_cons = NEXT_TX_BD(sw_tx_cons);
4647         }
4648
4649         if (txr->used_tx_bd == 0) {
4650                 /* Clear the TX timeout timer. */
4651                 txr->tx_watchdog.wd_timer = 0;
4652         }
4653
4654         /* Clear the tx hardware queue full flag. */
4655         if (txr->max_tx_bd - txr->used_tx_bd >= BCE_TX_SPARE_SPACE)
4656                 ifsq_clr_oactive(txr->ifsq);
4657         txr->tx_cons = sw_tx_cons;
4658 }
4659
4660
4661 /****************************************************************************/
4662 /* Disables interrupt generation.                                           */
4663 /*                                                                          */
4664 /* Returns:                                                                 */
4665 /*   Nothing.                                                               */
4666 /****************************************************************************/
4667 static void
4668 bce_disable_intr(struct bce_softc *sc)
4669 {
4670         int i;
4671
4672         for (i = 0; i < sc->rx_ring_cnt; ++i) {
4673                 REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
4674                     (sc->rx_rings[i].idx << 24) |
4675                     BCE_PCICFG_INT_ACK_CMD_MASK_INT);
4676         }
4677         REG_RD(sc, BCE_PCICFG_INT_ACK_CMD);
4678
4679         callout_stop(&sc->bce_ckmsi_callout);
4680         sc->bce_msi_maylose = FALSE;
4681         sc->bce_check_rx_cons = 0;
4682         sc->bce_check_tx_cons = 0;
4683         sc->bce_check_status_idx = 0xffff;
4684
4685         for (i = 0; i < sc->rx_ring_cnt; ++i)
4686                 lwkt_serialize_handler_disable(sc->bce_msix[i].msix_serialize);
4687 }
4688
4689
4690 /****************************************************************************/
4691 /* Enables interrupt generation.                                            */
4692 /*                                                                          */
4693 /* Returns:                                                                 */
4694 /*   Nothing.                                                               */
4695 /****************************************************************************/
4696 static void
4697 bce_enable_intr(struct bce_softc *sc)
4698 {
4699         int i;
4700
4701         for (i = 0; i < sc->rx_ring_cnt; ++i)
4702                 lwkt_serialize_handler_enable(sc->bce_msix[i].msix_serialize);
4703
4704         for (i = 0; i < sc->rx_ring_cnt; ++i) {
4705                 struct bce_rx_ring *rxr = &sc->rx_rings[i];
4706
4707                 REG_WR(sc, BCE_PCICFG_INT_ACK_CMD, (rxr->idx << 24) |
4708                        BCE_PCICFG_INT_ACK_CMD_INDEX_VALID |
4709                        BCE_PCICFG_INT_ACK_CMD_MASK_INT |
4710                        rxr->last_status_idx);
4711                 REG_WR(sc, BCE_PCICFG_INT_ACK_CMD, (rxr->idx << 24) |
4712                        BCE_PCICFG_INT_ACK_CMD_INDEX_VALID |
4713                        rxr->last_status_idx);
4714         }
4715         REG_WR(sc, BCE_HC_COMMAND, sc->hc_command | BCE_HC_COMMAND_COAL_NOW);
4716
4717         if (sc->bce_flags & BCE_CHECK_MSI_FLAG) {
4718                 sc->bce_msi_maylose = FALSE;
4719                 sc->bce_check_rx_cons = 0;
4720                 sc->bce_check_tx_cons = 0;
4721                 sc->bce_check_status_idx = 0xffff;
4722
4723                 if (bootverbose)
4724                         if_printf(&sc->arpcom.ac_if, "check msi\n");
4725
4726                 callout_reset_bycpu(&sc->bce_ckmsi_callout, BCE_MSI_CKINTVL,
4727                     bce_check_msi, sc, sc->bce_msix[0].msix_cpuid);
4728         }
4729 }
4730
4731
4732 /****************************************************************************/
4733 /* Reenables interrupt generation during interrupt handling.                */
4734 /*                                                                          */
4735 /* Returns:                                                                 */
4736 /*   Nothing.                                                               */
4737 /****************************************************************************/
4738 static void
4739 bce_reenable_intr(struct bce_rx_ring *rxr)
4740 {
4741         REG_WR(rxr->sc, BCE_PCICFG_INT_ACK_CMD, (rxr->idx << 24) |
4742                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID | rxr->last_status_idx);
4743 }
4744
4745
4746 /****************************************************************************/
4747 /* Handles controller initialization.                                       */
4748 /*                                                                          */
4749 /* Returns:                                                                 */
4750 /*   Nothing.                                                               */
4751 /****************************************************************************/
4752 static void
4753 bce_init(void *xsc)
4754 {
4755         struct bce_softc *sc = xsc;
4756         struct ifnet *ifp = &sc->arpcom.ac_if;
4757         uint32_t ether_mtu;
4758         int error, i;
4759         boolean_t polling;
4760
4761         ASSERT_IFNET_SERIALIZED_ALL(ifp);
4762
4763         /* Check if the driver is still running and bail out if it is. */
4764         if (ifp->if_flags & IFF_RUNNING)
4765                 return;
4766
4767         bce_stop(sc);
4768
4769         error = bce_reset(sc, BCE_DRV_MSG_CODE_RESET);
4770         if (error) {
4771                 if_printf(ifp, "Controller reset failed!\n");
4772                 goto back;
4773         }
4774
4775         error = bce_chipinit(sc);
4776         if (error) {
4777                 if_printf(ifp, "Controller initialization failed!\n");
4778                 goto back;
4779         }
4780
4781         error = bce_blockinit(sc);
4782         if (error) {
4783                 if_printf(ifp, "Block initialization failed!\n");
4784                 goto back;
4785         }
4786
4787         /* Load our MAC address. */
4788         bcopy(IF_LLADDR(ifp), sc->eaddr, ETHER_ADDR_LEN);
4789         bce_set_mac_addr(sc);
4790
4791         /* Calculate and program the Ethernet MTU size. */
4792         ether_mtu = ETHER_HDR_LEN + EVL_ENCAPLEN + ifp->if_mtu + ETHER_CRC_LEN;
4793
4794         /*
4795          * Program the mtu, enabling jumbo frame
4796          * support if necessary.  Also set the mbuf
4797          * allocation count for RX frames.
4798          */
4799         if (ether_mtu > ETHER_MAX_LEN + EVL_ENCAPLEN) {
4800 #ifdef notyet
4801                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE,
4802                        min(ether_mtu, BCE_MAX_JUMBO_ETHER_MTU) |
4803                        BCE_EMAC_RX_MTU_SIZE_JUMBO_ENA);
4804 #else
4805                 panic("jumbo buffer is not supported yet");
4806 #endif
4807         } else {
4808                 REG_WR(sc, BCE_EMAC_RX_MTU_SIZE, ether_mtu);
4809         }
4810
4811         /* Program appropriate promiscuous/multicast filtering. */
4812         bce_set_rx_mode(sc);
4813
4814         /*
4815          * Init RX buffer descriptor chain.
4816          */
4817         REG_WR(sc, BCE_RLUP_RSS_CONFIG, 0);
4818         bce_reg_wr_ind(sc, BCE_RXP_SCRATCH_RSS_TBL_SZ, 0);
4819
4820         for (i = 0; i < sc->rx_ring_cnt; ++i)
4821                 bce_init_rx_chain(&sc->rx_rings[i]);    /* XXX return value */
4822
4823         if (sc->rx_ring_cnt > 1)
4824                 bce_init_rss(sc);
4825
4826         /*
4827          * Init TX buffer descriptor chain.
4828          */
4829         REG_WR(sc, BCE_TSCH_TSS_CFG, 0);
4830
4831         for (i = 0; i < sc->tx_ring_cnt; ++i)
4832                 bce_init_tx_chain(&sc->tx_rings[i]);
4833
4834         if (sc->tx_ring_cnt > 1) {
4835                 REG_WR(sc, BCE_TSCH_TSS_CFG,
4836                     ((sc->tx_ring_cnt - 1) << 24) | (TX_TSS_CID << 7));
4837         }
4838
4839         polling = FALSE;
4840 #ifdef IFPOLL_ENABLE
4841         if (ifp->if_flags & IFF_NPOLLING)
4842                 polling = TRUE;
4843 #endif
4844
4845         if (polling) {
4846                 /* Disable interrupts if we are polling. */
4847                 bce_disable_intr(sc);
4848
4849                 /* Change coalesce parameters */
4850                 bce_npoll_coal_change(sc);
4851         } else {
4852                 /* Enable host interrupts. */
4853                 bce_enable_intr(sc);
4854         }
4855         bce_set_timer_cpuid(sc, polling);
4856
4857         bce_ifmedia_upd(ifp);
4858
4859         ifp->if_flags |= IFF_RUNNING;
4860         for (i = 0; i < sc->tx_ring_cnt; ++i) {
4861                 ifsq_clr_oactive(sc->tx_rings[i].ifsq);
4862                 ifsq_watchdog_start(&sc->tx_rings[i].tx_watchdog);
4863         }
4864
4865         callout_reset_bycpu(&sc->bce_tick_callout, hz, bce_tick, sc,
4866             sc->bce_timer_cpuid);
4867 back:
4868         if (error)
4869                 bce_stop(sc);
4870 }
4871
4872
4873 /****************************************************************************/
4874 /* Initialize the controller just enough so that any management firmware    */
4875 /* running on the device will continue to operate corectly.                 */
4876 /*                                                                          */
4877 /* Returns:                                                                 */
4878 /*   Nothing.                                                               */
4879 /****************************************************************************/
4880 static void
4881 bce_mgmt_init(struct bce_softc *sc)
4882 {
4883         struct ifnet *ifp = &sc->arpcom.ac_if;
4884
4885         /* Bail out if management firmware is not running. */
4886         if (!(sc->bce_flags & BCE_MFW_ENABLE_FLAG))
4887                 return;
4888
4889         /* Enable all critical blocks in the MAC. */
4890         if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709 ||
4891             BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5716) {
4892                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS,
4893                     BCE_MISC_ENABLE_DEFAULT_XI);
4894         } else {
4895                 REG_WR(sc, BCE_MISC_ENABLE_SET_BITS, BCE_MISC_ENABLE_DEFAULT);
4896         }
4897         REG_RD(sc, BCE_MISC_ENABLE_SET_BITS);
4898         DELAY(20);
4899
4900         bce_ifmedia_upd(ifp);
4901 }
4902
4903
4904 /****************************************************************************/
4905 /* Encapsultes an mbuf cluster into the tx_bd chain structure and makes the */
4906 /* memory visible to the controller.                                        */
4907 /*                                                                          */
4908 /* Returns:                                                                 */
4909 /*   0 for success, positive value for failure.                             */
4910 /****************************************************************************/
4911 static int
4912 bce_encap(struct bce_tx_ring *txr, struct mbuf **m_head, int *nsegs_used)
4913 {
4914         bus_dma_segment_t segs[BCE_MAX_SEGMENTS];
4915         bus_dmamap_t map, tmp_map;
4916         struct mbuf *m0 = *m_head;
4917         struct tx_bd *txbd = NULL;
4918         uint16_t vlan_tag = 0, flags = 0, mss = 0;
4919         uint16_t chain_prod, chain_prod_start, prod;
4920         uint32_t prod_bseq;
4921         int i, error, maxsegs, nsegs;
4922
4923         /* Transfer any checksum offload flags to the bd. */
4924         if (m0->m_pkthdr.csum_flags & CSUM_TSO) {
4925                 error = bce_tso_setup(txr, m_head, &flags, &mss);
4926                 if (error)
4927                         return ENOBUFS;
4928                 m0 = *m_head;
4929         } else if (m0->m_pkthdr.csum_flags & BCE_CSUM_FEATURES) {
4930                 if (m0->m_pkthdr.csum_flags & CSUM_IP)
4931                         flags |= TX_BD_FLAGS_IP_CKSUM;
4932                 if (m0->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
4933                         flags |= TX_BD_FLAGS_TCP_UDP_CKSUM;
4934         }
4935
4936         /* Transfer any VLAN tags to the bd. */
4937         if (m0->m_flags & M_VLANTAG) {
4938                 flags |= TX_BD_FLAGS_VLAN_TAG;
4939                 vlan_tag = m0->m_pkthdr.ether_vlantag;
4940         }
4941
4942         prod = txr->tx_prod;
4943         chain_prod_start = chain_prod = TX_CHAIN_IDX(txr, prod);
4944
4945         /* Map the mbuf into DMAable memory. */
4946         map = txr->tx_mbuf_map[chain_prod_start];
4947
4948         maxsegs = txr->max_tx_bd - txr->used_tx_bd;
4949         KASSERT(maxsegs >= BCE_TX_SPARE_SPACE,
4950                 ("not enough segments %d", maxsegs));
4951         if (maxsegs > BCE_MAX_SEGMENTS)
4952                 maxsegs = BCE_MAX_SEGMENTS;
4953
4954         /* Map the mbuf into our DMA address space. */
4955         error = bus_dmamap_load_mbuf_defrag(txr->tx_mbuf_tag, map, m_head,
4956                         segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
4957         if (error)
4958                 goto back;
4959         bus_dmamap_sync(txr->tx_mbuf_tag, map, BUS_DMASYNC_PREWRITE);
4960
4961         *nsegs_used += nsegs;
4962
4963         /* Reset m0 */
4964         m0 = *m_head;
4965
4966         /* prod points to an empty tx_bd at this point. */
4967         prod_bseq  = txr->tx_prod_bseq;
4968
4969         /*
4970          * Cycle through each mbuf segment that makes up
4971          * the outgoing frame, gathering the mapping info
4972          * for that segment and creating a tx_bd to for
4973          * the mbuf.
4974          */
4975         for (i = 0; i < nsegs; i++) {
4976                 chain_prod = TX_CHAIN_IDX(txr, prod);
4977                 txbd =
4978                 &txr->tx_bd_chain[TX_PAGE(chain_prod)][TX_IDX(chain_prod)];
4979
4980                 txbd->tx_bd_haddr_lo = htole32(BCE_ADDR_LO(segs[i].ds_addr));
4981                 txbd->tx_bd_haddr_hi = htole32(BCE_ADDR_HI(segs[i].ds_addr));
4982                 txbd->tx_bd_mss_nbytes = htole32(mss << 16) |
4983                     htole16(segs[i].ds_len);
4984                 txbd->tx_bd_vlan_tag = htole16(vlan_tag);
4985                 txbd->tx_bd_flags = htole16(flags);
4986
4987                 prod_bseq += segs[i].ds_len;
4988                 if (i == 0)
4989                         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_START);
4990                 prod = NEXT_TX_BD(prod);
4991         }
4992
4993         /* Set the END flag on the last TX buffer descriptor. */
4994         txbd->tx_bd_flags |= htole16(TX_BD_FLAGS_END);
4995
4996         /*
4997          * Ensure that the mbuf pointer for this transmission
4998          * is placed at the array index of the last
4999          * descriptor in this chain.  This is done
5000          * because a single map is used for all
5001          * segments of the mbuf and we don't want to
5002          * unload the map before all of the segments
5003          * have been freed.
5004          */
5005         txr->tx_mbuf_ptr[chain_prod] = m0;
5006
5007         tmp_map = txr->tx_mbuf_map[chain_prod];
5008         txr->tx_mbuf_map[chain_prod] = map;
5009         txr->tx_mbuf_map[chain_prod_start] = tmp_map;
5010
5011         txr->used_tx_bd += nsegs;
5012
5013         /* prod points to the next free tx_bd at this point. */
5014         txr->tx_prod = prod;
5015         txr->tx_prod_bseq = prod_bseq;
5016 back:
5017         if (error) {
5018                 m_freem(*m_head);
5019                 *m_head = NULL;
5020         }
5021         return error;
5022 }
5023
5024
5025 static void
5026 bce_xmit(struct bce_tx_ring *txr)
5027 {
5028         /* Start the transmit. */
5029         REG_WR16(txr->sc, MB_GET_CID_ADDR(txr->tx_cid) + BCE_L2CTX_TX_HOST_BIDX,
5030             txr->tx_prod);
5031         REG_WR(txr->sc, MB_GET_CID_ADDR(txr->tx_cid) + BCE_L2CTX_TX_HOST_BSEQ,
5032             txr->tx_prod_bseq);
5033 }
5034
5035
5036 /****************************************************************************/
5037 /* Main transmit routine when called from another routine with a lock.      */
5038 /*                                                                          */
5039 /* Returns:                                                                 */
5040 /*   Nothing.                                                               */
5041 /****************************************************************************/
5042 static void
5043 bce_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
5044 {
5045         struct bce_softc *sc = ifp->if_softc;
5046         struct bce_tx_ring *txr = ifsq_get_priv(ifsq);
5047         int count = 0;
5048
5049         KKASSERT(txr->ifsq == ifsq);
5050         ASSERT_SERIALIZED(&txr->tx_serialize);
5051
5052         /* If there's no link or the transmit queue is empty then just exit. */
5053         if (!sc->bce_link) {
5054                 ifsq_purge(ifsq);
5055                 return;
5056         }
5057
5058         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
5059                 return;
5060
5061         for (;;) {
5062                 struct mbuf *m_head;
5063
5064                 /*
5065                  * We keep BCE_TX_SPARE_SPACE entries, so bce_encap() is
5066                  * unlikely to fail.
5067                  */
5068                 if (txr->max_tx_bd - txr->used_tx_bd < BCE_TX_SPARE_SPACE) {
5069                         ifsq_set_oactive(ifsq);
5070                         break;
5071                 }
5072
5073                 /* Check for any frames to send. */
5074                 m_head = ifsq_dequeue(ifsq, NULL);
5075                 if (m_head == NULL)
5076                         break;
5077
5078                 /*
5079                  * Pack the data into the transmit ring. If we
5080                  * don't have room, place the mbuf back at the
5081                  * head of the queue and set the OACTIVE flag
5082                  * to wait for the NIC to drain the chain.
5083                  */
5084                 if (bce_encap(txr, &m_head, &count)) {
5085                         IFNET_STAT_INC(ifp, oerrors, 1);
5086                         if (txr->used_tx_bd == 0) {
5087                                 continue;
5088                         } else {
5089                                 ifsq_set_oactive(ifsq);
5090                                 break;
5091                         }
5092                 }
5093
5094                 if (count >= txr->tx_wreg) {
5095                         bce_xmit(txr);
5096                         count = 0;
5097                 }
5098
5099                 /* Send a copy of the frame to any BPF listeners. */
5100                 ETHER_BPF_MTAP(ifp, m_head);
5101
5102                 /* Set the tx timeout. */
5103                 txr->tx_watchdog.wd_timer = BCE_TX_TIMEOUT;
5104         }
5105         if (count > 0)
5106                 bce_xmit(txr);
5107 }
5108
5109
5110 /****************************************************************************/
5111 /* Handles any IOCTL calls from the operating system.                       */
5112 /*                                                                          */
5113 /* Returns:                                                                 */
5114 /*   0 for success, positive value for failure.                             */
5115 /****************************************************************************/
5116 static int
5117 bce_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
5118 {
5119         struct bce_softc *sc = ifp->if_softc;
5120         struct ifreq *ifr = (struct ifreq *)data;
5121         struct mii_data *mii;
5122         int mask, error = 0;
5123
5124         ASSERT_IFNET_SERIALIZED_ALL(ifp);
5125
5126         switch(command) {
5127         case SIOCSIFMTU:
5128                 /* Check that the MTU setting is supported. */
5129                 if (ifr->ifr_mtu < BCE_MIN_MTU ||
5130 #ifdef notyet
5131                     ifr->ifr_mtu > BCE_MAX_JUMBO_MTU
5132 #else
5133                     ifr->ifr_mtu > ETHERMTU
5134 #endif
5135                    ) {
5136                         error = EINVAL;
5137                         break;
5138                 }
5139
5140                 ifp->if_mtu = ifr->ifr_mtu;
5141                 ifp->if_flags &= ~IFF_RUNNING;  /* Force reinitialize */
5142                 bce_init(sc);
5143                 break;
5144
5145         case SIOCSIFFLAGS:
5146                 if (ifp->if_flags & IFF_UP) {
5147                         if (ifp->if_flags & IFF_RUNNING) {
5148                                 mask = ifp->if_flags ^ sc->bce_if_flags;
5149
5150                                 if (mask & (IFF_PROMISC | IFF_ALLMULTI))
5151                                         bce_set_rx_mode(sc);
5152                         } else {
5153                                 bce_init(sc);
5154                         }
5155                 } else if (ifp->if_flags & IFF_RUNNING) {
5156                         bce_stop(sc);
5157
5158                         /* If MFW is running, restart the controller a bit. */
5159                         if (sc->bce_flags & BCE_MFW_ENABLE_FLAG) {
5160                                 bce_reset(sc, BCE_DRV_MSG_CODE_RESET);
5161                                 bce_chipinit(sc);
5162                                 bce_mgmt_init(sc);
5163                         }
5164                 }
5165                 sc->bce_if_flags = ifp->if_flags;
5166                 break;
5167
5168         case SIOCADDMULTI:
5169         case SIOCDELMULTI:
5170                 if (ifp->if_flags & IFF_RUNNING)
5171                         bce_set_rx_mode(sc);
5172                 break;
5173
5174         case SIOCSIFMEDIA:
5175         case SIOCGIFMEDIA:
5176                 mii = device_get_softc(sc->bce_miibus);
5177                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
5178                 break;
5179
5180         case SIOCSIFCAP:
5181                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
5182                 if (mask & IFCAP_HWCSUM) {
5183                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
5184                         if (ifp->if_capenable & IFCAP_TXCSUM)
5185                                 ifp->if_hwassist |= BCE_CSUM_FEATURES;
5186                         else
5187                                 ifp->if_hwassist &= ~BCE_CSUM_FEATURES;
5188                 }
5189                 if (mask & IFCAP_TSO) {
5190                         ifp->if_capenable ^= IFCAP_TSO;
5191                         if (ifp->if_capenable & IFCAP_TSO)
5192                                 ifp->if_hwassist |= CSUM_TSO;
5193                         else
5194                                 ifp->if_hwassist &= ~CSUM_TSO;
5195                 }
5196                 break;
5197
5198         default:
5199                 error = ether_ioctl(ifp, command, data);
5200                 break;
5201         }
5202         return error;
5203 }
5204
5205
5206 /****************************************************************************/
5207 /* Transmit timeout handler.                                                */
5208 /*                                                                          */
5209 /* Returns:                                                                 */
5210 /*   Nothing.                                                               */
5211 /****************************************************************************/
5212 static void
5213 bce_watchdog(struct ifaltq_subque *ifsq)
5214 {
5215         struct ifnet *ifp = ifsq_get_ifp(ifsq);
5216         struct bce_softc *sc = ifp->if_softc;
5217         int i;
5218
5219         ASSERT_IFNET_SERIALIZED_ALL(ifp);
5220
5221         /*
5222          * If we are in this routine because of pause frames, then
5223          * don't reset the hardware.
5224          */
5225         if (REG_RD(sc, BCE_EMAC_TX_STATUS) & BCE_EMAC_TX_STATUS_XOFFED)
5226                 return;
5227
5228         if_printf(ifp, "Watchdog timeout occurred, resetting!\n");
5229
5230         ifp->if_flags &= ~IFF_RUNNING;  /* Force reinitialize */
5231         bce_init(sc);
5232
5233         IFNET_STAT_INC(ifp, oerrors, 1);
5234
5235         for (i = 0; i < sc->tx_ring_cnt; ++i)
5236                 ifsq_devstart_sched(sc->tx_rings[i].ifsq);
5237 }
5238
5239
5240 #ifdef IFPOLL_ENABLE
5241
5242 static void
5243 bce_npoll_status(struct ifnet *ifp)
5244 {
5245         struct bce_softc *sc = ifp->if_softc;
5246         struct status_block *sblk = sc->status_block;
5247         uint32_t status_attn_bits;
5248
5249         ASSERT_SERIALIZED(&sc->main_serialize);
5250
5251         status_attn_bits = sblk->status_attn_bits;
5252
5253         /* Was it a link change interrupt? */
5254         if ((status_attn_bits & STATUS_ATTN_BITS_LINK_STATE) !=
5255             (sblk->status_attn_bits_ack & STATUS_ATTN_BITS_LINK_STATE)) {
5256                 bce_phy_intr(sc);
5257
5258                 /*
5259                  * Clear any transient status updates during link state change.
5260                  */
5261                 REG_WR(sc, BCE_HC_COMMAND,
5262                     sc->hc_command | BCE_HC_COMMAND_COAL_NOW_WO_INT);
5263                 REG_RD(sc, BCE_HC_COMMAND);
5264         }
5265
5266         /*
5267          * If any other attention is asserted then the chip is toast.
5268          */
5269         if ((status_attn_bits & ~STATUS_ATTN_BITS_LINK_STATE) !=
5270              (sblk->status_attn_bits_ack & ~STATUS_ATTN_BITS_LINK_STATE)) {
5271                 if_printf(ifp, "Fatal attention detected: 0x%08X\n",
5272                     sblk->status_attn_bits);
5273                 bce_serialize_skipmain(sc);
5274                 bce_init(sc);
5275                 bce_deserialize_skipmain(sc);
5276         }
5277 }
5278
5279 static void
5280 bce_npoll_rx(struct ifnet *ifp, void *arg, int count)
5281 {
5282         struct bce_rx_ring *rxr = arg;
5283         uint16_t hw_rx_cons;
5284
5285         ASSERT_SERIALIZED(&rxr->rx_serialize);
5286
5287         /*
5288          * Save the status block index value for use when enabling
5289          * the interrupt.
5290          */
5291         rxr->last_status_idx = *rxr->hw_status_idx;
5292
5293         /* Make sure status index is extracted before RX/TX cons */
5294         cpu_lfence();
5295
5296         hw_rx_cons = bce_get_hw_rx_cons(rxr);
5297
5298         /* Check for any completed RX frames. */
5299         if (hw_rx_cons != rxr->rx_cons)
5300                 bce_rx_intr(rxr, count, hw_rx_cons);
5301 }
5302
5303 static void
5304 bce_npoll_rx_pack(struct ifnet *ifp, void *arg, int count)
5305 {
5306         struct bce_rx_ring *rxr = arg;
5307
5308         KASSERT(rxr->idx == 0, ("not the first RX ring, but %d", rxr->idx));
5309         bce_npoll_rx(ifp, rxr, count);
5310
5311         KASSERT(rxr->sc->rx_ring_cnt != rxr->sc->rx_ring_cnt2,
5312             ("RX ring count %d, count2 %d", rxr->sc->rx_ring_cnt,
5313              rxr->sc->rx_ring_cnt2));
5314
5315         /* Last ring carries packets whose masked hash is 0 */
5316         rxr = &rxr->sc->rx_rings[rxr->sc->rx_ring_cnt - 1];
5317
5318         lwkt_serialize_enter(&rxr->rx_serialize);
5319         bce_npoll_rx(ifp, rxr, count);
5320         lwkt_serialize_exit(&rxr->rx_serialize);
5321 }
5322
5323 static void
5324 bce_npoll_tx(struct ifnet *ifp, void *arg, int count __unused)
5325 {
5326         struct bce_tx_ring *txr = arg;
5327         uint16_t hw_tx_cons;
5328
5329         ASSERT_SERIALIZED(&txr->tx_serialize);
5330
5331         hw_tx_cons = bce_get_hw_tx_cons(txr);
5332
5333         /* Check for any completed TX frames. */
5334         if (hw_tx_cons != txr->tx_cons) {
5335                 bce_tx_intr(txr, hw_tx_cons);
5336                 if (!ifsq_is_empty(txr->ifsq))
5337                         ifsq_devstart(txr->ifsq);
5338         }
5339 }
5340
5341 static void
5342 bce_npoll(struct ifnet *ifp, struct ifpoll_info *info)
5343 {
5344         struct bce_softc *sc = ifp->if_softc;
5345         int i;
5346
5347         ASSERT_IFNET_SERIALIZED_ALL(ifp);
5348
5349         if (info != NULL) {
5350                 info->ifpi_status.status_func = bce_npoll_status;
5351                 info->ifpi_status.serializer = &sc->main_serialize;
5352
5353                 for (i = 0; i < sc->tx_ring_cnt; ++i) {
5354                         struct bce_tx_ring *txr = &sc->tx_rings[i];
5355                         int idx = i + sc->npoll_ofs;
5356
5357                         KKASSERT(idx < ncpus2);
5358                         info->ifpi_tx[idx].poll_func = bce_npoll_tx;
5359                         info->ifpi_tx[idx].arg = txr;
5360                         info->ifpi_tx[idx].serializer = &txr->tx_serialize;
5361                         ifsq_set_cpuid(txr->ifsq, idx);
5362                 }
5363
5364                 for (i = 0; i < sc->rx_ring_cnt2; ++i) {
5365                         struct bce_rx_ring *rxr = &sc->rx_rings[i];
5366                         int idx = i + sc->npoll_ofs;
5367
5368                         KKASSERT(idx < ncpus2);
5369                         if (i == 0 && sc->rx_ring_cnt2 != sc->rx_ring_cnt) {
5370                                 /*
5371                                  * If RSS is enabled, the packets whose
5372                                  * masked hash are 0 are queued to the
5373                                  * last RX ring; piggyback the last RX
5374                                  * ring's processing in the first RX
5375                                  * polling handler. (see also: comment
5376                                  * in bce_setup_ring_cnt())
5377                                  */
5378                                 if (bootverbose) {
5379                                         if_printf(ifp, "npoll pack last "
5380                                             "RX ring on cpu%d\n", idx);
5381                                 }
5382                                 info->ifpi_rx[idx].poll_func =
5383                                     bce_npoll_rx_pack;
5384                         } else {
5385                                 info->ifpi_rx[idx].poll_func = bce_npoll_rx;
5386                         }
5387                         info->ifpi_rx[idx].arg = rxr;
5388                         info->ifpi_rx[idx].serializer = &rxr->rx_serialize;
5389                 }
5390
5391                 if (ifp->if_flags & IFF_RUNNING) {
5392                         bce_set_timer_cpuid(sc, TRUE);
5393                         bce_disable_intr(sc);
5394                         bce_npoll_coal_change(sc);
5395                 }
5396         } else {
5397                 for (i = 0; i < sc->tx_ring_cnt; ++i) {
5398                         ifsq_set_cpuid(sc->tx_rings[i].ifsq,
5399                             sc->bce_msix[i].msix_cpuid);
5400                 }
5401
5402                 if (ifp->if_flags & IFF_RUNNING) {
5403                         bce_set_timer_cpuid(sc, FALSE);
5404                         bce_enable_intr(sc);
5405
5406                         sc->bce_coalchg_mask |= BCE_COALMASK_TX_BDS_INT |
5407                             BCE_COALMASK_RX_BDS_INT;
5408                         bce_coal_change(sc);
5409                 }
5410         }
5411 }
5412
5413 #endif  /* IFPOLL_ENABLE */
5414
5415
5416 /*
5417  * Interrupt handler.
5418  */
5419 /****************************************************************************/
5420 /* Main interrupt entry point.  Verifies that the controller generated the  */
5421 /* interrupt and then calls a separate routine for handle the various       */
5422 /* interrupt causes (PHY, TX, RX).                                          */
5423 /*                                                                          */
5424 /* Returns:                                                                 */
5425 /*   0 for success, positive value for failure.                             */
5426 /****************************************************************************/
5427 static void
5428 bce_intr(struct bce_softc *sc)
5429 {
5430         struct ifnet *ifp = &sc->arpcom.ac_if;
5431         struct status_block *sblk;
5432         uint16_t hw_rx_cons, hw_tx_cons;
5433         uint32_t status_attn_bits;
5434         struct bce_tx_ring *txr = &sc->tx_rings[0];
5435         struct bce_rx_ring *rxr = &sc->rx_rings[0];
5436
5437         ASSERT_SERIALIZED(&sc->main_serialize);
5438
5439         sblk = sc->status_block;
5440
5441         /*
5442          * Save the status block index value for use during
5443          * the next interrupt.
5444          */
5445         rxr->last_status_idx = *rxr->hw_status_idx;
5446
5447         /* Make sure status index is extracted before RX/TX cons */
5448         cpu_lfence();
5449
5450         /* Check if the hardware has finished any work. */
5451         hw_rx_cons = bce_get_hw_rx_cons(rxr);
5452         hw_tx_cons = bce_get_hw_tx_cons(txr);
5453
5454         status_attn_bits = sblk->status_attn_bits;
5455
5456         /* Was it a link change interrupt? */
5457         if ((status_attn_bits & STATUS_ATTN_BITS_LINK_STATE) !=
5458             (sblk->status_attn_bits_ack & STATUS_ATTN_BITS_LINK_STATE)) {
5459                 bce_phy_intr(sc);
5460
5461                 /*
5462                  * Clear any transient status updates during link state
5463                  * change.
5464                  */
5465                 REG_WR(sc, BCE_HC_COMMAND,
5466                     sc->hc_command | BCE_HC_COMMAND_COAL_NOW_WO_INT);
5467                 REG_RD(sc, BCE_HC_COMMAND);
5468         }
5469
5470         /*
5471          * If any other attention is asserted then
5472          * the chip is toast.
5473          */
5474         if ((status_attn_bits & ~STATUS_ATTN_BITS_LINK_STATE) !=
5475             (sblk->status_attn_bits_ack & ~STATUS_ATTN_BITS_LINK_STATE)) {
5476                 if_printf(ifp, "Fatal attention detected: 0x%08X\n",
5477                           sblk->status_attn_bits);
5478                 bce_serialize_skipmain(sc);
5479                 bce_init(sc);
5480                 bce_deserialize_skipmain(sc);
5481                 return;
5482         }
5483
5484         /* Check for any completed RX frames. */
5485         lwkt_serialize_enter(&rxr->rx_serialize);
5486         if (hw_rx_cons != rxr->rx_cons)
5487                 bce_rx_intr(rxr, -1, hw_rx_cons);
5488         lwkt_serialize_exit(&rxr->rx_serialize);
5489
5490         /* Check for any completed TX frames. */
5491         lwkt_serialize_enter(&txr->tx_serialize);
5492         if (hw_tx_cons != txr->tx_cons) {
5493                 bce_tx_intr(txr, hw_tx_cons);
5494                 if (!ifsq_is_empty(txr->ifsq))
5495                         ifsq_devstart(txr->ifsq);
5496         }
5497         lwkt_serialize_exit(&txr->tx_serialize);
5498 }
5499
5500 static void
5501 bce_intr_legacy(void *xsc)
5502 {
5503         struct bce_softc *sc = xsc;
5504         struct bce_rx_ring *rxr = &sc->rx_rings[0];
5505         struct status_block *sblk;
5506
5507         sblk = sc->status_block;
5508
5509         /*
5510          * If the hardware status block index matches the last value
5511          * read by the driver and we haven't asserted our interrupt
5512          * then there's nothing to do.
5513          */
5514         if (sblk->status_idx == rxr->last_status_idx &&
5515             (REG_RD(sc, BCE_PCICFG_MISC_STATUS) &
5516              BCE_PCICFG_MISC_STATUS_INTA_VALUE))
5517                 return;
5518
5519         /* Ack the interrupt and stop others from occuring. */
5520         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
5521                BCE_PCICFG_INT_ACK_CMD_USE_INT_HC_PARAM |
5522                BCE_PCICFG_INT_ACK_CMD_MASK_INT);
5523
5524         /*
5525          * Read back to deassert IRQ immediately to avoid too
5526          * many spurious interrupts.
5527          */
5528         REG_RD(sc, BCE_PCICFG_INT_ACK_CMD);
5529
5530         bce_intr(sc);
5531
5532         /* Re-enable interrupts. */
5533         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
5534                BCE_PCICFG_INT_ACK_CMD_INDEX_VALID |
5535                BCE_PCICFG_INT_ACK_CMD_MASK_INT | rxr->last_status_idx);
5536         bce_reenable_intr(rxr);
5537 }
5538
5539 static void
5540 bce_intr_msi(void *xsc)
5541 {
5542         struct bce_softc *sc = xsc;
5543
5544         /* Ack the interrupt and stop others from occuring. */
5545         REG_WR(sc, BCE_PCICFG_INT_ACK_CMD,
5546                BCE_PCICFG_INT_ACK_CMD_USE_INT_HC_PARAM |
5547                BCE_PCICFG_INT_ACK_CMD_MASK_INT);
5548
5549         bce_intr(sc);
5550
5551         /* Re-enable interrupts */
5552         bce_reenable_intr(&sc->rx_rings[0]);
5553 }
5554
5555 static void
5556 bce_intr_msi_oneshot(void *xsc)
5557 {
5558         struct bce_softc *sc = xsc;
5559
5560         bce_intr(sc);
5561
5562         /* Re-enable interrupts */
5563         bce_reenable_intr(&sc->rx_rings[0]);
5564 }
5565
5566 static void
5567 bce_intr_msix_rxtx(void *xrxr)
5568 {
5569         struct bce_rx_ring *rxr = xrxr;
5570         struct bce_tx_ring *txr;
5571         uint16_t hw_rx_cons, hw_tx_cons;
5572
5573         ASSERT_SERIALIZED(&rxr->rx_serialize);
5574
5575         KKASSERT(rxr->idx < rxr->sc->tx_ring_cnt);
5576         txr = &rxr->sc->tx_rings[rxr->idx];
5577
5578         /*
5579          * Save the status block index value for use during
5580          * the next interrupt.
5581          */
5582         rxr->last_status_idx = *rxr->hw_status_idx;
5583
5584         /* Make sure status index is extracted before RX/TX cons */
5585         cpu_lfence();
5586
5587         /* Check if the hardware has finished any work. */
5588         hw_rx_cons = bce_get_hw_rx_cons(rxr);
5589         if (hw_rx_cons != rxr->rx_cons)
5590                 bce_rx_intr(rxr, -1, hw_rx_cons);
5591
5592         /* Check for any completed TX frames. */
5593         hw_tx_cons = bce_get_hw_tx_cons(txr);
5594         lwkt_serialize_enter(&txr->tx_serialize);
5595         if (hw_tx_cons != txr->tx_cons) {
5596                 bce_tx_intr(txr, hw_tx_cons);
5597                 if (!ifsq_is_empty(txr->ifsq))
5598                         ifsq_devstart(txr->ifsq);
5599         }
5600         lwkt_serialize_exit(&txr->tx_serialize);
5601
5602         /* Re-enable interrupts */
5603         bce_reenable_intr(rxr);
5604 }
5605
5606 static void
5607 bce_intr_msix_rx(void *xrxr)
5608 {
5609         struct bce_rx_ring *rxr = xrxr;
5610         uint16_t hw_rx_cons;
5611
5612         ASSERT_SERIALIZED(&rxr->rx_serialize);
5613
5614         /*
5615          * Save the status block index value for use during
5616          * the next interrupt.
5617          */
5618         rxr->last_status_idx = *rxr->hw_status_idx;
5619
5620         /* Make sure status index is extracted before RX cons */
5621         cpu_lfence();
5622
5623         /* Check if the hardware has finished any work. */
5624         hw_rx_cons = bce_get_hw_rx_cons(rxr);
5625         if (hw_rx_cons != rxr->rx_cons)
5626                 bce_rx_intr(rxr, -1, hw_rx_cons);
5627
5628         /* Re-enable interrupts */
5629         bce_reenable_intr(rxr);
5630 }
5631
5632
5633 /****************************************************************************/
5634 /* Programs the various packet receive modes (broadcast and multicast).     */
5635 /*                                                                          */
5636 /* Returns:                                                                 */
5637 /*   Nothing.                                                               */
5638 /****************************************************************************/
5639 static void
5640 bce_set_rx_mode(struct bce_softc *sc)
5641 {
5642         struct ifnet *ifp = &sc->arpcom.ac_if;
5643         struct ifmultiaddr *ifma;
5644         uint32_t hashes[NUM_MC_HASH_REGISTERS] = { 0, 0, 0, 0, 0, 0, 0, 0 };
5645         uint32_t rx_mode, sort_mode;
5646         int h, i;
5647
5648         ASSERT_IFNET_SERIALIZED_ALL(ifp);
5649
5650         /* Initialize receive mode default settings. */
5651         rx_mode = sc->rx_mode &
5652                   ~(BCE_EMAC_RX_MODE_PROMISCUOUS |
5653                     BCE_EMAC_RX_MODE_KEEP_VLAN_TAG);
5654         sort_mode = 1 | BCE_RPM_SORT_USER0_BC_EN;
5655
5656         /*
5657          * ASF/IPMI/UMP firmware requires that VLAN tag stripping
5658          * be enbled.
5659          */
5660         if (!(BCE_IF_CAPABILITIES & IFCAP_VLAN_HWTAGGING) &&
5661             !(sc->bce_flags & BCE_MFW_ENABLE_FLAG))
5662                 rx_mode |= BCE_EMAC_RX_MODE_KEEP_VLAN_TAG;
5663
5664         /*
5665          * Check for promiscuous, all multicast, or selected
5666          * multicast address filtering.
5667          */
5668         if (ifp->if_flags & IFF_PROMISC) {
5669                 /* Enable promiscuous mode. */
5670                 rx_mode |= BCE_EMAC_RX_MODE_PROMISCUOUS;
5671                 sort_mode |= BCE_RPM_SORT_USER0_PROM_EN;
5672         } else if (ifp->if_flags & IFF_ALLMULTI) {
5673                 /* Enable all multicast addresses. */
5674                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++) {
5675                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4),
5676                                0xffffffff);
5677                 }
5678                 sort_mode |= BCE_RPM_SORT_USER0_MC_EN;
5679         } else {
5680                 /* Accept one or more multicast(s). */
5681                 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
5682                         if (ifma->ifma_addr->sa_family != AF_LINK)
5683                                 continue;
5684                         h = ether_crc32_le(
5685                             LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
5686                             ETHER_ADDR_LEN) & 0xFF;
5687                         hashes[(h & 0xE0) >> 5] |= 1 << (h & 0x1F);
5688                 }
5689
5690                 for (i = 0; i < NUM_MC_HASH_REGISTERS; i++) {
5691                         REG_WR(sc, BCE_EMAC_MULTICAST_HASH0 + (i * 4),
5692                                hashes[i]);
5693                 }
5694                 sort_mode |= BCE_RPM_SORT_USER0_MC_HSH_EN;
5695         }
5696
5697         /* Only make changes if the recive mode has actually changed. */
5698         if (rx_mode != sc->rx_mode) {
5699                 sc->rx_mode = rx_mode;
5700                 REG_WR(sc, BCE_EMAC_RX_MODE, rx_mode);
5701         }
5702
5703         /* Disable and clear the exisitng sort before enabling a new sort. */
5704         REG_WR(sc, BCE_RPM_SORT_USER0, 0x0);
5705         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode);
5706         REG_WR(sc, BCE_RPM_SORT_USER0, sort_mode | BCE_RPM_SORT_USER0_ENA);
5707 }
5708
5709
5710 /****************************************************************************/
5711 /* Called periodically to updates statistics from the controllers           */
5712 /* statistics block.                                                        */
5713 /*                                                                          */
5714 /* Returns:                                                                 */
5715 /*   Nothing.                                                               */
5716 /****************************************************************************/
5717 static void
5718 bce_stats_update(struct bce_softc *sc)
5719 {
5720         struct ifnet *ifp = &sc->arpcom.ac_if;
5721         struct statistics_block *stats = sc->stats_block;
5722
5723         ASSERT_SERIALIZED(&sc->main_serialize);
5724
5725         /*
5726          * Certain controllers don't report carrier sense errors correctly.
5727          * See errata E11_5708CA0_1165.
5728          */
5729         if (!(BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5706) &&
5730             !(BCE_CHIP_ID(sc) == BCE_CHIP_ID_5708_A0)) {
5731                 IFNET_STAT_INC(ifp, oerrors,
5732                         (u_long)stats->stat_Dot3StatsCarrierSenseErrors);
5733         }
5734
5735         /*
5736          * Update the sysctl statistics from the hardware statistics.
5737          */
5738         sc->stat_IfHCInOctets =
5739                 ((uint64_t)stats->stat_IfHCInOctets_hi << 32) +
5740                  (uint64_t)stats->stat_IfHCInOctets_lo;
5741
5742         sc->stat_IfHCInBadOctets =
5743                 ((uint64_t)stats->stat_IfHCInBadOctets_hi << 32) +
5744                  (uint64_t)stats->stat_IfHCInBadOctets_lo;
5745
5746         sc->stat_IfHCOutOctets =
5747                 ((uint64_t)stats->stat_IfHCOutOctets_hi << 32) +
5748                  (uint64_t)stats->stat_IfHCOutOctets_lo;
5749
5750         sc->stat_IfHCOutBadOctets =
5751                 ((uint64_t)stats->stat_IfHCOutBadOctets_hi << 32) +
5752                  (uint64_t)stats->stat_IfHCOutBadOctets_lo;
5753
5754         sc->stat_IfHCInUcastPkts =
5755                 ((uint64_t)stats->stat_IfHCInUcastPkts_hi << 32) +
5756                  (uint64_t)stats->stat_IfHCInUcastPkts_lo;
5757
5758         sc->stat_IfHCInMulticastPkts =
5759                 ((uint64_t)stats->stat_IfHCInMulticastPkts_hi << 32) +
5760                  (uint64_t)stats->stat_IfHCInMulticastPkts_lo;
5761
5762         sc->stat_IfHCInBroadcastPkts =
5763                 ((uint64_t)stats->stat_IfHCInBroadcastPkts_hi << 32) +
5764                  (uint64_t)stats->stat_IfHCInBroadcastPkts_lo;
5765
5766         sc->stat_IfHCOutUcastPkts =
5767                 ((uint64_t)stats->stat_IfHCOutUcastPkts_hi << 32) +
5768                  (uint64_t)stats->stat_IfHCOutUcastPkts_lo;
5769
5770         sc->stat_IfHCOutMulticastPkts =
5771                 ((uint64_t)stats->stat_IfHCOutMulticastPkts_hi << 32) +
5772                  (uint64_t)stats->stat_IfHCOutMulticastPkts_lo;
5773
5774         sc->stat_IfHCOutBroadcastPkts =
5775                 ((uint64_t)stats->stat_IfHCOutBroadcastPkts_hi << 32) +
5776                  (uint64_t)stats->stat_IfHCOutBroadcastPkts_lo;
5777
5778         sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors =
5779                 stats->stat_emac_tx_stat_dot3statsinternalmactransmiterrors;
5780
5781         sc->stat_Dot3StatsCarrierSenseErrors =
5782                 stats->stat_Dot3StatsCarrierSenseErrors;
5783
5784         sc->stat_Dot3StatsFCSErrors =
5785                 stats->stat_Dot3StatsFCSErrors;
5786
5787         sc->stat_Dot3StatsAlignmentErrors =
5788                 stats->stat_Dot3StatsAlignmentErrors;
5789
5790         sc->stat_Dot3StatsSingleCollisionFrames =
5791                 stats->stat_Dot3StatsSingleCollisionFrames;
5792
5793         sc->stat_Dot3StatsMultipleCollisionFrames =
5794                 stats->stat_Dot3StatsMultipleCollisionFrames;
5795
5796         sc->stat_Dot3StatsDeferredTransmissions =
5797                 stats->stat_Dot3StatsDeferredTransmissions;
5798
5799         sc->stat_Dot3StatsExcessiveCollisions =
5800                 stats->stat_Dot3StatsExcessiveCollisions;
5801
5802         sc->stat_Dot3StatsLateCollisions =
5803                 stats->stat_Dot3StatsLateCollisions;
5804
5805         sc->stat_EtherStatsCollisions =
5806                 stats->stat_EtherStatsCollisions;
5807
5808         sc->stat_EtherStatsFragments =
5809                 stats->stat_EtherStatsFragments;
5810
5811         sc->stat_EtherStatsJabbers =
5812                 stats->stat_EtherStatsJabbers;
5813
5814         sc->stat_EtherStatsUndersizePkts =
5815                 stats->stat_EtherStatsUndersizePkts;
5816
5817         sc->stat_EtherStatsOverrsizePkts =
5818                 stats->stat_EtherStatsOverrsizePkts;
5819
5820         sc->stat_EtherStatsPktsRx64Octets =
5821                 stats->stat_EtherStatsPktsRx64Octets;
5822
5823         sc->stat_EtherStatsPktsRx65Octetsto127Octets =
5824                 stats->stat_EtherStatsPktsRx65Octetsto127Octets;
5825
5826         sc->stat_EtherStatsPktsRx128Octetsto255Octets =
5827                 stats->stat_EtherStatsPktsRx128Octetsto255Octets;
5828
5829         sc->stat_EtherStatsPktsRx256Octetsto511Octets =
5830                 stats->stat_EtherStatsPktsRx256Octetsto511Octets;
5831
5832         sc->stat_EtherStatsPktsRx512Octetsto1023Octets =
5833                 stats->stat_EtherStatsPktsRx512Octetsto1023Octets;
5834
5835         sc->stat_EtherStatsPktsRx1024Octetsto1522Octets =
5836                 stats->stat_EtherStatsPktsRx1024Octetsto1522Octets;
5837
5838         sc->stat_EtherStatsPktsRx1523Octetsto9022Octets =
5839                 stats->stat_EtherStatsPktsRx1523Octetsto9022Octets;
5840
5841         sc->stat_EtherStatsPktsTx64Octets =
5842                 stats->stat_EtherStatsPktsTx64Octets;
5843
5844         sc->stat_EtherStatsPktsTx65Octetsto127Octets =
5845                 stats->stat_EtherStatsPktsTx65Octetsto127Octets;
5846
5847         sc->stat_EtherStatsPktsTx128Octetsto255Octets =
5848                 stats->stat_EtherStatsPktsTx128Octetsto255Octets;
5849
5850         sc->stat_EtherStatsPktsTx256Octetsto511Octets =
5851                 stats->stat_EtherStatsPktsTx256Octetsto511Octets;
5852
5853         sc->stat_EtherStatsPktsTx512Octetsto1023Octets =
5854                 stats->stat_EtherStatsPktsTx512Octetsto1023Octets;
5855
5856         sc->stat_EtherStatsPktsTx1024Octetsto1522Octets =
5857                 stats->stat_EtherStatsPktsTx1024Octetsto1522Octets;
5858
5859         sc->stat_EtherStatsPktsTx1523Octetsto9022Octets =
5860                 stats->stat_EtherStatsPktsTx1523Octetsto9022Octets;
5861
5862         sc->stat_XonPauseFramesReceived =
5863                 stats->stat_XonPauseFramesReceived;
5864
5865         sc->stat_XoffPauseFramesReceived =
5866                 stats->stat_XoffPauseFramesReceived;
5867
5868         sc->stat_OutXonSent =
5869                 stats->stat_OutXonSent;
5870
5871         sc->stat_OutXoffSent =
5872                 stats->stat_OutXoffSent;
5873
5874         sc->stat_FlowControlDone =
5875                 stats->stat_FlowControlDone;
5876
5877         sc->stat_MacControlFramesReceived =
5878                 stats->stat_MacControlFramesReceived;
5879
5880         sc->stat_XoffStateEntered =
5881                 stats->stat_XoffStateEntered;
5882
5883         sc->stat_IfInFramesL2FilterDiscards =
5884                 stats->stat_IfInFramesL2FilterDiscards;
5885
5886         sc->stat_IfInRuleCheckerDiscards =
5887                 stats->stat_IfInRuleCheckerDiscards;
5888
5889         sc->stat_IfInFTQDiscards =
5890                 stats->stat_IfInFTQDiscards;
5891
5892         sc->stat_IfInMBUFDiscards =
5893                 stats->stat_IfInMBUFDiscards;
5894
5895         sc->stat_IfInRuleCheckerP4Hit =
5896                 stats->stat_IfInRuleCheckerP4Hit;
5897
5898         sc->stat_CatchupInRuleCheckerDiscards =
5899                 stats->stat_CatchupInRuleCheckerDiscards;
5900
5901         sc->stat_CatchupInFTQDiscards =
5902                 stats->stat_CatchupInFTQDiscards;
5903
5904         sc->stat_CatchupInMBUFDiscards =
5905                 stats->stat_CatchupInMBUFDiscards;
5906
5907         sc->stat_CatchupInRuleCheckerP4Hit =
5908                 stats->stat_CatchupInRuleCheckerP4Hit;
5909
5910         sc->com_no_buffers = REG_RD_IND(sc, 0x120084);
5911
5912         /*
5913          * Update the interface statistics from the
5914          * hardware statistics.
5915          */
5916         IFNET_STAT_SET(ifp, collisions, (u_long)sc->stat_EtherStatsCollisions);
5917
5918         IFNET_STAT_SET(ifp, ierrors, (u_long)sc->stat_EtherStatsUndersizePkts +
5919             (u_long)sc->stat_EtherStatsOverrsizePkts +
5920             (u_long)sc->stat_IfInMBUFDiscards +
5921             (u_long)sc->stat_Dot3StatsAlignmentErrors +
5922             (u_long)sc->stat_Dot3StatsFCSErrors +
5923             (u_long)sc->stat_IfInRuleCheckerDiscards +
5924             (u_long)sc->stat_IfInFTQDiscards +
5925             (u_long)sc->com_no_buffers);
5926
5927         IFNET_STAT_SET(ifp, oerrors,
5928             (u_long)sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors +
5929             (u_long)sc->stat_Dot3StatsExcessiveCollisions +
5930             (u_long)sc->stat_Dot3StatsLateCollisions);
5931 }
5932
5933
5934 /****************************************************************************/
5935 /* Periodic function to notify the bootcode that the driver is still        */
5936 /* present.                                                                 */
5937 /*                                                                          */
5938 /* Returns:                                                                 */
5939 /*   Nothing.                                                               */
5940 /****************************************************************************/
5941 static void
5942 bce_pulse(void *xsc)
5943 {
5944         struct bce_softc *sc = xsc;
5945         struct ifnet *ifp = &sc->arpcom.ac_if;
5946         uint32_t msg;
5947
5948         lwkt_serialize_enter(&sc->main_serialize);
5949
5950         /* Tell the firmware that the driver is still running. */
5951         msg = (uint32_t)++sc->bce_fw_drv_pulse_wr_seq;
5952         bce_shmem_wr(sc, BCE_DRV_PULSE_MB, msg);
5953
5954         /* Update the bootcode condition. */
5955         sc->bc_state = bce_shmem_rd(sc, BCE_BC_STATE_CONDITION);
5956
5957         /* Report whether the bootcode still knows the driver is running. */
5958         if (!sc->bce_drv_cardiac_arrest) {
5959                 if (!(sc->bc_state & BCE_CONDITION_DRV_PRESENT)) {
5960                         sc->bce_drv_cardiac_arrest = 1;
5961                         if_printf(ifp, "Bootcode lost the driver pulse! "
5962                             "(bc_state = 0x%08X)\n", sc->bc_state);
5963                 }
5964         } else {
5965                 /*
5966                  * Not supported by all bootcode versions.
5967                  * (v5.0.11+ and v5.2.1+)  Older bootcode
5968                  * will require the driver to reset the
5969                  * controller to clear this condition.
5970                  */
5971                 if (sc->bc_state & BCE_CONDITION_DRV_PRESENT) {
5972                         sc->bce_drv_cardiac_arrest = 0;
5973                         if_printf(ifp, "Bootcode found the driver pulse! "
5974                             "(bc_state = 0x%08X)\n", sc->bc_state);
5975                 }
5976         }
5977
5978         /* Schedule the next pulse. */
5979         callout_reset_bycpu(&sc->bce_pulse_callout, hz, bce_pulse, sc,
5980             sc->bce_timer_cpuid);
5981
5982         lwkt_serialize_exit(&sc->main_serialize);
5983 }
5984
5985
5986 /****************************************************************************/
5987 /* Periodic function to check whether MSI is lost                           */
5988 /*                                                                          */
5989 /* Returns:                                                                 */
5990 /*   Nothing.                                                               */
5991 /****************************************************************************/
5992 static void
5993 bce_check_msi(void *xsc)
5994 {
5995         struct bce_softc *sc = xsc;
5996         struct ifnet *ifp = &sc->arpcom.ac_if;
5997         struct status_block *sblk = sc->status_block;
5998         struct bce_tx_ring *txr = &sc->tx_rings[0];
5999         struct bce_rx_ring *rxr = &sc->rx_rings[0];
6000
6001         lwkt_serialize_enter(&sc->main_serialize);
6002
6003         KKASSERT(mycpuid == sc->bce_msix[0].msix_cpuid);
6004
6005         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
6006                 lwkt_serialize_exit(&sc->main_serialize);
6007                 return;
6008         }
6009
6010         if (bce_get_hw_rx_cons(rxr) != rxr->rx_cons ||
6011             bce_get_hw_tx_cons(txr) != txr->tx_cons ||
6012             (sblk->status_attn_bits & STATUS_ATTN_BITS_LINK_STATE) !=
6013             (sblk->status_attn_bits_ack & STATUS_ATTN_BITS_LINK_STATE)) {
6014                 if (sc->bce_check_rx_cons == rxr->rx_cons &&
6015                     sc->bce_check_tx_cons == txr->tx_cons &&
6016                     sc->bce_check_status_idx == rxr->last_status_idx) {
6017                         uint32_t msi_ctrl;
6018
6019                         if (!sc->bce_msi_maylose) {
6020                                 sc->bce_msi_maylose = TRUE;
6021                                 goto done;
6022                         }
6023
6024                         msi_ctrl = REG_RD(sc, BCE_PCICFG_MSI_CONTROL);
6025                         if (msi_ctrl & BCE_PCICFG_MSI_CONTROL_ENABLE) {
6026                                 if (bootverbose)
6027                                         if_printf(ifp, "lost MSI\n");
6028
6029                                 REG_WR(sc, BCE_PCICFG_MSI_CONTROL,
6030                                     msi_ctrl & ~BCE_PCICFG_MSI_CONTROL_ENABLE);
6031                                 REG_WR(sc, BCE_PCICFG_MSI_CONTROL, msi_ctrl);
6032
6033                                 bce_intr_msi(sc);
6034                         } else if (bootverbose) {
6035                                 if_printf(ifp, "MSI may be lost\n");
6036                         }
6037                 }
6038         }
6039         sc->bce_msi_maylose = FALSE;
6040         sc->bce_check_rx_cons = rxr->rx_cons;
6041         sc->bce_check_tx_cons = txr->tx_cons;
6042         sc->bce_check_status_idx = rxr->last_status_idx;
6043
6044 done:
6045         callout_reset(&sc->bce_ckmsi_callout, BCE_MSI_CKINTVL,
6046             bce_check_msi, sc);
6047         lwkt_serialize_exit(&sc->main_serialize);
6048 }
6049
6050
6051 /****************************************************************************/
6052 /* Periodic function to perform maintenance tasks.                          */
6053 /*                                                                          */
6054 /* Returns:                                                                 */
6055 /*   Nothing.                                                               */
6056 /****************************************************************************/
6057 static void
6058 bce_tick_serialized(struct bce_softc *sc)
6059 {
6060         struct mii_data *mii;
6061
6062         ASSERT_SERIALIZED(&sc->main_serialize);
6063
6064         /* Update the statistics from the hardware statistics block. */
6065         bce_stats_update(sc);
6066
6067         /* Schedule the next tick. */
6068         callout_reset_bycpu(&sc->bce_tick_callout, hz, bce_tick, sc,
6069             sc->bce_timer_cpuid);
6070
6071         /* If link is up already up then we're done. */
6072         if (sc->bce_link)
6073                 return;
6074
6075         mii = device_get_softc(sc->bce_miibus);
6076         mii_tick(mii);
6077
6078         /* Check if the link has come up. */
6079         if ((mii->mii_media_status & IFM_ACTIVE) &&
6080             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
6081                 int i;
6082
6083                 sc->bce_link++;
6084                 /* Now that link is up, handle any outstanding TX traffic. */
6085                 for (i = 0; i < sc->tx_ring_cnt; ++i)
6086                         ifsq_devstart_sched(sc->tx_rings[i].ifsq);
6087         }
6088 }
6089
6090
6091 static void
6092 bce_tick(void *xsc)
6093 {
6094         struct bce_softc *sc = xsc;
6095
6096         lwkt_serialize_enter(&sc->main_serialize);
6097         bce_tick_serialized(sc);
6098         lwkt_serialize_exit(&sc->main_serialize);
6099 }
6100
6101
6102 /****************************************************************************/
6103 /* Adds any sysctl parameters for tuning or debugging purposes.             */
6104 /*                                                                          */
6105 /* Returns:                                                                 */
6106 /*   0 for success, positive value for failure.                             */
6107 /****************************************************************************/
6108 static void
6109 bce_add_sysctls(struct bce_softc *sc)
6110 {
6111         struct sysctl_ctx_list *ctx;
6112         struct sysctl_oid_list *children;
6113 #if defined(BCE_TSS_DEBUG) || defined(BCE_RSS_DEBUG)
6114         char node[32];
6115         int i;
6116 #endif
6117
6118         sysctl_ctx_init(&sc->bce_sysctl_ctx);
6119         sc->bce_sysctl_tree = SYSCTL_ADD_NODE(&sc->bce_sysctl_ctx,
6120                                               SYSCTL_STATIC_CHILDREN(_hw),
6121                                               OID_AUTO,
6122                                               device_get_nameunit(sc->bce_dev),
6123                                               CTLFLAG_RD, 0, "");
6124         if (sc->bce_sysctl_tree == NULL) {
6125                 device_printf(sc->bce_dev, "can't add sysctl node\n");
6126                 return;
6127         }
6128
6129         ctx = &sc->bce_sysctl_ctx;
6130         children = SYSCTL_CHILDREN(sc->bce_sysctl_tree);
6131
6132         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_bds_int",
6133                         CTLTYPE_INT | CTLFLAG_RW,
6134                         sc, 0, bce_sysctl_tx_bds_int, "I",
6135                         "Send max coalesced BD count during interrupt");
6136         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_bds",
6137                         CTLTYPE_INT | CTLFLAG_RW,
6138                         sc, 0, bce_sysctl_tx_bds, "I",
6139                         "Send max coalesced BD count");
6140         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_ticks_int",
6141                         CTLTYPE_INT | CTLFLAG_RW,
6142                         sc, 0, bce_sysctl_tx_ticks_int, "I",
6143                         "Send coalescing ticks during interrupt");
6144         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_ticks",
6145                         CTLTYPE_INT | CTLFLAG_RW,
6146                         sc, 0, bce_sysctl_tx_ticks, "I",
6147                         "Send coalescing ticks");
6148
6149         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_bds_int",
6150                         CTLTYPE_INT | CTLFLAG_RW,
6151                         sc, 0, bce_sysctl_rx_bds_int, "I",
6152                         "Receive max coalesced BD count during interrupt");
6153         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_bds",
6154                         CTLTYPE_INT | CTLFLAG_RW,
6155                         sc, 0, bce_sysctl_rx_bds, "I",
6156                         "Receive max coalesced BD count");
6157         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_ticks_int",
6158                         CTLTYPE_INT | CTLFLAG_RW,
6159                         sc, 0, bce_sysctl_rx_ticks_int, "I",
6160                         "Receive coalescing ticks during interrupt");
6161         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rx_ticks",
6162                         CTLTYPE_INT | CTLFLAG_RW,
6163                         sc, 0, bce_sysctl_rx_ticks, "I",
6164                         "Receive coalescing ticks");
6165
6166         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_rings",
6167                 CTLFLAG_RD, &sc->rx_ring_cnt, 0, "# of RX rings");
6168         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_pages",
6169                 CTLFLAG_RD, &sc->rx_rings[0].rx_pages, 0, "# of RX pages");
6170
6171         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_rings",
6172                 CTLFLAG_RD, &sc->tx_ring_cnt, 0, "# of TX rings");
6173         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_pages",
6174                 CTLFLAG_RD, &sc->tx_rings[0].tx_pages, 0, "# of TX pages");
6175
6176         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_wreg",
6177                 CTLFLAG_RW, &sc->tx_rings[0].tx_wreg, 0,
6178                 "# segments before write to hardware registers");
6179
6180 #ifdef IFPOLL_ENABLE
6181         SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "npoll_offset",
6182             CTLTYPE_INT|CTLFLAG_RW, sc, 0, bce_sysctl_npoll_offset,
6183             "I", "NPOLLING cpu offset");
6184 #endif
6185
6186 #ifdef BCE_RSS_DEBUG
6187         SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rss_debug",
6188             CTLFLAG_RW, &sc->rss_debug, 0, "RSS debug level");
6189         for (i = 0; i < sc->rx_ring_cnt; ++i) {
6190                 ksnprintf(node, sizeof(node), "rx%d_pkt", i);
6191                 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, node,
6192                     CTLFLAG_RW, &sc->rx_rings[i].rx_pkts,
6193                     "RXed packets");
6194         }
6195 #endif
6196
6197 #ifdef BCE_TSS_DEBUG
6198         for (i = 0; i < sc->tx_ring_cnt; ++i) {
6199                 ksnprintf(node, sizeof(node), "tx%d_pkt", i);
6200                 SYSCTL_ADD_ULONG(ctx, children, OID_AUTO, node,
6201                     CTLFLAG_RW, &sc->tx_rings[i].tx_pkts,
6202                     "TXed packets");
6203         }
6204 #endif
6205
6206         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6207                 "stat_IfHCInOctets",
6208                 CTLFLAG_RD, &sc->stat_IfHCInOctets,
6209                 "Bytes received");
6210
6211         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6212                 "stat_IfHCInBadOctets",
6213                 CTLFLAG_RD, &sc->stat_IfHCInBadOctets,
6214                 "Bad bytes received");
6215
6216         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6217                 "stat_IfHCOutOctets",
6218                 CTLFLAG_RD, &sc->stat_IfHCOutOctets,
6219                 "Bytes sent");
6220
6221         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6222                 "stat_IfHCOutBadOctets",
6223                 CTLFLAG_RD, &sc->stat_IfHCOutBadOctets,
6224                 "Bad bytes sent");
6225
6226         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6227                 "stat_IfHCInUcastPkts",
6228                 CTLFLAG_RD, &sc->stat_IfHCInUcastPkts,
6229                 "Unicast packets received");
6230
6231         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6232                 "stat_IfHCInMulticastPkts",
6233                 CTLFLAG_RD, &sc->stat_IfHCInMulticastPkts,
6234                 "Multicast packets received");
6235
6236         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6237                 "stat_IfHCInBroadcastPkts",
6238                 CTLFLAG_RD, &sc->stat_IfHCInBroadcastPkts,
6239                 "Broadcast packets received");
6240
6241         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6242                 "stat_IfHCOutUcastPkts",
6243                 CTLFLAG_RD, &sc->stat_IfHCOutUcastPkts,
6244                 "Unicast packets sent");
6245
6246         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6247                 "stat_IfHCOutMulticastPkts",
6248                 CTLFLAG_RD, &sc->stat_IfHCOutMulticastPkts,
6249                 "Multicast packets sent");
6250
6251         SYSCTL_ADD_ULONG(ctx, children, OID_AUTO,
6252                 "stat_IfHCOutBroadcastPkts",
6253                 CTLFLAG_RD, &sc->stat_IfHCOutBroadcastPkts,
6254                 "Broadcast packets sent");
6255
6256         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6257                 "stat_emac_tx_stat_dot3statsinternalmactransmiterrors",
6258                 CTLFLAG_RD, &sc->stat_emac_tx_stat_dot3statsinternalmactransmiterrors,
6259                 0, "Internal MAC transmit errors");
6260
6261         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6262                 "stat_Dot3StatsCarrierSenseErrors",
6263                 CTLFLAG_RD, &sc->stat_Dot3StatsCarrierSenseErrors,
6264                 0, "Carrier sense errors");
6265
6266         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6267                 "stat_Dot3StatsFCSErrors",
6268                 CTLFLAG_RD, &sc->stat_Dot3StatsFCSErrors,
6269                 0, "Frame check sequence errors");
6270
6271         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6272                 "stat_Dot3StatsAlignmentErrors",
6273                 CTLFLAG_RD, &sc->stat_Dot3StatsAlignmentErrors,
6274                 0, "Alignment errors");
6275
6276         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6277                 "stat_Dot3StatsSingleCollisionFrames",
6278                 CTLFLAG_RD, &sc->stat_Dot3StatsSingleCollisionFrames,
6279                 0, "Single Collision Frames");
6280
6281         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6282                 "stat_Dot3StatsMultipleCollisionFrames",
6283                 CTLFLAG_RD, &sc->stat_Dot3StatsMultipleCollisionFrames,
6284                 0, "Multiple Collision Frames");
6285
6286         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6287                 "stat_Dot3StatsDeferredTransmissions",
6288                 CTLFLAG_RD, &sc->stat_Dot3StatsDeferredTransmissions,
6289                 0, "Deferred Transmissions");
6290
6291         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6292                 "stat_Dot3StatsExcessiveCollisions",
6293                 CTLFLAG_RD, &sc->stat_Dot3StatsExcessiveCollisions,
6294                 0, "Excessive Collisions");
6295
6296         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6297                 "stat_Dot3StatsLateCollisions",
6298                 CTLFLAG_RD, &sc->stat_Dot3StatsLateCollisions,
6299                 0, "Late Collisions");
6300
6301         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6302                 "stat_EtherStatsCollisions",
6303                 CTLFLAG_RD, &sc->stat_EtherStatsCollisions,
6304                 0, "Collisions");
6305
6306         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6307                 "stat_EtherStatsFragments",
6308                 CTLFLAG_RD, &sc->stat_EtherStatsFragments,
6309                 0, "Fragments");
6310
6311         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6312                 "stat_EtherStatsJabbers",
6313                 CTLFLAG_RD, &sc->stat_EtherStatsJabbers,
6314                 0, "Jabbers");
6315
6316         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6317                 "stat_EtherStatsUndersizePkts",
6318                 CTLFLAG_RD, &sc->stat_EtherStatsUndersizePkts,
6319                 0, "Undersize packets");
6320
6321         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6322                 "stat_EtherStatsOverrsizePkts",
6323                 CTLFLAG_RD, &sc->stat_EtherStatsOverrsizePkts,
6324                 0, "stat_EtherStatsOverrsizePkts");
6325
6326         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6327                 "stat_EtherStatsPktsRx64Octets",
6328                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx64Octets,
6329                 0, "Bytes received in 64 byte packets");
6330
6331         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6332                 "stat_EtherStatsPktsRx65Octetsto127Octets",
6333                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx65Octetsto127Octets,
6334                 0, "Bytes received in 65 to 127 byte packets");
6335
6336         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6337                 "stat_EtherStatsPktsRx128Octetsto255Octets",
6338                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx128Octetsto255Octets,
6339                 0, "Bytes received in 128 to 255 byte packets");
6340
6341         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6342                 "stat_EtherStatsPktsRx256Octetsto511Octets",
6343                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx256Octetsto511Octets,
6344                 0, "Bytes received in 256 to 511 byte packets");
6345
6346         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6347                 "stat_EtherStatsPktsRx512Octetsto1023Octets",
6348                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx512Octetsto1023Octets,
6349                 0, "Bytes received in 512 to 1023 byte packets");
6350
6351         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6352                 "stat_EtherStatsPktsRx1024Octetsto1522Octets",
6353                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1024Octetsto1522Octets,
6354                 0, "Bytes received in 1024 t0 1522 byte packets");
6355
6356         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6357                 "stat_EtherStatsPktsRx1523Octetsto9022Octets",
6358                 CTLFLAG_RD, &sc->stat_EtherStatsPktsRx1523Octetsto9022Octets,
6359                 0, "Bytes received in 1523 to 9022 byte packets");
6360
6361         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6362                 "stat_EtherStatsPktsTx64Octets",
6363                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx64Octets,
6364                 0, "Bytes sent in 64 byte packets");
6365
6366         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6367                 "stat_EtherStatsPktsTx65Octetsto127Octets",
6368                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx65Octetsto127Octets,
6369                 0, "Bytes sent in 65 to 127 byte packets");
6370
6371         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6372                 "stat_EtherStatsPktsTx128Octetsto255Octets",
6373                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx128Octetsto255Octets,
6374                 0, "Bytes sent in 128 to 255 byte packets");
6375
6376         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6377                 "stat_EtherStatsPktsTx256Octetsto511Octets",
6378                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx256Octetsto511Octets,
6379                 0, "Bytes sent in 256 to 511 byte packets");
6380
6381         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6382                 "stat_EtherStatsPktsTx512Octetsto1023Octets",
6383                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx512Octetsto1023Octets,
6384                 0, "Bytes sent in 512 to 1023 byte packets");
6385
6386         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6387                 "stat_EtherStatsPktsTx1024Octetsto1522Octets",
6388                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1024Octetsto1522Octets,
6389                 0, "Bytes sent in 1024 to 1522 byte packets");
6390
6391         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6392                 "stat_EtherStatsPktsTx1523Octetsto9022Octets",
6393                 CTLFLAG_RD, &sc->stat_EtherStatsPktsTx1523Octetsto9022Octets,
6394                 0, "Bytes sent in 1523 to 9022 byte packets");
6395
6396         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6397                 "stat_XonPauseFramesReceived",
6398                 CTLFLAG_RD, &sc->stat_XonPauseFramesReceived,
6399                 0, "XON pause frames receved");
6400
6401         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6402                 "stat_XoffPauseFramesReceived",
6403                 CTLFLAG_RD, &sc->stat_XoffPauseFramesReceived,
6404                 0, "XOFF pause frames received");
6405
6406         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6407                 "stat_OutXonSent",
6408                 CTLFLAG_RD, &sc->stat_OutXonSent,
6409                 0, "XON pause frames sent");
6410
6411         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6412                 "stat_OutXoffSent",
6413                 CTLFLAG_RD, &sc->stat_OutXoffSent,
6414                 0, "XOFF pause frames sent");
6415
6416         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6417                 "stat_FlowControlDone",
6418                 CTLFLAG_RD, &sc->stat_FlowControlDone,
6419                 0, "Flow control done");
6420
6421         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6422                 "stat_MacControlFramesReceived",
6423                 CTLFLAG_RD, &sc->stat_MacControlFramesReceived,
6424                 0, "MAC control frames received");
6425
6426         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6427                 "stat_XoffStateEntered",
6428                 CTLFLAG_RD, &sc->stat_XoffStateEntered,
6429                 0, "XOFF state entered");
6430
6431         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6432                 "stat_IfInFramesL2FilterDiscards",
6433                 CTLFLAG_RD, &sc->stat_IfInFramesL2FilterDiscards,
6434                 0, "Received L2 packets discarded");
6435
6436         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6437                 "stat_IfInRuleCheckerDiscards",
6438                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerDiscards,
6439                 0, "Received packets discarded by rule");
6440
6441         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6442                 "stat_IfInFTQDiscards",
6443                 CTLFLAG_RD, &sc->stat_IfInFTQDiscards,
6444                 0, "Received packet FTQ discards");
6445
6446         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6447                 "stat_IfInMBUFDiscards",
6448                 CTLFLAG_RD, &sc->stat_IfInMBUFDiscards,
6449                 0, "Received packets discarded due to lack of controller buffer memory");
6450
6451         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6452                 "stat_IfInRuleCheckerP4Hit",
6453                 CTLFLAG_RD, &sc->stat_IfInRuleCheckerP4Hit,
6454                 0, "Received packets rule checker hits");
6455
6456         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6457                 "stat_CatchupInRuleCheckerDiscards",
6458                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerDiscards,
6459                 0, "Received packets discarded in Catchup path");
6460
6461         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6462                 "stat_CatchupInFTQDiscards",
6463                 CTLFLAG_RD, &sc->stat_CatchupInFTQDiscards,
6464                 0, "Received packets discarded in FTQ in Catchup path");
6465
6466         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6467                 "stat_CatchupInMBUFDiscards",
6468                 CTLFLAG_RD, &sc->stat_CatchupInMBUFDiscards,
6469                 0, "Received packets discarded in controller buffer memory in Catchup path");
6470
6471         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6472                 "stat_CatchupInRuleCheckerP4Hit",
6473                 CTLFLAG_RD, &sc->stat_CatchupInRuleCheckerP4Hit,
6474                 0, "Received packets rule checker hits in Catchup path");
6475
6476         SYSCTL_ADD_UINT(ctx, children, OID_AUTO,
6477                 "com_no_buffers",
6478                 CTLFLAG_RD, &sc->com_no_buffers,
6479                 0, "Valid packets received but no RX buffers available");
6480 }
6481
6482 static int
6483 bce_sysctl_tx_bds_int(SYSCTL_HANDLER_ARGS)
6484 {
6485         struct bce_softc *sc = arg1;
6486
6487         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6488                         &sc->bce_tx_quick_cons_trip_int,
6489                         BCE_COALMASK_TX_BDS_INT);
6490 }
6491
6492 static int
6493 bce_sysctl_tx_bds(SYSCTL_HANDLER_ARGS)
6494 {
6495         struct bce_softc *sc = arg1;
6496
6497         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6498                         &sc->bce_tx_quick_cons_trip,
6499                         BCE_COALMASK_TX_BDS);
6500 }
6501
6502 static int
6503 bce_sysctl_tx_ticks_int(SYSCTL_HANDLER_ARGS)
6504 {
6505         struct bce_softc *sc = arg1;
6506
6507         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6508                         &sc->bce_tx_ticks_int,
6509                         BCE_COALMASK_TX_TICKS_INT);
6510 }
6511
6512 static int
6513 bce_sysctl_tx_ticks(SYSCTL_HANDLER_ARGS)
6514 {
6515         struct bce_softc *sc = arg1;
6516
6517         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6518                         &sc->bce_tx_ticks,
6519                         BCE_COALMASK_TX_TICKS);
6520 }
6521
6522 static int
6523 bce_sysctl_rx_bds_int(SYSCTL_HANDLER_ARGS)
6524 {
6525         struct bce_softc *sc = arg1;
6526
6527         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6528                         &sc->bce_rx_quick_cons_trip_int,
6529                         BCE_COALMASK_RX_BDS_INT);
6530 }
6531
6532 static int
6533 bce_sysctl_rx_bds(SYSCTL_HANDLER_ARGS)
6534 {
6535         struct bce_softc *sc = arg1;
6536
6537         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6538                         &sc->bce_rx_quick_cons_trip,
6539                         BCE_COALMASK_RX_BDS);
6540 }
6541
6542 static int
6543 bce_sysctl_rx_ticks_int(SYSCTL_HANDLER_ARGS)
6544 {
6545         struct bce_softc *sc = arg1;
6546
6547         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6548                         &sc->bce_rx_ticks_int,
6549                         BCE_COALMASK_RX_TICKS_INT);
6550 }
6551
6552 static int
6553 bce_sysctl_rx_ticks(SYSCTL_HANDLER_ARGS)
6554 {
6555         struct bce_softc *sc = arg1;
6556
6557         return bce_sysctl_coal_change(oidp, arg1, arg2, req,
6558                         &sc->bce_rx_ticks,
6559                         BCE_COALMASK_RX_TICKS);
6560 }
6561
6562 static int
6563 bce_sysctl_coal_change(SYSCTL_HANDLER_ARGS, uint32_t *coal,
6564     uint32_t coalchg_mask)
6565 {
6566         struct bce_softc *sc = arg1;
6567         struct ifnet *ifp = &sc->arpcom.ac_if;
6568         int error = 0, v;
6569
6570         ifnet_serialize_all(ifp);
6571
6572         v = *coal;
6573         error = sysctl_handle_int(oidp, &v, 0, req);
6574         if (!error && req->newptr != NULL) {
6575                 if (v < 0) {
6576                         error = EINVAL;
6577                 } else {
6578                         *coal = v;
6579                         sc->bce_coalchg_mask |= coalchg_mask;
6580
6581                         /* Commit changes */
6582                         bce_coal_change(sc);
6583                 }
6584         }
6585
6586         ifnet_deserialize_all(ifp);
6587         return error;
6588 }
6589
6590 static void
6591 bce_coal_change(struct bce_softc *sc)
6592 {
6593         struct ifnet *ifp = &sc->arpcom.ac_if;
6594         int i;
6595
6596         ASSERT_SERIALIZED(&sc->main_serialize);
6597
6598         if ((ifp->if_flags & IFF_RUNNING) == 0) {
6599                 sc->bce_coalchg_mask = 0;
6600                 return;
6601         }
6602
6603         if (sc->bce_coalchg_mask &
6604             (BCE_COALMASK_TX_BDS | BCE_COALMASK_TX_BDS_INT)) {
6605                 REG_WR(sc, BCE_HC_TX_QUICK_CONS_TRIP,
6606                        (sc->bce_tx_quick_cons_trip_int << 16) |
6607                        sc->bce_tx_quick_cons_trip);
6608                 for (i = 1; i < sc->rx_ring_cnt; ++i) {
6609                         uint32_t base;
6610
6611                         base = ((i - 1) * BCE_HC_SB_CONFIG_SIZE) +
6612                             BCE_HC_SB_CONFIG_1;
6613                         REG_WR(sc, base + BCE_HC_TX_QUICK_CONS_TRIP_OFF,
6614                             (sc->bce_tx_quick_cons_trip_int << 16) |
6615                             sc->bce_tx_quick_cons_trip);
6616                 }
6617                 if (bootverbose) {
6618                         if_printf(ifp, "tx_bds %u, tx_bds_int %u\n",
6619                                   sc->bce_tx_quick_cons_trip,
6620                                   sc->bce_tx_quick_cons_trip_int);
6621                 }
6622         }
6623
6624         if (sc->bce_coalchg_mask &
6625             (BCE_COALMASK_TX_TICKS | BCE_COALMASK_TX_TICKS_INT)) {
6626                 REG_WR(sc, BCE_HC_TX_TICKS,
6627                        (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
6628                 for (i = 1; i < sc->rx_ring_cnt; ++i) {
6629                         uint32_t base;
6630
6631                         base = ((i - 1) * BCE_HC_SB_CONFIG_SIZE) +
6632                             BCE_HC_SB_CONFIG_1;
6633                         REG_WR(sc, base + BCE_HC_TX_TICKS_OFF,
6634                             (sc->bce_tx_ticks_int << 16) | sc->bce_tx_ticks);
6635                 }
6636                 if (bootverbose) {
6637                         if_printf(ifp, "tx_ticks %u, tx_ticks_int %u\n",
6638                                   sc->bce_tx_ticks, sc->bce_tx_ticks_int);
6639                 }
6640         }
6641
6642         if (sc->bce_coalchg_mask &
6643             (BCE_COALMASK_RX_BDS | BCE_COALMASK_RX_BDS_INT)) {
6644                 REG_WR(sc, BCE_HC_RX_QUICK_CONS_TRIP,
6645                        (sc->bce_rx_quick_cons_trip_int << 16) |
6646                        sc->bce_rx_quick_cons_trip);
6647                 for (i = 1; i < sc->rx_ring_cnt; ++i) {
6648                         uint32_t base;
6649
6650                         base = ((i - 1) * BCE_HC_SB_CONFIG_SIZE) +
6651                             BCE_HC_SB_CONFIG_1;
6652                         REG_WR(sc, base + BCE_HC_RX_QUICK_CONS_TRIP_OFF,
6653                             (sc->bce_rx_quick_cons_trip_int << 16) |
6654                             sc->bce_rx_quick_cons_trip);
6655                 }
6656                 if (bootverbose) {
6657                         if_printf(ifp, "rx_bds %u, rx_bds_int %u\n",
6658                                   sc->bce_rx_quick_cons_trip,
6659                                   sc->bce_rx_quick_cons_trip_int);
6660                 }
6661         }
6662
6663         if (sc->bce_coalchg_mask &
6664             (BCE_COALMASK_RX_TICKS | BCE_COALMASK_RX_TICKS_INT)) {
6665                 REG_WR(sc, BCE_HC_RX_TICKS,
6666                        (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
6667                 for (i = 1; i < sc->rx_ring_cnt; ++i) {
6668                         uint32_t base;
6669
6670                         base = ((i - 1) * BCE_HC_SB_CONFIG_SIZE) +
6671                             BCE_HC_SB_CONFIG_1;
6672                         REG_WR(sc, base + BCE_HC_RX_TICKS_OFF,
6673                             (sc->bce_rx_ticks_int << 16) | sc->bce_rx_ticks);
6674                 }
6675                 if (bootverbose) {
6676                         if_printf(ifp, "rx_ticks %u, rx_ticks_int %u\n",
6677                                   sc->bce_rx_ticks, sc->bce_rx_ticks_int);
6678                 }
6679         }
6680
6681         sc->bce_coalchg_mask = 0;
6682 }
6683
6684 static int
6685 bce_tso_setup(struct bce_tx_ring *txr, struct mbuf **mp,
6686     uint16_t *flags0, uint16_t *mss0)
6687 {
6688         struct mbuf *m;
6689         uint16_t flags;
6690         int thoff, iphlen, hoff;
6691
6692         m = *mp;
6693         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
6694
6695         hoff = m->m_pkthdr.csum_lhlen;
6696         iphlen = m->m_pkthdr.csum_iphlen;
6697         thoff = m->m_pkthdr.csum_thlen;
6698
6699         KASSERT(hoff >= sizeof(struct ether_header),
6700             ("invalid ether header len %d", hoff));
6701         KASSERT(iphlen >= sizeof(struct ip),
6702             ("invalid ip header len %d", iphlen));
6703         KASSERT(thoff >= sizeof(struct tcphdr),
6704             ("invalid tcp header len %d", thoff));
6705
6706         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
6707                 m = m_pullup(m, hoff + iphlen + thoff);
6708                 if (m == NULL) {
6709                         *mp = NULL;
6710                         return ENOBUFS;
6711                 }
6712                 *mp = m;
6713         }
6714
6715         /* Set the LSO flag in the TX BD */
6716         flags = TX_BD_FLAGS_SW_LSO;
6717
6718         /* Set the length of IP + TCP options (in 32 bit words) */
6719         flags |= (((iphlen + thoff -
6720             sizeof(struct ip) - sizeof(struct tcphdr)) >> 2) << 8);
6721
6722         *mss0 = htole16(m->m_pkthdr.tso_segsz);
6723         *flags0 = flags;
6724
6725         return 0;
6726 }
6727
6728 static void
6729 bce_setup_serialize(struct bce_softc *sc)
6730 {
6731         int i, j;
6732
6733         /*
6734          * Allocate serializer array
6735          */
6736
6737         /* Main + TX + RX */
6738         sc->serialize_cnt = 1 + sc->tx_ring_cnt + sc->rx_ring_cnt;
6739
6740         sc->serializes =
6741             kmalloc(sc->serialize_cnt * sizeof(struct lwkt_serialize *),
6742                 M_DEVBUF, M_WAITOK | M_ZERO);
6743
6744         /*
6745          * Setup serializers
6746          *
6747          * NOTE: Order is critical
6748          */
6749
6750         i = 0;
6751         KKASSERT(i < sc->serialize_cnt);
6752         sc->serializes[i++] = &sc->main_serialize;
6753
6754         sc->rx_serialize = i;
6755         for (j = 0; j < sc->rx_ring_cnt; ++j) {
6756                 KKASSERT(i < sc->serialize_cnt);
6757                 sc->serializes[i++] = &sc->rx_rings[j].rx_serialize;
6758         }
6759
6760         sc->tx_serialize = i;
6761         for (j = 0; j < sc->tx_ring_cnt; ++j) {
6762                 KKASSERT(i < sc->serialize_cnt);
6763                 sc->serializes[i++] = &sc->tx_rings[j].tx_serialize;
6764         }
6765
6766         KKASSERT(i == sc->serialize_cnt);
6767 }
6768
6769 static void
6770 bce_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
6771 {
6772         struct bce_softc *sc = ifp->if_softc;
6773
6774         ifnet_serialize_array_enter(sc->serializes, sc->serialize_cnt,
6775             sc->tx_serialize, sc->rx_serialize, slz);
6776 }
6777
6778 static void
6779 bce_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
6780 {
6781         struct bce_softc *sc = ifp->if_softc;
6782
6783         ifnet_serialize_array_exit(sc->serializes, sc->serialize_cnt,
6784             sc->tx_serialize, sc->rx_serialize, slz);
6785 }
6786
6787 static int
6788 bce_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
6789 {
6790         struct bce_softc *sc = ifp->if_softc;
6791
6792         return ifnet_serialize_array_try(sc->serializes, sc->serialize_cnt,
6793             sc->tx_serialize, sc->rx_serialize, slz);
6794 }
6795
6796 #ifdef INVARIANTS
6797
6798 static void
6799 bce_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
6800     boolean_t serialized)
6801 {
6802         struct bce_softc *sc = ifp->if_softc;
6803
6804         ifnet_serialize_array_assert(sc->serializes, sc->serialize_cnt,
6805             sc->tx_serialize, sc->rx_serialize, slz, serialized);
6806 }
6807
6808 #endif  /* INVARIANTS */
6809
6810 static void
6811 bce_serialize_skipmain(struct bce_softc *sc)
6812 {
6813         lwkt_serialize_array_enter(sc->serializes, sc->serialize_cnt, 1);
6814 }
6815
6816 static void
6817 bce_deserialize_skipmain(struct bce_softc *sc)
6818 {
6819         lwkt_serialize_array_exit(sc->serializes, sc->serialize_cnt, 1);
6820 }
6821
6822 #ifdef IFPOLL_ENABLE
6823
6824 static int
6825 bce_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS)
6826 {
6827         struct bce_softc *sc = (void *)arg1;
6828         struct ifnet *ifp = &sc->arpcom.ac_if;
6829         int error, off;
6830
6831         off = sc->npoll_ofs;
6832         error = sysctl_handle_int(oidp, &off, 0, req);
6833         if (error || req->newptr == NULL)
6834                 return error;
6835         if (off < 0)
6836                 return EINVAL;
6837
6838         ifnet_serialize_all(ifp);
6839         if (off >= ncpus2 || off % sc->rx_ring_cnt2 != 0) {
6840                 error = EINVAL;
6841         } else {
6842                 error = 0;
6843                 sc->npoll_ofs = off;
6844         }
6845         ifnet_deserialize_all(ifp);
6846
6847         return error;
6848 }
6849
6850 #endif  /* IFPOLL_ENABLE */
6851
6852 static void
6853 bce_set_timer_cpuid(struct bce_softc *sc, boolean_t polling)
6854 {
6855         if (polling)
6856                 sc->bce_timer_cpuid = 0; /* XXX */
6857         else
6858                 sc->bce_timer_cpuid = sc->bce_msix[0].msix_cpuid;
6859 }
6860
6861 static int
6862 bce_alloc_intr(struct bce_softc *sc)
6863 {
6864         u_int irq_flags;
6865
6866         bce_try_alloc_msix(sc);
6867         if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX)
6868                 return 0;
6869
6870         sc->bce_irq_type = pci_alloc_1intr(sc->bce_dev, bce_msi_enable,
6871             &sc->bce_irq_rid, &irq_flags);
6872
6873         sc->bce_res_irq = bus_alloc_resource_any(sc->bce_dev, SYS_RES_IRQ,
6874             &sc->bce_irq_rid, irq_flags);
6875         if (sc->bce_res_irq == NULL) {
6876                 device_printf(sc->bce_dev, "PCI map interrupt failed\n");
6877                 return ENXIO;
6878         }
6879         return 0;
6880 }
6881
6882 static void
6883 bce_try_alloc_msix(struct bce_softc *sc)
6884 {
6885         struct bce_msix_data *msix;
6886         int offset, i, error;
6887         boolean_t setup = FALSE;
6888
6889         if (sc->rx_ring_cnt == 1)
6890                 return;
6891
6892         if (sc->rx_ring_cnt2 == ncpus2) {
6893                 offset = 0;
6894         } else {
6895                 int offset_def =
6896                     (sc->rx_ring_cnt2 * device_get_unit(sc->bce_dev)) % ncpus2;
6897
6898                 offset = device_getenv_int(sc->bce_dev,
6899                     "msix.offset", offset_def);
6900                 if (offset >= ncpus2 || offset % sc->rx_ring_cnt2 != 0) {
6901                         device_printf(sc->bce_dev,
6902                             "invalid msix.offset %d, use %d\n",
6903                             offset, offset_def);
6904                         offset = offset_def;
6905                 }
6906         }
6907
6908         msix = &sc->bce_msix[0];
6909         msix->msix_serialize = &sc->main_serialize;
6910         msix->msix_func = bce_intr_msi_oneshot;
6911         msix->msix_arg = sc;
6912         KKASSERT(offset < ncpus2);
6913         msix->msix_cpuid = offset;
6914         ksnprintf(msix->msix_desc, sizeof(msix->msix_desc), "%s combo",
6915             device_get_nameunit(sc->bce_dev));
6916
6917         for (i = 1; i < sc->rx_ring_cnt; ++i) {
6918                 struct bce_rx_ring *rxr = &sc->rx_rings[i];
6919
6920                 msix = &sc->bce_msix[i];
6921
6922                 msix->msix_serialize = &rxr->rx_serialize;
6923                 msix->msix_arg = rxr;
6924                 msix->msix_cpuid = offset + (i % sc->rx_ring_cnt2);
6925                 KKASSERT(msix->msix_cpuid < ncpus2);
6926
6927                 if (i < sc->tx_ring_cnt) {
6928                         msix->msix_func = bce_intr_msix_rxtx;
6929                         ksnprintf(msix->msix_desc, sizeof(msix->msix_desc),
6930                             "%s rxtx%d", device_get_nameunit(sc->bce_dev), i);
6931                 } else {
6932                         msix->msix_func = bce_intr_msix_rx;
6933                         ksnprintf(msix->msix_desc, sizeof(msix->msix_desc),
6934                             "%s rx%d", device_get_nameunit(sc->bce_dev), i);
6935                 }
6936         }
6937
6938         /*
6939          * Setup MSI-X table
6940          */
6941         bce_setup_msix_table(sc);
6942         REG_WR(sc, BCE_PCI_MSIX_CONTROL, BCE_MSIX_MAX - 1);
6943         REG_WR(sc, BCE_PCI_MSIX_TBL_OFF_BIR, BCE_PCI_GRC_WINDOW2_BASE);
6944         REG_WR(sc, BCE_PCI_MSIX_PBA_OFF_BIT, BCE_PCI_GRC_WINDOW3_BASE);
6945         /* Flush */
6946         REG_RD(sc, BCE_PCI_MSIX_CONTROL);
6947
6948         error = pci_setup_msix(sc->bce_dev);
6949         if (error) {
6950                 device_printf(sc->bce_dev, "Setup MSI-X failed\n");
6951                 goto back;
6952         }
6953         setup = TRUE;
6954
6955         for (i = 0; i < sc->rx_ring_cnt; ++i) {
6956                 msix = &sc->bce_msix[i];
6957
6958                 error = pci_alloc_msix_vector(sc->bce_dev, i, &msix->msix_rid,
6959                     msix->msix_cpuid);
6960                 if (error) {
6961                         device_printf(sc->bce_dev,
6962                             "Unable to allocate MSI-X %d on cpu%d\n",
6963                             i, msix->msix_cpuid);
6964                         goto back;
6965                 }
6966
6967                 msix->msix_res = bus_alloc_resource_any(sc->bce_dev,
6968                     SYS_RES_IRQ, &msix->msix_rid, RF_ACTIVE);
6969                 if (msix->msix_res == NULL) {
6970                         device_printf(sc->bce_dev,
6971                             "Unable to allocate MSI-X %d resource\n", i);
6972                         error = ENOMEM;
6973                         goto back;
6974                 }
6975         }
6976
6977         pci_enable_msix(sc->bce_dev);
6978         sc->bce_irq_type = PCI_INTR_TYPE_MSIX;
6979 back:
6980         if (error)
6981                 bce_free_msix(sc, setup);
6982 }
6983
6984 static void
6985 bce_setup_ring_cnt(struct bce_softc *sc)
6986 {
6987         int msix_enable, ring_max, msix_cnt2, msix_cnt, i;
6988
6989         sc->rx_ring_cnt = 1;
6990         sc->rx_ring_cnt2 = 1;
6991         sc->tx_ring_cnt = 1;
6992
6993         if (BCE_CHIP_NUM(sc) != BCE_CHIP_NUM_5709)
6994                 return;
6995
6996         msix_enable = device_getenv_int(sc->bce_dev, "msix.enable",
6997             bce_msix_enable);
6998         if (!msix_enable)
6999                 return;
7000
7001         if (ncpus2 == 1)
7002                 return;
7003
7004         msix_cnt = pci_msix_count(sc->bce_dev);
7005         if (msix_cnt <= 1)
7006                 return;
7007
7008         i = 0;
7009         while ((1 << (i + 1)) <= msix_cnt)
7010                 ++i;
7011         msix_cnt2 = 1 << i;
7012
7013         /*
7014          * One extra RX ring will be needed (see below), so make sure
7015          * that there are enough MSI-X vectors.
7016          */
7017         if (msix_cnt == msix_cnt2) {
7018                 /*
7019                  * XXX
7020                  * This probably will not happen; 5709/5716
7021                  * come with 9 MSI-X vectors.
7022                  */
7023                 msix_cnt2 >>= 1;
7024                 if (msix_cnt2 <= 1) {
7025                         device_printf(sc->bce_dev,
7026                             "MSI-X count %d could not be used\n", msix_cnt);
7027                         return;
7028                 }
7029                 device_printf(sc->bce_dev, "MSI-X count %d is power of 2\n",
7030                     msix_cnt);
7031         }
7032
7033         /*
7034          * Setup RX ring count
7035          */
7036         ring_max = BCE_RX_RING_MAX;
7037         if (ring_max > msix_cnt2)
7038                 ring_max = msix_cnt2;
7039         sc->rx_ring_cnt2 = device_getenv_int(sc->bce_dev, "rx_rings",
7040             bce_rx_rings);
7041         sc->rx_ring_cnt2 = if_ring_count2(sc->rx_ring_cnt2, ring_max);
7042
7043         /*
7044          * One extra RX ring is allocated, since the first RX ring
7045          * could not be used for RSS hashed packets whose masked
7046          * hash is 0.  The first RX ring is only used for packets
7047          * whose RSS hash could not be calculated, e.g. ARP packets.
7048          * This extra RX ring will be used for packets whose masked
7049          * hash is 0.  The effective RX ring count involved in RSS
7050          * is still sc->rx_ring_cnt2.
7051          */
7052         KKASSERT(sc->rx_ring_cnt2 + 1 <= msix_cnt);
7053         sc->rx_ring_cnt = sc->rx_ring_cnt2 + 1;
7054
7055         /*
7056          * Setup TX ring count
7057          *
7058          * NOTE:
7059          * TX ring count must be less than the effective RSS RX ring
7060          * count, since we use RX ring software data struct to save
7061          * status index and various other MSI-X related stuffs.
7062          */
7063         ring_max = BCE_TX_RING_MAX;
7064         if (ring_max > msix_cnt2)
7065                 ring_max = msix_cnt2;
7066         if (ring_max > sc->rx_ring_cnt2)
7067                 ring_max = sc->rx_ring_cnt2;
7068         sc->tx_ring_cnt = device_getenv_int(sc->bce_dev, "tx_rings",
7069             bce_tx_rings);
7070         sc->tx_ring_cnt = if_ring_count2(sc->tx_ring_cnt, ring_max);
7071 }
7072
7073 static void
7074 bce_free_msix(struct bce_softc *sc, boolean_t setup)
7075 {
7076         int i;
7077
7078         KKASSERT(sc->rx_ring_cnt > 1);
7079
7080         for (i = 0; i < sc->rx_ring_cnt; ++i) {
7081                 struct bce_msix_data *msix = &sc->bce_msix[i];
7082
7083                 if (msix->msix_res != NULL) {
7084                         bus_release_resource(sc->bce_dev, SYS_RES_IRQ,
7085                             msix->msix_rid, msix->msix_res);
7086                 }
7087                 if (msix->msix_rid >= 0)
7088                         pci_release_msix_vector(sc->bce_dev, msix->msix_rid);
7089         }
7090         if (setup)
7091                 pci_teardown_msix(sc->bce_dev);
7092 }
7093
7094 static void
7095 bce_free_intr(struct bce_softc *sc)
7096 {
7097         if (sc->bce_irq_type != PCI_INTR_TYPE_MSIX) {
7098                 if (sc->bce_res_irq != NULL) {
7099                         bus_release_resource(sc->bce_dev, SYS_RES_IRQ,
7100                             sc->bce_irq_rid, sc->bce_res_irq);
7101                 }
7102                 if (sc->bce_irq_type == PCI_INTR_TYPE_MSI)
7103                         pci_release_msi(sc->bce_dev);
7104         } else {
7105                 bce_free_msix(sc, TRUE);
7106         }
7107 }
7108
7109 static void
7110 bce_setup_msix_table(struct bce_softc *sc)
7111 {
7112         REG_WR(sc, BCE_PCI_GRC_WINDOW_ADDR, BCE_PCI_GRC_WINDOW_ADDR_SEP_WIN);
7113         REG_WR(sc, BCE_PCI_GRC_WINDOW2_ADDR, BCE_MSIX_TABLE_ADDR);
7114         REG_WR(sc, BCE_PCI_GRC_WINDOW3_ADDR, BCE_MSIX_PBA_ADDR);
7115 }
7116
7117 static int
7118 bce_setup_intr(struct bce_softc *sc)
7119 {
7120         void (*irq_handle)(void *);
7121         int error;
7122
7123         if (sc->bce_irq_type == PCI_INTR_TYPE_MSIX)
7124                 return bce_setup_msix(sc);
7125
7126         if (sc->bce_irq_type == PCI_INTR_TYPE_LEGACY) {
7127                 irq_handle = bce_intr_legacy;
7128         } else if (sc->bce_irq_type == PCI_INTR_TYPE_MSI) {
7129                 if (BCE_CHIP_NUM(sc) == BCE_CHIP_NUM_5709) {
7130                         irq_handle = bce_intr_msi_oneshot;
7131                         sc->bce_flags |= BCE_ONESHOT_MSI_FLAG;
7132                 } else {
7133                         irq_handle = bce_intr_msi;
7134                         sc->bce_flags |= BCE_CHECK_MSI_FLAG;
7135                 }
7136         } else {
7137                 panic("%s: unsupported intr type %d",
7138                     device_get_nameunit(sc->bce_dev), sc->bce_irq_type);
7139         }
7140
7141         error = bus_setup_intr(sc->bce_dev, sc->bce_res_irq, INTR_MPSAFE,
7142             irq_handle, sc, &sc->bce_intrhand, &sc->main_serialize);
7143         if (error != 0) {
7144                 device_printf(sc->bce_dev, "Failed to setup IRQ!\n");
7145                 return error;
7146         }
7147         sc->bce_msix[0].msix_cpuid = rman_get_cpuid(sc->bce_res_irq);
7148         sc->bce_msix[0].msix_serialize = &sc->main_serialize;
7149
7150         return 0;
7151 }
7152
7153 static void
7154 bce_teardown_intr(struct bce_softc *sc)
7155 {
7156         if (sc->bce_irq_type != PCI_INTR_TYPE_MSIX)
7157                 bus_teardown_intr(sc->bce_dev, sc->bce_res_irq, sc->bce_intrhand);
7158         else
7159                 bce_teardown_msix(sc, sc->rx_ring_cnt);
7160 }
7161
7162 static int
7163 bce_setup_msix(struct bce_softc *sc)
7164 {
7165         int i;
7166
7167         for (i = 0; i < sc->rx_ring_cnt; ++i) {
7168                 struct bce_msix_data *msix = &sc->bce_msix[i];
7169                 int error;
7170
7171                 error = bus_setup_intr_descr(sc->bce_dev, msix->msix_res,
7172                     INTR_MPSAFE, msix->msix_func, msix->msix_arg,
7173                     &msix->msix_handle, msix->msix_serialize, msix->msix_desc);
7174                 if (error) {
7175                         device_printf(sc->bce_dev, "could not set up %s "
7176                             "interrupt handler.\n", msix->msix_desc);
7177                         bce_teardown_msix(sc, i);
7178                         return error;
7179                 }
7180         }
7181         return 0;
7182 }
7183
7184 static void
7185 bce_teardown_msix(struct bce_softc *sc, int msix_cnt)
7186 {
7187         int i;
7188
7189         for (i = 0; i < msix_cnt; ++i) {
7190                 struct bce_msix_data *msix = &sc->bce_msix[i];
7191
7192                 bus_teardown_intr(sc->bce_dev, msix->msix_res,
7193                     msix->msix_handle);
7194         }
7195 }
7196
7197 static void
7198 bce_init_rss(struct bce_softc *sc)
7199 {
7200         uint8_t key[BCE_RLUP_RSS_KEY_CNT * BCE_RLUP_RSS_KEY_SIZE];
7201         uint32_t tbl = 0;
7202         int i;
7203
7204         KKASSERT(sc->rx_ring_cnt > 2);
7205
7206         /*
7207          * Configure RSS keys
7208          */
7209         toeplitz_get_key(key, sizeof(key));
7210         for (i = 0; i < BCE_RLUP_RSS_KEY_CNT; ++i) {
7211                 uint32_t rss_key;
7212
7213                 rss_key = BCE_RLUP_RSS_KEYVAL(key, i);
7214                 BCE_RSS_DPRINTF(sc, 1, "rss_key%d 0x%08x\n", i, rss_key);
7215
7216                 REG_WR(sc, BCE_RLUP_RSS_KEY(i), rss_key);
7217         }
7218
7219         /*
7220          * Configure the redirect table
7221          *
7222          * NOTE:
7223          * - The "queue ID" in redirect table is the software RX ring's
7224          *   index _minus_ one.
7225          * - The last RX ring, whose "queue ID" is (sc->rx_ring_cnt - 2)
7226          *   will be used for packets whose masked hash is 0.
7227          *   (see also: comment in bce_setup_ring_cnt())
7228          *
7229          * The redirect table is configured in following fashion, except
7230          * for the masked hash 0, which is noted above:
7231          * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
7232          */
7233         for (i = 0; i < BCE_RXP_SCRATCH_RSS_TBL_MAX_ENTRIES; i++) {
7234                 int shift = (i % 8) << 2, qid;
7235
7236                 qid = i % sc->rx_ring_cnt2;
7237                 if (qid > 0)
7238                         --qid;
7239                 else
7240                         qid = sc->rx_ring_cnt - 2;
7241                 KKASSERT(qid < (sc->rx_ring_cnt - 1));
7242
7243                 tbl |= qid << shift;
7244                 if (i % 8 == 7) {
7245                         BCE_RSS_DPRINTF(sc, 1, "tbl 0x%08x\n", tbl);
7246                         REG_WR(sc, BCE_RLUP_RSS_DATA, tbl);
7247                         REG_WR(sc, BCE_RLUP_RSS_COMMAND, (i >> 3) |
7248                             BCE_RLUP_RSS_COMMAND_RSS_WRITE_MASK |
7249                             BCE_RLUP_RSS_COMMAND_WRITE |
7250                             BCE_RLUP_RSS_COMMAND_HASH_MASK);
7251                         tbl = 0;
7252                 }
7253         }
7254         REG_WR(sc, BCE_RLUP_RSS_CONFIG,
7255             BCE_RLUP_RSS_CONFIG_IPV4_RSS_TYPE_ALL_XI);
7256 }
7257
7258 static void
7259 bce_npoll_coal_change(struct bce_softc *sc)
7260 {
7261         uint32_t old_rx_cons, old_tx_cons;
7262
7263         old_rx_cons = sc->bce_rx_quick_cons_trip_int;
7264         old_tx_cons = sc->bce_tx_quick_cons_trip_int;
7265         sc->bce_rx_quick_cons_trip_int = 1;
7266         sc->bce_tx_quick_cons_trip_int = 1;
7267
7268         sc->bce_coalchg_mask |= BCE_COALMASK_TX_BDS_INT |
7269             BCE_COALMASK_RX_BDS_INT;
7270         bce_coal_change(sc);
7271
7272         sc->bce_rx_quick_cons_trip_int = old_rx_cons;
7273         sc->bce_tx_quick_cons_trip_int = old_tx_cons;
7274 }