sys/dev/netif/bge/if_bge.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  */
  35
  36 /*
  37  * Broadcom BCM570x family gigabit ethernet driver for FreeBSD.
  38  *
  39  * Written by Bill Paul <wpaul@windriver.com>
  40  * Senior Engineer, Wind River Systems
  41  */
  42
  43 /*
  44  * The Broadcom BCM5700 is based on technology originally developed by
  45  * Alteon Networks as part of the Tigon I and Tigon II gigabit ethernet
  46  * MAC chips. The BCM5700, sometimes refered to as the Tigon III, has
  47  * two on-board MIPS R4000 CPUs and can have as much as 16MB of external
  48  * SSRAM. The BCM5700 supports TCP, UDP and IP checksum offload, jumbo
  49  * frames, highly configurable RX filtering, and 16 RX and TX queues
  50  * (which, along with RX filter rules, can be used for QOS applications).
  51  * Other features, such as TCP segmentation, may be available as part
  52  * of value-added firmware updates. Unlike the Tigon I and Tigon II,
  53  * firmware images can be stored in hardware and need not be compiled
  54  * into the driver.
  55  *
  56  * The BCM5700 supports the PCI v2.2 and PCI-X v1.0 standards, and will
  57  * function in a 32-bit/64-bit 33/66Mhz bus, or a 64-bit/133Mhz bus.
  58  *
  59  * The BCM5701 is a single-chip solution incorporating both the BCM5700
  60  * MAC and a BCM5401 10/100/1000 PHY. Unlike the BCM5700, the BCM5701
  61  * does not support external SSRAM.
  62  *
  63  * Broadcom also produces a variation of the BCM5700 under the "Altima"
  64  * brand name, which is functionally similar but lacks PCI-X support.
  65  *
  66  * Without external SSRAM, you can only have at most 4 TX rings,
  67  * and the use of the mini RX ring is disabled. This seems to imply
  68  * that these features are simply not available on the BCM5701. As a
  69  * result, this driver does not implement any support for the mini RX
  70  * ring.
  71  */
  72
  73 #include "opt_ifpoll.h"
  74
  75 #include <sys/param.h>
  76 #include <sys/bus.h>
  77 #include <sys/endian.h>
  78 #include <sys/kernel.h>
  79 #include <sys/ktr.h>
  80 #include <sys/interrupt.h>
  81 #include <sys/mbuf.h>
  82 #include <sys/malloc.h>
  83 #include <sys/queue.h>
  84 #include <sys/rman.h>
  85 #include <sys/serialize.h>
  86 #include <sys/socket.h>
  87 #include <sys/sockio.h>
  88 #include <sys/sysctl.h>
  89
  90 #include <netinet/ip.h>
  91 #include <netinet/tcp.h>
  92
  93 #include <net/bpf.h>
  94 #include <net/ethernet.h>
  95 #include <net/if.h>
  96 #include <net/if_arp.h>
  97 #include <net/if_dl.h>
  98 #include <net/if_media.h>
  99 #include <net/if_poll.h>
 100 #include <net/if_types.h>
 101 #include <net/ifq_var.h>
 102 #include <net/vlan/if_vlan_var.h>
 103 #include <net/vlan/if_vlan_ether.h>
 104
 105 #include <dev/netif/mii_layer/mii.h>
 106 #include <dev/netif/mii_layer/miivar.h>
 107 #include <dev/netif/mii_layer/brgphyreg.h>
 108
 109 #include "pcidevs.h"
 110 #include <bus/pci/pcireg.h>
 111 #include <bus/pci/pcivar.h>
 112
 113 #include <dev/netif/bge/if_bgereg.h>
 114 #include <dev/netif/bge/if_bgevar.h>
 115
 116 /* "device miibus" required.  See GENERIC if you get errors here. */
 117 #include "miibus_if.h"
 118
 119 #define BGE_CSUM_FEATURES       (CSUM_IP | CSUM_TCP)
 120
 121 #define BGE_RESET_SHUTDOWN      0
 122 #define BGE_RESET_START         1
 123 #define BGE_RESET_SUSPEND       2
 124
 125 static const struct bge_type {
 126         uint16_t                bge_vid;
 127         uint16_t                bge_did;
 128         char                    *bge_name;
 129 } bge_devs[] = {
 130         { PCI_VENDOR_3COM, PCI_PRODUCT_3COM_3C996,
 131                 "3COM 3C996 Gigabit Ethernet" },
 132
 133         { PCI_VENDOR_ALTEON, PCI_PRODUCT_ALTEON_BCM5700,
 134                 "Alteon BCM5700 Gigabit Ethernet" },
 135         { PCI_VENDOR_ALTEON, PCI_PRODUCT_ALTEON_BCM5701,
 136                 "Alteon BCM5701 Gigabit Ethernet" },
 137
 138         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC1000,
 139                 "Altima AC1000 Gigabit Ethernet" },
 140         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC1001,
 141                 "Altima AC1002 Gigabit Ethernet" },
 142         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC9100,
 143                 "Altima AC9100 Gigabit Ethernet" },
 144
 145         { PCI_VENDOR_APPLE, PCI_PRODUCT_APPLE_BCM5701,
 146                 "Apple BCM5701 Gigabit Ethernet" },
 147
 148         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5700,
 149                 "Broadcom BCM5700 Gigabit Ethernet" },
 150         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5701,
 151                 "Broadcom BCM5701 Gigabit Ethernet" },
 152         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702,
 153                 "Broadcom BCM5702 Gigabit Ethernet" },
 154         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702X,
 155                 "Broadcom BCM5702X Gigabit Ethernet" },
 156         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702_ALT,
 157                 "Broadcom BCM5702 Gigabit Ethernet" },
 158         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703,
 159                 "Broadcom BCM5703 Gigabit Ethernet" },
 160         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703X,
 161                 "Broadcom BCM5703X Gigabit Ethernet" },
 162         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703A3,
 163                 "Broadcom BCM5703 Gigabit Ethernet" },
 164         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704C,
 165                 "Broadcom BCM5704C Dual Gigabit Ethernet" },
 166         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704S,
 167                 "Broadcom BCM5704S Dual Gigabit Ethernet" },
 168         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704S_ALT,
 169                 "Broadcom BCM5704S Dual Gigabit Ethernet" },
 170         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705,
 171                 "Broadcom BCM5705 Gigabit Ethernet" },
 172         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705F,
 173                 "Broadcom BCM5705F Gigabit Ethernet" },
 174         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705K,
 175                 "Broadcom BCM5705K Gigabit Ethernet" },
 176         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705M,
 177                 "Broadcom BCM5705M Gigabit Ethernet" },
 178         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705M_ALT,
 179                 "Broadcom BCM5705M Gigabit Ethernet" },
 180         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5714,
 181                 "Broadcom BCM5714C Gigabit Ethernet" },
 182         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5714S,
 183                 "Broadcom BCM5714S Gigabit Ethernet" },
 184         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5715,
 185                 "Broadcom BCM5715 Gigabit Ethernet" },
 186         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5715S,
 187                 "Broadcom BCM5715S Gigabit Ethernet" },
 188         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720,
 189                 "Broadcom BCM5720 Gigabit Ethernet" },
 190         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5721,
 191                 "Broadcom BCM5721 Gigabit Ethernet" },
 192         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5722,
 193                 "Broadcom BCM5722 Gigabit Ethernet" },
 194         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5723,
 195                 "Broadcom BCM5723 Gigabit Ethernet" },
 196         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5750,
 197                 "Broadcom BCM5750 Gigabit Ethernet" },
 198         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5750M,
 199                 "Broadcom BCM5750M Gigabit Ethernet" },
 200         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751,
 201                 "Broadcom BCM5751 Gigabit Ethernet" },
 202         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751F,
 203                 "Broadcom BCM5751F Gigabit Ethernet" },
 204         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751M,
 205                 "Broadcom BCM5751M Gigabit Ethernet" },
 206         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5752,
 207                 "Broadcom BCM5752 Gigabit Ethernet" },
 208         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5752M,
 209                 "Broadcom BCM5752M Gigabit Ethernet" },
 210         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753,
 211                 "Broadcom BCM5753 Gigabit Ethernet" },
 212         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753F,
 213                 "Broadcom BCM5753F Gigabit Ethernet" },
 214         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753M,
 215                 "Broadcom BCM5753M Gigabit Ethernet" },
 216         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5754,
 217                 "Broadcom BCM5754 Gigabit Ethernet" },
 218         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5754M,
 219                 "Broadcom BCM5754M Gigabit Ethernet" },
 220         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5755,
 221                 "Broadcom BCM5755 Gigabit Ethernet" },
 222         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5755M,
 223                 "Broadcom BCM5755M Gigabit Ethernet" },
 224         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5756,
 225                 "Broadcom BCM5756 Gigabit Ethernet" },
 226         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5761,
 227                 "Broadcom BCM5761 Gigabit Ethernet" },
 228         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5761E,
 229                 "Broadcom BCM5761E Gigabit Ethernet" },
 230         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5761S,
 231                 "Broadcom BCM5761S Gigabit Ethernet" },
 232         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5761SE,
 233                 "Broadcom BCM5761SE Gigabit Ethernet" },
 234         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5764,
 235                 "Broadcom BCM5764 Gigabit Ethernet" },
 236         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5780,
 237                 "Broadcom BCM5780 Gigabit Ethernet" },
 238         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5780S,
 239                 "Broadcom BCM5780S Gigabit Ethernet" },
 240         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5781,
 241                 "Broadcom BCM5781 Gigabit Ethernet" },
 242         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5782,
 243                 "Broadcom BCM5782 Gigabit Ethernet" },
 244         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5784,
 245                 "Broadcom BCM5784 Gigabit Ethernet" },
 246         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5785F,
 247                 "Broadcom BCM5785F Gigabit Ethernet" },
 248         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5785G,
 249                 "Broadcom BCM5785G Gigabit Ethernet" },
 250         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5786,
 251                 "Broadcom BCM5786 Gigabit Ethernet" },
 252         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787,
 253                 "Broadcom BCM5787 Gigabit Ethernet" },
 254         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787F,
 255                 "Broadcom BCM5787F Gigabit Ethernet" },
 256         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787M,
 257                 "Broadcom BCM5787M Gigabit Ethernet" },
 258         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5788,
 259                 "Broadcom BCM5788 Gigabit Ethernet" },
 260         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5789,
 261                 "Broadcom BCM5789 Gigabit Ethernet" },
 262         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5901,
 263                 "Broadcom BCM5901 Fast Ethernet" },
 264         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5901A2,
 265                 "Broadcom BCM5901A2 Fast Ethernet" },
 266         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5903M,
 267                 "Broadcom BCM5903M Fast Ethernet" },
 268         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5906,
 269                 "Broadcom BCM5906 Fast Ethernet"},
 270         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5906M,
 271                 "Broadcom BCM5906M Fast Ethernet"},
 272         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57760,
 273                 "Broadcom BCM57760 Gigabit Ethernet"},
 274         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57780,
 275                 "Broadcom BCM57780 Gigabit Ethernet"},
 276         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57788,
 277                 "Broadcom BCM57788 Gigabit Ethernet"},
 278         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57790,
 279                 "Broadcom BCM57790 Gigabit Ethernet"},
 280         { PCI_VENDOR_SCHNEIDERKOCH, PCI_PRODUCT_SCHNEIDERKOCH_SK_9DX1,
 281                 "SysKonnect Gigabit Ethernet" },
 282
 283         { 0, 0, NULL }
 284 };
 285
 286 #define BGE_IS_JUMBO_CAPABLE(sc)        ((sc)->bge_flags & BGE_FLAG_JUMBO)
 287 #define BGE_IS_5700_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5700_FAMILY)
 288 #define BGE_IS_5705_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5705_PLUS)
 289 #define BGE_IS_5714_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5714_FAMILY)
 290 #define BGE_IS_575X_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_575X_PLUS)
 291 #define BGE_IS_5755_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5755_PLUS)
 292 #define BGE_IS_5788(sc)                 ((sc)->bge_flags & BGE_FLAG_5788)
 293
 294 #define BGE_IS_CRIPPLED(sc)             \
 295         (BGE_IS_5788((sc)) || (sc)->bge_asicrev == BGE_ASICREV_BCM5700)
 296
 297 typedef int     (*bge_eaddr_fcn_t)(struct bge_softc *, uint8_t[]);
 298
 299 static int      bge_probe(device_t);
 300 static int      bge_attach(device_t);
 301 static int      bge_detach(device_t);
 302 static void     bge_txeof(struct bge_softc *, uint16_t);
 303 static void     bge_rxeof(struct bge_softc *, uint16_t, int);
 304
 305 static void     bge_tick(void *);
 306 static void     bge_stats_update(struct bge_softc *);
 307 static void     bge_stats_update_regs(struct bge_softc *);
 308 static struct mbuf *
 309                 bge_defrag_shortdma(struct mbuf *);
 310 static int      bge_encap(struct bge_softc *, struct mbuf **,
 311                     uint32_t *, int *);
 312 static void     bge_xmit(struct bge_softc *, uint32_t);
 313 static int      bge_setup_tso(struct bge_softc *, struct mbuf **,
 314                     uint16_t *, uint16_t *);
 315
 316 #ifdef IFPOLL_ENABLE
 317 static void     bge_npoll(struct ifnet *, struct ifpoll_info *);
 318 static void     bge_npoll_compat(struct ifnet *, void *, int );
 319 #endif
 320 static void     bge_intr_crippled(void *);
 321 static void     bge_intr_legacy(void *);
 322 static void     bge_msi(void *);
 323 static void     bge_msi_oneshot(void *);
 324 static void     bge_intr(struct bge_softc *);
 325 static void     bge_enable_intr(struct bge_softc *);
 326 static void     bge_disable_intr(struct bge_softc *);
 327 static void     bge_start(struct ifnet *, struct ifaltq_subque *);
 328 static int      bge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 329 static void     bge_init(void *);
 330 static void     bge_stop(struct bge_softc *);
 331 static void     bge_watchdog(struct ifnet *);
 332 static void     bge_shutdown(device_t);
 333 static int      bge_suspend(device_t);
 334 static int      bge_resume(device_t);
 335 static int      bge_ifmedia_upd(struct ifnet *);
 336 static void     bge_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 337
 338 static uint8_t  bge_nvram_getbyte(struct bge_softc *, int, uint8_t *);
 339 static int      bge_read_nvram(struct bge_softc *, caddr_t, int, int);
 340
 341 static uint8_t  bge_eeprom_getbyte(struct bge_softc *, uint32_t, uint8_t *);
 342 static int      bge_read_eeprom(struct bge_softc *, caddr_t, uint32_t, size_t);
 343
 344 static void     bge_setmulti(struct bge_softc *);
 345 static void     bge_setpromisc(struct bge_softc *);
 346 static void     bge_enable_msi(struct bge_softc *sc);
 347
 348 static int      bge_alloc_jumbo_mem(struct bge_softc *);
 349 static void     bge_free_jumbo_mem(struct bge_softc *);
 350 static struct bge_jslot
 351                 *bge_jalloc(struct bge_softc *);
 352 static void     bge_jfree(void *);
 353 static void     bge_jref(void *);
 354 static int      bge_newbuf_std(struct bge_softc *, int, int);
 355 static int      bge_newbuf_jumbo(struct bge_softc *, int, int);
 356 static void     bge_setup_rxdesc_std(struct bge_softc *, int);
 357 static void     bge_setup_rxdesc_jumbo(struct bge_softc *, int);
 358 static int      bge_init_rx_ring_std(struct bge_softc *);
 359 static void     bge_free_rx_ring_std(struct bge_softc *);
 360 static int      bge_init_rx_ring_jumbo(struct bge_softc *);
 361 static void     bge_free_rx_ring_jumbo(struct bge_softc *);
 362 static void     bge_free_tx_ring(struct bge_softc *);
 363 static int      bge_init_tx_ring(struct bge_softc *);
 364
 365 static int      bge_chipinit(struct bge_softc *);
 366 static int      bge_blockinit(struct bge_softc *);
 367 static void     bge_stop_block(struct bge_softc *, bus_size_t, uint32_t);
 368
 369 static uint32_t bge_readmem_ind(struct bge_softc *, uint32_t);
 370 static void     bge_writemem_ind(struct bge_softc *, uint32_t, uint32_t);
 371 #ifdef notdef
 372 static uint32_t bge_readreg_ind(struct bge_softc *, uint32_t);
 373 #endif
 374 static void     bge_writereg_ind(struct bge_softc *, uint32_t, uint32_t);
 375 static void     bge_writemem_direct(struct bge_softc *, uint32_t, uint32_t);
 376 static void     bge_writembx(struct bge_softc *, int, int);
 377
 378 static int      bge_miibus_readreg(device_t, int, int);
 379 static int      bge_miibus_writereg(device_t, int, int, int);
 380 static void     bge_miibus_statchg(device_t);
 381 static void     bge_bcm5700_link_upd(struct bge_softc *, uint32_t);
 382 static void     bge_tbi_link_upd(struct bge_softc *, uint32_t);
 383 static void     bge_copper_link_upd(struct bge_softc *, uint32_t);
 384 static void     bge_autopoll_link_upd(struct bge_softc *, uint32_t);
 385 static void     bge_link_poll(struct bge_softc *);
 386
 387 static void     bge_reset(struct bge_softc *);
 388
 389 static int      bge_dma_alloc(struct bge_softc *);
 390 static void     bge_dma_free(struct bge_softc *);
 391 static int      bge_dma_block_alloc(struct bge_softc *, bus_size_t,
 392                                     bus_dma_tag_t *, bus_dmamap_t *,
 393                                     void **, bus_addr_t *);
 394 static void     bge_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 395
 396 static int      bge_get_eaddr_mem(struct bge_softc *, uint8_t[]);
 397 static int      bge_get_eaddr_nvram(struct bge_softc *, uint8_t[]);
 398 static int      bge_get_eaddr_eeprom(struct bge_softc *, uint8_t[]);
 399 static int      bge_get_eaddr(struct bge_softc *, uint8_t[]);
 400
 401 static void     bge_coal_change(struct bge_softc *);
 402 static int      bge_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 403 static int      bge_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 404 static int      bge_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
 405 static int      bge_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
 406 static int      bge_sysctl_rx_coal_ticks_int(SYSCTL_HANDLER_ARGS);
 407 static int      bge_sysctl_tx_coal_ticks_int(SYSCTL_HANDLER_ARGS);
 408 static int      bge_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 409 static int      bge_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 410 static int      bge_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
 411                     int, int, uint32_t);
 412
 413 static void     bge_sig_post_reset(struct bge_softc *, int);
 414 static void     bge_sig_legacy(struct bge_softc *, int);
 415 static void     bge_sig_pre_reset(struct bge_softc *, int);
 416 static void     bge_stop_fw(struct bge_softc *);
 417 static void     bge_asf_driver_up(struct bge_softc *);
 418
 419 static void     bge_ape_lock_init(struct bge_softc *);
 420 static void     bge_ape_read_fw_ver(struct bge_softc *);
 421 static int      bge_ape_lock(struct bge_softc *, int);
 422 static void     bge_ape_unlock(struct bge_softc *, int);
 423 static void     bge_ape_send_event(struct bge_softc *, uint32_t);
 424 static void     bge_ape_driver_state_change(struct bge_softc *, int);
 425
 426 /*
 427  * Set following tunable to 1 for some IBM blade servers with the DNLK
 428  * switch module. Auto negotiation is broken for those configurations.
 429  */
 430 static int      bge_fake_autoneg = 0;
 431 TUNABLE_INT("hw.bge.fake_autoneg", &bge_fake_autoneg);
 432
 433 static int      bge_msi_enable = 1;
 434 TUNABLE_INT("hw.bge.msi.enable", &bge_msi_enable);
 435
 436 static int      bge_allow_asf = 1;
 437 TUNABLE_INT("hw.bge.allow_asf", &bge_allow_asf);
 438
 439 #if !defined(KTR_IF_BGE)
 440 #define KTR_IF_BGE      KTR_ALL
 441 #endif
 442 KTR_INFO_MASTER(if_bge);
 443 KTR_INFO(KTR_IF_BGE, if_bge, intr, 0, "intr");
 444 KTR_INFO(KTR_IF_BGE, if_bge, rx_pkt, 1, "rx_pkt");
 445 KTR_INFO(KTR_IF_BGE, if_bge, tx_pkt, 2, "tx_pkt");
 446 #define logif(name)     KTR_LOG(if_bge_ ## name)
 447
 448 static device_method_t bge_methods[] = {
 449         /* Device interface */
 450         DEVMETHOD(device_probe,         bge_probe),
 451         DEVMETHOD(device_attach,        bge_attach),
 452         DEVMETHOD(device_detach,        bge_detach),
 453         DEVMETHOD(device_shutdown,      bge_shutdown),
 454         DEVMETHOD(device_suspend,       bge_suspend),
 455         DEVMETHOD(device_resume,        bge_resume),
 456
 457         /* bus interface */
 458         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 459         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 460
 461         /* MII interface */
 462         DEVMETHOD(miibus_readreg,       bge_miibus_readreg),
 463         DEVMETHOD(miibus_writereg,      bge_miibus_writereg),
 464         DEVMETHOD(miibus_statchg,       bge_miibus_statchg),
 465
 466         DEVMETHOD_END
 467 };
 468
 469 static DEFINE_CLASS_0(bge, bge_driver, bge_methods, sizeof(struct bge_softc));
 470 static devclass_t bge_devclass;
 471
 472 DECLARE_DUMMY_MODULE(if_bge);
 473 DRIVER_MODULE(if_bge, pci, bge_driver, bge_devclass, NULL, NULL);
 474 DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, NULL, NULL);
 475
 476 static uint32_t
 477 bge_readmem_ind(struct bge_softc *sc, uint32_t off)
 478 {
 479         device_t dev = sc->bge_dev;
 480         uint32_t val;
 481
 482         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
 483             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 484                 return 0;
 485
 486         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 487         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 488         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 489         return (val);
 490 }
 491
 492 static void
 493 bge_writemem_ind(struct bge_softc *sc, uint32_t off, uint32_t val)
 494 {
 495         device_t dev = sc->bge_dev;
 496
 497         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
 498             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 499                 return;
 500
 501         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 502         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 503         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 504 }
 505
 506 #ifdef notdef
 507 static uint32_t
 508 bge_readreg_ind(struct bge_softc *sc, uin32_t off)
 509 {
 510         device_t dev = sc->bge_dev;
 511
 512         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 513         return(pci_read_config(dev, BGE_PCI_REG_DATA, 4));
 514 }
 515 #endif
 516
 517 static void
 518 bge_writereg_ind(struct bge_softc *sc, uint32_t off, uint32_t val)
 519 {
 520         device_t dev = sc->bge_dev;
 521
 522         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 523         pci_write_config(dev, BGE_PCI_REG_DATA, val, 4);
 524 }
 525
 526 static void
 527 bge_writemem_direct(struct bge_softc *sc, uint32_t off, uint32_t val)
 528 {
 529         CSR_WRITE_4(sc, off, val);
 530 }
 531
 532 static void
 533 bge_writembx(struct bge_softc *sc, int off, int val)
 534 {
 535         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
 536                 off += BGE_LPMBX_IRQ0_HI - BGE_MBX_IRQ0_HI;
 537
 538         CSR_WRITE_4(sc, off, val);
 539         if (sc->bge_mbox_reorder)
 540                 CSR_READ_4(sc, off);
 541 }
 542
 543 static uint8_t
 544 bge_nvram_getbyte(struct bge_softc *sc, int addr, uint8_t *dest)
 545 {
 546         uint32_t access, byte = 0;
 547         int i;
 548
 549         /* Lock. */
 550         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
 551         for (i = 0; i < 8000; i++) {
 552                 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1)
 553                         break;
 554                 DELAY(20);
 555         }
 556         if (i == 8000)
 557                 return (1);
 558
 559         /* Enable access. */
 560         access = CSR_READ_4(sc, BGE_NVRAM_ACCESS);
 561         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access | BGE_NVRAMACC_ENABLE);
 562
 563         CSR_WRITE_4(sc, BGE_NVRAM_ADDR, addr & 0xfffffffc);
 564         CSR_WRITE_4(sc, BGE_NVRAM_CMD, BGE_NVRAM_READCMD);
 565         for (i = 0; i < BGE_TIMEOUT * 10; i++) {
 566                 DELAY(10);
 567                 if (CSR_READ_4(sc, BGE_NVRAM_CMD) & BGE_NVRAMCMD_DONE) {
 568                         DELAY(10);
 569                         break;
 570                 }
 571         }
 572
 573         if (i == BGE_TIMEOUT * 10) {
 574                 if_printf(&sc->arpcom.ac_if, "nvram read timed out\n");
 575                 return (1);
 576         }
 577
 578         /* Get result. */
 579         byte = CSR_READ_4(sc, BGE_NVRAM_RDDATA);
 580
 581         *dest = (bswap32(byte) >> ((addr % 4) * 8)) & 0xFF;
 582
 583         /* Disable access. */
 584         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access);
 585
 586         /* Unlock. */
 587         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_CLR1);
 588         CSR_READ_4(sc, BGE_NVRAM_SWARB);
 589
 590         return (0);
 591 }
 592
 593 /*
 594  * Read a sequence of bytes from NVRAM.
 595  */
 596 static int
 597 bge_read_nvram(struct bge_softc *sc, caddr_t dest, int off, int cnt)
 598 {
 599         int err = 0, i;
 600         uint8_t byte = 0;
 601
 602         if (sc->bge_asicrev != BGE_ASICREV_BCM5906)
 603                 return (1);
 604
 605         for (i = 0; i < cnt; i++) {
 606                 err = bge_nvram_getbyte(sc, off + i, &byte);
 607                 if (err)
 608                         break;
 609                 *(dest + i) = byte;
 610         }
 611
 612         return (err ? 1 : 0);
 613 }
 614
 615 /*
 616  * Read a byte of data stored in the EEPROM at address 'addr.' The
 617  * BCM570x supports both the traditional bitbang interface and an
 618  * auto access interface for reading the EEPROM. We use the auto
 619  * access method.
 620  */
 621 static uint8_t
 622 bge_eeprom_getbyte(struct bge_softc *sc, uint32_t addr, uint8_t *dest)
 623 {
 624         int i;
 625         uint32_t byte = 0;
 626
 627         /*
 628          * Enable use of auto EEPROM access so we can avoid
 629          * having to use the bitbang method.
 630          */
 631         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 632
 633         /* Reset the EEPROM, load the clock period. */
 634         CSR_WRITE_4(sc, BGE_EE_ADDR,
 635             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 636         DELAY(20);
 637
 638         /* Issue the read EEPROM command. */
 639         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 640
 641         /* Wait for completion */
 642         for(i = 0; i < BGE_TIMEOUT * 10; i++) {
 643                 DELAY(10);
 644                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 645                         break;
 646         }
 647
 648         if (i == BGE_TIMEOUT) {
 649                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 650                 return(1);
 651         }
 652
 653         /* Get result. */
 654         byte = CSR_READ_4(sc, BGE_EE_DATA);
 655
 656         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 657
 658         return(0);
 659 }
 660
 661 /*
 662  * Read a sequence of bytes from the EEPROM.
 663  */
 664 static int
 665 bge_read_eeprom(struct bge_softc *sc, caddr_t dest, uint32_t off, size_t len)
 666 {
 667         size_t i;
 668         int err;
 669         uint8_t byte;
 670
 671         for (byte = 0, err = 0, i = 0; i < len; i++) {
 672                 err = bge_eeprom_getbyte(sc, off + i, &byte);
 673                 if (err)
 674                         break;
 675                 *(dest + i) = byte;
 676         }
 677
 678         return(err ? 1 : 0);
 679 }
 680
 681 static int
 682 bge_miibus_readreg(device_t dev, int phy, int reg)
 683 {
 684         struct bge_softc *sc = device_get_softc(dev);
 685         uint32_t val;
 686         int i;
 687
 688         KASSERT(phy == sc->bge_phyno,
 689             ("invalid phyno %d, should be %d", phy, sc->bge_phyno));
 690
 691         if (bge_ape_lock(sc, sc->bge_phy_ape_lock) != 0)
 692                 return 0;
 693
 694         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 695         if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
 696                 CSR_WRITE_4(sc, BGE_MI_MODE,
 697                     sc->bge_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 698                 DELAY(80);
 699         }
 700
 701         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
 702             BGE_MIPHY(phy) | BGE_MIREG(reg));
 703
 704         /* Poll for the PHY register access to complete. */
 705         for (i = 0; i < BGE_TIMEOUT; i++) {
 706                 DELAY(10);
 707                 val = CSR_READ_4(sc, BGE_MI_COMM);
 708                 if ((val & BGE_MICOMM_BUSY) == 0) {
 709                         DELAY(5);
 710                         val = CSR_READ_4(sc, BGE_MI_COMM);
 711                         break;
 712                 }
 713         }
 714         if (i == BGE_TIMEOUT) {
 715                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
 716                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 717                 val = 0;
 718         }
 719
 720         /* Restore the autopoll bit if necessary. */
 721         if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
 722                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
 723                 DELAY(80);
 724         }
 725
 726         bge_ape_unlock(sc, sc->bge_phy_ape_lock);
 727
 728         if (val & BGE_MICOMM_READFAIL)
 729                 return 0;
 730
 731         return (val & 0xFFFF);
 732 }
 733
 734 static int
 735 bge_miibus_writereg(device_t dev, int phy, int reg, int val)
 736 {
 737         struct bge_softc *sc = device_get_softc(dev);
 738         int i;
 739
 740         KASSERT(phy == sc->bge_phyno,
 741             ("invalid phyno %d, should be %d", phy, sc->bge_phyno));
 742
 743         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
 744             (reg == BRGPHY_MII_1000CTL || reg == BRGPHY_MII_AUXCTL))
 745                return 0;
 746
 747         if (bge_ape_lock(sc, sc->bge_phy_ape_lock) != 0)
 748                 return 0;
 749
 750         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 751         if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
 752                 CSR_WRITE_4(sc, BGE_MI_MODE,
 753                     sc->bge_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 754                 DELAY(80);
 755         }
 756
 757         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
 758             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
 759
 760         for (i = 0; i < BGE_TIMEOUT; i++) {
 761                 DELAY(10);
 762                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 763                         DELAY(5);
 764                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 765                         break;
 766                 }
 767         }
 768         if (i == BGE_TIMEOUT) {
 769                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 770                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
 771         }
 772
 773         /* Restore the autopoll bit if necessary. */
 774         if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
 775                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
 776                 DELAY(80);
 777         }
 778
 779         bge_ape_unlock(sc, sc->bge_phy_ape_lock);
 780
 781         return 0;
 782 }
 783
 784 static void
 785 bge_miibus_statchg(device_t dev)
 786 {
 787         struct bge_softc *sc;
 788         struct mii_data *mii;
 789         uint32_t mac_mode;
 790
 791         sc = device_get_softc(dev);
 792         if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0)
 793                 return;
 794
 795         mii = device_get_softc(sc->bge_miibus);
 796
 797         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 798             (IFM_ACTIVE | IFM_AVALID)) {
 799                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 800                 case IFM_10_T:
 801                 case IFM_100_TX:
 802                         sc->bge_link = 1;
 803                         break;
 804                 case IFM_1000_T:
 805                 case IFM_1000_SX:
 806                 case IFM_2500_SX:
 807                         if (sc->bge_asicrev != BGE_ASICREV_BCM5906)
 808                                 sc->bge_link = 1;
 809                         else
 810                                 sc->bge_link = 0;
 811                         break;
 812                 default:
 813                         sc->bge_link = 0;
 814                         break;
 815                 }
 816         } else {
 817                 sc->bge_link = 0;
 818         }
 819         if (sc->bge_link == 0)
 820                 return;
 821
 822         /*
 823          * APE firmware touches these registers to keep the MAC
 824          * connected to the outside world.  Try to keep the
 825          * accesses atomic.
 826          */
 827
 828         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) &
 829             ~(BGE_MACMODE_PORTMODE | BGE_MACMODE_HALF_DUPLEX);
 830
 831         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
 832             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX)
 833                 mac_mode |= BGE_PORTMODE_GMII;
 834         else
 835                 mac_mode |= BGE_PORTMODE_MII;
 836
 837         if ((mii->mii_media_active & IFM_GMASK) != IFM_FDX)
 838                 mac_mode |= BGE_MACMODE_HALF_DUPLEX;
 839
 840         CSR_WRITE_4(sc, BGE_MAC_MODE, mac_mode);
 841         DELAY(40);
 842 }
 843
 844 /*
 845  * Memory management for jumbo frames.
 846  */
 847 static int
 848 bge_alloc_jumbo_mem(struct bge_softc *sc)
 849 {
 850         struct ifnet *ifp = &sc->arpcom.ac_if;
 851         struct bge_jslot *entry;
 852         uint8_t *ptr;
 853         bus_addr_t paddr;
 854         int i, error;
 855
 856         /*
 857          * Create tag for jumbo mbufs.
 858          * This is really a bit of a kludge. We allocate a special
 859          * jumbo buffer pool which (thanks to the way our DMA
 860          * memory allocation works) will consist of contiguous
 861          * pages. This means that even though a jumbo buffer might
 862          * be larger than a page size, we don't really need to
 863          * map it into more than one DMA segment. However, the
 864          * default mbuf tag will result in multi-segment mappings,
 865          * so we have to create a special jumbo mbuf tag that
 866          * lets us get away with mapping the jumbo buffers as
 867          * a single segment. I think eventually the driver should
 868          * be changed so that it uses ordinary mbufs and cluster
 869          * buffers, i.e. jumbo frames can span multiple DMA
 870          * descriptors. But that's a project for another day.
 871          */
 872
 873         /*
 874          * Create DMA stuffs for jumbo RX ring.
 875          */
 876         error = bge_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 877                                     &sc->bge_cdata.bge_rx_jumbo_ring_tag,
 878                                     &sc->bge_cdata.bge_rx_jumbo_ring_map,
 879                                     (void *)&sc->bge_ldata.bge_rx_jumbo_ring,
 880                                     &sc->bge_ldata.bge_rx_jumbo_ring_paddr);
 881         if (error) {
 882                 if_printf(ifp, "could not create jumbo RX ring\n");
 883                 return error;
 884         }
 885
 886         /*
 887          * Create DMA stuffs for jumbo buffer block.
 888          */
 889         error = bge_dma_block_alloc(sc, BGE_JMEM,
 890                                     &sc->bge_cdata.bge_jumbo_tag,
 891                                     &sc->bge_cdata.bge_jumbo_map,
 892                                     (void **)&sc->bge_ldata.bge_jumbo_buf,
 893                                     &paddr);
 894         if (error) {
 895                 if_printf(ifp, "could not create jumbo buffer\n");
 896                 return error;
 897         }
 898
 899         SLIST_INIT(&sc->bge_jfree_listhead);
 900
 901         /*
 902          * Now divide it up into 9K pieces and save the addresses
 903          * in an array. Note that we play an evil trick here by using
 904          * the first few bytes in the buffer to hold the the address
 905          * of the softc structure for this interface. This is because
 906          * bge_jfree() needs it, but it is called by the mbuf management
 907          * code which will not pass it to us explicitly.
 908          */
 909         for (i = 0, ptr = sc->bge_ldata.bge_jumbo_buf; i < BGE_JSLOTS; i++) {
 910                 entry = &sc->bge_cdata.bge_jslots[i];
 911                 entry->bge_sc = sc;
 912                 entry->bge_buf = ptr;
 913                 entry->bge_paddr = paddr;
 914                 entry->bge_inuse = 0;
 915                 entry->bge_slot = i;
 916                 SLIST_INSERT_HEAD(&sc->bge_jfree_listhead, entry, jslot_link);
 917
 918                 ptr += BGE_JLEN;
 919                 paddr += BGE_JLEN;
 920         }
 921         return 0;
 922 }
 923
 924 static void
 925 bge_free_jumbo_mem(struct bge_softc *sc)
 926 {
 927         /* Destroy jumbo RX ring. */
 928         bge_dma_block_free(sc->bge_cdata.bge_rx_jumbo_ring_tag,
 929                            sc->bge_cdata.bge_rx_jumbo_ring_map,
 930                            sc->bge_ldata.bge_rx_jumbo_ring);
 931
 932         /* Destroy jumbo buffer block. */
 933         bge_dma_block_free(sc->bge_cdata.bge_jumbo_tag,
 934                            sc->bge_cdata.bge_jumbo_map,
 935                            sc->bge_ldata.bge_jumbo_buf);
 936 }
 937
 938 /*
 939  * Allocate a jumbo buffer.
 940  */
 941 static struct bge_jslot *
 942 bge_jalloc(struct bge_softc *sc)
 943 {
 944         struct bge_jslot *entry;
 945
 946         lwkt_serialize_enter(&sc->bge_jslot_serializer);
 947         entry = SLIST_FIRST(&sc->bge_jfree_listhead);
 948         if (entry) {
 949                 SLIST_REMOVE_HEAD(&sc->bge_jfree_listhead, jslot_link);
 950                 entry->bge_inuse = 1;
 951         } else {
 952                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 953         }
 954         lwkt_serialize_exit(&sc->bge_jslot_serializer);
 955         return(entry);
 956 }
 957
 958 /*
 959  * Adjust usage count on a jumbo buffer.
 960  */
 961 static void
 962 bge_jref(void *arg)
 963 {
 964         struct bge_jslot *entry = (struct bge_jslot *)arg;
 965         struct bge_softc *sc = entry->bge_sc;
 966
 967         if (sc == NULL)
 968                 panic("bge_jref: can't find softc pointer!");
 969
 970         if (&sc->bge_cdata.bge_jslots[entry->bge_slot] != entry) {
 971                 panic("bge_jref: asked to reference buffer "
 972                     "that we don't manage!");
 973         } else if (entry->bge_inuse == 0) {
 974                 panic("bge_jref: buffer already free!");
 975         } else {
 976                 atomic_add_int(&entry->bge_inuse, 1);
 977         }
 978 }
 979
 980 /*
 981  * Release a jumbo buffer.
 982  */
 983 static void
 984 bge_jfree(void *arg)
 985 {
 986         struct bge_jslot *entry = (struct bge_jslot *)arg;
 987         struct bge_softc *sc = entry->bge_sc;
 988
 989         if (sc == NULL)
 990                 panic("bge_jfree: can't find softc pointer!");
 991
 992         if (&sc->bge_cdata.bge_jslots[entry->bge_slot] != entry) {
 993                 panic("bge_jfree: asked to free buffer that we don't manage!");
 994         } else if (entry->bge_inuse == 0) {
 995                 panic("bge_jfree: buffer already free!");
 996         } else {
 997                 /*
 998                  * Possible MP race to 0, use the serializer.  The atomic insn
 999                  * is still needed for races against bge_jref().
1000                  */
1001                 lwkt_serialize_enter(&sc->bge_jslot_serializer);
1002                 atomic_subtract_int(&entry->bge_inuse, 1);
1003                 if (entry->bge_inuse == 0) {
1004                         SLIST_INSERT_HEAD(&sc->bge_jfree_listhead,
1005                                           entry, jslot_link);
1006                 }
1007                 lwkt_serialize_exit(&sc->bge_jslot_serializer);
1008         }
1009 }
1010
1011
1012 /*
1013  * Intialize a standard receive ring descriptor.
1014  */
1015 static int
1016 bge_newbuf_std(struct bge_softc *sc, int i, int init)
1017 {
1018         struct mbuf *m_new = NULL;
1019         bus_dma_segment_t seg;
1020         bus_dmamap_t map;
1021         int error, nsegs;
1022
1023         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
1024         if (m_new == NULL)
1025                 return ENOBUFS;
1026         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
1027
1028         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0)
1029                 m_adj(m_new, ETHER_ALIGN);
1030
1031         error = bus_dmamap_load_mbuf_segment(sc->bge_cdata.bge_rx_mtag,
1032                         sc->bge_cdata.bge_rx_tmpmap, m_new,
1033                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
1034         if (error) {
1035                 m_freem(m_new);
1036                 return error;
1037         }
1038
1039         if (!init) {
1040                 bus_dmamap_sync(sc->bge_cdata.bge_rx_mtag,
1041                                 sc->bge_cdata.bge_rx_std_dmamap[i],
1042                                 BUS_DMASYNC_POSTREAD);
1043                 bus_dmamap_unload(sc->bge_cdata.bge_rx_mtag,
1044                         sc->bge_cdata.bge_rx_std_dmamap[i]);
1045         }
1046
1047         map = sc->bge_cdata.bge_rx_tmpmap;
1048         sc->bge_cdata.bge_rx_tmpmap = sc->bge_cdata.bge_rx_std_dmamap[i];
1049         sc->bge_cdata.bge_rx_std_dmamap[i] = map;
1050
1051         sc->bge_cdata.bge_rx_std_chain[i].bge_mbuf = m_new;
1052         sc->bge_cdata.bge_rx_std_chain[i].bge_paddr = seg.ds_addr;
1053
1054         bge_setup_rxdesc_std(sc, i);
1055         return 0;
1056 }
1057
1058 static void
1059 bge_setup_rxdesc_std(struct bge_softc *sc, int i)
1060 {
1061         struct bge_rxchain *rc;
1062         struct bge_rx_bd *r;
1063
1064         rc = &sc->bge_cdata.bge_rx_std_chain[i];
1065         r = &sc->bge_ldata.bge_rx_std_ring[i];
1066
1067         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bge_paddr);
1068         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bge_paddr);
1069         r->bge_len = rc->bge_mbuf->m_len;
1070         r->bge_idx = i;
1071         r->bge_flags = BGE_RXBDFLAG_END;
1072 }
1073
1074 /*
1075  * Initialize a jumbo receive ring descriptor. This allocates
1076  * a jumbo buffer from the pool managed internally by the driver.
1077  */
1078 static int
1079 bge_newbuf_jumbo(struct bge_softc *sc, int i, int init)
1080 {
1081         struct mbuf *m_new = NULL;
1082         struct bge_jslot *buf;
1083         bus_addr_t paddr;
1084
1085         /* Allocate the mbuf. */
1086         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
1087         if (m_new == NULL)
1088                 return ENOBUFS;
1089
1090         /* Allocate the jumbo buffer */
1091         buf = bge_jalloc(sc);
1092         if (buf == NULL) {
1093                 m_freem(m_new);
1094                 return ENOBUFS;
1095         }
1096
1097         /* Attach the buffer to the mbuf. */
1098         m_new->m_ext.ext_arg = buf;
1099         m_new->m_ext.ext_buf = buf->bge_buf;
1100         m_new->m_ext.ext_free = bge_jfree;
1101         m_new->m_ext.ext_ref = bge_jref;
1102         m_new->m_ext.ext_size = BGE_JUMBO_FRAMELEN;
1103
1104         m_new->m_flags |= M_EXT;
1105
1106         m_new->m_data = m_new->m_ext.ext_buf;
1107         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
1108
1109         paddr = buf->bge_paddr;
1110         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0) {
1111                 m_adj(m_new, ETHER_ALIGN);
1112                 paddr += ETHER_ALIGN;
1113         }
1114
1115         /* Save necessary information */
1116         sc->bge_cdata.bge_rx_jumbo_chain[i].bge_mbuf = m_new;
1117         sc->bge_cdata.bge_rx_jumbo_chain[i].bge_paddr = paddr;
1118
1119         /* Set up the descriptor. */
1120         bge_setup_rxdesc_jumbo(sc, i);
1121         return 0;
1122 }
1123
1124 static void
1125 bge_setup_rxdesc_jumbo(struct bge_softc *sc, int i)
1126 {
1127         struct bge_rx_bd *r;
1128         struct bge_rxchain *rc;
1129
1130         r = &sc->bge_ldata.bge_rx_jumbo_ring[i];
1131         rc = &sc->bge_cdata.bge_rx_jumbo_chain[i];
1132
1133         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bge_paddr);
1134         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bge_paddr);
1135         r->bge_len = rc->bge_mbuf->m_len;
1136         r->bge_idx = i;
1137         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
1138 }
1139
1140 static int
1141 bge_init_rx_ring_std(struct bge_softc *sc)
1142 {
1143         int i, error;
1144
1145         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
1146                 error = bge_newbuf_std(sc, i, 1);
1147                 if (error)
1148                         return error;
1149         }
1150
1151         sc->bge_std = BGE_STD_RX_RING_CNT - 1;
1152         bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std);
1153
1154         return(0);
1155 }
1156
1157 static void
1158 bge_free_rx_ring_std(struct bge_softc *sc)
1159 {
1160         int i;
1161
1162         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
1163                 struct bge_rxchain *rc = &sc->bge_cdata.bge_rx_std_chain[i];
1164
1165                 if (rc->bge_mbuf != NULL) {
1166                         bus_dmamap_unload(sc->bge_cdata.bge_rx_mtag,
1167                                           sc->bge_cdata.bge_rx_std_dmamap[i]);
1168                         m_freem(rc->bge_mbuf);
1169                         rc->bge_mbuf = NULL;
1170                 }
1171                 bzero(&sc->bge_ldata.bge_rx_std_ring[i],
1172                     sizeof(struct bge_rx_bd));
1173         }
1174 }
1175
1176 static int
1177 bge_init_rx_ring_jumbo(struct bge_softc *sc)
1178 {
1179         struct bge_rcb *rcb;
1180         int i, error;
1181
1182         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1183                 error = bge_newbuf_jumbo(sc, i, 1);
1184                 if (error)
1185                         return error;
1186         }
1187
1188         sc->bge_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
1189
1190         rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
1191         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
1192         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1193
1194         bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo);
1195
1196         return(0);
1197 }
1198
1199 static void
1200 bge_free_rx_ring_jumbo(struct bge_softc *sc)
1201 {
1202         int i;
1203
1204         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1205                 struct bge_rxchain *rc = &sc->bge_cdata.bge_rx_jumbo_chain[i];
1206
1207                 if (rc->bge_mbuf != NULL) {
1208                         m_freem(rc->bge_mbuf);
1209                         rc->bge_mbuf = NULL;
1210                 }
1211                 bzero(&sc->bge_ldata.bge_rx_jumbo_ring[i],
1212                     sizeof(struct bge_rx_bd));
1213         }
1214 }
1215
1216 static void
1217 bge_free_tx_ring(struct bge_softc *sc)
1218 {
1219         int i;
1220
1221         for (i = 0; i < BGE_TX_RING_CNT; i++) {
1222                 if (sc->bge_cdata.bge_tx_chain[i] != NULL) {
1223                         bus_dmamap_unload(sc->bge_cdata.bge_tx_mtag,
1224                                           sc->bge_cdata.bge_tx_dmamap[i]);
1225                         m_freem(sc->bge_cdata.bge_tx_chain[i]);
1226                         sc->bge_cdata.bge_tx_chain[i] = NULL;
1227                 }
1228                 bzero(&sc->bge_ldata.bge_tx_ring[i],
1229                     sizeof(struct bge_tx_bd));
1230         }
1231 }
1232
1233 static int
1234 bge_init_tx_ring(struct bge_softc *sc)
1235 {
1236         sc->bge_txcnt = 0;
1237         sc->bge_tx_saved_considx = 0;
1238         sc->bge_tx_prodidx = 0;
1239
1240         /* Initialize transmit producer index for host-memory send ring. */
1241         bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bge_tx_prodidx);
1242
1243         /* 5700 b2 errata */
1244         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1245                 bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bge_tx_prodidx);
1246
1247         bge_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1248         /* 5700 b2 errata */
1249         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1250                 bge_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1251
1252         return(0);
1253 }
1254
1255 static void
1256 bge_setmulti(struct bge_softc *sc)
1257 {
1258         struct ifnet *ifp;
1259         struct ifmultiaddr *ifma;
1260         uint32_t hashes[4] = { 0, 0, 0, 0 };
1261         int h, i;
1262
1263         ifp = &sc->arpcom.ac_if;
1264
1265         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1266                 for (i = 0; i < 4; i++)
1267                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1268                 return;
1269         }
1270
1271         /* First, zot all the existing filters. */
1272         for (i = 0; i < 4; i++)
1273                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1274
1275         /* Now program new ones. */
1276         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1277                 if (ifma->ifma_addr->sa_family != AF_LINK)
1278                         continue;
1279                 h = ether_crc32_le(
1280                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1281                     ETHER_ADDR_LEN) & 0x7f;
1282                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1283         }
1284
1285         for (i = 0; i < 4; i++)
1286                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1287 }
1288
1289 /*
1290  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1291  * self-test results.
1292  */
1293 static int
1294 bge_chipinit(struct bge_softc *sc)
1295 {
1296         int i;
1297         uint32_t dma_rw_ctl, mode_ctl;
1298         uint16_t val;
1299
1300         /* Set endian type before we access any non-PCI registers. */
1301         pci_write_config(sc->bge_dev, BGE_PCI_MISC_CTL,
1302             BGE_INIT | sc->bge_pci_miscctl, 4);
1303
1304         /*
1305          * Clear the MAC statistics block in the NIC's
1306          * internal memory.
1307          */
1308         for (i = BGE_STATS_BLOCK;
1309             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1310                 BGE_MEMWIN_WRITE(sc, i, 0);
1311
1312         for (i = BGE_STATUS_BLOCK;
1313             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1314                 BGE_MEMWIN_WRITE(sc, i, 0);
1315
1316         if (sc->bge_chiprev == BGE_CHIPREV_5704_BX) {
1317                 /*
1318                  * Fix data corruption caused by non-qword write with WB.
1319                  * Fix master abort in PCI mode.
1320                  * Fix PCI latency timer.
1321                  */
1322                 val = pci_read_config(sc->bge_dev, BGE_PCI_MSI_DATA + 2, 2);
1323                 val |= (1 << 10) | (1 << 12) | (1 << 13);
1324                 pci_write_config(sc->bge_dev, BGE_PCI_MSI_DATA + 2, val, 2);
1325         }
1326
1327         /* Set up the PCI DMA control register. */
1328         dma_rw_ctl = BGE_PCI_READ_CMD | BGE_PCI_WRITE_CMD;
1329         if (sc->bge_flags & BGE_FLAG_PCIE) {
1330                 /* PCI-E bus */
1331                 /* DMA read watermark not used on PCI-E */
1332                 dma_rw_ctl |= (0x3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1333         } else if (sc->bge_flags & BGE_FLAG_PCIX) {
1334                 /* PCI-X bus */
1335                 if (sc->bge_asicrev == BGE_ASICREV_BCM5780) {
1336                         dma_rw_ctl |= (0x4 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1337                             (0x2 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1338                         dma_rw_ctl |= BGE_PCIDMARWCTL_ONEDMA_ATONCE_GLOBAL;
1339                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5714) {
1340                         dma_rw_ctl |= (0x4 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1341                             (0x2 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1342                         dma_rw_ctl |= BGE_PCIDMARWCTL_ONEDMA_ATONCE_LOCAL;
1343                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1344                     sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1345                         uint32_t rd_wat = 0x7;
1346                         uint32_t clkctl;
1347
1348                         clkctl = CSR_READ_4(sc, BGE_PCI_CLKCTL) & 0x1f;
1349                         if ((sc->bge_flags & BGE_FLAG_MAXADDR_40BIT) &&
1350                             sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1351                                 dma_rw_ctl |=
1352                                     BGE_PCIDMARWCTL_ONEDMA_ATONCE_LOCAL;
1353                         } else if (clkctl == 0x6 || clkctl == 0x7) {
1354                                 dma_rw_ctl |=
1355                                     BGE_PCIDMARWCTL_ONEDMA_ATONCE_GLOBAL;
1356                         }
1357                         if (sc->bge_asicrev == BGE_ASICREV_BCM5703)
1358                                 rd_wat = 0x4;
1359
1360                         dma_rw_ctl |= (rd_wat << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1361                             (3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1362                         dma_rw_ctl |= BGE_PCIDMARWCTL_ASRT_ALL_BE;
1363                 } else {
1364                         dma_rw_ctl |= (0x3 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1365                             (0x3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1366                         dma_rw_ctl |= 0xf;
1367                 }
1368         } else {
1369                 /* Conventional PCI bus */
1370                 dma_rw_ctl |= (0x7 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1371                     (0x7 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1372                 if (sc->bge_asicrev != BGE_ASICREV_BCM5705 &&
1373                     sc->bge_asicrev != BGE_ASICREV_BCM5750)
1374                         dma_rw_ctl |= 0xf;
1375         }
1376
1377         if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1378             sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1379                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_MINDMA;
1380         } else if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
1381             sc->bge_asicrev == BGE_ASICREV_BCM5701) {
1382                 dma_rw_ctl |= BGE_PCIDMARWCTL_USE_MRM |
1383                     BGE_PCIDMARWCTL_ASRT_ALL_BE;
1384         }
1385         pci_write_config(sc->bge_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1386
1387         /*
1388          * Set up general mode register.
1389          */
1390         mode_ctl = BGE_DMA_SWAP_OPTIONS|
1391             BGE_MODECTL_MAC_ATTN_INTR|BGE_MODECTL_HOST_SEND_BDS|
1392             BGE_MODECTL_TX_NO_PHDR_CSUM;
1393
1394         /*
1395          * BCM5701 B5 have a bug causing data corruption when using
1396          * 64-bit DMA reads, which can be terminated early and then
1397          * completed later as 32-bit accesses, in combination with
1398          * certain bridges.
1399          */
1400         if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
1401             sc->bge_chipid == BGE_CHIPID_BCM5701_B5)
1402                 mode_ctl |= BGE_MODECTL_FORCE_PCI32;
1403
1404         /*
1405          * Tell the firmware the driver is running
1406          */
1407         if (sc->bge_asf_mode & ASF_STACKUP)
1408                 mode_ctl |= BGE_MODECTL_STACKUP;
1409
1410         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1411
1412         /*
1413          * Disable memory write invalidate.  Apparently it is not supported
1414          * properly by these devices.  Also ensure that INTx isn't disabled,
1415          * as these chips need it even when using MSI.
1416          */
1417         PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD,
1418             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1419
1420         /* Set the timer prescaler (always 66Mhz) */
1421         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1422
1423         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
1424                 DELAY(40);      /* XXX */
1425
1426                 /* Put PHY into ready state */
1427                 BGE_CLRBIT(sc, BGE_MISC_CFG, BGE_MISCCFG_EPHY_IDDQ);
1428                 CSR_READ_4(sc, BGE_MISC_CFG); /* Flush */
1429                 DELAY(40);
1430         }
1431
1432         return(0);
1433 }
1434
1435 static int
1436 bge_blockinit(struct bge_softc *sc)
1437 {
1438         struct bge_rcb *rcb;
1439         bus_size_t vrcb;
1440         bge_hostaddr taddr;
1441         uint32_t val;
1442         int i, limit;
1443
1444         /*
1445          * Initialize the memory window pointer register so that
1446          * we can access the first 32K of internal NIC RAM. This will
1447          * allow us to set up the TX send ring RCBs and the RX return
1448          * ring RCBs, plus other things which live in NIC memory.
1449          */
1450         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1451
1452         /* Note: the BCM5704 has a smaller mbuf space than other chips. */
1453
1454         if (!BGE_IS_5705_PLUS(sc)) {
1455                 /* Configure mbuf memory pool */
1456                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_BASEADDR, BGE_BUFFPOOL_1);
1457                 if (sc->bge_asicrev == BGE_ASICREV_BCM5704)
1458                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x10000);
1459                 else
1460                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x18000);
1461
1462                 /* Configure DMA resource pool */
1463                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_BASEADDR,
1464                     BGE_DMA_DESCRIPTORS);
1465                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LEN, 0x2000);
1466         }
1467
1468         /* Configure mbuf pool watermarks */
1469         if (!BGE_IS_5705_PLUS(sc)) {
1470                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x50);
1471                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x20);
1472                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1473         } else if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
1474                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1475                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x04);
1476                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x10);
1477         } else {
1478                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1479                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1480                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1481         }
1482
1483         /* Configure DMA resource watermarks */
1484         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1485         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1486
1487         /* Enable buffer manager */
1488         CSR_WRITE_4(sc, BGE_BMAN_MODE,
1489             BGE_BMANMODE_ENABLE|BGE_BMANMODE_LOMBUF_ATTN);
1490
1491         /* Poll for buffer manager start indication */
1492         for (i = 0; i < BGE_TIMEOUT; i++) {
1493                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1494                         break;
1495                 DELAY(10);
1496         }
1497
1498         if (i == BGE_TIMEOUT) {
1499                 if_printf(&sc->arpcom.ac_if,
1500                           "buffer manager failed to start\n");
1501                 return(ENXIO);
1502         }
1503
1504         /* Enable flow-through queues */
1505         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1506         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1507
1508         /* Wait until queue initialization is complete */
1509         for (i = 0; i < BGE_TIMEOUT; i++) {
1510                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1511                         break;
1512                 DELAY(10);
1513         }
1514
1515         if (i == BGE_TIMEOUT) {
1516                 if_printf(&sc->arpcom.ac_if,
1517                           "flow-through queue init failed\n");
1518                 return(ENXIO);
1519         }
1520
1521         /*
1522          * Summary of rings supported by the controller:
1523          *
1524          * Standard Receive Producer Ring
1525          * - This ring is used to feed receive buffers for "standard"
1526          *   sized frames (typically 1536 bytes) to the controller.
1527          *
1528          * Jumbo Receive Producer Ring
1529          * - This ring is used to feed receive buffers for jumbo sized
1530          *   frames (i.e. anything bigger than the "standard" frames)
1531          *   to the controller.
1532          *
1533          * Mini Receive Producer Ring
1534          * - This ring is used to feed receive buffers for "mini"
1535          *   sized frames to the controller.
1536          * - This feature required external memory for the controller
1537          *   but was never used in a production system.  Should always
1538          *   be disabled.
1539          *
1540          * Receive Return Ring
1541          * - After the controller has placed an incoming frame into a
1542          *   receive buffer that buffer is moved into a receive return
1543          *   ring.  The driver is then responsible to passing the
1544          *   buffer up to the stack.  Many versions of the controller
1545          *   support multiple RR rings.
1546          *
1547          * Send Ring
1548          * - This ring is used for outgoing frames.  Many versions of
1549          *   the controller support multiple send rings.
1550          */
1551
1552         /* Initialize the standard receive producer ring control block. */
1553         rcb = &sc->bge_ldata.bge_info.bge_std_rx_rcb;
1554         rcb->bge_hostaddr.bge_addr_lo =
1555             BGE_ADDR_LO(sc->bge_ldata.bge_rx_std_ring_paddr);
1556         rcb->bge_hostaddr.bge_addr_hi =
1557             BGE_ADDR_HI(sc->bge_ldata.bge_rx_std_ring_paddr);
1558         if (BGE_IS_5705_PLUS(sc)) {
1559                 /*
1560                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1561                  * Bits 15-2 : Reserved (should be 0)
1562                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1563                  * Bit 0     : Reserved
1564                  */
1565                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1566         } else {
1567                 /*
1568                  * Ring size is always XXX entries
1569                  * Bits 31-16: Maximum RX frame size
1570                  * Bits 15-2 : Reserved (should be 0)
1571                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1572                  * Bit 0     : Reserved
1573                  */
1574                 rcb->bge_maxlen_flags =
1575                     BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN, 0);
1576         }
1577         rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1578         /* Write the standard receive producer ring control block. */
1579         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1580         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1581         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1582         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1583         /* Reset the standard receive producer ring producer index. */
1584         bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1585
1586         /*
1587          * Initialize the jumbo RX producer ring control
1588          * block.  We set the 'ring disabled' bit in the
1589          * flags field until we're actually ready to start
1590          * using this ring (i.e. once we set the MTU
1591          * high enough to require it).
1592          */
1593         if (BGE_IS_JUMBO_CAPABLE(sc)) {
1594                 rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
1595                 /* Get the jumbo receive producer ring RCB parameters. */
1596                 rcb->bge_hostaddr.bge_addr_lo =
1597                     BGE_ADDR_LO(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
1598                 rcb->bge_hostaddr.bge_addr_hi =
1599                     BGE_ADDR_HI(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
1600                 rcb->bge_maxlen_flags =
1601                     BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN,
1602                     BGE_RCB_FLAG_RING_DISABLED);
1603                 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1604                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1605                     rcb->bge_hostaddr.bge_addr_hi);
1606                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1607                     rcb->bge_hostaddr.bge_addr_lo);
1608                 /* Program the jumbo receive producer ring RCB parameters. */
1609                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1610                     rcb->bge_maxlen_flags);
1611                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1612                 /* Reset the jumbo receive producer ring producer index. */
1613                 bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1614         }
1615
1616         /* Disable the mini receive producer ring RCB. */
1617         if (BGE_IS_5700_FAMILY(sc)) {
1618                 rcb = &sc->bge_ldata.bge_info.bge_mini_rx_rcb;
1619                 rcb->bge_maxlen_flags =
1620                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED);
1621                 CSR_WRITE_4(sc, BGE_RX_MINI_RCB_MAXLEN_FLAGS,
1622                     rcb->bge_maxlen_flags);
1623                 /* Reset the mini receive producer ring producer index. */
1624                 bge_writembx(sc, BGE_MBX_RX_MINI_PROD_LO, 0);
1625         }
1626
1627         /* Choose de-pipeline mode for BCM5906 A0, A1 and A2. */
1628         if (sc->bge_asicrev == BGE_ASICREV_BCM5906 &&
1629             (sc->bge_chipid == BGE_CHIPID_BCM5906_A0 ||
1630              sc->bge_chipid == BGE_CHIPID_BCM5906_A1 ||
1631              sc->bge_chipid == BGE_CHIPID_BCM5906_A2)) {
1632                 CSR_WRITE_4(sc, BGE_ISO_PKT_TX,
1633                     (CSR_READ_4(sc, BGE_ISO_PKT_TX) & ~3) | 2);
1634         }
1635
1636         /*
1637          * The BD ring replenish thresholds control how often the
1638          * hardware fetches new BD's from the producer rings in host
1639          * memory.  Setting the value too low on a busy system can
1640          * starve the hardware and recue the throughpout.
1641          *
1642          * Set the BD ring replentish thresholds. The recommended
1643          * values are 1/8th the number of descriptors allocated to
1644          * each ring.
1645          */
1646         if (BGE_IS_5705_PLUS(sc))
1647                 val = 8;
1648         else
1649                 val = BGE_STD_RX_RING_CNT / 8;
1650         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1651         if (BGE_IS_JUMBO_CAPABLE(sc)) {
1652                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1653                     BGE_JUMBO_RX_RING_CNT/8);
1654         }
1655
1656         /*
1657          * Disable all send rings by setting the 'ring disabled' bit
1658          * in the flags field of all the TX send ring control blocks,
1659          * located in NIC memory.
1660          */
1661         if (!BGE_IS_5705_PLUS(sc)) {
1662                 /* 5700 to 5704 had 16 send rings. */
1663                 limit = BGE_TX_RINGS_EXTSSRAM_MAX;
1664         } else {
1665                 limit = 1;
1666         }
1667         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1668         for (i = 0; i < limit; i++) {
1669                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1670                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1671                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1672                 vrcb += sizeof(struct bge_rcb);
1673         }
1674
1675         /* Configure send ring RCB 0 (we use only the first ring) */
1676         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1677         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_tx_ring_paddr);
1678         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1679         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1680         RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1681             BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1682         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1683             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1684
1685         /*
1686          * Disable all receive return rings by setting the
1687          * 'ring diabled' bit in the flags field of all the receive
1688          * return ring control blocks, located in NIC memory.
1689          */
1690         if (!BGE_IS_5705_PLUS(sc))
1691                 limit = BGE_RX_RINGS_MAX;
1692         else if (sc->bge_asicrev == BGE_ASICREV_BCM5755)
1693                 limit = 4;
1694         else
1695                 limit = 1;
1696         /* Disable all receive return rings. */
1697         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1698         for (i = 0; i < limit; i++) {
1699                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1700                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1701                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1702                     BGE_RCB_FLAG_RING_DISABLED);
1703                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1704                 bge_writembx(sc, BGE_MBX_RX_CONS0_LO +
1705                     (i * (sizeof(uint64_t))), 0);
1706                 vrcb += sizeof(struct bge_rcb);
1707         }
1708
1709         /*
1710          * Set up receive return ring 0.  Note that the NIC address
1711          * for RX return rings is 0x0.  The return rings live entirely
1712          * within the host, so the nicaddr field in the RCB isn't used.
1713          */
1714         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1715         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_rx_return_ring_paddr);
1716         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1717         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1718         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1719         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1720             BGE_RCB_MAXLEN_FLAGS(sc->bge_return_ring_cnt, 0));
1721
1722         /* Set random backoff seed for TX */
1723         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1724             (sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1725              sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1726              sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5]) &
1727             BGE_TX_BACKOFF_SEED_MASK);
1728
1729         /* Set inter-packet gap */
1730         CSR_WRITE_4(sc, BGE_TX_LENGTHS, 0x2620);
1731
1732         /*
1733          * Specify which ring to use for packets that don't match
1734          * any RX rules.
1735          */
1736         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1737
1738         /*
1739          * Configure number of RX lists. One interrupt distribution
1740          * list, sixteen active lists, one bad frames class.
1741          */
1742         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1743
1744         /* Inialize RX list placement stats mask. */
1745         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1746         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1747
1748         /* Disable host coalescing until we get it set up */
1749         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1750
1751         /* Poll to make sure it's shut down. */
1752         for (i = 0; i < BGE_TIMEOUT; i++) {
1753                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1754                         break;
1755                 DELAY(10);
1756         }
1757
1758         if (i == BGE_TIMEOUT) {
1759                 if_printf(&sc->arpcom.ac_if,
1760                           "host coalescing engine failed to idle\n");
1761                 return(ENXIO);
1762         }
1763
1764         /* Set up host coalescing defaults */
1765         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bge_rx_coal_ticks);
1766         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bge_tx_coal_ticks);
1767         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bge_rx_coal_bds);
1768         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bge_tx_coal_bds);
1769         if (!BGE_IS_5705_PLUS(sc)) {
1770                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS_INT,
1771                     sc->bge_rx_coal_ticks_int);
1772                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS_INT,
1773                     sc->bge_tx_coal_ticks_int);
1774         }
1775         /*
1776          * NOTE:
1777          * The datasheet (57XX-PG105-R) says BCM5705+ do not
1778          * have following two registers; obviously it is wrong.
1779          */
1780         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, sc->bge_rx_coal_bds_int);
1781         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, sc->bge_tx_coal_bds_int);
1782
1783         /* Set up address of statistics block */
1784         if (!BGE_IS_5705_PLUS(sc)) {
1785                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_HI,
1786                     BGE_ADDR_HI(sc->bge_ldata.bge_stats_paddr));
1787                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_LO,
1788                     BGE_ADDR_LO(sc->bge_ldata.bge_stats_paddr));
1789
1790                 CSR_WRITE_4(sc, BGE_HCC_STATS_BASEADDR, BGE_STATS_BLOCK);
1791                 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_BASEADDR, BGE_STATUS_BLOCK);
1792                 CSR_WRITE_4(sc, BGE_HCC_STATS_TICKS, sc->bge_stat_ticks);
1793         }
1794
1795         /* Set up address of status block */
1796         bzero(sc->bge_ldata.bge_status_block, BGE_STATUS_BLK_SZ);
1797         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1798             BGE_ADDR_HI(sc->bge_ldata.bge_status_block_paddr));
1799         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1800             BGE_ADDR_LO(sc->bge_ldata.bge_status_block_paddr));
1801
1802         /*
1803          * Set up status block partail update size.
1804          *
1805          * Because only single TX ring, RX produce ring and Rx return ring
1806          * are used, ask device to update only minimum part of status block
1807          * except for BCM5700 AX/BX, whose status block partial update size
1808          * can't be configured.
1809          */
1810         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
1811             sc->bge_chipid != BGE_CHIPID_BCM5700_C0) {
1812                 /* XXX Actually reserved on BCM5700 AX/BX */
1813                 val = BGE_STATBLKSZ_FULL;
1814         } else {
1815                 val = BGE_STATBLKSZ_32BYTE;
1816         }
1817 #if 0
1818         /*
1819          * Does not seem to have visible effect in both
1820          * bulk data (1472B UDP datagram) and tiny data
1821          * (18B UDP datagram) TX tests.
1822          */
1823         if (!BGE_IS_CRIPPLED(sc))
1824                 val |= BGE_HCCMODE_CLRTICK_TX;
1825 #endif
1826
1827         /* Turn on host coalescing state machine */
1828         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1829
1830         /* Turn on RX BD completion state machine and enable attentions */
1831         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1832             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1833
1834         /* Turn on RX list placement state machine */
1835         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1836
1837         /* Turn on RX list selector state machine. */
1838         if (!BGE_IS_5705_PLUS(sc))
1839                 CSR_WRITE_4(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
1840
1841         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1842             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1843             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1844             BGE_MACMODE_FRMHDR_DMA_ENB;
1845
1846         if (sc->bge_flags & BGE_FLAG_TBI)
1847                 val |= BGE_PORTMODE_TBI;
1848         else if (sc->bge_flags & BGE_FLAG_MII_SERDES)
1849                 val |= BGE_PORTMODE_GMII;
1850         else
1851                 val |= BGE_PORTMODE_MII;
1852
1853         /* Allow APE to send/receive frames. */
1854         if (sc->bge_mfw_flags & BGE_MFW_ON_APE)
1855                 val |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
1856
1857         /* Turn on DMA, clear stats */
1858         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1859         DELAY(40);
1860
1861         /* Set misc. local control, enable interrupts on attentions */
1862         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1863
1864 #ifdef notdef
1865         /* Assert GPIO pins for PHY reset */
1866         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1867             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1868         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1869             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1870 #endif
1871
1872         /* Turn on DMA completion state machine */
1873         if (!BGE_IS_5705_PLUS(sc))
1874                 CSR_WRITE_4(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
1875
1876         /* Turn on write DMA state machine */
1877         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1878         if (BGE_IS_5755_PLUS(sc)) {
1879                 /* Enable host coalescing bug fix. */
1880                 val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1881         }
1882         if (sc->bge_asicrev == BGE_ASICREV_BCM5785) {
1883                 /* Request larger DMA burst size to get better performance. */
1884                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1885         }
1886         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1887         DELAY(40);
1888
1889         if (sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
1890             sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
1891             sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
1892             sc->bge_asicrev == BGE_ASICREV_BCM57780) {
1893                 /*
1894                  * Enable fix for read DMA FIFO overruns.
1895                  * The fix is to limit the number of RX BDs
1896                  * the hardware would fetch at a fime.
1897                  */
1898                 val = CSR_READ_4(sc, BGE_RDMA_RSRVCTRL);
1899                 CSR_WRITE_4(sc, BGE_RDMA_RSRVCTRL,
1900                     val| BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1901         }
1902
1903         /* Turn on read DMA state machine */
1904         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1905         if (sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
1906             sc->bge_asicrev == BGE_ASICREV_BCM5785 ||
1907             sc->bge_asicrev == BGE_ASICREV_BCM57780)
1908                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1909                   BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1910                   BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1911         if (sc->bge_flags & BGE_FLAG_PCIE)
1912                 val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1913         if (sc->bge_flags & BGE_FLAG_TSO)
1914                 val |= BGE_RDMAMODE_TSO4_ENABLE;
1915         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1916         DELAY(40);
1917
1918         /* Turn on RX data completion state machine */
1919         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1920
1921         /* Turn on RX BD initiator state machine */
1922         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1923
1924         /* Turn on RX data and RX BD initiator state machine */
1925         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1926
1927         /* Turn on Mbuf cluster free state machine */
1928         if (!BGE_IS_5705_PLUS(sc))
1929                 CSR_WRITE_4(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
1930
1931         /* Turn on send BD completion state machine */
1932         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1933
1934         /* Turn on send data completion state machine */
1935         val = BGE_SDCMODE_ENABLE;
1936         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
1937                 val |= BGE_SDCMODE_CDELAY;
1938         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1939
1940         /* Turn on send data initiator state machine */
1941         if (sc->bge_flags & BGE_FLAG_TSO)
1942                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
1943                     BGE_SDIMODE_HW_LSO_PRE_DMA);
1944         else
1945                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1946
1947         /* Turn on send BD initiator state machine */
1948         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1949
1950         /* Turn on send BD selector state machine */
1951         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1952
1953         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1954         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1955             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1956
1957         /* ack/clear link change events */
1958         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1959             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1960             BGE_MACSTAT_LINK_CHANGED);
1961         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1962
1963         /*
1964          * Enable attention when the link has changed state for
1965          * devices that use auto polling.
1966          */
1967         if (sc->bge_flags & BGE_FLAG_TBI) {
1968                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1969         } else {
1970                 if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
1971                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bge_mi_mode);
1972                         DELAY(80);
1973                 }
1974                 if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
1975                     sc->bge_chipid != BGE_CHIPID_BCM5700_B2) {
1976                         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB,
1977                             BGE_EVTENB_MI_INTERRUPT);
1978                 }
1979         }
1980
1981         /*
1982          * Clear any pending link state attention.
1983          * Otherwise some link state change events may be lost until attention
1984          * is cleared by bge_intr() -> bge_softc.bge_link_upd() sequence.
1985          * It's not necessary on newer BCM chips - perhaps enabling link
1986          * state change attentions implies clearing pending attention.
1987          */
1988         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1989             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1990             BGE_MACSTAT_LINK_CHANGED);
1991
1992         /* Enable link state change attentions. */
1993         BGE_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1994
1995         return(0);
1996 }
1997
1998 /*
1999  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
2000  * against our list and return its name if we find a match. Note
2001  * that since the Broadcom controller contains VPD support, we
2002  * can get the device name string from the controller itself instead
2003  * of the compiled-in string. This is a little slow, but it guarantees
2004  * we'll always announce the right product name.
2005  */
2006 static int
2007 bge_probe(device_t dev)
2008 {
2009         const struct bge_type *t;
2010         uint16_t product, vendor;
2011
2012         product = pci_get_device(dev);
2013         vendor = pci_get_vendor(dev);
2014
2015         for (t = bge_devs; t->bge_name != NULL; t++) {
2016                 if (vendor == t->bge_vid && product == t->bge_did)
2017                         break;
2018         }
2019         if (t->bge_name == NULL)
2020                 return(ENXIO);
2021
2022         device_set_desc(dev, t->bge_name);
2023         return(0);
2024 }
2025
2026 static int
2027 bge_attach(device_t dev)
2028 {
2029         struct ifnet *ifp;
2030         struct bge_softc *sc;
2031         uint32_t hwcfg = 0, misccfg;
2032         int error = 0, rid, capmask;
2033         uint8_t ether_addr[ETHER_ADDR_LEN];
2034         uint16_t product, vendor;
2035         driver_intr_t *intr_func;
2036         uintptr_t mii_priv = 0;
2037         u_int intr_flags;
2038         int msi_enable;
2039
2040         sc = device_get_softc(dev);
2041         sc->bge_dev = dev;
2042         callout_init_mp(&sc->bge_stat_timer);
2043         lwkt_serialize_init(&sc->bge_jslot_serializer);
2044
2045         sc->bge_func_addr = pci_get_function(dev);
2046         product = pci_get_device(dev);
2047         vendor = pci_get_vendor(dev);
2048
2049 #ifndef BURN_BRIDGES
2050         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
2051                 uint32_t irq, mem;
2052
2053                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
2054                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
2055
2056                 device_printf(dev, "chip is in D%d power mode "
2057                     "-- setting to D0\n", pci_get_powerstate(dev));
2058
2059                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
2060
2061                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
2062                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
2063         }
2064 #endif  /* !BURN_BRIDGE */
2065
2066         /*
2067          * Map control/status registers.
2068          */
2069         pci_enable_busmaster(dev);
2070
2071         rid = BGE_PCI_BAR0;
2072         sc->bge_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
2073             RF_ACTIVE);
2074
2075         if (sc->bge_res == NULL) {
2076                 device_printf(dev, "couldn't map memory\n");
2077                 return ENXIO;
2078         }
2079
2080         sc->bge_btag = rman_get_bustag(sc->bge_res);
2081         sc->bge_bhandle = rman_get_bushandle(sc->bge_res);
2082
2083         /* Save various chip information */
2084         sc->bge_chipid =
2085             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
2086             BGE_PCIMISCCTL_ASICREV_SHIFT;
2087         if (BGE_ASICREV(sc->bge_chipid) == BGE_ASICREV_USE_PRODID_REG) {
2088                 /* All chips, which use BGE_PCI_PRODID_ASICREV, have CPMU */
2089                 sc->bge_flags |= BGE_FLAG_CPMU;
2090                 sc->bge_chipid = pci_read_config(dev, BGE_PCI_PRODID_ASICREV, 4);
2091         }
2092         sc->bge_asicrev = BGE_ASICREV(sc->bge_chipid);
2093         sc->bge_chiprev = BGE_CHIPREV(sc->bge_chipid);
2094
2095         /* Save chipset family. */
2096         switch (sc->bge_asicrev) {
2097         case BGE_ASICREV_BCM5755:
2098         case BGE_ASICREV_BCM5761:
2099         case BGE_ASICREV_BCM5784:
2100         case BGE_ASICREV_BCM5785:
2101         case BGE_ASICREV_BCM5787:
2102         case BGE_ASICREV_BCM57780:
2103             sc->bge_flags |= BGE_FLAG_5755_PLUS | BGE_FLAG_575X_PLUS |
2104                 BGE_FLAG_5705_PLUS;
2105             break;
2106
2107         case BGE_ASICREV_BCM5700:
2108         case BGE_ASICREV_BCM5701:
2109         case BGE_ASICREV_BCM5703:
2110         case BGE_ASICREV_BCM5704:
2111                 sc->bge_flags |= BGE_FLAG_5700_FAMILY | BGE_FLAG_JUMBO;
2112                 break;
2113
2114         case BGE_ASICREV_BCM5714_A0:
2115         case BGE_ASICREV_BCM5780:
2116         case BGE_ASICREV_BCM5714:
2117                 sc->bge_flags |= BGE_FLAG_5714_FAMILY;
2118                 /* Fall through */
2119
2120         case BGE_ASICREV_BCM5750:
2121         case BGE_ASICREV_BCM5752:
2122         case BGE_ASICREV_BCM5906:
2123                 sc->bge_flags |= BGE_FLAG_575X_PLUS;
2124                 /* Fall through */
2125
2126         case BGE_ASICREV_BCM5705:
2127                 sc->bge_flags |= BGE_FLAG_5705_PLUS;
2128                 break;
2129         }
2130
2131         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
2132                 sc->bge_flags |= BGE_FLAG_NO_EEPROM;
2133
2134         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
2135                 sc->bge_flags |= BGE_FLAG_APE;
2136
2137         misccfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID_MASK;
2138         if (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
2139             (misccfg == BGE_MISCCFG_BOARD_ID_5788 ||
2140              misccfg == BGE_MISCCFG_BOARD_ID_5788M))
2141                 sc->bge_flags |= BGE_FLAG_5788;
2142
2143         /* BCM5755 or higher and BCM5906 have short DMA bug. */
2144         if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
2145                 sc->bge_flags |= BGE_FLAG_SHORTDMA;
2146
2147         /*
2148          * Increase STD RX ring prod index by at most 8 for BCM5750,
2149          * BCM5752 and BCM5755 to workaround hardware errata.
2150          */
2151         if (sc->bge_asicrev == BGE_ASICREV_BCM5750 ||
2152             sc->bge_asicrev == BGE_ASICREV_BCM5752 ||
2153             sc->bge_asicrev == BGE_ASICREV_BCM5755)
2154                 sc->bge_rx_wreg = 8;
2155
2156         /*
2157          * Check if this is a PCI-X or PCI Express device.
2158          */
2159         if (BGE_IS_5705_PLUS(sc)) {
2160                 if (pci_is_pcie(dev)) {
2161                         sc->bge_flags |= BGE_FLAG_PCIE;
2162                         sc->bge_pciecap = pci_get_pciecap_ptr(sc->bge_dev);
2163                         pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
2164                 }
2165         } else {
2166                 /*
2167                  * Check if the device is in PCI-X Mode.
2168                  * (This bit is not valid on PCI Express controllers.)
2169                  */
2170                 if ((pci_read_config(sc->bge_dev, BGE_PCI_PCISTATE, 4) &
2171                     BGE_PCISTATE_PCI_BUSMODE) == 0) {
2172                         sc->bge_flags |= BGE_FLAG_PCIX;
2173                         sc->bge_pcixcap = pci_get_pcixcap_ptr(sc->bge_dev);
2174                         sc->bge_mbox_reorder = device_getenv_int(sc->bge_dev,
2175                             "mbox_reorder", 0);
2176                 }
2177         }
2178         device_printf(dev, "CHIP ID 0x%08x; "
2179                       "ASIC REV 0x%02x; CHIP REV 0x%02x; %s\n",
2180                       sc->bge_chipid, sc->bge_asicrev, sc->bge_chiprev,
2181                       (sc->bge_flags & BGE_FLAG_PCIX) ? "PCI-X"
2182                       : ((sc->bge_flags & BGE_FLAG_PCIE) ?
2183                         "PCI-E" : "PCI"));
2184
2185         /*
2186          * The 40bit DMA bug applies to the 5714/5715 controllers and is
2187          * not actually a MAC controller bug but an issue with the embedded
2188          * PCIe to PCI-X bridge in the device. Use 40bit DMA workaround.
2189          */
2190         if ((sc->bge_flags & BGE_FLAG_PCIX) &&
2191             (BGE_IS_5714_FAMILY(sc) || device_getenv_int(dev, "dma40b", 0)))
2192                 sc->bge_flags |= BGE_FLAG_MAXADDR_40BIT;
2193
2194         /*
2195          * When using the BCM5701 in PCI-X mode, data corruption has
2196          * been observed in the first few bytes of some received packets.
2197          * Aligning the packet buffer in memory eliminates the corruption.
2198          * Unfortunately, this misaligns the packet payloads.  On platforms
2199          * which do not support unaligned accesses, we will realign the
2200          * payloads by copying the received packets.
2201          */
2202         if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
2203             (sc->bge_flags & BGE_FLAG_PCIX))
2204                 sc->bge_flags |= BGE_FLAG_RX_ALIGNBUG;
2205
2206         if (!BGE_IS_CRIPPLED(sc)) {
2207                 if (device_getenv_int(dev, "status_tag", 1)) {
2208                         sc->bge_flags |= BGE_FLAG_STATUS_TAG;
2209                         sc->bge_pci_miscctl = BGE_PCIMISCCTL_TAGGED_STATUS;
2210                         if (bootverbose)
2211                                 device_printf(dev, "enable status tag\n");
2212                 }
2213         }
2214
2215         if (BGE_IS_5755_PLUS(sc)) {
2216                 /*
2217                  * BCM5754 and BCM5787 shares the same ASIC id so
2218                  * explicit device id check is required.
2219                  * Due to unknown reason TSO does not work on BCM5755M.
2220                  */
2221                 if (product != PCI_PRODUCT_BROADCOM_BCM5754 &&
2222                     product != PCI_PRODUCT_BROADCOM_BCM5754M &&
2223                     product != PCI_PRODUCT_BROADCOM_BCM5755M)
2224                         sc->bge_flags |= BGE_FLAG_TSO;
2225         }
2226
2227         /*
2228          * Set various PHY quirk flags.
2229          */
2230
2231         if ((sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
2232              sc->bge_asicrev == BGE_ASICREV_BCM5701) &&
2233             pci_get_subvendor(dev) == PCI_VENDOR_DELL)
2234                 mii_priv |= BRGPHY_FLAG_NO_3LED;
2235
2236         capmask = MII_CAPMASK_DEFAULT;
2237         if ((sc->bge_asicrev == BGE_ASICREV_BCM5703 &&
2238              (misccfg == 0x4000 || misccfg == 0x8000)) ||
2239             (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
2240              vendor == PCI_VENDOR_BROADCOM &&
2241              (product == PCI_PRODUCT_BROADCOM_BCM5901 ||
2242               product == PCI_PRODUCT_BROADCOM_BCM5901A2 ||
2243               product == PCI_PRODUCT_BROADCOM_BCM5705F)) ||
2244             (vendor == PCI_VENDOR_BROADCOM &&
2245              (product == PCI_PRODUCT_BROADCOM_BCM5751F ||
2246               product == PCI_PRODUCT_BROADCOM_BCM5753F ||
2247               product == PCI_PRODUCT_BROADCOM_BCM5787F)) ||
2248             product == PCI_PRODUCT_BROADCOM_BCM57790 ||
2249             sc->bge_asicrev == BGE_ASICREV_BCM5906) {
2250                 /* 10/100 only */
2251                 capmask &= ~BMSR_EXTSTAT;
2252         }
2253
2254         mii_priv |= BRGPHY_FLAG_WIRESPEED;
2255         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
2256             (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
2257              (sc->bge_chipid != BGE_CHIPID_BCM5705_A0 &&
2258               sc->bge_chipid != BGE_CHIPID_BCM5705_A1)) ||
2259             sc->bge_asicrev == BGE_ASICREV_BCM5906)
2260                 mii_priv &= ~BRGPHY_FLAG_WIRESPEED;
2261
2262         if (sc->bge_chipid == BGE_CHIPID_BCM5701_A0 ||
2263             sc->bge_chipid == BGE_CHIPID_BCM5701_B0)
2264                 mii_priv |= BRGPHY_FLAG_CRC_BUG;
2265
2266         if (sc->bge_chiprev == BGE_CHIPREV_5703_AX ||
2267             sc->bge_chiprev == BGE_CHIPREV_5704_AX)
2268                 mii_priv |= BRGPHY_FLAG_ADC_BUG;
2269
2270         if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0)
2271                 mii_priv |= BRGPHY_FLAG_5704_A0;
2272
2273         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
2274                 mii_priv |= BRGPHY_FLAG_5906;
2275
2276         if (BGE_IS_5705_PLUS(sc) &&
2277             sc->bge_asicrev != BGE_ASICREV_BCM5906 &&
2278             /* sc->bge_asicrev != BGE_ASICREV_BCM5717 && */
2279             sc->bge_asicrev != BGE_ASICREV_BCM5785 &&
2280             /* sc->bge_asicrev != BGE_ASICREV_BCM57765 && */
2281             sc->bge_asicrev != BGE_ASICREV_BCM57780) {
2282                 if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
2283                     sc->bge_asicrev == BGE_ASICREV_BCM5761 ||
2284                     sc->bge_asicrev == BGE_ASICREV_BCM5784 ||
2285                     sc->bge_asicrev == BGE_ASICREV_BCM5787) {
2286                         if (product != PCI_PRODUCT_BROADCOM_BCM5722 &&
2287                             product != PCI_PRODUCT_BROADCOM_BCM5756)
2288                                 mii_priv |= BRGPHY_FLAG_JITTER_BUG;
2289                         if (product == PCI_PRODUCT_BROADCOM_BCM5755M)
2290                                 mii_priv |= BRGPHY_FLAG_ADJUST_TRIM;
2291                 } else {
2292                         mii_priv |= BRGPHY_FLAG_BER_BUG;
2293                 }
2294         }
2295
2296         /*
2297          * Chips with APE need BAR2 access for APE registers/memory.
2298          */
2299         if (sc->bge_flags & BGE_FLAG_APE) {
2300                 uint32_t pcistate;
2301
2302                 rid = PCIR_BAR(2);
2303                 sc->bge_res2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
2304                     RF_ACTIVE);
2305                 if (sc->bge_res2 == NULL) {
2306                         device_printf(dev, "couldn't map BAR2 memory\n");
2307                         error = ENXIO;
2308                         goto fail;
2309                 }
2310
2311                 /* Enable APE register/memory access by host driver. */
2312                 pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2313                 pcistate |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
2314                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
2315                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
2316                 pci_write_config(dev, BGE_PCI_PCISTATE, pcistate, 4);
2317
2318                 bge_ape_lock_init(sc);
2319                 bge_ape_read_fw_ver(sc);
2320         }
2321
2322         /*
2323          * Allocate interrupt
2324          */
2325         msi_enable = bge_msi_enable;
2326         if ((sc->bge_flags & BGE_FLAG_STATUS_TAG) == 0) {
2327                 /* If "tagged status" is disabled, don't enable MSI */
2328                 msi_enable = 0;
2329         } else if (msi_enable) {
2330                 msi_enable = 0; /* Disable by default */
2331                 if (BGE_IS_575X_PLUS(sc)) {
2332                         msi_enable = 1;
2333                         /* XXX we filter all 5714 chips */
2334                         if (sc->bge_asicrev == BGE_ASICREV_BCM5714 ||
2335                             (sc->bge_asicrev == BGE_ASICREV_BCM5750 &&
2336                              (sc->bge_chiprev == BGE_CHIPREV_5750_AX ||
2337                               sc->bge_chiprev == BGE_CHIPREV_5750_BX)))
2338                                 msi_enable = 0;
2339                         else if (BGE_IS_5755_PLUS(sc) ||
2340                             sc->bge_asicrev == BGE_ASICREV_BCM5906)
2341                                 sc->bge_flags |= BGE_FLAG_ONESHOT_MSI;
2342                 }
2343         }
2344         if (msi_enable) {
2345                 if (pci_find_extcap(dev, PCIY_MSI, &sc->bge_msicap)) {
2346                         device_printf(dev, "no MSI capability\n");
2347                         msi_enable = 0;
2348                 }
2349         }
2350
2351         sc->bge_irq_type = pci_alloc_1intr(dev, msi_enable, &sc->bge_irq_rid,
2352             &intr_flags);
2353
2354         sc->bge_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bge_irq_rid,
2355             intr_flags);
2356         if (sc->bge_irq == NULL) {
2357                 device_printf(dev, "couldn't map interrupt\n");
2358                 error = ENXIO;
2359                 goto fail;
2360         }
2361
2362         if (sc->bge_irq_type == PCI_INTR_TYPE_MSI)
2363                 bge_enable_msi(sc);
2364         else
2365                 sc->bge_flags &= ~BGE_FLAG_ONESHOT_MSI;
2366
2367         /* Initialize if_name earlier, so if_printf could be used */
2368         ifp = &sc->arpcom.ac_if;
2369         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2370
2371         sc->bge_asf_mode = 0;
2372         /* No ASF if APE present. */
2373         if ((sc->bge_flags & BGE_FLAG_APE) == 0) {
2374                 if (bge_allow_asf && (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) ==
2375                     BGE_SRAM_DATA_SIG_MAGIC)) {
2376                         if (bge_readmem_ind(sc, BGE_SRAM_DATA_CFG) &
2377                             BGE_HWCFG_ASF) {
2378                                 sc->bge_asf_mode |= ASF_ENABLE;
2379                                 sc->bge_asf_mode |= ASF_STACKUP;
2380                                 if (BGE_IS_575X_PLUS(sc))
2381                                         sc->bge_asf_mode |= ASF_NEW_HANDSHAKE;
2382                         }
2383                 }
2384         }
2385
2386         /*
2387          * Try to reset the chip.
2388          */
2389         bge_stop_fw(sc);
2390         bge_sig_pre_reset(sc, BGE_RESET_SHUTDOWN);
2391         bge_reset(sc);
2392         bge_sig_legacy(sc, BGE_RESET_SHUTDOWN);
2393         bge_sig_post_reset(sc, BGE_RESET_SHUTDOWN);
2394
2395         if (bge_chipinit(sc)) {
2396                 device_printf(dev, "chip initialization failed\n");
2397                 error = ENXIO;
2398                 goto fail;
2399         }
2400
2401         /*
2402          * Get station address
2403          */
2404         error = bge_get_eaddr(sc, ether_addr);
2405         if (error) {
2406                 device_printf(dev, "failed to read station address\n");
2407                 goto fail;
2408         }
2409
2410         /* 5705/5750 limits RX return ring to 512 entries. */
2411         if (BGE_IS_5705_PLUS(sc))
2412                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT_5705;
2413         else
2414                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT;
2415
2416         error = bge_dma_alloc(sc);
2417         if (error)
2418                 goto fail;
2419
2420         /* Set default tuneable values. */
2421         sc->bge_stat_ticks = BGE_TICKS_PER_SEC;
2422         sc->bge_rx_coal_ticks = BGE_RX_COAL_TICKS_DEF;
2423         sc->bge_tx_coal_ticks = BGE_TX_COAL_TICKS_DEF;
2424         sc->bge_rx_coal_bds = BGE_RX_COAL_BDS_DEF;
2425         sc->bge_tx_coal_bds = BGE_TX_COAL_BDS_DEF;
2426         if (sc->bge_flags & BGE_FLAG_STATUS_TAG) {
2427                 sc->bge_rx_coal_ticks_int = BGE_RX_COAL_TICKS_DEF;
2428                 sc->bge_tx_coal_ticks_int = BGE_TX_COAL_TICKS_DEF;
2429                 sc->bge_rx_coal_bds_int = BGE_RX_COAL_BDS_DEF;
2430                 sc->bge_tx_coal_bds_int = BGE_TX_COAL_BDS_DEF;
2431         } else {
2432                 sc->bge_rx_coal_ticks_int = BGE_RX_COAL_TICKS_MIN;
2433                 sc->bge_tx_coal_ticks_int = BGE_TX_COAL_TICKS_MIN;
2434                 sc->bge_rx_coal_bds_int = BGE_RX_COAL_BDS_MIN;
2435                 sc->bge_tx_coal_bds_int = BGE_TX_COAL_BDS_MIN;
2436         }
2437         sc->bge_tx_wreg = BGE_TX_WREG_NSEGS;
2438
2439         /* Set up TX spare and reserved descriptor count */
2440         if (sc->bge_flags & BGE_FLAG_TSO) {
2441                 sc->bge_txspare = BGE_NSEG_SPARE_TSO;
2442                 sc->bge_txrsvd = BGE_NSEG_RSVD_TSO;
2443         } else {
2444                 sc->bge_txspare = BGE_NSEG_SPARE;
2445                 sc->bge_txrsvd = BGE_NSEG_RSVD;
2446         }
2447
2448         /* Set up ifnet structure */
2449         ifp->if_softc = sc;
2450         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2451         ifp->if_ioctl = bge_ioctl;
2452         ifp->if_start = bge_start;
2453 #ifdef IFPOLL_ENABLE
2454         ifp->if_npoll = bge_npoll;
2455 #endif
2456         ifp->if_watchdog = bge_watchdog;
2457         ifp->if_init = bge_init;
2458         ifp->if_mtu = ETHERMTU;
2459         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2460         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
2461         ifq_set_ready(&ifp->if_snd);
2462
2463         /*
2464          * 5700 B0 chips do not support checksumming correctly due
2465          * to hardware bugs.
2466          */
2467         if (sc->bge_chipid != BGE_CHIPID_BCM5700_B0) {
2468                 ifp->if_capabilities |= IFCAP_HWCSUM;
2469                 ifp->if_hwassist |= BGE_CSUM_FEATURES;
2470         }
2471         if (sc->bge_flags & BGE_FLAG_TSO) {
2472                 ifp->if_capabilities |= IFCAP_TSO;
2473                 ifp->if_hwassist |= CSUM_TSO;
2474         }
2475         ifp->if_capenable = ifp->if_capabilities;
2476
2477         /*
2478          * Figure out what sort of media we have by checking the
2479          * hardware config word in the first 32k of NIC internal memory,
2480          * or fall back to examining the EEPROM if necessary.
2481          * Note: on some BCM5700 cards, this value appears to be unset.
2482          * If that's the case, we have to rely on identifying the NIC
2483          * by its PCI subsystem ID, as we do below for the SysKonnect
2484          * SK-9D41.
2485          */
2486         if (bge_readmem_ind(sc, BGE_SRAM_DATA_SIG) == BGE_SRAM_DATA_SIG_MAGIC) {
2487                 hwcfg = bge_readmem_ind(sc, BGE_SRAM_DATA_CFG);
2488         } else {
2489                 if (bge_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
2490                                     sizeof(hwcfg))) {
2491                         device_printf(dev, "failed to read EEPROM\n");
2492                         error = ENXIO;
2493                         goto fail;
2494                 }
2495                 hwcfg = ntohl(hwcfg);
2496         }
2497
2498         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
2499         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
2500             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER) {
2501                 if (BGE_IS_5714_FAMILY(sc))
2502                         sc->bge_flags |= BGE_FLAG_MII_SERDES;
2503                 else
2504                         sc->bge_flags |= BGE_FLAG_TBI;
2505         }
2506
2507         /* Setup MI MODE */
2508         if (sc->bge_flags & BGE_FLAG_CPMU)
2509                 sc->bge_mi_mode = BGE_MIMODE_500KHZ_CONST;
2510         else
2511                 sc->bge_mi_mode = BGE_MIMODE_BASE;
2512         if (BGE_IS_5700_FAMILY(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5705) {
2513                 /* Enable auto polling for BCM570[0-5]. */
2514                 sc->bge_mi_mode |= BGE_MIMODE_AUTOPOLL;
2515         }
2516
2517         /* Setup link status update stuffs */
2518         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
2519             sc->bge_chipid != BGE_CHIPID_BCM5700_B2) {
2520                 sc->bge_link_upd = bge_bcm5700_link_upd;
2521                 sc->bge_link_chg = BGE_MACSTAT_MI_INTERRUPT;
2522         } else if (sc->bge_flags & BGE_FLAG_TBI) {
2523                 sc->bge_link_upd = bge_tbi_link_upd;
2524                 sc->bge_link_chg = BGE_MACSTAT_LINK_CHANGED;
2525         } else if (sc->bge_mi_mode & BGE_MIMODE_AUTOPOLL) {
2526                 sc->bge_link_upd = bge_autopoll_link_upd;
2527                 sc->bge_link_chg = BGE_MACSTAT_LINK_CHANGED;
2528         } else {
2529                 sc->bge_link_upd = bge_copper_link_upd;
2530                 sc->bge_link_chg = BGE_MACSTAT_LINK_CHANGED;
2531         }
2532
2533         /*
2534          * Broadcom's own driver always assumes the internal
2535          * PHY is at GMII address 1.  On some chips, the PHY responds
2536          * to accesses at all addresses, which could cause us to
2537          * bogusly attach the PHY 32 times at probe type.  Always
2538          * restricting the lookup to address 1 is simpler than
2539          * trying to figure out which chips revisions should be
2540          * special-cased.
2541          */
2542         sc->bge_phyno = 1;
2543
2544         if (sc->bge_flags & BGE_FLAG_TBI) {
2545                 ifmedia_init(&sc->bge_ifmedia, IFM_IMASK,
2546                     bge_ifmedia_upd, bge_ifmedia_sts);
2547                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2548                 ifmedia_add(&sc->bge_ifmedia,
2549                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2550                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2551                 ifmedia_set(&sc->bge_ifmedia, IFM_ETHER|IFM_AUTO);
2552                 sc->bge_ifmedia.ifm_media = sc->bge_ifmedia.ifm_cur->ifm_media;
2553         } else {
2554                 struct mii_probe_args mii_args;
2555                 int tries;
2556
2557                 /*
2558                  * Do transceiver setup and tell the firmware the
2559                  * driver is down so we can try to get access the
2560                  * probe if ASF is running.  Retry a couple of times
2561                  * if we get a conflict with the ASF firmware accessing
2562                  * the PHY.
2563                  */
2564                 tries = 0;
2565                 BGE_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
2566 again:
2567                 bge_asf_driver_up(sc);
2568
2569                 mii_probe_args_init(&mii_args, bge_ifmedia_upd, bge_ifmedia_sts);
2570                 mii_args.mii_probemask = 1 << sc->bge_phyno;
2571                 mii_args.mii_capmask = capmask;
2572                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2573                 mii_args.mii_priv = mii_priv;
2574
2575                 error = mii_probe(dev, &sc->bge_miibus, &mii_args);
2576                 if (error) {
2577                         if (tries++ < 4) {
2578                                 device_printf(sc->bge_dev, "Probe MII again\n");
2579                                 bge_miibus_writereg(sc->bge_dev,
2580                                     sc->bge_phyno, MII_BMCR, BMCR_RESET);
2581                                 goto again;
2582                         }
2583                         device_printf(dev, "MII without any PHY!\n");
2584                         goto fail;
2585                 }
2586
2587                 /*
2588                  * Now tell the firmware we are going up after probing the PHY
2589                  */
2590                 if (sc->bge_asf_mode & ASF_STACKUP)
2591                         BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
2592         }
2593
2594         /*
2595          * Create sysctl nodes.
2596          */
2597         sysctl_ctx_init(&sc->bge_sysctl_ctx);
2598         sc->bge_sysctl_tree = SYSCTL_ADD_NODE(&sc->bge_sysctl_ctx,
2599                                               SYSCTL_STATIC_CHILDREN(_hw),
2600                                               OID_AUTO,
2601                                               device_get_nameunit(dev),
2602                                               CTLFLAG_RD, 0, "");
2603         if (sc->bge_sysctl_tree == NULL) {
2604                 device_printf(dev, "can't add sysctl node\n");
2605                 error = ENXIO;
2606                 goto fail;
2607         }
2608
2609         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2610                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2611                         OID_AUTO, "rx_coal_ticks",
2612                         CTLTYPE_INT | CTLFLAG_RW,
2613                         sc, 0, bge_sysctl_rx_coal_ticks, "I",
2614                         "Receive coalescing ticks (usec).");
2615         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2616                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2617                         OID_AUTO, "tx_coal_ticks",
2618                         CTLTYPE_INT | CTLFLAG_RW,
2619                         sc, 0, bge_sysctl_tx_coal_ticks, "I",
2620                         "Transmit coalescing ticks (usec).");
2621         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2622                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2623                         OID_AUTO, "rx_coal_bds",
2624                         CTLTYPE_INT | CTLFLAG_RW,
2625                         sc, 0, bge_sysctl_rx_coal_bds, "I",
2626                         "Receive max coalesced BD count.");
2627         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2628                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2629                         OID_AUTO, "tx_coal_bds",
2630                         CTLTYPE_INT | CTLFLAG_RW,
2631                         sc, 0, bge_sysctl_tx_coal_bds, "I",
2632                         "Transmit max coalesced BD count.");
2633
2634         SYSCTL_ADD_INT(&sc->bge_sysctl_ctx,
2635                        SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2636                        OID_AUTO, "tx_wreg", CTLFLAG_RW,
2637                        &sc->bge_tx_wreg, 0,
2638                        "# of segments before writing to hardware register");
2639
2640         if (sc->bge_flags & BGE_FLAG_PCIE) {
2641                 /*
2642                  * A common design characteristic for many Broadcom
2643                  * client controllers is that they only support a
2644                  * single outstanding DMA read operation on the PCIe
2645                  * bus. This means that it will take twice as long to
2646                  * fetch a TX frame that is split into header and
2647                  * payload buffers as it does to fetch a single,
2648                  * contiguous TX frame (2 reads vs. 1 read). For these
2649                  * controllers, coalescing buffers to reduce the number
2650                  * of memory reads is effective way to get maximum
2651                  * performance(about 940Mbps).  Without collapsing TX
2652                  * buffers the maximum TCP bulk transfer performance
2653                  * is about 850Mbps. However forcing coalescing mbufs
2654                  * consumes a lot of CPU cycles, so leave it off by
2655                  * default.
2656                  */
2657                 SYSCTL_ADD_INT(&sc->bge_sysctl_ctx,
2658                                SYSCTL_CHILDREN(sc->bge_sysctl_tree),
2659                                OID_AUTO, "force_defrag", CTLFLAG_RW,
2660                                &sc->bge_force_defrag, 0,
2661                                "Force defragment on TX path");
2662         }
2663         if (sc->bge_flags & BGE_FLAG_STATUS_TAG) {
2664                 if (!BGE_IS_5705_PLUS(sc)) {
2665                         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2666                             SYSCTL_CHILDREN(sc->bge_sysctl_tree), OID_AUTO,
2667                             "rx_coal_ticks_int", CTLTYPE_INT | CTLFLAG_RW,
2668                             sc, 0, bge_sysctl_rx_coal_ticks_int, "I",
2669                             "Receive coalescing ticks "
2670                             "during interrupt (usec).");
2671                         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2672                             SYSCTL_CHILDREN(sc->bge_sysctl_tree), OID_AUTO,
2673                             "tx_coal_ticks_int", CTLTYPE_INT | CTLFLAG_RW,
2674                             sc, 0, bge_sysctl_tx_coal_ticks_int, "I",
2675                             "Transmit coalescing ticks "
2676                             "during interrupt (usec).");
2677                 }
2678                 SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2679                     SYSCTL_CHILDREN(sc->bge_sysctl_tree), OID_AUTO,
2680                     "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2681                     sc, 0, bge_sysctl_rx_coal_bds_int, "I",
2682                     "Receive max coalesced BD count during interrupt.");
2683                 SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
2684                     SYSCTL_CHILDREN(sc->bge_sysctl_tree), OID_AUTO,
2685                     "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2686                     sc, 0, bge_sysctl_tx_coal_bds_int, "I",
2687                     "Transmit max coalesced BD count during interrupt.");
2688         }
2689
2690         /*
2691          * Call MI attach routine.
2692          */
2693         ether_ifattach(ifp, ether_addr, NULL);
2694
2695         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->bge_irq));
2696
2697 #ifdef IFPOLL_ENABLE
2698         /* Polling setup */
2699         ifpoll_compat_setup(&sc->bge_npoll,
2700             &sc->bge_sysctl_ctx, sc->bge_sysctl_tree, device_get_unit(dev),
2701             ifp->if_serializer);
2702 #endif
2703
2704         if (sc->bge_irq_type == PCI_INTR_TYPE_MSI) {
2705                 if (sc->bge_flags & BGE_FLAG_ONESHOT_MSI) {
2706                         intr_func = bge_msi_oneshot;
2707                         if (bootverbose)
2708                                 device_printf(dev, "oneshot MSI\n");
2709                 } else {
2710                         intr_func = bge_msi;
2711                 }
2712         } else if (sc->bge_flags & BGE_FLAG_STATUS_TAG) {
2713                 intr_func = bge_intr_legacy;
2714         } else {
2715                 intr_func = bge_intr_crippled;
2716         }
2717         error = bus_setup_intr(dev, sc->bge_irq, INTR_MPSAFE, intr_func, sc,
2718             &sc->bge_intrhand, ifp->if_serializer);
2719         if (error) {
2720                 ether_ifdetach(ifp);
2721                 device_printf(dev, "couldn't set up irq\n");
2722                 goto fail;
2723         }
2724
2725         return(0);
2726 fail:
2727         bge_detach(dev);
2728         return(error);
2729 }
2730
2731 static int
2732 bge_detach(device_t dev)
2733 {
2734         struct bge_softc *sc = device_get_softc(dev);
2735
2736         if (device_is_attached(dev)) {
2737                 struct ifnet *ifp = &sc->arpcom.ac_if;
2738
2739                 lwkt_serialize_enter(ifp->if_serializer);
2740                 bge_stop(sc);
2741                 bus_teardown_intr(dev, sc->bge_irq, sc->bge_intrhand);
2742                 lwkt_serialize_exit(ifp->if_serializer);
2743
2744                 ether_ifdetach(ifp);
2745         }
2746
2747         if (sc->bge_flags & BGE_FLAG_TBI)
2748                 ifmedia_removeall(&sc->bge_ifmedia);
2749         if (sc->bge_miibus)
2750                 device_delete_child(dev, sc->bge_miibus);
2751         bus_generic_detach(dev);
2752
2753         if (sc->bge_irq != NULL) {
2754                 bus_release_resource(dev, SYS_RES_IRQ, sc->bge_irq_rid,
2755                     sc->bge_irq);
2756         }
2757         if (sc->bge_irq_type == PCI_INTR_TYPE_MSI)
2758                 pci_release_msi(dev);
2759
2760         if (sc->bge_res != NULL) {
2761                 bus_release_resource(dev, SYS_RES_MEMORY,
2762                     BGE_PCI_BAR0, sc->bge_res);
2763         }
2764         if (sc->bge_res2 != NULL) {
2765                 bus_release_resource(dev, SYS_RES_MEMORY,
2766                     PCIR_BAR(2), sc->bge_res2);
2767         }
2768
2769         if (sc->bge_sysctl_tree != NULL)
2770                 sysctl_ctx_free(&sc->bge_sysctl_ctx);
2771
2772         bge_dma_free(sc);
2773
2774         return 0;
2775 }
2776
2777 static void
2778 bge_reset(struct bge_softc *sc)
2779 {
2780         device_t dev = sc->bge_dev;
2781         uint32_t cachesize, command, reset, mac_mode, mac_mode_mask;
2782         void (*write_op)(struct bge_softc *, uint32_t, uint32_t);
2783         int i, val = 0;
2784
2785         mac_mode_mask = BGE_MACMODE_HALF_DUPLEX | BGE_MACMODE_PORTMODE;
2786         if (sc->bge_mfw_flags & BGE_MFW_ON_APE)
2787                 mac_mode_mask |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
2788         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & mac_mode_mask;
2789
2790         if (BGE_IS_575X_PLUS(sc) && !BGE_IS_5714_FAMILY(sc) &&
2791             sc->bge_asicrev != BGE_ASICREV_BCM5906) {
2792                 if (sc->bge_flags & BGE_FLAG_PCIE)
2793                         write_op = bge_writemem_direct;
2794                 else
2795                         write_op = bge_writemem_ind;
2796         } else {
2797                 write_op = bge_writereg_ind;
2798         }
2799
2800         if (sc->bge_asicrev != BGE_ASICREV_BCM5700 &&
2801             sc->bge_asicrev != BGE_ASICREV_BCM5701) {
2802                 CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
2803                 for (i = 0; i < 8000; i++) {
2804                         if (CSR_READ_4(sc, BGE_NVRAM_SWARB) &
2805                             BGE_NVRAMSWARB_GNT1)
2806                                 break;
2807                         DELAY(20);
2808                 }
2809                 if (i == 8000) {
2810                         if (bootverbose) {
2811                                 if_printf(&sc->arpcom.ac_if,
2812                                     "NVRAM lock timedout!\n");
2813                         }
2814                 }
2815         }
2816         /* Take APE lock when performing reset. */
2817         bge_ape_lock(sc, BGE_APE_LOCK_GRC);
2818
2819         /* Save some important PCI state. */
2820         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2821         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2822
2823         pci_write_config(dev, BGE_PCI_MISC_CTL,
2824             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2825             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2826             sc->bge_pci_miscctl, 4);
2827
2828         /* Disable fastboot on controllers that support it. */
2829         if (sc->bge_asicrev == BGE_ASICREV_BCM5752 ||
2830             BGE_IS_5755_PLUS(sc)) {
2831                 if (bootverbose)
2832                         if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2833                 CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2834         }
2835
2836         /*
2837          * Write the magic number to SRAM at offset 0xB50.
2838          * When firmware finishes its initialization it will
2839          * write ~BGE_SRAM_FW_MB_MAGIC to the same location.
2840          */
2841         bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
2842
2843         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2844
2845         /* XXX: Broadcom Linux driver. */
2846         if (sc->bge_flags & BGE_FLAG_PCIE) {
2847                 /* Force PCI-E 1.0a mode */
2848                 if (sc->bge_asicrev != BGE_ASICREV_BCM5785 &&
2849                     CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2850                     (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2851                      BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2852                         CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2853                             BGE_PCIE_PHY_TSTCTL_PSCRAM);
2854                 }
2855                 if (sc->bge_chipid != BGE_CHIPID_BCM5750_A0) {
2856                         /* Prevent PCIE link training during global reset */
2857                         CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2858                         reset |= (1<<29);
2859                 }
2860         }
2861
2862         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
2863                 uint32_t status, ctrl;
2864
2865                 status = CSR_READ_4(sc, BGE_VCPU_STATUS);
2866                 CSR_WRITE_4(sc, BGE_VCPU_STATUS,
2867                     status | BGE_VCPU_STATUS_DRV_RESET);
2868                 ctrl = CSR_READ_4(sc, BGE_VCPU_EXT_CTRL);
2869                 CSR_WRITE_4(sc, BGE_VCPU_EXT_CTRL,
2870                     ctrl & ~BGE_VCPU_EXT_CTRL_HALT_CPU);
2871         }
2872
2873         /*
2874          * Set GPHY Power Down Override to leave GPHY
2875          * powered up in D0 uninitialized.
2876          */
2877         if (BGE_IS_5705_PLUS(sc) && (sc->bge_flags & BGE_FLAG_CPMU) == 0)
2878                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2879
2880         /* Issue global reset */
2881         write_op(sc, BGE_MISC_CFG, reset);
2882
2883         if (sc->bge_flags & BGE_FLAG_PCIE)
2884                 DELAY(100 * 1000);
2885         else
2886                 DELAY(1000);
2887
2888         /* XXX: Broadcom Linux driver. */
2889         if (sc->bge_flags & BGE_FLAG_PCIE) {
2890                 uint16_t devctl;
2891
2892                 if (sc->bge_chipid == BGE_CHIPID_BCM5750_A0) {
2893                         uint32_t v;
2894
2895                         DELAY(500000); /* wait for link training to complete */
2896                         v = pci_read_config(dev, 0xc4, 4);
2897                         pci_write_config(dev, 0xc4, v | (1<<15), 4);
2898                 }
2899
2900                 devctl = pci_read_config(dev,
2901                     sc->bge_pciecap + PCIER_DEVCTRL, 2);
2902
2903                 /* Disable no snoop and disable relaxed ordering. */
2904                 devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2905
2906                 /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2907                 if ((sc->bge_flags & BGE_FLAG_CPMU) == 0) {
2908                         devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2909                         devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2910                 }
2911
2912                 pci_write_config(dev, sc->bge_pciecap + PCIER_DEVCTRL,
2913                     devctl, 2);
2914
2915                 /* Clear error status. */
2916                 pci_write_config(dev, sc->bge_pciecap + PCIER_DEVSTS,
2917                     PCIEM_DEVSTS_CORR_ERR |
2918                     PCIEM_DEVSTS_NFATAL_ERR |
2919                     PCIEM_DEVSTS_FATAL_ERR |
2920                     PCIEM_DEVSTS_UNSUPP_REQ, 2);
2921         }
2922
2923         /* Reset some of the PCI state that got zapped by reset */
2924         pci_write_config(dev, BGE_PCI_MISC_CTL,
2925             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2926             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2927             sc->bge_pci_miscctl, 4);
2928         val = BGE_PCISTATE_ROM_ENABLE | BGE_PCISTATE_ROM_RETRY_ENABLE;
2929         if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0 &&
2930             (sc->bge_flags & BGE_FLAG_PCIX))
2931                 val |= BGE_PCISTATE_RETRY_SAME_DMA;
2932         if (sc->bge_mfw_flags & BGE_MFW_ON_APE) {
2933                 val |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
2934                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
2935                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
2936         }
2937         pci_write_config(dev, BGE_PCI_PCISTATE, val, 4);
2938         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2939         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2940
2941         /*
2942          * Disable PCI-X relaxed ordering to ensure status block update
2943          * comes first then packet buffer DMA. Otherwise driver may
2944          * read stale status block.
2945          */
2946         if (sc->bge_flags & BGE_FLAG_PCIX) {
2947                 uint16_t devctl;
2948
2949                 devctl = pci_read_config(dev,
2950                     sc->bge_pcixcap + PCIXR_COMMAND, 2);
2951                 devctl &= ~PCIXM_COMMAND_ERO;
2952                 if (sc->bge_asicrev == BGE_ASICREV_BCM5703) {
2953                         devctl &= ~PCIXM_COMMAND_MAX_READ;
2954                         devctl |= PCIXM_COMMAND_MAX_READ_2048;
2955                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
2956                         devctl &= ~(PCIXM_COMMAND_MAX_SPLITS |
2957                             PCIXM_COMMAND_MAX_READ);
2958                         devctl |= PCIXM_COMMAND_MAX_READ_2048;
2959                 }
2960                 pci_write_config(dev, sc->bge_pcixcap + PCIXR_COMMAND,
2961                     devctl, 2);
2962         }
2963
2964         /*
2965          * Enable memory arbiter and re-enable MSI if necessary.
2966          */
2967         if (BGE_IS_5714_FAMILY(sc)) {
2968                 uint32_t val;
2969
2970                 if (sc->bge_irq_type == PCI_INTR_TYPE_MSI) {
2971                         /*
2972                          * Resetting BCM5714 family will clear MSI
2973                          * enable bit; restore it after resetting.
2974                          */
2975                         PCI_SETBIT(sc->bge_dev, sc->bge_msicap + PCIR_MSI_CTRL,
2976                             PCIM_MSICTRL_MSI_ENABLE, 2);
2977                         BGE_SETBIT(sc, BGE_MSI_MODE, BGE_MSIMODE_ENABLE);
2978                 }
2979                 val = CSR_READ_4(sc, BGE_MARB_MODE);
2980                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE | val);
2981         } else {
2982                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2983         }
2984
2985         /* Fix up byte swapping. */
2986         CSR_WRITE_4(sc, BGE_MODE_CTL, BGE_DMA_SWAP_OPTIONS |
2987             BGE_MODECTL_BYTESWAP_DATA);
2988
2989         val = CSR_READ_4(sc, BGE_MAC_MODE);
2990         val = (val & ~mac_mode_mask) | mac_mode;
2991         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
2992         DELAY(40);
2993
2994         bge_ape_unlock(sc, BGE_APE_LOCK_GRC);
2995
2996         if (sc->bge_asicrev == BGE_ASICREV_BCM5906) {
2997                 for (i = 0; i < BGE_TIMEOUT; i++) {
2998                         val = CSR_READ_4(sc, BGE_VCPU_STATUS);
2999                         if (val & BGE_VCPU_STATUS_INIT_DONE)
3000                                 break;
3001                         DELAY(100);
3002                 }
3003                 if (i == BGE_TIMEOUT) {
3004                         if_printf(&sc->arpcom.ac_if, "reset timed out\n");
3005                         return;
3006                 }
3007         } else {
3008                 int delay_us = 10;
3009
3010                 if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
3011                         delay_us = 100;
3012
3013                 /*
3014                  * Poll until we see the 1's complement of the magic number.
3015                  * This indicates that the firmware initialization
3016                  * is complete.
3017                  */
3018                 for (i = 0; i < BGE_FIRMWARE_TIMEOUT; i++) {
3019                         val = bge_readmem_ind(sc, BGE_SRAM_FW_MB);
3020                         if (val == ~BGE_SRAM_FW_MB_MAGIC)
3021                                 break;
3022                         DELAY(delay_us);
3023                 }
3024                 if (i == BGE_FIRMWARE_TIMEOUT) {
3025                         if_printf(&sc->arpcom.ac_if, "firmware handshake "
3026                                   "timed out, found 0x%08x\n", val);
3027                 }
3028         }
3029
3030         /*
3031          * The 5704 in TBI mode apparently needs some special
3032          * adjustment to insure the SERDES drive level is set
3033          * to 1.2V.
3034          */
3035         if (sc->bge_asicrev == BGE_ASICREV_BCM5704 &&
3036             (sc->bge_flags & BGE_FLAG_TBI)) {
3037                 uint32_t serdescfg;
3038
3039                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
3040                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
3041                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
3042         }
3043
3044         /* XXX: Broadcom Linux driver. */
3045         if ((sc->bge_flags & BGE_FLAG_PCIE) &&
3046             sc->bge_chipid != BGE_CHIPID_BCM5750_A0 &&
3047             sc->bge_asicrev != BGE_ASICREV_BCM5785) {
3048                 uint32_t v;
3049
3050                 /* Enable Data FIFO protection. */
3051                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
3052                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
3053         }
3054
3055         DELAY(10000);
3056 }
3057
3058 /*
3059  * Frame reception handling. This is called if there's a frame
3060  * on the receive return list.
3061  *
3062  * Note: we have to be able to handle two possibilities here:
3063  * 1) the frame is from the jumbo recieve ring
3064  * 2) the frame is from the standard receive ring
3065  */
3066
3067 static void
3068 bge_rxeof(struct bge_softc *sc, uint16_t rx_prod, int count)
3069 {
3070         struct ifnet *ifp;
3071         int stdcnt = 0, jumbocnt = 0;
3072
3073         ifp = &sc->arpcom.ac_if;
3074
3075         while (sc->bge_rx_saved_considx != rx_prod && count != 0) {
3076                 struct bge_rx_bd        *cur_rx;
3077                 uint32_t                rxidx;
3078                 struct mbuf             *m = NULL;
3079                 uint16_t                vlan_tag = 0;
3080                 int                     have_tag = 0;
3081
3082                 --count;
3083
3084                 cur_rx =
3085             &sc->bge_ldata.bge_rx_return_ring[sc->bge_rx_saved_considx];
3086
3087                 rxidx = cur_rx->bge_idx;
3088                 BGE_INC(sc->bge_rx_saved_considx, sc->bge_return_ring_cnt);
3089                 logif(rx_pkt);
3090
3091                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
3092                         have_tag = 1;
3093                         vlan_tag = cur_rx->bge_vlan_tag;
3094                 }
3095
3096                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
3097                         BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT);
3098                         jumbocnt++;
3099
3100                         if (rxidx != sc->bge_jumbo) {
3101                                 IFNET_STAT_INC(ifp, ierrors, 1);
3102                                 if_printf(ifp, "sw jumbo index(%d) "
3103                                     "and hw jumbo index(%d) mismatch, drop!\n",
3104                                     sc->bge_jumbo, rxidx);
3105                                 bge_setup_rxdesc_jumbo(sc, rxidx);
3106                                 continue;
3107                         }
3108
3109                         m = sc->bge_cdata.bge_rx_jumbo_chain[rxidx].bge_mbuf;
3110                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
3111                                 IFNET_STAT_INC(ifp, ierrors, 1);
3112                                 bge_setup_rxdesc_jumbo(sc, sc->bge_jumbo);
3113                                 continue;
3114                         }
3115                         if (bge_newbuf_jumbo(sc, sc->bge_jumbo, 0)) {
3116                                 IFNET_STAT_INC(ifp, ierrors, 1);
3117                                 bge_setup_rxdesc_jumbo(sc, sc->bge_jumbo);
3118                                 continue;
3119                         }
3120                 } else {
3121                         int discard = 0;
3122
3123                         BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT);
3124                         stdcnt++;
3125
3126                         if (rxidx != sc->bge_std) {
3127                                 IFNET_STAT_INC(ifp, ierrors, 1);
3128                                 if_printf(ifp, "sw std index(%d) "
3129                                     "and hw std index(%d) mismatch, drop!\n",
3130                                     sc->bge_std, rxidx);
3131                                 bge_setup_rxdesc_std(sc, rxidx);
3132                                 discard = 1;
3133                                 goto refresh_rx;
3134                         }
3135
3136                         m = sc->bge_cdata.bge_rx_std_chain[rxidx].bge_mbuf;
3137                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
3138                                 IFNET_STAT_INC(ifp, ierrors, 1);
3139                                 bge_setup_rxdesc_std(sc, sc->bge_std);
3140                                 discard = 1;
3141                                 goto refresh_rx;
3142                         }
3143                         if (bge_newbuf_std(sc, sc->bge_std, 0)) {
3144                                 IFNET_STAT_INC(ifp, ierrors, 1);
3145                                 bge_setup_rxdesc_std(sc, sc->bge_std);
3146                                 discard = 1;
3147                         }
3148 refresh_rx:
3149                         if (sc->bge_rx_wreg > 0 && stdcnt >= sc->bge_rx_wreg) {
3150                                 bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO,
3151                                     sc->bge_std);
3152                                 stdcnt = 0;
3153                         }
3154                         if (discard)
3155                                 continue;
3156                 }
3157
3158                 IFNET_STAT_INC(ifp, ipackets, 1);
3159 #if !defined(__i386__) && !defined(__x86_64__)
3160                 /*
3161                  * The x86 allows unaligned accesses, but for other
3162                  * platforms we must make sure the payload is aligned.
3163                  */
3164                 if (sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) {
3165                         bcopy(m->m_data, m->m_data + ETHER_ALIGN,
3166                             cur_rx->bge_len);
3167                         m->m_data += ETHER_ALIGN;
3168                 }
3169 #endif
3170                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
3171                 m->m_pkthdr.rcvif = ifp;
3172
3173                 if (ifp->if_capenable & IFCAP_RXCSUM) {
3174                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
3175                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
3176                                 if ((cur_rx->bge_ip_csum ^ 0xffff) == 0)
3177                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3178                         }
3179                         if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) &&
3180                             m->m_pkthdr.len >= BGE_MIN_FRAMELEN) {
3181                                 m->m_pkthdr.csum_data =
3182                                         cur_rx->bge_tcp_udp_csum;
3183                                 m->m_pkthdr.csum_flags |=
3184                                         CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
3185                         }
3186                 }
3187
3188                 /*
3189                  * If we received a packet with a vlan tag, pass it
3190                  * to vlan_input() instead of ether_input().
3191                  */
3192                 if (have_tag) {
3193                         m->m_flags |= M_VLANTAG;
3194                         m->m_pkthdr.ether_vlantag = vlan_tag;
3195                 }
3196                 ifp->if_input(ifp, m, NULL, -1);
3197         }
3198
3199         bge_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bge_rx_saved_considx);
3200         if (stdcnt)
3201                 bge_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std);
3202         if (jumbocnt)
3203                 bge_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo);
3204 }
3205
3206 static void
3207 bge_txeof(struct bge_softc *sc, uint16_t tx_cons)
3208 {
3209         struct ifnet *ifp;
3210
3211         ifp = &sc->arpcom.ac_if;
3212
3213         /*
3214          * Go through our tx ring and free mbufs for those
3215          * frames that have been sent.
3216          */
3217         while (sc->bge_tx_saved_considx != tx_cons) {
3218                 uint32_t idx = 0;
3219
3220                 idx = sc->bge_tx_saved_considx;
3221                 if (sc->bge_cdata.bge_tx_chain[idx] != NULL) {
3222                         IFNET_STAT_INC(ifp, opackets, 1);
3223                         bus_dmamap_unload(sc->bge_cdata.bge_tx_mtag,
3224                             sc->bge_cdata.bge_tx_dmamap[idx]);
3225                         m_freem(sc->bge_cdata.bge_tx_chain[idx]);
3226                         sc->bge_cdata.bge_tx_chain[idx] = NULL;
3227                 }
3228                 sc->bge_txcnt--;
3229                 BGE_INC(sc->bge_tx_saved_considx, BGE_TX_RING_CNT);
3230                 logif(tx_pkt);
3231         }
3232
3233         if ((BGE_TX_RING_CNT - sc->bge_txcnt) >=
3234             (sc->bge_txrsvd + sc->bge_txspare))
3235                 ifq_clr_oactive(&ifp->if_snd);
3236
3237         if (sc->bge_txcnt == 0)
3238                 ifp->if_timer = 0;
3239
3240         if (!ifq_is_empty(&ifp->if_snd))
3241                 if_devstart(ifp);
3242 }
3243
3244 #ifdef IFPOLL_ENABLE
3245
3246 static void
3247 bge_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycles)
3248 {
3249         struct bge_softc *sc = ifp->if_softc;
3250         struct bge_status_block *sblk = sc->bge_ldata.bge_status_block;
3251         uint16_t rx_prod, tx_cons;
3252
3253         ASSERT_SERIALIZED(ifp->if_serializer);
3254
3255         if (sc->bge_npoll.ifpc_stcount-- == 0) {
3256                 sc->bge_npoll.ifpc_stcount = sc->bge_npoll.ifpc_stfrac;
3257                 /*
3258                  * Process link state changes.
3259                  */
3260                 bge_link_poll(sc);
3261         }
3262
3263         if (sc->bge_flags & BGE_FLAG_STATUS_TAG) {
3264                 sc->bge_status_tag = sblk->bge_status_tag;
3265                 /*
3266                  * Use a load fence to ensure that status_tag
3267                  * is saved  before rx_prod and tx_cons.
3268                  */
3269                 cpu_lfence();
3270         }
3271
3272         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
3273         if (sc->bge_rx_saved_considx != rx_prod)
3274                 bge_rxeof(sc, rx_prod, cycles);
3275
3276         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
3277         if (sc->bge_tx_saved_considx != tx_cons)
3278                 bge_txeof(sc, tx_cons);
3279
3280         if (sc->bge_flags & BGE_FLAG_STATUS_TAG)
3281                 bge_writembx(sc, BGE_MBX_IRQ0_LO, sc->bge_status_tag << 24);
3282
3283         if (sc->bge_coal_chg)
3284                 bge_coal_change(sc);
3285 }
3286
3287 static void
3288 bge_npoll(struct ifnet *ifp, struct ifpoll_info *info)
3289 {
3290         struct bge_softc *sc = ifp->if_softc;
3291
3292         ASSERT_SERIALIZED(ifp->if_serializer);
3293
3294         if (info != NULL) {
3295                 int cpuid = sc->bge_npoll.ifpc_cpuid;
3296
3297                 info->ifpi_rx[cpuid].poll_func = bge_npoll_compat;
3298                 info->ifpi_rx[cpuid].arg = NULL;
3299                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
3300
3301                 if (ifp->if_flags & IFF_RUNNING)
3302                         bge_disable_intr(sc);
3303                 ifq_set_cpuid(&ifp->if_snd, cpuid);
3304         } else {
3305                 if (ifp->if_flags & IFF_RUNNING)
3306                         bge_enable_intr(sc);
3307                 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->bge_irq));
3308         }
3309 }
3310
3311 #endif  /* IFPOLL_ENABLE */
3312
3313 static void
3314 bge_intr_crippled(void *xsc)
3315 {
3316         struct bge_softc *sc = xsc;
3317         struct ifnet *ifp = &sc->arpcom.ac_if;
3318
3319         logif(intr);
3320
3321         /*
3322          * Ack the interrupt by writing something to BGE_MBX_IRQ0_LO.  Don't
3323          * disable interrupts by writing nonzero like we used to, since with
3324          * our current organization this just gives complications and
3325          * pessimizations for re-enabling interrupts.  We used to have races
3326          * instead of the necessary complications.  Disabling interrupts
3327          * would just reduce the chance of a status update while we are
3328          * running (by switching to the interrupt-mode coalescence
3329          * parameters), but this chance is already very low so it is more
3330          * efficient to get another interrupt than prevent it.
3331          *
3332          * We do the ack first to ensure another interrupt if there is a
3333          * status update after the ack.  We don't check for the status
3334          * changing later because it is more efficient to get another
3335          * interrupt than prevent it, not quite as above (not checking is
3336          * a smaller optimization than not toggling the interrupt enable,
3337          * since checking doesn't involve PCI accesses and toggling require
3338          * the status check).  So toggling would probably be a pessimization
3339          * even with MSI.  It would only be needed for using a task queue.
3340          */
3341         bge_writembx(sc, BGE_MBX_IRQ0_LO, 0);
3342
3343         /*
3344          * Process link state changes.
3345          */
3346         bge_link_poll(sc);
3347
3348         if (ifp->if_flags & IFF_RUNNING) {
3349                 struct bge_status_block *sblk = sc->bge_ldata.bge_status_block;
3350                 uint16_t rx_prod, tx_cons;
3351
3352                 rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
3353                 if (sc->bge_rx_saved_considx != rx_prod)
3354                         bge_rxeof(sc, rx_prod, -1);
3355
3356                 tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
3357                 if (sc->bge_tx_saved_considx != tx_cons)
3358                         bge_txeof(sc, tx_cons);
3359         }
3360
3361         if (sc->bge_coal_chg)
3362                 bge_coal_change(sc);
3363 }
3364
3365 static void
3366 bge_intr_legacy(void *xsc)
3367 {
3368         struct bge_softc *sc = xsc;
3369         struct bge_status_block *sblk = sc->bge_ldata.bge_status_block;
3370
3371         if (sc->bge_status_tag == sblk->bge_status_tag) {
3372                 uint32_t val;
3373
3374                 val = pci_read_config(sc->bge_dev, BGE_PCI_PCISTATE, 4);
3375                 if (val & BGE_PCISTAT_INTR_NOTACT)
3376                         return;
3377         }
3378
3379         /*
3380          * NOTE:
3381          * Interrupt will have to be disabled if tagged status
3382          * is used, else interrupt will always be asserted on
3383          * certain chips (at least on BCM5750 AX/BX).
3384          */
3385         bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3386
3387         bge_intr(sc);
3388 }
3389
3390 static void
3391 bge_msi(void *xsc)
3392 {
3393         struct bge_softc *sc = xsc;
3394
3395         /* Disable interrupt first */
3396         bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3397         bge_intr(sc);
3398 }
3399
3400 static void
3401 bge_msi_oneshot(void *xsc)
3402 {
3403         bge_intr(xsc);
3404 }
3405
3406 static void
3407 bge_intr(struct bge_softc *sc)
3408 {
3409         struct ifnet *ifp = &sc->arpcom.ac_if;
3410         struct bge_status_block *sblk = sc->bge_ldata.bge_status_block;
3411         uint16_t rx_prod, tx_cons;
3412         uint32_t status;
3413
3414         sc->bge_status_tag = sblk->bge_status_tag;
3415         /*
3416          * Use a load fence to ensure that status_tag is saved
3417          * before rx_prod, tx_cons and status.
3418          */
3419         cpu_lfence();
3420
3421         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
3422         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
3423         status = sblk->bge_status;
3424
3425         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bge_link_evt)
3426                 bge_link_poll(sc);
3427
3428         if (ifp->if_flags & IFF_RUNNING) {
3429                 if (sc->bge_rx_saved_considx != rx_prod)
3430                         bge_rxeof(sc, rx_prod, -1);
3431
3432                 if (sc->bge_tx_saved_considx != tx_cons)
3433                         bge_txeof(sc, tx_cons);
3434         }
3435
3436         bge_writembx(sc, BGE_MBX_IRQ0_LO, sc->bge_status_tag << 24);
3437
3438         if (sc->bge_coal_chg)
3439                 bge_coal_change(sc);
3440 }
3441
3442 static void
3443 bge_tick(void *xsc)
3444 {
3445         struct bge_softc *sc = xsc;
3446         struct ifnet *ifp = &sc->arpcom.ac_if;
3447
3448         lwkt_serialize_enter(ifp->if_serializer);
3449
3450         if (BGE_IS_5705_PLUS(sc))
3451                 bge_stats_update_regs(sc);
3452         else
3453                 bge_stats_update(sc);
3454
3455         if (sc->bge_flags & BGE_FLAG_TBI) {
3456                 /*
3457                  * Since in TBI mode auto-polling can't be used we should poll
3458                  * link status manually. Here we register pending link event
3459                  * and trigger interrupt.
3460                  */
3461                 sc->bge_link_evt++;
3462                 if (BGE_IS_CRIPPLED(sc))
3463                         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
3464                 else
3465                         BGE_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3466         } else if (!sc->bge_link) {
3467                 mii_tick(device_get_softc(sc->bge_miibus));
3468         }
3469
3470         bge_asf_driver_up(sc);
3471
3472         callout_reset(&sc->bge_stat_timer, hz, bge_tick, sc);
3473
3474         lwkt_serialize_exit(ifp->if_serializer);
3475 }
3476
3477 static void
3478 bge_stats_update_regs(struct bge_softc *sc)
3479 {
3480         struct ifnet *ifp = &sc->arpcom.ac_if;
3481         struct bge_mac_stats_regs stats;
3482         uint32_t *s;
3483         int i;
3484
3485         s = (uint32_t *)&stats;
3486         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
3487                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
3488                 s++;
3489         }
3490
3491         IFNET_STAT_SET(ifp, collisions,
3492            (stats.dot3StatsSingleCollisionFrames +
3493            stats.dot3StatsMultipleCollisionFrames +
3494            stats.dot3StatsExcessiveCollisions +
3495            stats.dot3StatsLateCollisions));
3496 }
3497
3498 static void
3499 bge_stats_update(struct bge_softc *sc)
3500 {
3501         struct ifnet *ifp = &sc->arpcom.ac_if;
3502         bus_size_t stats;
3503
3504         stats = BGE_MEMWIN_START + BGE_STATS_BLOCK;
3505
3506 #define READ_STAT(sc, stats, stat)      \
3507         CSR_READ_4(sc, stats + offsetof(struct bge_stats, stat))
3508
3509         IFNET_STAT_SET(ifp, collisions,
3510            (READ_STAT(sc, stats,
3511                 txstats.dot3StatsSingleCollisionFrames.bge_addr_lo) +
3512             READ_STAT(sc, stats,
3513                 txstats.dot3StatsMultipleCollisionFrames.bge_addr_lo) +
3514             READ_STAT(sc, stats,
3515                 txstats.dot3StatsExcessiveCollisions.bge_addr_lo) +
3516             READ_STAT(sc, stats,
3517                 txstats.dot3StatsLateCollisions.bge_addr_lo)));
3518
3519 #undef READ_STAT
3520
3521 #ifdef notdef
3522         IFNET_STAT_SET(ifp, collisions,
3523            (sc->bge_rdata->bge_info.bge_stats.dot3StatsSingleCollisionFrames +
3524            sc->bge_rdata->bge_info.bge_stats.dot3StatsMultipleCollisionFrames +
3525            sc->bge_rdata->bge_info.bge_stats.dot3StatsExcessiveCollisions +
3526            sc->bge_rdata->bge_info.bge_stats.dot3StatsLateCollisions));
3527 #endif
3528 }
3529
3530 /*
3531  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
3532  * pointers to descriptors.
3533  */
3534 static int
3535 bge_encap(struct bge_softc *sc, struct mbuf **m_head0, uint32_t *txidx,
3536     int *segs_used)
3537 {
3538         struct bge_tx_bd *d = NULL, *last_d;
3539         uint16_t csum_flags = 0, mss = 0;
3540         bus_dma_segment_t segs[BGE_NSEG_NEW];
3541         bus_dmamap_t map;
3542         int error, maxsegs, nsegs, idx, i;
3543         struct mbuf *m_head = *m_head0, *m_new;
3544
3545         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3546                 error = bge_setup_tso(sc, m_head0, &mss, &csum_flags);
3547                 if (error)
3548                         return ENOBUFS;
3549                 m_head = *m_head0;
3550         } else if (m_head->m_pkthdr.csum_flags & BGE_CSUM_FEATURES) {
3551                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
3552                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
3553                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
3554                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
3555                 if (m_head->m_flags & M_LASTFRAG)
3556                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
3557                 else if (m_head->m_flags & M_FRAG)
3558                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
3559         }
3560
3561         idx = *txidx;
3562         map = sc->bge_cdata.bge_tx_dmamap[idx];
3563
3564         maxsegs = (BGE_TX_RING_CNT - sc->bge_txcnt) - sc->bge_txrsvd;
3565         KASSERT(maxsegs >= sc->bge_txspare,
3566                 ("not enough segments %d", maxsegs));
3567
3568         if (maxsegs > BGE_NSEG_NEW)
3569                 maxsegs = BGE_NSEG_NEW;
3570
3571         /*
3572          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
3573          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
3574          * but when such padded frames employ the bge IP/TCP checksum
3575          * offload, the hardware checksum assist gives incorrect results
3576          * (possibly from incorporating its own padding into the UDP/TCP
3577          * checksum; who knows).  If we pad such runts with zeros, the
3578          * onboard checksum comes out correct.
3579          */
3580         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
3581             m_head->m_pkthdr.len < BGE_MIN_FRAMELEN) {
3582                 error = m_devpad(m_head, BGE_MIN_FRAMELEN);
3583                 if (error)
3584                         goto back;
3585         }
3586
3587         if ((sc->bge_flags & BGE_FLAG_SHORTDMA) && m_head->m_next != NULL) {
3588                 m_new = bge_defrag_shortdma(m_head);
3589                 if (m_new == NULL) {
3590                         error = ENOBUFS;
3591                         goto back;
3592                 }
3593                 *m_head0 = m_head = m_new;
3594         }
3595         if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 &&
3596             sc->bge_force_defrag && (sc->bge_flags & BGE_FLAG_PCIE) &&
3597             m_head->m_next != NULL) {
3598                 /*
3599                  * Forcefully defragment mbuf chain to overcome hardware
3600                  * limitation which only support a single outstanding
3601                  * DMA read operation.  If it fails, keep moving on using
3602                  * the original mbuf chain.
3603                  */
3604                 m_new = m_defrag(m_head, MB_DONTWAIT);
3605                 if (m_new != NULL)
3606                         *m_head0 = m_head = m_new;
3607         }
3608
3609         error = bus_dmamap_load_mbuf_defrag(sc->bge_cdata.bge_tx_mtag, map,
3610                         m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
3611         if (error)
3612                 goto back;
3613         *segs_used += nsegs;
3614
3615         m_head = *m_head0;
3616         bus_dmamap_sync(sc->bge_cdata.bge_tx_mtag, map, BUS_DMASYNC_PREWRITE);
3617
3618         for (i = 0; ; i++) {
3619                 d = &sc->bge_ldata.bge_tx_ring[idx];
3620
3621                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
3622                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
3623                 d->bge_len = segs[i].ds_len;
3624                 d->bge_flags = csum_flags;
3625                 d->bge_mss = mss;
3626
3627                 if (i == nsegs - 1)
3628                         break;
3629                 BGE_INC(idx, BGE_TX_RING_CNT);
3630         }
3631         last_d = d;
3632
3633         /* Set vlan tag to the first segment of the packet. */
3634         d = &sc->bge_ldata.bge_tx_ring[*txidx];
3635         if (m_head->m_flags & M_VLANTAG) {
3636                 d->bge_flags |= BGE_TXBDFLAG_VLAN_TAG;
3637                 d->bge_vlan_tag = m_head->m_pkthdr.ether_vlantag;
3638         } else {
3639                 d->bge_vlan_tag = 0;
3640         }
3641
3642         /* Mark the last segment as end of packet... */
3643         last_d->bge_flags |= BGE_TXBDFLAG_END;
3644
3645         /*
3646          * Insure that the map for this transmission is placed at
3647          * the array index of the last descriptor in this chain.
3648          */
3649         sc->bge_cdata.bge_tx_dmamap[*txidx] = sc->bge_cdata.bge_tx_dmamap[idx];
3650         sc->bge_cdata.bge_tx_dmamap[idx] = map;
3651         sc->bge_cdata.bge_tx_chain[idx] = m_head;
3652         sc->bge_txcnt += nsegs;
3653
3654         BGE_INC(idx, BGE_TX_RING_CNT);
3655         *txidx = idx;
3656 back:
3657         if (error) {
3658                 m_freem(*m_head0);
3659                 *m_head0 = NULL;
3660         }
3661         return error;
3662 }
3663
3664 static void
3665 bge_xmit(struct bge_softc *sc, uint32_t prodidx)
3666 {
3667         /* Transmit */
3668         bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
3669         /* 5700 b2 errata */
3670         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
3671                 bge_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
3672 }
3673
3674 /*
3675  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
3676  * to the mbuf data regions directly in the transmit descriptors.
3677  */
3678 static void
3679 bge_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3680 {
3681         struct bge_softc *sc = ifp->if_softc;
3682         struct mbuf *m_head = NULL;
3683         uint32_t prodidx;
3684         int nsegs = 0;
3685
3686         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
3687
3688         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
3689                 return;
3690
3691         prodidx = sc->bge_tx_prodidx;
3692
3693         while (sc->bge_cdata.bge_tx_chain[prodidx] == NULL) {
3694                 m_head = ifq_dequeue(&ifp->if_snd);
3695                 if (m_head == NULL)
3696                         break;
3697
3698                 /*
3699                  * XXX
3700                  * The code inside the if() block is never reached since we
3701                  * must mark CSUM_IP_FRAGS in our if_hwassist to start getting
3702                  * requests to checksum TCP/UDP in a fragmented packet.
3703                  *
3704                  * XXX
3705                  * safety overkill.  If this is a fragmented packet chain
3706                  * with delayed TCP/UDP checksums, then only encapsulate
3707                  * it if we have enough descriptors to handle the entire
3708                  * chain at once.
3709                  * (paranoia -- may not actually be needed)
3710                  */
3711                 if ((m_head->m_flags & M_FIRSTFRAG) &&
3712                     (m_head->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) {
3713                         if ((BGE_TX_RING_CNT - sc->bge_txcnt) <
3714                             m_head->m_pkthdr.csum_data + sc->bge_txrsvd) {
3715                                 ifq_set_oactive(&ifp->if_snd);
3716                                 ifq_prepend(&ifp->if_snd, m_head);
3717                                 break;
3718                         }
3719                 }
3720
3721                 /*
3722                  * Sanity check: avoid coming within bge_txrsvd
3723                  * descriptors of the end of the ring.  Also make
3724                  * sure there are bge_txspare descriptors for
3725                  * jumbo buffers' defragmentation.
3726                  */
3727                 if ((BGE_TX_RING_CNT - sc->bge_txcnt) <
3728                     (sc->bge_txrsvd + sc->bge_txspare)) {
3729                         ifq_set_oactive(&ifp->if_snd);
3730                         ifq_prepend(&ifp->if_snd, m_head);
3731                         break;
3732                 }
3733
3734                 /*
3735                  * Pack the data into the transmit ring. If we
3736                  * don't have room, set the OACTIVE flag and wait
3737                  * for the NIC to drain the ring.
3738                  */
3739                 if (bge_encap(sc, &m_head, &prodidx, &nsegs)) {
3740                         ifq_set_oactive(&ifp->if_snd);
3741                         IFNET_STAT_INC(ifp, oerrors, 1);
3742                         break;
3743                 }
3744
3745                 if (nsegs >= sc->bge_tx_wreg) {
3746                         bge_xmit(sc, prodidx);
3747                         nsegs = 0;
3748                 }
3749
3750                 ETHER_BPF_MTAP(ifp, m_head);
3751
3752                 /*
3753                  * Set a timeout in case the chip goes out to lunch.
3754                  */
3755                 ifp->if_timer = 5;
3756         }
3757
3758         if (nsegs > 0)
3759                 bge_xmit(sc, prodidx);
3760         sc->bge_tx_prodidx = prodidx;
3761 }
3762
3763 static void
3764 bge_init(void *xsc)
3765 {
3766         struct bge_softc *sc = xsc;
3767         struct ifnet *ifp = &sc->arpcom.ac_if;
3768         uint16_t *m;
3769         uint32_t mode;
3770
3771         ASSERT_SERIALIZED(ifp->if_serializer);
3772
3773         /* Cancel pending I/O and flush buffers. */
3774         bge_stop(sc);
3775
3776         bge_stop_fw(sc);
3777         bge_sig_pre_reset(sc, BGE_RESET_START);
3778         bge_reset(sc);
3779         bge_sig_legacy(sc, BGE_RESET_START);
3780         bge_sig_post_reset(sc, BGE_RESET_START);
3781
3782         bge_chipinit(sc);
3783
3784         /*
3785          * Init the various state machines, ring
3786          * control blocks and firmware.
3787          */
3788         if (bge_blockinit(sc)) {
3789                 if_printf(ifp, "initialization failure\n");
3790                 bge_stop(sc);
3791                 return;
3792         }
3793
3794         /* Specify MTU. */
3795         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
3796             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
3797
3798         /* Load our MAC address. */
3799         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
3800         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
3801         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
3802
3803         /* Enable or disable promiscuous mode as needed. */
3804         bge_setpromisc(sc);
3805
3806         /* Program multicast filter. */
3807         bge_setmulti(sc);
3808
3809         /* Init RX ring. */
3810         if (bge_init_rx_ring_std(sc)) {
3811                 if_printf(ifp, "RX ring initialization failed\n");
3812                 bge_stop(sc);
3813                 return;
3814         }
3815
3816         /*
3817          * Workaround for a bug in 5705 ASIC rev A0. Poll the NIC's
3818          * memory to insure that the chip has in fact read the first
3819          * entry of the ring.
3820          */
3821         if (sc->bge_chipid == BGE_CHIPID_BCM5705_A0) {
3822                 uint32_t                v, i;
3823                 for (i = 0; i < 10; i++) {
3824                         DELAY(20);
3825                         v = bge_readmem_ind(sc, BGE_STD_RX_RINGS + 8);
3826                         if (v == (MCLBYTES - ETHER_ALIGN))
3827                                 break;
3828                 }
3829                 if (i == 10)
3830                         if_printf(ifp, "5705 A0 chip failed to load RX ring\n");
3831         }
3832
3833         /* Init jumbo RX ring. */
3834         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
3835                 if (bge_init_rx_ring_jumbo(sc)) {
3836                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
3837                         bge_stop(sc);
3838                         return;
3839                 }
3840         }
3841
3842         /* Init our RX return ring index */
3843         sc->bge_rx_saved_considx = 0;
3844
3845         /* Init TX ring. */
3846         bge_init_tx_ring(sc);
3847
3848         /* Enable TX MAC state machine lockup fix. */
3849         mode = CSR_READ_4(sc, BGE_TX_MODE);
3850         if (BGE_IS_5755_PLUS(sc) || sc->bge_asicrev == BGE_ASICREV_BCM5906)
3851                 mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3852         /* Turn on transmitter */
3853         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3854         DELAY(100);
3855
3856         /* Turn on receiver */
3857         mode = CSR_READ_4(sc, BGE_RX_MODE);
3858         if (BGE_IS_5755_PLUS(sc))
3859                 mode |= BGE_RXMODE_IPV6_ENABLE;
3860         CSR_WRITE_4(sc, BGE_RX_MODE, mode | BGE_RXMODE_ENABLE);
3861         DELAY(10);
3862
3863         /*
3864          * Set the number of good frames to receive after RX MBUF
3865          * Low Watermark has been reached.  After the RX MAC receives
3866          * this number of frames, it will drop subsequent incoming
3867          * frames until the MBUF High Watermark is reached.
3868          */
3869         CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3870
3871         if (sc->bge_irq_type == PCI_INTR_TYPE_MSI) {
3872                 if (bootverbose) {
3873                         if_printf(ifp, "MSI_MODE: %#x\n",
3874                             CSR_READ_4(sc, BGE_MSI_MODE));
3875                 }
3876
3877                 /*
3878                  * XXX
3879                  * Linux driver turns it on for all chips supporting MSI?!
3880                  */
3881                 if (sc->bge_flags & BGE_FLAG_ONESHOT_MSI) {
3882                         /*
3883                          * XXX
3884                          * According to 5722-PG101-R,
3885                          * BGE_PCIE_TRANSACT_ONESHOT_MSI applies only to
3886                          * BCM5906.
3887                          */
3888                         BGE_SETBIT(sc, BGE_PCIE_TRANSACT,
3889                             BGE_PCIE_TRANSACT_ONESHOT_MSI);
3890                 }
3891         }
3892
3893         /* Tell firmware we're alive. */
3894         BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3895
3896         /* Enable host interrupts if polling(4) is not enabled. */
3897         PCI_SETBIT(sc->bge_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3898 #ifdef IFPOLL_ENABLE
3899         if (ifp->if_flags & IFF_NPOLLING)
3900                 bge_disable_intr(sc);
3901         else
3902 #endif
3903         bge_enable_intr(sc);
3904
3905         ifp->if_flags |= IFF_RUNNING;
3906         ifq_clr_oactive(&ifp->if_snd);
3907
3908         bge_ifmedia_upd(ifp);
3909
3910         callout_reset(&sc->bge_stat_timer, hz, bge_tick, sc);
3911 }
3912
3913 /*
3914  * Set media options.
3915  */
3916 static int
3917 bge_ifmedia_upd(struct ifnet *ifp)
3918 {
3919         struct bge_softc *sc = ifp->if_softc;
3920
3921         /* If this is a 1000baseX NIC, enable the TBI port. */
3922         if (sc->bge_flags & BGE_FLAG_TBI) {
3923                 struct ifmedia *ifm = &sc->bge_ifmedia;
3924
3925                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3926                         return(EINVAL);
3927
3928                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3929                 case IFM_AUTO:
3930                         /*
3931                          * The BCM5704 ASIC appears to have a special
3932                          * mechanism for programming the autoneg
3933                          * advertisement registers in TBI mode.
3934                          */
3935                         if (!bge_fake_autoneg &&
3936                             sc->bge_asicrev == BGE_ASICREV_BCM5704) {
3937                                 uint32_t sgdig;
3938
3939                                 CSR_WRITE_4(sc, BGE_TX_TBI_AUTONEG, 0);
3940                                 sgdig = CSR_READ_4(sc, BGE_SGDIG_CFG);
3941                                 sgdig |= BGE_SGDIGCFG_AUTO |
3942                                          BGE_SGDIGCFG_PAUSE_CAP |
3943                                          BGE_SGDIGCFG_ASYM_PAUSE;
3944                                 CSR_WRITE_4(sc, BGE_SGDIG_CFG,
3945                                             sgdig | BGE_SGDIGCFG_SEND);
3946                                 DELAY(5);
3947                                 CSR_WRITE_4(sc, BGE_SGDIG_CFG, sgdig);
3948                         }
3949                         break;
3950                 case IFM_1000_SX:
3951                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3952                                 BGE_CLRBIT(sc, BGE_MAC_MODE,
3953                                     BGE_MACMODE_HALF_DUPLEX);
3954                         } else {
3955                                 BGE_SETBIT(sc, BGE_MAC_MODE,
3956                                     BGE_MACMODE_HALF_DUPLEX);
3957                         }
3958                         DELAY(40);
3959                         break;
3960                 default:
3961                         return(EINVAL);
3962                 }
3963         } else {
3964                 struct mii_data *mii = device_get_softc(sc->bge_miibus);
3965
3966                 sc->bge_link_evt++;
3967                 sc->bge_link = 0;
3968                 if (mii->mii_instance) {
3969                         struct mii_softc *miisc;
3970
3971                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3972                                 mii_phy_reset(miisc);
3973                 }
3974                 mii_mediachg(mii);
3975
3976                 /*
3977                  * Force an interrupt so that we will call bge_link_upd
3978                  * if needed and clear any pending link state attention.
3979                  * Without this we are not getting any further interrupts
3980                  * for link state changes and thus will not UP the link and
3981                  * not be able to send in bge_start.  The only way to get
3982                  * things working was to receive a packet and get an RX
3983                  * intr.
3984                  *
3985                  * bge_tick should help for fiber cards and we might not
3986                  * need to do this here if BGE_FLAG_TBI is set but as
3987                  * we poll for fiber anyway it should not harm.
3988                  */
3989                 if (BGE_IS_CRIPPLED(sc))
3990                         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
3991                 else
3992                         BGE_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3993         }
3994         return(0);
3995 }
3996
3997 /*
3998  * Report current media status.
3999  */
4000 static void
4001 bge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
4002 {
4003         struct bge_softc *sc = ifp->if_softc;
4004
4005         if ((ifp->if_flags & IFF_RUNNING) == 0)
4006                 return;
4007
4008         if (sc->bge_flags & BGE_FLAG_TBI) {
4009                 ifmr->ifm_status = IFM_AVALID;
4010                 ifmr->ifm_active = IFM_ETHER;
4011                 if (CSR_READ_4(sc, BGE_MAC_STS) &
4012                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
4013                         ifmr->ifm_status |= IFM_ACTIVE;
4014                 } else {
4015                         ifmr->ifm_active |= IFM_NONE;
4016                         return;
4017                 }
4018
4019                 ifmr->ifm_active |= IFM_1000_SX;
4020                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
4021                         ifmr->ifm_active |= IFM_HDX;
4022                 else
4023                         ifmr->ifm_active |= IFM_FDX;
4024         } else {
4025                 struct mii_data *mii = device_get_softc(sc->bge_miibus);
4026
4027                 mii_pollstat(mii);
4028                 ifmr->ifm_active = mii->mii_media_active;
4029                 ifmr->ifm_status = mii->mii_media_status;
4030         }
4031 }
4032
4033 static int
4034 bge_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
4035 {
4036         struct bge_softc *sc = ifp->if_softc;
4037         struct ifreq *ifr = (struct ifreq *)data;
4038         int mask, error = 0;
4039
4040         ASSERT_SERIALIZED(ifp->if_serializer);
4041
4042         switch (command) {
4043         case SIOCSIFMTU:
4044                 if ((!BGE_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
4045                     (BGE_IS_JUMBO_CAPABLE(sc) &&
4046                      ifr->ifr_mtu > BGE_JUMBO_MTU)) {
4047                         error = EINVAL;
4048                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
4049                         ifp->if_mtu = ifr->ifr_mtu;
4050                         if (ifp->if_flags & IFF_RUNNING)
4051                                 bge_init(sc);
4052                 }
4053                 break;
4054         case SIOCSIFFLAGS:
4055                 if (ifp->if_flags & IFF_UP) {
4056                         if (ifp->if_flags & IFF_RUNNING) {
4057                                 mask = ifp->if_flags ^ sc->bge_if_flags;
4058
4059                                 /*
4060                                  * If only the state of the PROMISC flag
4061                                  * changed, then just use the 'set promisc
4062                                  * mode' command instead of reinitializing
4063                                  * the entire NIC. Doing a full re-init
4064                                  * means reloading the firmware and waiting
4065                                  * for it to start up, which may take a
4066                                  * second or two.  Similarly for ALLMULTI.
4067                                  */
4068                                 if (mask & IFF_PROMISC)
4069                                         bge_setpromisc(sc);
4070                                 if (mask & IFF_ALLMULTI)
4071                                         bge_setmulti(sc);
4072                         } else {
4073                                 bge_init(sc);
4074                         }
4075                 } else if (ifp->if_flags & IFF_RUNNING) {
4076                         bge_stop(sc);
4077                 }
4078                 sc->bge_if_flags = ifp->if_flags;
4079                 break;
4080         case SIOCADDMULTI:
4081         case SIOCDELMULTI:
4082                 if (ifp->if_flags & IFF_RUNNING)
4083                         bge_setmulti(sc);
4084                 break;
4085         case SIOCSIFMEDIA:
4086         case SIOCGIFMEDIA:
4087                 if (sc->bge_flags & BGE_FLAG_TBI) {
4088                         error = ifmedia_ioctl(ifp, ifr,
4089                             &sc->bge_ifmedia, command);
4090                 } else {
4091                         struct mii_data *mii;
4092
4093                         mii = device_get_softc(sc->bge_miibus);
4094                         error = ifmedia_ioctl(ifp, ifr,
4095                                               &mii->mii_media, command);
4096                 }
4097                 break;
4098         case SIOCSIFCAP:
4099                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
4100                 if (mask & IFCAP_HWCSUM) {
4101                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
4102                         if (ifp->if_capenable & IFCAP_TXCSUM)
4103                                 ifp->if_hwassist |= BGE_CSUM_FEATURES;
4104                         else
4105                                 ifp->if_hwassist &= ~BGE_CSUM_FEATURES;
4106                 }
4107                 if (mask & IFCAP_TSO) {
4108                         ifp->if_capenable ^= IFCAP_TSO;
4109                         if (ifp->if_capenable & IFCAP_TSO)
4110                                 ifp->if_hwassist |= CSUM_TSO;
4111                         else
4112                                 ifp->if_hwassist &= ~CSUM_TSO;
4113                 }
4114                 break;
4115         default:
4116                 error = ether_ioctl(ifp, command, data);
4117                 break;
4118         }
4119         return error;
4120 }
4121
4122 static void
4123 bge_watchdog(struct ifnet *ifp)
4124 {
4125         struct bge_softc *sc = ifp->if_softc;
4126
4127         if_printf(ifp, "watchdog timeout -- resetting\n");
4128
4129         bge_init(sc);
4130
4131         IFNET_STAT_INC(ifp, oerrors, 1);
4132
4133         if (!ifq_is_empty(&ifp->if_snd))
4134                 if_devstart(ifp);
4135 }
4136
4137 /*
4138  * Stop the adapter and free any mbufs allocated to the
4139  * RX and TX lists.
4140  */
4141 static void
4142 bge_stop(struct bge_softc *sc)
4143 {
4144         struct ifnet *ifp = &sc->arpcom.ac_if;
4145
4146         ASSERT_SERIALIZED(ifp->if_serializer);
4147
4148         callout_stop(&sc->bge_stat_timer);
4149
4150         /* Disable host interrupts. */
4151         bge_disable_intr(sc);
4152
4153         /*
4154          * Tell firmware we're shutting down.
4155          */
4156         bge_stop_fw(sc);
4157         bge_sig_pre_reset(sc, BGE_RESET_SHUTDOWN);
4158
4159         /*
4160          * Disable all of the receiver blocks
4161          */
4162         bge_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
4163         bge_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
4164         bge_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
4165         if (BGE_IS_5700_FAMILY(sc))
4166                 bge_stop_block(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
4167         bge_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
4168         bge_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
4169         bge_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
4170
4171         /*
4172          * Disable all of the transmit blocks
4173          */
4174         bge_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
4175         bge_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
4176         bge_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
4177         bge_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
4178         bge_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
4179         if (BGE_IS_5700_FAMILY(sc))
4180                 bge_stop_block(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
4181         bge_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
4182
4183         /*
4184          * Shut down all of the memory managers and related
4185          * state machines.
4186          */
4187         bge_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
4188         bge_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
4189         if (BGE_IS_5700_FAMILY(sc))
4190                 bge_stop_block(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
4191         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
4192         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
4193         if (!BGE_IS_5705_PLUS(sc)) {
4194                 BGE_CLRBIT(sc, BGE_BMAN_MODE, BGE_BMANMODE_ENABLE);
4195                 BGE_CLRBIT(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
4196         }
4197
4198         bge_reset(sc);
4199         bge_sig_legacy(sc, BGE_RESET_SHUTDOWN);
4200         bge_sig_post_reset(sc, BGE_RESET_SHUTDOWN);
4201
4202         /*
4203          * Keep the ASF firmware running if up.
4204          */
4205         if (sc->bge_asf_mode & ASF_STACKUP)
4206                 BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
4207         else
4208                 BGE_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
4209
4210         /* Free the RX lists. */
4211         bge_free_rx_ring_std(sc);
4212
4213         /* Free jumbo RX list. */
4214         if (BGE_IS_JUMBO_CAPABLE(sc))
4215                 bge_free_rx_ring_jumbo(sc);
4216
4217         /* Free TX buffers. */
4218         bge_free_tx_ring(sc);
4219
4220         sc->bge_status_tag = 0;
4221         sc->bge_link = 0;
4222         sc->bge_coal_chg = 0;
4223
4224         sc->bge_tx_saved_considx = BGE_TXCONS_UNSET;
4225
4226         ifp->if_flags &= ~IFF_RUNNING;
4227         ifq_clr_oactive(&ifp->if_snd);
4228         ifp->if_timer = 0;
4229 }
4230
4231 /*
4232  * Stop all chip I/O so that the kernel's probe routines don't
4233  * get confused by errant DMAs when rebooting.
4234  */
4235 static void
4236 bge_shutdown(device_t dev)
4237 {
4238         struct bge_softc *sc = device_get_softc(dev);
4239         struct ifnet *ifp = &sc->arpcom.ac_if;
4240
4241         lwkt_serialize_enter(ifp->if_serializer);
4242         bge_stop(sc);
4243         lwkt_serialize_exit(ifp->if_serializer);
4244 }
4245
4246 static int
4247 bge_suspend(device_t dev)
4248 {
4249         struct bge_softc *sc = device_get_softc(dev);
4250         struct ifnet *ifp = &sc->arpcom.ac_if;
4251
4252         lwkt_serialize_enter(ifp->if_serializer);
4253         bge_stop(sc);
4254         lwkt_serialize_exit(ifp->if_serializer);
4255
4256         return 0;
4257 }
4258
4259 static int
4260 bge_resume(device_t dev)
4261 {
4262         struct bge_softc *sc = device_get_softc(dev);
4263         struct ifnet *ifp = &sc->arpcom.ac_if;
4264
4265         lwkt_serialize_enter(ifp->if_serializer);
4266
4267         if (ifp->if_flags & IFF_UP) {
4268                 bge_init(sc);
4269
4270                 if (!ifq_is_empty(&ifp->if_snd))
4271                         if_devstart(ifp);
4272         }
4273
4274         lwkt_serialize_exit(ifp->if_serializer);
4275
4276         return 0;
4277 }
4278
4279 static void
4280 bge_setpromisc(struct bge_softc *sc)
4281 {
4282         struct ifnet *ifp = &sc->arpcom.ac_if;
4283
4284         if (ifp->if_flags & IFF_PROMISC)
4285                 BGE_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
4286         else
4287                 BGE_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
4288 }
4289
4290 static void
4291 bge_dma_free(struct bge_softc *sc)
4292 {
4293         int i;
4294
4295         /* Destroy RX mbuf DMA stuffs. */
4296         if (sc->bge_cdata.bge_rx_mtag != NULL) {
4297                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
4298                         bus_dmamap_destroy(sc->bge_cdata.bge_rx_mtag,
4299                             sc->bge_cdata.bge_rx_std_dmamap[i]);
4300                 }
4301                 bus_dmamap_destroy(sc->bge_cdata.bge_rx_mtag,
4302                                    sc->bge_cdata.bge_rx_tmpmap);
4303                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_mtag);
4304         }
4305
4306         /* Destroy TX mbuf DMA stuffs. */
4307         if (sc->bge_cdata.bge_tx_mtag != NULL) {
4308                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
4309                         bus_dmamap_destroy(sc->bge_cdata.bge_tx_mtag,
4310                             sc->bge_cdata.bge_tx_dmamap[i]);
4311                 }
4312                 bus_dma_tag_destroy(sc->bge_cdata.bge_tx_mtag);
4313         }
4314
4315         /* Destroy standard RX ring */
4316         bge_dma_block_free(sc->bge_cdata.bge_rx_std_ring_tag,
4317                            sc->bge_cdata.bge_rx_std_ring_map,
4318                            sc->bge_ldata.bge_rx_std_ring);
4319
4320         if (BGE_IS_JUMBO_CAPABLE(sc))
4321                 bge_free_jumbo_mem(sc);
4322
4323         /* Destroy RX return ring */
4324         bge_dma_block_free(sc->bge_cdata.bge_rx_return_ring_tag,
4325                            sc->bge_cdata.bge_rx_return_ring_map,
4326                            sc->bge_ldata.bge_rx_return_ring);
4327
4328         /* Destroy TX ring */
4329         bge_dma_block_free(sc->bge_cdata.bge_tx_ring_tag,
4330                            sc->bge_cdata.bge_tx_ring_map,
4331                            sc->bge_ldata.bge_tx_ring);
4332
4333         /* Destroy status block */
4334         bge_dma_block_free(sc->bge_cdata.bge_status_tag,
4335                            sc->bge_cdata.bge_status_map,
4336                            sc->bge_ldata.bge_status_block);
4337
4338         /* Destroy statistics block */
4339         bge_dma_block_free(sc->bge_cdata.bge_stats_tag,
4340                            sc->bge_cdata.bge_stats_map,
4341                            sc->bge_ldata.bge_stats);
4342
4343         /* Destroy the parent tag */
4344         if (sc->bge_cdata.bge_parent_tag != NULL)
4345                 bus_dma_tag_destroy(sc->bge_cdata.bge_parent_tag);
4346 }
4347
4348 static int
4349 bge_dma_alloc(struct bge_softc *sc)
4350 {
4351         struct ifnet *ifp = &sc->arpcom.ac_if;
4352         int i, error;
4353         bus_addr_t lowaddr;
4354         bus_size_t txmaxsz;
4355
4356         lowaddr = BUS_SPACE_MAXADDR;
4357         if (sc->bge_flags & BGE_FLAG_MAXADDR_40BIT)
4358                 lowaddr = BGE_DMA_MAXADDR_40BIT;
4359
4360         /*
4361          * Allocate the parent bus DMA tag appropriate for PCI.
4362          *
4363          * All of the NetExtreme/NetLink controllers have 4GB boundary
4364          * DMA bug.
4365          * Whenever an address crosses a multiple of the 4GB boundary
4366          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
4367          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
4368          * state machine will lockup and cause the device to hang.
4369          */
4370         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
4371                                    lowaddr, BUS_SPACE_MAXADDR,
4372                                    NULL, NULL,
4373                                    BUS_SPACE_MAXSIZE_32BIT, 0,
4374                                    BUS_SPACE_MAXSIZE_32BIT,
4375                                    0, &sc->bge_cdata.bge_parent_tag);
4376         if (error) {
4377                 if_printf(ifp, "could not allocate parent dma tag\n");
4378                 return error;
4379         }
4380
4381         /*
4382          * Create DMA tag and maps for RX mbufs.
4383          */
4384         error = bus_dma_tag_create(sc->bge_cdata.bge_parent_tag, 1, 0,
4385                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4386                                    NULL, NULL, MCLBYTES, 1, MCLBYTES,
4387                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK,
4388                                    &sc->bge_cdata.bge_rx_mtag);
4389         if (error) {
4390                 if_printf(ifp, "could not allocate RX mbuf dma tag\n");
4391                 return error;
4392         }
4393
4394         error = bus_dmamap_create(sc->bge_cdata.bge_rx_mtag,
4395                                   BUS_DMA_WAITOK, &sc->bge_cdata.bge_rx_tmpmap);
4396         if (error) {
4397                 bus_dma_tag_destroy(sc->bge_cdata.bge_rx_mtag);
4398                 sc->bge_cdata.bge_rx_mtag = NULL;
4399                 return error;
4400         }
4401
4402         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
4403                 error = bus_dmamap_create(sc->bge_cdata.bge_rx_mtag,
4404                                           BUS_DMA_WAITOK,
4405                                           &sc->bge_cdata.bge_rx_std_dmamap[i]);
4406                 if (error) {
4407                         int j;
4408
4409                         for (j = 0; j < i; ++j) {
4410                                 bus_dmamap_destroy(sc->bge_cdata.bge_rx_mtag,
4411                                         sc->bge_cdata.bge_rx_std_dmamap[j]);
4412                         }
4413                         bus_dma_tag_destroy(sc->bge_cdata.bge_rx_mtag);
4414                         sc->bge_cdata.bge_rx_mtag = NULL;
4415
4416                         if_printf(ifp, "could not create DMA map for RX\n");
4417                         return error;
4418                 }
4419         }
4420
4421         /*
4422          * Create DMA tag and maps for TX mbufs.
4423          */
4424         if (sc->bge_flags & BGE_FLAG_TSO)
4425                 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header);
4426         else
4427                 txmaxsz = BGE_JUMBO_FRAMELEN;
4428         error = bus_dma_tag_create(sc->bge_cdata.bge_parent_tag, 1, 0,
4429                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4430                                    NULL, NULL,
4431                                    txmaxsz, BGE_NSEG_NEW, PAGE_SIZE,
4432                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK |
4433                                    BUS_DMA_ONEBPAGE,
4434                                    &sc->bge_cdata.bge_tx_mtag);
4435         if (error) {
4436                 if_printf(ifp, "could not allocate TX mbuf dma tag\n");
4437                 return error;
4438         }
4439
4440         for (i = 0; i < BGE_TX_RING_CNT; i++) {
4441                 error = bus_dmamap_create(sc->bge_cdata.bge_tx_mtag,
4442                                           BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
4443                                           &sc->bge_cdata.bge_tx_dmamap[i]);
4444                 if (error) {
4445                         int j;
4446
4447                         for (j = 0; j < i; ++j) {
4448                                 bus_dmamap_destroy(sc->bge_cdata.bge_tx_mtag,
4449                                         sc->bge_cdata.bge_tx_dmamap[j]);
4450                         }
4451                         bus_dma_tag_destroy(sc->bge_cdata.bge_tx_mtag);
4452                         sc->bge_cdata.bge_tx_mtag = NULL;
4453
4454                         if_printf(ifp, "could not create DMA map for TX\n");
4455                         return error;
4456                 }
4457         }
4458
4459         /*
4460          * Create DMA stuffs for standard RX ring.
4461          */
4462         error = bge_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
4463                                     &sc->bge_cdata.bge_rx_std_ring_tag,
4464                                     &sc->bge_cdata.bge_rx_std_ring_map,
4465                                     (void *)&sc->bge_ldata.bge_rx_std_ring,
4466                                     &sc->bge_ldata.bge_rx_std_ring_paddr);
4467         if (error) {
4468                 if_printf(ifp, "could not create std RX ring\n");
4469                 return error;
4470         }
4471
4472         /*
4473          * Create jumbo buffer pool.
4474          */
4475         if (BGE_IS_JUMBO_CAPABLE(sc)) {
4476                 error = bge_alloc_jumbo_mem(sc);
4477                 if (error) {
4478                         if_printf(ifp, "could not create jumbo buffer pool\n");
4479                         return error;
4480                 }
4481         }
4482
4483         /*
4484          * Create DMA stuffs for RX return ring.
4485          */
4486         error = bge_dma_block_alloc(sc,
4487             BGE_RX_RTN_RING_SZ(sc->bge_return_ring_cnt),
4488             &sc->bge_cdata.bge_rx_return_ring_tag,
4489             &sc->bge_cdata.bge_rx_return_ring_map,
4490             (void *)&sc->bge_ldata.bge_rx_return_ring,
4491             &sc->bge_ldata.bge_rx_return_ring_paddr);
4492         if (error) {
4493                 if_printf(ifp, "could not create RX ret ring\n");
4494                 return error;
4495         }
4496
4497         /*
4498          * Create DMA stuffs for TX ring.
4499          */
4500         error = bge_dma_block_alloc(sc, BGE_TX_RING_SZ,
4501                                     &sc->bge_cdata.bge_tx_ring_tag,
4502                                     &sc->bge_cdata.bge_tx_ring_map,
4503                                     (void *)&sc->bge_ldata.bge_tx_ring,
4504                                     &sc->bge_ldata.bge_tx_ring_paddr);
4505         if (error) {
4506                 if_printf(ifp, "could not create TX ring\n");
4507                 return error;
4508         }
4509
4510         /*
4511          * Create DMA stuffs for status block.
4512          */
4513         error = bge_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
4514                                     &sc->bge_cdata.bge_status_tag,
4515                                     &sc->bge_cdata.bge_status_map,
4516                                     (void *)&sc->bge_ldata.bge_status_block,
4517                                     &sc->bge_ldata.bge_status_block_paddr);
4518         if (error) {
4519                 if_printf(ifp, "could not create status block\n");
4520                 return error;
4521         }
4522
4523         /*
4524          * Create DMA stuffs for statistics block.
4525          */
4526         error = bge_dma_block_alloc(sc, BGE_STATS_SZ,
4527                                     &sc->bge_cdata.bge_stats_tag,
4528                                     &sc->bge_cdata.bge_stats_map,
4529                                     (void *)&sc->bge_ldata.bge_stats,
4530                                     &sc->bge_ldata.bge_stats_paddr);
4531         if (error) {
4532                 if_printf(ifp, "could not create stats block\n");
4533                 return error;
4534         }
4535         return 0;
4536 }
4537
4538 static int
4539 bge_dma_block_alloc(struct bge_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
4540                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
4541 {
4542         bus_dmamem_t dmem;
4543         int error;
4544
4545         error = bus_dmamem_coherent(sc->bge_cdata.bge_parent_tag, PAGE_SIZE, 0,
4546                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4547                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
4548         if (error)
4549                 return error;
4550
4551         *tag = dmem.dmem_tag;
4552         *map = dmem.dmem_map;
4553         *addr = dmem.dmem_addr;
4554         *paddr = dmem.dmem_busaddr;
4555
4556         return 0;
4557 }
4558
4559 static void
4560 bge_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
4561 {
4562         if (tag != NULL) {
4563                 bus_dmamap_unload(tag, map);
4564                 bus_dmamem_free(tag, addr, map);
4565                 bus_dma_tag_destroy(tag);
4566         }
4567 }
4568
4569 /*
4570  * Grrr. The link status word in the status block does
4571  * not work correctly on the BCM5700 rev AX and BX chips,
4572  * according to all available information. Hence, we have
4573  * to enable MII interrupts in order to properly obtain
4574  * async link changes. Unfortunately, this also means that
4575  * we have to read the MAC status register to detect link
4576  * changes, thereby adding an additional register access to
4577  * the interrupt handler.
4578  *
4579  * XXX: perhaps link state detection procedure used for
4580  * BGE_CHIPID_BCM5700_B2 can be used for others BCM5700 revisions.
4581  */
4582 static void
4583 bge_bcm5700_link_upd(struct bge_softc *sc, uint32_t status __unused)
4584 {
4585         struct ifnet *ifp = &sc->arpcom.ac_if;
4586         struct mii_data *mii = device_get_softc(sc->bge_miibus);
4587
4588         mii_pollstat(mii);
4589
4590         if (!sc->bge_link &&
4591             (mii->mii_media_status & IFM_ACTIVE) &&
4592             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
4593                 sc->bge_link++;
4594                 if (bootverbose)
4595                         if_printf(ifp, "link UP\n");
4596         } else if (sc->bge_link &&
4597             (!(mii->mii_media_status & IFM_ACTIVE) ||
4598             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
4599                 sc->bge_link = 0;
4600                 if (bootverbose)
4601                         if_printf(ifp, "link DOWN\n");
4602         }
4603
4604         /* Clear the interrupt. */
4605         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_MI_INTERRUPT);
4606         bge_miibus_readreg(sc->bge_dev, 1, BRGPHY_MII_ISR);
4607         bge_miibus_writereg(sc->bge_dev, 1, BRGPHY_MII_IMR, BRGPHY_INTRS);
4608 }
4609
4610 static void
4611 bge_tbi_link_upd(struct bge_softc *sc, uint32_t status)
4612 {
4613         struct ifnet *ifp = &sc->arpcom.ac_if;
4614
4615 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
4616
4617         /*
4618          * Sometimes PCS encoding errors are detected in
4619          * TBI mode (on fiber NICs), and for some reason
4620          * the chip will signal them as link changes.
4621          * If we get a link change event, but the 'PCS
4622          * encoding error' bit in the MAC status register
4623          * is set, don't bother doing a link check.
4624          * This avoids spurious "gigabit link up" messages
4625          * that sometimes appear on fiber NICs during
4626          * periods of heavy traffic.
4627          */
4628         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
4629                 if (!sc->bge_link) {
4630                         sc->bge_link++;
4631                         if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
4632                                 BGE_CLRBIT(sc, BGE_MAC_MODE,
4633                                     BGE_MACMODE_TBI_SEND_CFGS);
4634                                 DELAY(40);
4635                         }
4636                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
4637
4638                         if (bootverbose)
4639                                 if_printf(ifp, "link UP\n");
4640
4641                         ifp->if_link_state = LINK_STATE_UP;
4642                         if_link_state_change(ifp);
4643                 }
4644         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
4645                 if (sc->bge_link) {
4646                         sc->bge_link = 0;
4647
4648                         if (bootverbose)
4649                                 if_printf(ifp, "link DOWN\n");
4650
4651                         ifp->if_link_state = LINK_STATE_DOWN;
4652                         if_link_state_change(ifp);
4653                 }
4654         }
4655
4656 #undef PCS_ENCODE_ERR
4657
4658         /* Clear the attention. */
4659         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4660             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4661             BGE_MACSTAT_LINK_CHANGED);
4662 }
4663
4664 static void
4665 bge_copper_link_upd(struct bge_softc *sc, uint32_t status __unused)
4666 {
4667         struct ifnet *ifp = &sc->arpcom.ac_if;
4668         struct mii_data *mii = device_get_softc(sc->bge_miibus);
4669
4670         mii_pollstat(mii);
4671         bge_miibus_statchg(sc->bge_dev);
4672
4673         if (bootverbose) {
4674                 if (sc->bge_link)
4675                         if_printf(ifp, "link UP\n");
4676                 else
4677                         if_printf(ifp, "link DOWN\n");
4678         }
4679
4680         /* Clear the attention. */
4681         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4682             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4683             BGE_MACSTAT_LINK_CHANGED);
4684 }
4685
4686 static void
4687 bge_autopoll_link_upd(struct bge_softc *sc, uint32_t status __unused)
4688 {
4689         struct ifnet *ifp = &sc->arpcom.ac_if;
4690         struct mii_data *mii = device_get_softc(sc->bge_miibus);
4691
4692         mii_pollstat(mii);
4693
4694         if (!sc->bge_link &&
4695             (mii->mii_media_status & IFM_ACTIVE) &&
4696             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
4697                 sc->bge_link++;
4698                 if (bootverbose)
4699                         if_printf(ifp, "link UP\n");
4700         } else if (sc->bge_link &&
4701             (!(mii->mii_media_status & IFM_ACTIVE) ||
4702             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
4703                 sc->bge_link = 0;
4704                 if (bootverbose)
4705                         if_printf(ifp, "link DOWN\n");
4706         }
4707
4708         /* Clear the attention. */
4709         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4710             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4711             BGE_MACSTAT_LINK_CHANGED);
4712 }
4713
4714 static int
4715 bge_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
4716 {
4717         struct bge_softc *sc = arg1;
4718
4719         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4720             &sc->bge_rx_coal_ticks,
4721             BGE_RX_COAL_TICKS_MIN, BGE_RX_COAL_TICKS_MAX,
4722             BGE_RX_COAL_TICKS_CHG);
4723 }
4724
4725 static int
4726 bge_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
4727 {
4728         struct bge_softc *sc = arg1;
4729
4730         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4731             &sc->bge_tx_coal_ticks,
4732             BGE_TX_COAL_TICKS_MIN, BGE_TX_COAL_TICKS_MAX,
4733             BGE_TX_COAL_TICKS_CHG);
4734 }
4735
4736 static int
4737 bge_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
4738 {
4739         struct bge_softc *sc = arg1;
4740
4741         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4742             &sc->bge_rx_coal_bds,
4743             BGE_RX_COAL_BDS_MIN, BGE_RX_COAL_BDS_MAX,
4744             BGE_RX_COAL_BDS_CHG);
4745 }
4746
4747 static int
4748 bge_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
4749 {
4750         struct bge_softc *sc = arg1;
4751
4752         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4753             &sc->bge_tx_coal_bds,
4754             BGE_TX_COAL_BDS_MIN, BGE_TX_COAL_BDS_MAX,
4755             BGE_TX_COAL_BDS_CHG);
4756 }
4757
4758 static int
4759 bge_sysctl_rx_coal_ticks_int(SYSCTL_HANDLER_ARGS)
4760 {
4761         struct bge_softc *sc = arg1;
4762
4763         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4764             &sc->bge_rx_coal_ticks_int,
4765             BGE_RX_COAL_TICKS_MIN, BGE_RX_COAL_TICKS_MAX,
4766             BGE_RX_COAL_TICKS_INT_CHG);
4767 }
4768
4769 static int
4770 bge_sysctl_tx_coal_ticks_int(SYSCTL_HANDLER_ARGS)
4771 {
4772         struct bge_softc *sc = arg1;
4773
4774         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4775             &sc->bge_tx_coal_ticks_int,
4776             BGE_TX_COAL_TICKS_MIN, BGE_TX_COAL_TICKS_MAX,
4777             BGE_TX_COAL_TICKS_INT_CHG);
4778 }
4779
4780 static int
4781 bge_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4782 {
4783         struct bge_softc *sc = arg1;
4784
4785         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4786             &sc->bge_rx_coal_bds_int,
4787             BGE_RX_COAL_BDS_MIN, BGE_RX_COAL_BDS_MAX,
4788             BGE_RX_COAL_BDS_INT_CHG);
4789 }
4790
4791 static int
4792 bge_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4793 {
4794         struct bge_softc *sc = arg1;
4795
4796         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
4797             &sc->bge_tx_coal_bds_int,
4798             BGE_TX_COAL_BDS_MIN, BGE_TX_COAL_BDS_MAX,
4799             BGE_TX_COAL_BDS_INT_CHG);
4800 }
4801
4802 static int
4803 bge_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
4804     int coal_min, int coal_max, uint32_t coal_chg_mask)
4805 {
4806         struct bge_softc *sc = arg1;
4807         struct ifnet *ifp = &sc->arpcom.ac_if;
4808         int error = 0, v;
4809
4810         lwkt_serialize_enter(ifp->if_serializer);
4811
4812         v = *coal;
4813         error = sysctl_handle_int(oidp, &v, 0, req);
4814         if (!error && req->newptr != NULL) {
4815                 if (v < coal_min || v > coal_max) {
4816                         error = EINVAL;
4817                 } else {
4818                         *coal = v;
4819                         sc->bge_coal_chg |= coal_chg_mask;
4820                 }
4821         }
4822
4823         lwkt_serialize_exit(ifp->if_serializer);
4824         return error;
4825 }
4826
4827 static void
4828 bge_coal_change(struct bge_softc *sc)
4829 {
4830         struct ifnet *ifp = &sc->arpcom.ac_if;
4831
4832         ASSERT_SERIALIZED(ifp->if_serializer);
4833
4834         if (sc->bge_coal_chg & BGE_RX_COAL_TICKS_CHG) {
4835                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
4836                             sc->bge_rx_coal_ticks);
4837                 DELAY(10);
4838                 CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS);
4839
4840                 if (bootverbose) {
4841                         if_printf(ifp, "rx_coal_ticks -> %u\n",
4842                                   sc->bge_rx_coal_ticks);
4843                 }
4844         }
4845
4846         if (sc->bge_coal_chg & BGE_TX_COAL_TICKS_CHG) {
4847                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
4848                             sc->bge_tx_coal_ticks);
4849                 DELAY(10);
4850                 CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS);
4851
4852                 if (bootverbose) {
4853                         if_printf(ifp, "tx_coal_ticks -> %u\n",
4854                                   sc->bge_tx_coal_ticks);
4855                 }
4856         }
4857
4858         if (sc->bge_coal_chg & BGE_RX_COAL_BDS_CHG) {
4859                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
4860                             sc->bge_rx_coal_bds);
4861                 DELAY(10);
4862                 CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS);
4863
4864                 if (bootverbose) {
4865                         if_printf(ifp, "rx_coal_bds -> %u\n",
4866                                   sc->bge_rx_coal_bds);
4867                 }
4868         }
4869
4870         if (sc->bge_coal_chg & BGE_TX_COAL_BDS_CHG) {
4871                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS,
4872                             sc->bge_tx_coal_bds);
4873                 DELAY(10);
4874                 CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS);
4875
4876                 if (bootverbose) {
4877                         if_printf(ifp, "tx_max_coal_bds -> %u\n",
4878                                   sc->bge_tx_coal_bds);
4879                 }
4880         }
4881
4882         if (sc->bge_coal_chg & BGE_RX_COAL_TICKS_INT_CHG) {
4883                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS_INT,
4884                     sc->bge_rx_coal_ticks_int);
4885                 DELAY(10);
4886                 CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS_INT);
4887
4888                 if (bootverbose) {
4889                         if_printf(ifp, "rx_coal_ticks_int -> %u\n",
4890                             sc->bge_rx_coal_ticks_int);
4891                 }
4892         }
4893
4894         if (sc->bge_coal_chg & BGE_TX_COAL_TICKS_INT_CHG) {
4895                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS_INT,
4896                     sc->bge_tx_coal_ticks_int);
4897                 DELAY(10);
4898                 CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS_INT);
4899
4900                 if (bootverbose) {
4901                         if_printf(ifp, "tx_coal_ticks_int -> %u\n",
4902                             sc->bge_tx_coal_ticks_int);
4903                 }
4904         }
4905
4906         if (sc->bge_coal_chg & BGE_RX_COAL_BDS_INT_CHG) {
4907                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
4908                     sc->bge_rx_coal_bds_int);
4909                 DELAY(10);
4910                 CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT);
4911
4912                 if (bootverbose) {
4913                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
4914                             sc->bge_rx_coal_bds_int);
4915                 }
4916         }
4917
4918         if (sc->bge_coal_chg & BGE_TX_COAL_BDS_INT_CHG) {
4919                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
4920                     sc->bge_tx_coal_bds_int);
4921                 DELAY(10);
4922                 CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT);
4923
4924                 if (bootverbose) {
4925                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
4926                             sc->bge_tx_coal_bds_int);
4927                 }
4928         }
4929
4930         sc->bge_coal_chg = 0;
4931 }
4932
4933 static void
4934 bge_enable_intr(struct bge_softc *sc)
4935 {
4936         struct ifnet *ifp = &sc->arpcom.ac_if;
4937
4938         lwkt_serialize_handler_enable(ifp->if_serializer);
4939
4940         /*
4941          * Enable interrupt.
4942          */
4943         bge_writembx(sc, BGE_MBX_IRQ0_LO, sc->bge_status_tag << 24);
4944         if (sc->bge_flags & BGE_FLAG_ONESHOT_MSI) {
4945                 /* XXX Linux driver */
4946                 bge_writembx(sc, BGE_MBX_IRQ0_LO, sc->bge_status_tag << 24);
4947         }
4948
4949         /*
4950          * Unmask the interrupt when we stop polling.
4951          */
4952         PCI_CLRBIT(sc->bge_dev, BGE_PCI_MISC_CTL,
4953             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4954
4955         /*
4956          * Trigger another interrupt, since above writing
4957          * to interrupt mailbox0 may acknowledge pending
4958          * interrupt.
4959          */
4960         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
4961 }
4962
4963 static void
4964 bge_disable_intr(struct bge_softc *sc)
4965 {
4966         struct ifnet *ifp = &sc->arpcom.ac_if;
4967
4968         /*
4969          * Mask the interrupt when we start polling.
4970          */
4971         PCI_SETBIT(sc->bge_dev, BGE_PCI_MISC_CTL,
4972             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4973
4974         /*
4975          * Acknowledge possible asserted interrupt.
4976          */
4977         bge_writembx(sc, BGE_MBX_IRQ0_LO, 1);
4978
4979         sc->bge_npoll.ifpc_stcount = 0;
4980
4981         lwkt_serialize_handler_disable(ifp->if_serializer);
4982 }
4983
4984 static int
4985 bge_get_eaddr_mem(struct bge_softc *sc, uint8_t ether_addr[])
4986 {
4987         uint32_t mac_addr;
4988         int ret = 1;
4989
4990         mac_addr = bge_readmem_ind(sc, 0x0c14);
4991         if ((mac_addr >> 16) == 0x484b) {
4992                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
4993                 ether_addr[1] = (uint8_t)mac_addr;
4994                 mac_addr = bge_readmem_ind(sc, 0x0c18);
4995                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
4996                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
4997                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
4998                 ether_addr[5] = (uint8_t)mac_addr;
4999                 ret = 0;
5000         }
5001         return ret;
5002 }
5003
5004 static int
5005 bge_get_eaddr_nvram(struct bge_softc *sc, uint8_t ether_addr[])
5006 {
5007         int mac_offset = BGE_EE_MAC_OFFSET;
5008
5009         if (sc->bge_asicrev == BGE_ASICREV_BCM5906)
5010                 mac_offset = BGE_EE_MAC_OFFSET_5906;
5011
5012         return bge_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN);
5013 }
5014
5015 static int
5016 bge_get_eaddr_eeprom(struct bge_softc *sc, uint8_t ether_addr[])
5017 {
5018         if (sc->bge_flags & BGE_FLAG_NO_EEPROM)
5019                 return 1;
5020
5021         return bge_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
5022                                ETHER_ADDR_LEN);
5023 }
5024
5025 static int
5026 bge_get_eaddr(struct bge_softc *sc, uint8_t eaddr[])
5027 {
5028         static const bge_eaddr_fcn_t bge_eaddr_funcs[] = {
5029                 /* NOTE: Order is critical */
5030                 bge_get_eaddr_mem,
5031                 bge_get_eaddr_nvram,
5032                 bge_get_eaddr_eeprom,
5033                 NULL
5034         };
5035         const bge_eaddr_fcn_t *func;
5036
5037         for (func = bge_eaddr_funcs; *func != NULL; ++func) {
5038                 if ((*func)(sc, eaddr) == 0)
5039                         break;
5040         }
5041         return (*func == NULL ? ENXIO : 0);
5042 }
5043
5044 /*
5045  * NOTE: 'm' is not freed upon failure
5046  */
5047 struct mbuf *
5048 bge_defrag_shortdma(struct mbuf *m)
5049 {
5050         struct mbuf *n;
5051         int found;
5052
5053         /*
5054          * If device receive two back-to-back send BDs with less than
5055          * or equal to 8 total bytes then the device may hang.  The two
5056          * back-to-back send BDs must in the same frame for this failure
5057          * to occur.  Scan mbuf chains and see whether two back-to-back
5058          * send BDs are there.  If this is the case, allocate new mbuf
5059          * and copy the frame to workaround the silicon bug.
5060          */
5061         for (n = m, found = 0; n != NULL; n = n->m_next) {
5062                 if (n->m_len < 8) {
5063                         found++;
5064                         if (found > 1)
5065                                 break;
5066                         continue;
5067                 }
5068                 found = 0;
5069         }
5070
5071         if (found > 1)
5072                 n = m_defrag(m, MB_DONTWAIT);
5073         else
5074                 n = m;
5075         return n;
5076 }
5077
5078 static void
5079 bge_stop_block(struct bge_softc *sc, bus_size_t reg, uint32_t bit)
5080 {
5081         int i;
5082
5083         BGE_CLRBIT(sc, reg, bit);
5084         for (i = 0; i < BGE_TIMEOUT; i++) {
5085                 if ((CSR_READ_4(sc, reg) & bit) == 0)
5086                         return;
5087                 DELAY(100);
5088         }
5089 }
5090
5091 static void
5092 bge_link_poll(struct bge_softc *sc)
5093 {
5094         uint32_t status;
5095
5096         status = CSR_READ_4(sc, BGE_MAC_STS);
5097         if ((status & sc->bge_link_chg) || sc->bge_link_evt) {
5098                 sc->bge_link_evt = 0;
5099                 sc->bge_link_upd(sc, status);
5100         }
5101 }
5102
5103 static void
5104 bge_enable_msi(struct bge_softc *sc)
5105 {
5106         uint32_t msi_mode;
5107
5108         msi_mode = CSR_READ_4(sc, BGE_MSI_MODE);
5109         msi_mode |= BGE_MSIMODE_ENABLE;
5110         if (sc->bge_flags & BGE_FLAG_ONESHOT_MSI) {
5111                 /*
5112                  * According to all of the datasheets that are publicly
5113                  * available, bit 5 of the MSI_MODE is defined to be
5114                  * "MSI FIFO Underrun Attn" for BCM5755+ and BCM5906, on
5115                  * which "oneshot MSI" is enabled.  However, it is always
5116                  * safe to clear it here.
5117                  */
5118                 msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE;
5119         }
5120         CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode);
5121 }
5122
5123 static int
5124 bge_setup_tso(struct bge_softc *sc, struct mbuf **mp,
5125     uint16_t *mss0, uint16_t *flags0)
5126 {
5127         struct mbuf *m;
5128         struct ip *ip;
5129         struct tcphdr *th;
5130         int thoff, iphlen, hoff, hlen;
5131         uint16_t flags, mss;
5132
5133         m = *mp;
5134         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5135
5136         hoff = m->m_pkthdr.csum_lhlen;
5137         iphlen = m->m_pkthdr.csum_iphlen;
5138         thoff = m->m_pkthdr.csum_thlen;
5139
5140         KASSERT(hoff > 0, ("invalid ether header len"));
5141         KASSERT(iphlen > 0, ("invalid ip header len"));
5142         KASSERT(thoff > 0, ("invalid tcp header len"));
5143
5144         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5145                 m = m_pullup(m, hoff + iphlen + thoff);
5146                 if (m == NULL) {
5147                         *mp = NULL;
5148                         return ENOBUFS;
5149                 }
5150                 *mp = m;
5151         }
5152         ip = mtodoff(m, struct ip *, hoff);
5153         th = mtodoff(m, struct tcphdr *, hoff + iphlen);
5154
5155         mss = m->m_pkthdr.tso_segsz;
5156         flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA;
5157
5158         ip->ip_len = htons(mss + iphlen + thoff);
5159         th->th_sum = 0;
5160
5161         hlen = (iphlen + thoff) >> 2;
5162         mss |= (hlen << 11);
5163
5164         *mss0 = mss;
5165         *flags0 = flags;
5166
5167         return 0;
5168 }
5169
5170 static void
5171 bge_stop_fw(struct bge_softc *sc)
5172 {
5173         int i;
5174
5175         if (sc->bge_asf_mode) {
5176                 bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB, BGE_FW_CMD_PAUSE);
5177                 CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
5178                     CSR_READ_4(sc, BGE_RX_CPU_EVENT) | BGE_RX_CPU_DRV_EVENT);
5179
5180                 for (i = 0; i < 100; i++ ) {
5181                         if (!(CSR_READ_4(sc, BGE_RX_CPU_EVENT) &
5182                             BGE_RX_CPU_DRV_EVENT))
5183                                 break;
5184                         DELAY(10);
5185                 }
5186         }
5187 }
5188
5189 static void
5190 bge_sig_pre_reset(struct bge_softc *sc, int type)
5191 {
5192         /*
5193          * Some chips don't like this so only do this if ASF is enabled
5194          */
5195         if (sc->bge_asf_mode)
5196                 bge_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
5197
5198         if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
5199                 switch (type) {
5200                 case BGE_RESET_START:
5201                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5202                             BGE_FW_DRV_STATE_START);
5203                         break;
5204                 case BGE_RESET_SHUTDOWN:
5205                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5206                             BGE_FW_DRV_STATE_UNLOAD);
5207                         break;
5208                 case BGE_RESET_SUSPEND:
5209                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5210                             BGE_FW_DRV_STATE_SUSPEND);
5211                         break;
5212                 }
5213         }
5214
5215         if (type == BGE_RESET_START || type == BGE_RESET_SUSPEND)
5216                 bge_ape_driver_state_change(sc, type);
5217 }
5218
5219 static void
5220 bge_sig_legacy(struct bge_softc *sc, int type)
5221 {
5222         if (sc->bge_asf_mode) {
5223                 switch (type) {
5224                 case BGE_RESET_START:
5225                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5226                             BGE_FW_DRV_STATE_START);
5227                         break;
5228                 case BGE_RESET_SHUTDOWN:
5229                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5230                             BGE_FW_DRV_STATE_UNLOAD);
5231                         break;
5232                 }
5233         }
5234 }
5235
5236 static void
5237 bge_sig_post_reset(struct bge_softc *sc, int type)
5238 {
5239         if (sc->bge_asf_mode & ASF_NEW_HANDSHAKE) {
5240                 switch (type) {
5241                 case BGE_RESET_START:
5242                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5243                             BGE_FW_DRV_STATE_START_DONE);
5244                         /* START DONE */
5245                         break;
5246                 case BGE_RESET_SHUTDOWN:
5247                         bge_writemem_ind(sc, BGE_SRAM_FW_DRV_STATE_MB,
5248                             BGE_FW_DRV_STATE_UNLOAD_DONE);
5249                         break;
5250                 }
5251         }
5252         if (type == BGE_RESET_SHUTDOWN)
5253                 bge_ape_driver_state_change(sc, type);
5254 }
5255
5256 static void
5257 bge_asf_driver_up(struct bge_softc *sc)
5258 {
5259         if (sc->bge_asf_mode & ASF_STACKUP) {
5260                 /* Send ASF heartbeat aprox. every 2s */
5261                 if (sc->bge_asf_count)
5262                         sc->bge_asf_count --;
5263                 else {
5264                         sc->bge_asf_count = 2;
5265                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_MB,
5266                             BGE_FW_CMD_DRV_ALIVE);
5267                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_LEN_MB, 4);
5268                         bge_writemem_ind(sc, BGE_SRAM_FW_CMD_DATA_MB,
5269                             BGE_FW_HB_TIMEOUT_SEC);
5270                         CSR_WRITE_4(sc, BGE_RX_CPU_EVENT,
5271                             CSR_READ_4(sc, BGE_RX_CPU_EVENT) |
5272                             BGE_RX_CPU_DRV_EVENT);
5273                 }
5274         }
5275 }
5276
5277 /*
5278  * Clear all stale locks and select the lock for this driver instance.
5279  */
5280 static void
5281 bge_ape_lock_init(struct bge_softc *sc)
5282 {
5283         uint32_t bit, regbase;
5284         int i;
5285
5286         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
5287                 regbase = BGE_APE_LOCK_GRANT;
5288         else
5289                 regbase = BGE_APE_PER_LOCK_GRANT;
5290
5291         /* Clear any stale locks. */
5292         for (i = BGE_APE_LOCK_PHY0; i <= BGE_APE_LOCK_GPIO; i++) {
5293                 switch (i) {
5294                 case BGE_APE_LOCK_PHY0:
5295                 case BGE_APE_LOCK_PHY1:
5296                 case BGE_APE_LOCK_PHY2:
5297                 case BGE_APE_LOCK_PHY3:
5298                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
5299                         break;
5300                 default:
5301                         if (sc->bge_func_addr == 0)
5302                                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
5303                         else
5304                                 bit = (1 << sc->bge_func_addr);
5305                 }
5306                 APE_WRITE_4(sc, regbase + 4 * i, bit);
5307         }
5308
5309         /* Select the PHY lock based on the device's function number. */
5310         switch (sc->bge_func_addr) {
5311         case 0:
5312                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY0;
5313                 break;
5314         case 1:
5315                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY1;
5316                 break;
5317         case 2:
5318                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY2;
5319                 break;
5320         case 3:
5321                 sc->bge_phy_ape_lock = BGE_APE_LOCK_PHY3;
5322                 break;
5323         default:
5324                 device_printf(sc->bge_dev,
5325                     "PHY lock not supported on this function\n");
5326         }
5327 }
5328
5329 /*
5330  * Check for APE firmware, set flags, and print version info.
5331  */
5332 static void
5333 bge_ape_read_fw_ver(struct bge_softc *sc)
5334 {
5335         const char *fwtype;
5336         uint32_t apedata, features;
5337
5338         /* Check for a valid APE signature in shared memory. */
5339         apedata = APE_READ_4(sc, BGE_APE_SEG_SIG);
5340         if (apedata != BGE_APE_SEG_SIG_MAGIC) {
5341                 device_printf(sc->bge_dev, "no APE signature\n");
5342                 sc->bge_mfw_flags &= ~BGE_MFW_ON_APE;
5343                 return;
5344         }
5345
5346         /* Check if APE firmware is running. */
5347         apedata = APE_READ_4(sc, BGE_APE_FW_STATUS);
5348         if ((apedata & BGE_APE_FW_STATUS_READY) == 0) {
5349                 device_printf(sc->bge_dev, "APE signature found "
5350                     "but FW status not ready! 0x%08x\n", apedata);
5351                 return;
5352         }
5353
5354         sc->bge_mfw_flags |= BGE_MFW_ON_APE;
5355
5356         /* Fetch the APE firwmare type and version. */
5357         apedata = APE_READ_4(sc, BGE_APE_FW_VERSION);
5358         features = APE_READ_4(sc, BGE_APE_FW_FEATURES);
5359         if ((features & BGE_APE_FW_FEATURE_NCSI) != 0) {
5360                 sc->bge_mfw_flags |= BGE_MFW_TYPE_NCSI;
5361                 fwtype = "NCSI";
5362         } else if ((features & BGE_APE_FW_FEATURE_DASH) != 0) {
5363                 sc->bge_mfw_flags |= BGE_MFW_TYPE_DASH;
5364                 fwtype = "DASH";
5365         } else
5366                 fwtype = "UNKN";
5367
5368         /* Print the APE firmware version. */
5369         device_printf(sc->bge_dev, "APE FW version: %s v%d.%d.%d.%d\n",
5370             fwtype,
5371             (apedata & BGE_APE_FW_VERSION_MAJMSK) >> BGE_APE_FW_VERSION_MAJSFT,
5372             (apedata & BGE_APE_FW_VERSION_MINMSK) >> BGE_APE_FW_VERSION_MINSFT,
5373             (apedata & BGE_APE_FW_VERSION_REVMSK) >> BGE_APE_FW_VERSION_REVSFT,
5374             (apedata & BGE_APE_FW_VERSION_BLDMSK));
5375 }
5376
5377 static int
5378 bge_ape_lock(struct bge_softc *sc, int locknum)
5379 {
5380         uint32_t bit, gnt, req, status;
5381         int i, off;
5382
5383         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
5384                 return (0);
5385
5386         /* Lock request/grant registers have different bases. */
5387         if (sc->bge_asicrev == BGE_ASICREV_BCM5761) {
5388                 req = BGE_APE_LOCK_REQ;
5389                 gnt = BGE_APE_LOCK_GRANT;
5390         } else {
5391                 req = BGE_APE_PER_LOCK_REQ;
5392                 gnt = BGE_APE_PER_LOCK_GRANT;
5393         }
5394
5395         off = 4 * locknum;
5396
5397         switch (locknum) {
5398         case BGE_APE_LOCK_GPIO:
5399                 /* Lock required when using GPIO. */
5400                 if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
5401                         return (0);
5402                 if (sc->bge_func_addr == 0)
5403                         bit = BGE_APE_LOCK_REQ_DRIVER0;
5404                 else
5405                         bit = (1 << sc->bge_func_addr);
5406                 break;
5407         case BGE_APE_LOCK_GRC:
5408                 /* Lock required to reset the device. */
5409                 if (sc->bge_func_addr == 0)
5410                         bit = BGE_APE_LOCK_REQ_DRIVER0;
5411                 else
5412                         bit = (1 << sc->bge_func_addr);
5413                 break;
5414         case BGE_APE_LOCK_MEM:
5415                 /* Lock required when accessing certain APE memory. */
5416                 if (sc->bge_func_addr == 0)
5417                         bit = BGE_APE_LOCK_REQ_DRIVER0;
5418                 else
5419                         bit = (1 << sc->bge_func_addr);
5420                 break;
5421         case BGE_APE_LOCK_PHY0:
5422         case BGE_APE_LOCK_PHY1:
5423         case BGE_APE_LOCK_PHY2:
5424         case BGE_APE_LOCK_PHY3:
5425                 /* Lock required when accessing PHYs. */
5426                 bit = BGE_APE_LOCK_REQ_DRIVER0;
5427                 break;
5428         default:
5429                 return (EINVAL);
5430         }
5431
5432         /* Request a lock. */
5433         APE_WRITE_4(sc, req + off, bit);
5434
5435         /* Wait up to 1 second to acquire lock. */
5436         for (i = 0; i < 20000; i++) {
5437                 status = APE_READ_4(sc, gnt + off);
5438                 if (status == bit)
5439                         break;
5440                 DELAY(50);
5441         }
5442
5443         /* Handle any errors. */
5444         if (status != bit) {
5445                 device_printf(sc->bge_dev, "APE lock %d request failed! "
5446                     "request = 0x%04x[0x%04x], status = 0x%04x[0x%04x]\n",
5447                     locknum, req + off, bit & 0xFFFF, gnt + off,
5448                     status & 0xFFFF);
5449                 /* Revoke the lock request. */
5450                 APE_WRITE_4(sc, gnt + off, bit);
5451                 return (EBUSY);
5452         }
5453
5454         return (0);
5455 }
5456
5457 static void
5458 bge_ape_unlock(struct bge_softc *sc, int locknum)
5459 {
5460         uint32_t bit, gnt;
5461         int off;
5462
5463         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
5464                 return;
5465
5466         if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
5467                 gnt = BGE_APE_LOCK_GRANT;
5468         else
5469                 gnt = BGE_APE_PER_LOCK_GRANT;
5470
5471         off = 4 * locknum;
5472
5473         switch (locknum) {
5474         case BGE_APE_LOCK_GPIO:
5475                 if (sc->bge_asicrev == BGE_ASICREV_BCM5761)
5476                         return;
5477                 if (sc->bge_func_addr == 0)
5478                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
5479                 else
5480                         bit = (1 << sc->bge_func_addr);
5481                 break;
5482         case BGE_APE_LOCK_GRC:
5483                 if (sc->bge_func_addr == 0)
5484                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
5485                 else
5486                         bit = (1 << sc->bge_func_addr);
5487                 break;
5488         case BGE_APE_LOCK_MEM:
5489                 if (sc->bge_func_addr == 0)
5490                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
5491                 else
5492                         bit = (1 << sc->bge_func_addr);
5493                 break;
5494         case BGE_APE_LOCK_PHY0:
5495         case BGE_APE_LOCK_PHY1:
5496         case BGE_APE_LOCK_PHY2:
5497         case BGE_APE_LOCK_PHY3:
5498                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
5499                 break;
5500         default:
5501                 return;
5502         }
5503
5504         APE_WRITE_4(sc, gnt + off, bit);
5505 }
5506
5507 /*
5508  * Send an event to the APE firmware.
5509  */
5510 static void
5511 bge_ape_send_event(struct bge_softc *sc, uint32_t event)
5512 {
5513         uint32_t apedata;
5514         int i;
5515
5516         /* NCSI does not support APE events. */
5517         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
5518                 return;
5519
5520         /* Wait up to 1ms for APE to service previous event. */
5521         for (i = 10; i > 0; i--) {
5522                 if (bge_ape_lock(sc, BGE_APE_LOCK_MEM) != 0)
5523                         break;
5524                 apedata = APE_READ_4(sc, BGE_APE_EVENT_STATUS);
5525                 if ((apedata & BGE_APE_EVENT_STATUS_EVENT_PENDING) == 0) {
5526                         APE_WRITE_4(sc, BGE_APE_EVENT_STATUS, event |
5527                             BGE_APE_EVENT_STATUS_EVENT_PENDING);
5528                         bge_ape_unlock(sc, BGE_APE_LOCK_MEM);
5529                         APE_WRITE_4(sc, BGE_APE_EVENT, BGE_APE_EVENT_1);
5530                         break;
5531                 }
5532                 bge_ape_unlock(sc, BGE_APE_LOCK_MEM);
5533                 DELAY(100);
5534         }
5535         if (i == 0)
5536                 device_printf(sc->bge_dev, "APE event 0x%08x send timed out\n",
5537                     event);
5538 }
5539
5540 static void
5541 bge_ape_driver_state_change(struct bge_softc *sc, int kind)
5542 {
5543         uint32_t apedata, event;
5544
5545         if ((sc->bge_mfw_flags & BGE_MFW_ON_APE) == 0)
5546                 return;
5547
5548         switch (kind) {
5549         case BGE_RESET_START:
5550                 /* If this is the first load, clear the load counter. */
5551                 apedata = APE_READ_4(sc, BGE_APE_HOST_SEG_SIG);
5552                 if (apedata != BGE_APE_HOST_SEG_SIG_MAGIC)
5553                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, 0);
5554                 else {
5555                         apedata = APE_READ_4(sc, BGE_APE_HOST_INIT_COUNT);
5556                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, ++apedata);
5557                 }
5558                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_SIG,
5559                     BGE_APE_HOST_SEG_SIG_MAGIC);
5560                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_LEN,
5561                     BGE_APE_HOST_SEG_LEN_MAGIC);
5562
5563                 /* Add some version info if bge(4) supports it. */
5564                 APE_WRITE_4(sc, BGE_APE_HOST_DRIVER_ID,
5565                     BGE_APE_HOST_DRIVER_ID_MAGIC(1, 0));
5566                 APE_WRITE_4(sc, BGE_APE_HOST_BEHAVIOR,
5567                     BGE_APE_HOST_BEHAV_NO_PHYLOCK);
5568                 APE_WRITE_4(sc, BGE_APE_HOST_HEARTBEAT_INT_MS,
5569                     BGE_APE_HOST_HEARTBEAT_INT_DISABLE);
5570                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
5571                     BGE_APE_HOST_DRVR_STATE_START);
5572                 event = BGE_APE_EVENT_STATUS_STATE_START;
5573                 break;
5574         case BGE_RESET_SHUTDOWN:
5575                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
5576                     BGE_APE_HOST_DRVR_STATE_UNLOAD);
5577                 event = BGE_APE_EVENT_STATUS_STATE_UNLOAD;
5578                 break;
5579         case BGE_RESET_SUSPEND:
5580                 event = BGE_APE_EVENT_STATUS_STATE_SUSPEND;
5581                 break;
5582         default:
5583                 return;
5584         }
5585
5586         bge_ape_send_event(sc, event | BGE_APE_EVENT_STATUS_DRIVER_EVNT |
5587             BGE_APE_EVENT_STATUS_STATE_CHNGE);
5588 }