sys/dev/netif/bge/if_bge.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  * $DragonFly: src/sys/dev/netif/bge/if_bge.c,v 1.107 2008/09/17 08:51:29 sephe Exp $
  35  *
  36  */
  37
  38 /*
  39  * Broadcom BCM570x family gigabit ethernet driver for FreeBSD.
  40  *
  41  * Written by Bill Paul <wpaul@windriver.com>
  42  * Senior Engineer, Wind River Systems
  43  */
  44
  45 /*
  46  * The Broadcom BCM5700 is based on technology originally developed by
  47  * Alteon Networks as part of the Tigon I and Tigon II gigabit ethernet
  48  * MAC chips. The BCM5700, sometimes refered to as the Tigon III, has
  49  * two on-board MIPS R4000 CPUs and can have as much as 16MB of external
  50  * SSRAM. The BCM5700 supports TCP, UDP and IP checksum offload, jumbo
  51  * frames, highly configurable RX filtering, and 16 RX and TX queues
  52  * (which, along with RX filter rules, can be used for QOS applications).
  53  * Other features, such as TCP segmentation, may be available as part
  54  * of value-added firmware updates. Unlike the Tigon I and Tigon II,
  55  * firmware images can be stored in hardware and need not be compiled
  56  * into the driver.
  57  *
  58  * The BCM5700 supports the PCI v2.2 and PCI-X v1.0 standards, and will
  59  * function in a 32-bit/64-bit 33/66Mhz bus, or a 64-bit/133Mhz bus.
  60  *
  61  * The BCM5701 is a single-chip solution incorporating both the BCM5700
  62  * MAC and a BCM5401 10/100/1000 PHY. Unlike the BCM5700, the BCM5701
  63  * does not support external SSRAM.
  64  *
  65  * Broadcom also produces a variation of the BCM5700 under the "Altima"
  66  * brand name, which is functionally similar but lacks PCI-X support.
  67  *
  68  * Without external SSRAM, you can only have at most 4 TX rings,
  69  * and the use of the mini RX ring is disabled. This seems to imply
  70  * that these features are simply not available on the BCM5701. As a
  71  * result, this driver does not implement any support for the mini RX
  72  * ring.
  73  */
  74
  75 #include "opt_polling.h"
  76
  77 #include <sys/param.h>
  78 #include <sys/bus.h>
  79 #include <sys/endian.h>
  80 #include <sys/kernel.h>
  81 #include <sys/ktr.h>
  82 #include <sys/interrupt.h>
  83 #include <sys/mbuf.h>
  84 #include <sys/malloc.h>
  85 #include <sys/queue.h>
  86 #include <sys/rman.h>
  87 #include <sys/serialize.h>
  88 #include <sys/socket.h>
  89 #include <sys/sockio.h>
  90 #include <sys/sysctl.h>
  91
  92 #include <net/bpf.h>
  93 #include <net/ethernet.h>
  94 #include <net/if.h>
  95 #include <net/if_arp.h>
  96 #include <net/if_dl.h>
  97 #include <net/if_media.h>
  98 #include <net/if_types.h>
  99 #include <net/ifq_var.h>
 100 #include <net/vlan/if_vlan_var.h>
 101 #include <net/vlan/if_vlan_ether.h>
 102
 103 #include <dev/netif/mii_layer/mii.h>
 104 #include <dev/netif/mii_layer/miivar.h>
 105 #include <dev/netif/mii_layer/brgphyreg.h>
 106
 107 #include <bus/pci/pcidevs.h>
 108 #include <bus/pci/pcireg.h>
 109 #include <bus/pci/pcivar.h>
 110
 111 #include <dev/netif/bge/if_bgereg.h>
 112
 113 /* "device miibus" required.  See GENERIC if you get errors here. */
 114 #include "miibus_if.h"
 115
 116 #define BGE_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
 117 #define BGE_MIN_FRAME           60
 118
 119 static const struct bge_type bge_devs[] = {
 120         { PCI_VENDOR_3COM, PCI_PRODUCT_3COM_3C996,
 121                 "3COM 3C996 Gigabit Ethernet" },
 122
 123         { PCI_VENDOR_ALTEON, PCI_PRODUCT_ALTEON_BCM5700,
 124                 "Alteon BCM5700 Gigabit Ethernet" },
 125         { PCI_VENDOR_ALTEON, PCI_PRODUCT_ALTEON_BCM5701,
 126                 "Alteon BCM5701 Gigabit Ethernet" },
 127
 128         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC1000,
 129                 "Altima AC1000 Gigabit Ethernet" },
 130         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC1001,
 131                 "Altima AC1002 Gigabit Ethernet" },
 132         { PCI_VENDOR_ALTIMA, PCI_PRODUCT_ALTIMA_AC9100,
 133                 "Altima AC9100 Gigabit Ethernet" },
 134
 135         { PCI_VENDOR_APPLE, PCI_PRODUCT_APPLE_BCM5701,
 136                 "Apple BCM5701 Gigabit Ethernet" },
 137
 138         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5700,
 139                 "Broadcom BCM5700 Gigabit Ethernet" },
 140         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5701,
 141                 "Broadcom BCM5701 Gigabit Ethernet" },
 142         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702,
 143                 "Broadcom BCM5702 Gigabit Ethernet" },
 144         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702X,
 145                 "Broadcom BCM5702X Gigabit Ethernet" },
 146         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5702_ALT,
 147                 "Broadcom BCM5702 Gigabit Ethernet" },
 148         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703,
 149                 "Broadcom BCM5703 Gigabit Ethernet" },
 150         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703X,
 151                 "Broadcom BCM5703X Gigabit Ethernet" },
 152         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5703A3,
 153                 "Broadcom BCM5703 Gigabit Ethernet" },
 154         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704C,
 155                 "Broadcom BCM5704C Dual Gigabit Ethernet" },
 156         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704S,
 157                 "Broadcom BCM5704S Dual Gigabit Ethernet" },
 158         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5704S_ALT,
 159                 "Broadcom BCM5704S Dual Gigabit Ethernet" },
 160         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705,
 161                 "Broadcom BCM5705 Gigabit Ethernet" },
 162         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705F,
 163                 "Broadcom BCM5705F Gigabit Ethernet" },
 164         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705K,
 165                 "Broadcom BCM5705K Gigabit Ethernet" },
 166         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705M,
 167                 "Broadcom BCM5705M Gigabit Ethernet" },
 168         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5705M_ALT,
 169                 "Broadcom BCM5705M Gigabit Ethernet" },
 170         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5714,
 171                 "Broadcom BCM5714C Gigabit Ethernet" },
 172         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5714S,
 173                 "Broadcom BCM5714S Gigabit Ethernet" },
 174         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5715,
 175                 "Broadcom BCM5715 Gigabit Ethernet" },
 176         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5715S,
 177                 "Broadcom BCM5715S Gigabit Ethernet" },
 178         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720,
 179                 "Broadcom BCM5720 Gigabit Ethernet" },
 180         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5721,
 181                 "Broadcom BCM5721 Gigabit Ethernet" },
 182         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5722,
 183                 "Broadcom BCM5722 Gigabit Ethernet" },
 184         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5750,
 185                 "Broadcom BCM5750 Gigabit Ethernet" },
 186         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5750M,
 187                 "Broadcom BCM5750M Gigabit Ethernet" },
 188         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751,
 189                 "Broadcom BCM5751 Gigabit Ethernet" },
 190         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751F,
 191                 "Broadcom BCM5751F Gigabit Ethernet" },
 192         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5751M,
 193                 "Broadcom BCM5751M Gigabit Ethernet" },
 194         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5752,
 195                 "Broadcom BCM5752 Gigabit Ethernet" },
 196         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5752M,
 197                 "Broadcom BCM5752M Gigabit Ethernet" },
 198         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753,
 199                 "Broadcom BCM5753 Gigabit Ethernet" },
 200         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753F,
 201                 "Broadcom BCM5753F Gigabit Ethernet" },
 202         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5753M,
 203                 "Broadcom BCM5753M Gigabit Ethernet" },
 204         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5754,
 205                 "Broadcom BCM5754 Gigabit Ethernet" },
 206         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5754M,
 207                 "Broadcom BCM5754M Gigabit Ethernet" },
 208         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5755,
 209                 "Broadcom BCM5755 Gigabit Ethernet" },
 210         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5755M,
 211                 "Broadcom BCM5755M Gigabit Ethernet" },
 212         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5756,
 213                 "Broadcom BCM5756 Gigabit Ethernet" },
 214         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5780,
 215                 "Broadcom BCM5780 Gigabit Ethernet" },
 216         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5780S,
 217                 "Broadcom BCM5780S Gigabit Ethernet" },
 218         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5781,
 219                 "Broadcom BCM5781 Gigabit Ethernet" },
 220         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5782,
 221                 "Broadcom BCM5782 Gigabit Ethernet" },
 222         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5786,
 223                 "Broadcom BCM5786 Gigabit Ethernet" },
 224         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787,
 225                 "Broadcom BCM5787 Gigabit Ethernet" },
 226         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787F,
 227                 "Broadcom BCM5787F Gigabit Ethernet" },
 228         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5787M,
 229                 "Broadcom BCM5787M Gigabit Ethernet" },
 230         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5788,
 231                 "Broadcom BCM5788 Gigabit Ethernet" },
 232         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5789,
 233                 "Broadcom BCM5789 Gigabit Ethernet" },
 234         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5901,
 235                 "Broadcom BCM5901 Fast Ethernet" },
 236         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5901A2,
 237                 "Broadcom BCM5901A2 Fast Ethernet" },
 238         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5903M,
 239                 "Broadcom BCM5903M Fast Ethernet" },
 240
 241         { PCI_VENDOR_SCHNEIDERKOCH, PCI_PRODUCT_SCHNEIDERKOCH_SK_9DX1,
 242                 "SysKonnect Gigabit Ethernet" },
 243
 244         { 0, 0, NULL }
 245 };
 246
 247 #define BGE_IS_JUMBO_CAPABLE(sc)        ((sc)->bge_flags & BGE_FLAG_JUMBO)
 248 #define BGE_IS_5700_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5700_FAMILY)
 249 #define BGE_IS_5705_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_5705_PLUS)
 250 #define BGE_IS_5714_FAMILY(sc)          ((sc)->bge_flags & BGE_FLAG_5714_FAMILY)
 251 #define BGE_IS_575X_PLUS(sc)            ((sc)->bge_flags & BGE_FLAG_575X_PLUS)
 252
 253 static int      bge_probe(device_t);
 254 static int      bge_attach(device_t);
 255 static int      bge_detach(device_t);
 256 static void     bge_txeof(struct bge_softc *);
 257 static void     bge_rxeof(struct bge_softc *);
 258
 259 static void     bge_tick(void *);
 260 static void     bge_stats_update(struct bge_softc *);
 261 static void     bge_stats_update_regs(struct bge_softc *);
 262 static int      bge_encap(struct bge_softc *, struct mbuf **, uint32_t *);
 263
 264 #ifdef DEVICE_POLLING
 265 static void     bge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count);
 266 #endif
 267 static void     bge_intr(void *);
 268 static void     bge_enable_intr(struct bge_softc *);
 269 static void     bge_disable_intr(struct bge_softc *);
 270 static void     bge_start(struct ifnet *);
 271 static int      bge_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 272 static void     bge_init(void *);
 273 static void     bge_stop(struct bge_softc *);
 274 static void     bge_watchdog(struct ifnet *);
 275 static void     bge_shutdown(device_t);
 276 static int      bge_suspend(device_t);
 277 static int      bge_resume(device_t);
 278 static int      bge_ifmedia_upd(struct ifnet *);
 279 static void     bge_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 280
 281 static uint8_t  bge_eeprom_getbyte(struct bge_softc *, uint32_t, uint8_t *);
 282 static int      bge_read_eeprom(struct bge_softc *, caddr_t, uint32_t, size_t);
 283
 284 static void     bge_setmulti(struct bge_softc *);
 285 static void     bge_setpromisc(struct bge_softc *);
 286
 287 static int      bge_alloc_jumbo_mem(struct bge_softc *);
 288 static void     bge_free_jumbo_mem(struct bge_softc *);
 289 static struct bge_jslot
 290                 *bge_jalloc(struct bge_softc *);
 291 static void     bge_jfree(void *);
 292 static void     bge_jref(void *);
 293 static int      bge_newbuf_std(struct bge_softc *, int, struct mbuf *);
 294 static int      bge_newbuf_jumbo(struct bge_softc *, int, struct mbuf *);
 295 static int      bge_init_rx_ring_std(struct bge_softc *);
 296 static void     bge_free_rx_ring_std(struct bge_softc *);
 297 static int      bge_init_rx_ring_jumbo(struct bge_softc *);
 298 static void     bge_free_rx_ring_jumbo(struct bge_softc *);
 299 static void     bge_free_tx_ring(struct bge_softc *);
 300 static int      bge_init_tx_ring(struct bge_softc *);
 301
 302 static int      bge_chipinit(struct bge_softc *);
 303 static int      bge_blockinit(struct bge_softc *);
 304
 305 static uint32_t bge_readmem_ind(struct bge_softc *, uint32_t);
 306 static void     bge_writemem_ind(struct bge_softc *, uint32_t, uint32_t);
 307 #ifdef notdef
 308 static uint32_t bge_readreg_ind(struct bge_softc *, uint32_t);
 309 #endif
 310 static void     bge_writereg_ind(struct bge_softc *, uint32_t, uint32_t);
 311 static void     bge_writemem_direct(struct bge_softc *, uint32_t, uint32_t);
 312 static void     bge_set_max_readrq(struct bge_softc *);
 313
 314 static int      bge_miibus_readreg(device_t, int, int);
 315 static int      bge_miibus_writereg(device_t, int, int, int);
 316 static void     bge_miibus_statchg(device_t);
 317 static void     bge_bcm5700_link_upd(struct bge_softc *, uint32_t);
 318 static void     bge_tbi_link_upd(struct bge_softc *, uint32_t);
 319 static void     bge_copper_link_upd(struct bge_softc *, uint32_t);
 320
 321 static void     bge_reset(struct bge_softc *);
 322
 323 static void     bge_dma_map_addr(void *, bus_dma_segment_t *, int, int);
 324 static void     bge_dma_map_mbuf(void *, bus_dma_segment_t *, int,
 325                                  bus_size_t, int);
 326 static int      bge_dma_alloc(struct bge_softc *);
 327 static void     bge_dma_free(struct bge_softc *);
 328 static int      bge_dma_block_alloc(struct bge_softc *, bus_size_t,
 329                                     bus_dma_tag_t *, bus_dmamap_t *,
 330                                     void **, bus_addr_t *);
 331 static void     bge_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 332
 333 static void     bge_coal_change(struct bge_softc *);
 334 static int      bge_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 335 static int      bge_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 336 static int      bge_sysctl_rx_max_coal_bds(SYSCTL_HANDLER_ARGS);
 337 static int      bge_sysctl_tx_max_coal_bds(SYSCTL_HANDLER_ARGS);
 338 static int      bge_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *, uint32_t);
 339
 340 /*
 341  * Set following tunable to 1 for some IBM blade servers with the DNLK
 342  * switch module. Auto negotiation is broken for those configurations.
 343  */
 344 static int      bge_fake_autoneg = 0;
 345 TUNABLE_INT("hw.bge.fake_autoneg", &bge_fake_autoneg);
 346
 347 /* Interrupt moderation control variables. */
 348 static int      bge_rx_coal_ticks = 150;        /* usec */
 349 static int      bge_tx_coal_ticks = 1023;       /* usec */
 350 static int      bge_rx_max_coal_bds = 80;
 351 static int      bge_tx_max_coal_bds = 128;
 352
 353 TUNABLE_INT("hw.bge.rx_coal_ticks", &bge_rx_coal_ticks);
 354 TUNABLE_INT("hw.bge.tx_coal_ticks", &bge_tx_coal_ticks);
 355 TUNABLE_INT("hw.bge.rx_max_coal_bds", &bge_rx_max_coal_bds);
 356 TUNABLE_INT("hw.bge.tx_max_coal_bds", &bge_tx_max_coal_bds);
 357
 358 #if !defined(KTR_IF_BGE)
 359 #define KTR_IF_BGE      KTR_ALL
 360 #endif
 361 KTR_INFO_MASTER(if_bge);
 362 KTR_INFO(KTR_IF_BGE, if_bge, intr, 0, "intr", 0);
 363 KTR_INFO(KTR_IF_BGE, if_bge, rx_pkt, 1, "rx_pkt", 0);
 364 KTR_INFO(KTR_IF_BGE, if_bge, tx_pkt, 2, "tx_pkt", 0);
 365 #define logif(name)     KTR_LOG(if_bge_ ## name)
 366
 367 static device_method_t bge_methods[] = {
 368         /* Device interface */
 369         DEVMETHOD(device_probe,         bge_probe),
 370         DEVMETHOD(device_attach,        bge_attach),
 371         DEVMETHOD(device_detach,        bge_detach),
 372         DEVMETHOD(device_shutdown,      bge_shutdown),
 373         DEVMETHOD(device_suspend,       bge_suspend),
 374         DEVMETHOD(device_resume,        bge_resume),
 375
 376         /* bus interface */
 377         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 378         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 379
 380         /* MII interface */
 381         DEVMETHOD(miibus_readreg,       bge_miibus_readreg),
 382         DEVMETHOD(miibus_writereg,      bge_miibus_writereg),
 383         DEVMETHOD(miibus_statchg,       bge_miibus_statchg),
 384
 385         { 0, 0 }
 386 };
 387
 388 static DEFINE_CLASS_0(bge, bge_driver, bge_methods, sizeof(struct bge_softc));
 389 static devclass_t bge_devclass;
 390
 391 DECLARE_DUMMY_MODULE(if_bge);
 392 DRIVER_MODULE(if_bge, pci, bge_driver, bge_devclass, 0, 0);
 393 DRIVER_MODULE(miibus, bge, miibus_driver, miibus_devclass, 0, 0);
 394
 395 static uint32_t
 396 bge_readmem_ind(struct bge_softc *sc, uint32_t off)
 397 {
 398         device_t dev = sc->bge_dev;
 399         uint32_t val;
 400
 401         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 402         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 403         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 404         return (val);
 405 }
 406
 407 static void
 408 bge_writemem_ind(struct bge_softc *sc, uint32_t off, uint32_t val)
 409 {
 410         device_t dev = sc->bge_dev;
 411
 412         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 413         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 414         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 415 }
 416
 417 /*
 418  * PCI Express only
 419  */
 420 static void
 421 bge_set_max_readrq(struct bge_softc *sc)
 422 {
 423         device_t dev = sc->bge_dev;
 424         uint16_t val;
 425         uint8_t expr_ptr;
 426
 427         KKASSERT((sc->bge_flags & BGE_FLAG_PCIE) && sc->bge_expr_ptr != 0);
 428         expr_ptr = sc->bge_expr_ptr;
 429
 430         val = pci_read_config(dev, expr_ptr + PCIER_DEVCTRL, 2);
 431         if ((val & PCIEM_DEVCTL_MAX_READRQ_MASK) !=
 432             PCIEM_DEVCTL_MAX_READRQ_4096) {
 433                 device_printf(dev, "adjust device control 0x%04x ", val);
 434
 435                 val &= ~PCIEM_DEVCTL_MAX_READRQ_MASK;
 436                 val |= PCIEM_DEVCTL_MAX_READRQ_4096;
 437                 pci_write_config(dev, expr_ptr + PCIER_DEVCTRL, val, 2);
 438
 439                 kprintf("-> 0x%04x\n", val);
 440         }
 441 }
 442
 443 #ifdef notdef
 444 static uint32_t
 445 bge_readreg_ind(struct bge_softc *sc, uin32_t off)
 446 {
 447         device_t dev = sc->bge_dev;
 448
 449         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 450         return(pci_read_config(dev, BGE_PCI_REG_DATA, 4));
 451 }
 452 #endif
 453
 454 static void
 455 bge_writereg_ind(struct bge_softc *sc, uint32_t off, uint32_t val)
 456 {
 457         device_t dev = sc->bge_dev;
 458
 459         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 460         pci_write_config(dev, BGE_PCI_REG_DATA, val, 4);
 461 }
 462
 463 static void
 464 bge_writemem_direct(struct bge_softc *sc, uint32_t off, uint32_t val)
 465 {
 466         CSR_WRITE_4(sc, off, val);
 467 }
 468
 469 /*
 470  * Read a byte of data stored in the EEPROM at address 'addr.' The
 471  * BCM570x supports both the traditional bitbang interface and an
 472  * auto access interface for reading the EEPROM. We use the auto
 473  * access method.
 474  */
 475 static uint8_t
 476 bge_eeprom_getbyte(struct bge_softc *sc, uint32_t addr, uint8_t *dest)
 477 {
 478         int i;
 479         uint32_t byte = 0;
 480
 481         /*
 482          * Enable use of auto EEPROM access so we can avoid
 483          * having to use the bitbang method.
 484          */
 485         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 486
 487         /* Reset the EEPROM, load the clock period. */
 488         CSR_WRITE_4(sc, BGE_EE_ADDR,
 489             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 490         DELAY(20);
 491
 492         /* Issue the read EEPROM command. */
 493         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 494
 495         /* Wait for completion */
 496         for(i = 0; i < BGE_TIMEOUT * 10; i++) {
 497                 DELAY(10);
 498                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 499                         break;
 500         }
 501
 502         if (i == BGE_TIMEOUT) {
 503                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 504                 return(1);
 505         }
 506
 507         /* Get result. */
 508         byte = CSR_READ_4(sc, BGE_EE_DATA);
 509
 510         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 511
 512         return(0);
 513 }
 514
 515 /*
 516  * Read a sequence of bytes from the EEPROM.
 517  */
 518 static int
 519 bge_read_eeprom(struct bge_softc *sc, caddr_t dest, uint32_t off, size_t len)
 520 {
 521         size_t i;
 522         int err;
 523         uint8_t byte;
 524
 525         for (byte = 0, err = 0, i = 0; i < len; i++) {
 526                 err = bge_eeprom_getbyte(sc, off + i, &byte);
 527                 if (err)
 528                         break;
 529                 *(dest + i) = byte;
 530         }
 531
 532         return(err ? 1 : 0);
 533 }
 534
 535 static int
 536 bge_miibus_readreg(device_t dev, int phy, int reg)
 537 {
 538         struct bge_softc *sc = device_get_softc(dev);
 539         struct ifnet *ifp = &sc->arpcom.ac_if;
 540         uint32_t val, autopoll;
 541         int i;
 542
 543         /*
 544          * Broadcom's own driver always assumes the internal
 545          * PHY is at GMII address 1. On some chips, the PHY responds
 546          * to accesses at all addresses, which could cause us to
 547          * bogusly attach the PHY 32 times at probe type. Always
 548          * restricting the lookup to address 1 is simpler than
 549          * trying to figure out which chips revisions should be
 550          * special-cased.
 551          */
 552         if (phy != 1)
 553                 return(0);
 554
 555         /* Reading with autopolling on may trigger PCI errors */
 556         autopoll = CSR_READ_4(sc, BGE_MI_MODE);
 557         if (autopoll & BGE_MIMODE_AUTOPOLL) {
 558                 BGE_CLRBIT(sc, BGE_MI_MODE, BGE_MIMODE_AUTOPOLL);
 559                 DELAY(40);
 560         }
 561
 562         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ|BGE_MICOMM_BUSY|
 563             BGE_MIPHY(phy)|BGE_MIREG(reg));
 564
 565         for (i = 0; i < BGE_TIMEOUT; i++) {
 566                 DELAY(10);
 567                 val = CSR_READ_4(sc, BGE_MI_COMM);
 568                 if (!(val & BGE_MICOMM_BUSY))
 569                         break;
 570         }
 571
 572         if (i == BGE_TIMEOUT) {
 573                 if_printf(ifp, "PHY read timed out "
 574                           "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 575                 val = 0;
 576                 goto done;
 577         }
 578
 579         DELAY(5);
 580         val = CSR_READ_4(sc, BGE_MI_COMM);
 581
 582 done:
 583         if (autopoll & BGE_MIMODE_AUTOPOLL) {
 584                 BGE_SETBIT(sc, BGE_MI_MODE, BGE_MIMODE_AUTOPOLL);
 585                 DELAY(40);
 586         }
 587
 588         if (val & BGE_MICOMM_READFAIL)
 589                 return(0);
 590
 591         return(val & 0xFFFF);
 592 }
 593
 594 static int
 595 bge_miibus_writereg(device_t dev, int phy, int reg, int val)
 596 {
 597         struct bge_softc *sc = device_get_softc(dev);
 598         uint32_t autopoll;
 599         int i;
 600
 601         /*
 602          * See the related comment in bge_miibus_readreg()
 603          */
 604         if (phy != 1)
 605                 return(0);
 606
 607         /* Reading with autopolling on may trigger PCI errors */
 608         autopoll = CSR_READ_4(sc, BGE_MI_MODE);
 609         if (autopoll & BGE_MIMODE_AUTOPOLL) {
 610                 BGE_CLRBIT(sc, BGE_MI_MODE, BGE_MIMODE_AUTOPOLL);
 611                 DELAY(40);
 612         }
 613
 614         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE|BGE_MICOMM_BUSY|
 615             BGE_MIPHY(phy)|BGE_MIREG(reg)|val);
 616
 617         for (i = 0; i < BGE_TIMEOUT; i++) {
 618                 DELAY(10);
 619                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 620                         DELAY(5);
 621                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 622                         break;
 623                 }
 624         }
 625
 626         if (autopoll & BGE_MIMODE_AUTOPOLL) {
 627                 BGE_SETBIT(sc, BGE_MI_MODE, BGE_MIMODE_AUTOPOLL);
 628                 DELAY(40);
 629         }
 630
 631         if (i == BGE_TIMEOUT) {
 632                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 633                           "(phy %d, reg %d, val %d)\n", phy, reg, val);
 634                 return(0);
 635         }
 636
 637         return(0);
 638 }
 639
 640 static void
 641 bge_miibus_statchg(device_t dev)
 642 {
 643         struct bge_softc *sc;
 644         struct mii_data *mii;
 645
 646         sc = device_get_softc(dev);
 647         mii = device_get_softc(sc->bge_miibus);
 648
 649         BGE_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
 650         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T) {
 651                 BGE_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
 652         } else {
 653                 BGE_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
 654         }
 655
 656         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
 657                 BGE_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 658         } else {
 659                 BGE_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 660         }
 661 }
 662
 663 /*
 664  * Memory management for jumbo frames.
 665  */
 666 static int
 667 bge_alloc_jumbo_mem(struct bge_softc *sc)
 668 {
 669         struct ifnet *ifp = &sc->arpcom.ac_if;
 670         struct bge_jslot *entry;
 671         uint8_t *ptr;
 672         bus_addr_t paddr;
 673         int i, error;
 674
 675         /*
 676          * Create tag for jumbo mbufs.
 677          * This is really a bit of a kludge. We allocate a special
 678          * jumbo buffer pool which (thanks to the way our DMA
 679          * memory allocation works) will consist of contiguous
 680          * pages. This means that even though a jumbo buffer might
 681          * be larger than a page size, we don't really need to
 682          * map it into more than one DMA segment. However, the
 683          * default mbuf tag will result in multi-segment mappings,
 684          * so we have to create a special jumbo mbuf tag that
 685          * lets us get away with mapping the jumbo buffers as
 686          * a single segment. I think eventually the driver should
 687          * be changed so that it uses ordinary mbufs and cluster
 688          * buffers, i.e. jumbo frames can span multiple DMA
 689          * descriptors. But that's a project for another day.
 690          */
 691
 692         /*
 693          * Create DMA stuffs for jumbo RX ring.
 694          */
 695         error = bge_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 696                                     &sc->bge_cdata.bge_rx_jumbo_ring_tag,
 697                                     &sc->bge_cdata.bge_rx_jumbo_ring_map,
 698                                     (void **)&sc->bge_ldata.bge_rx_jumbo_ring,
 699                                     &sc->bge_ldata.bge_rx_jumbo_ring_paddr);
 700         if (error) {
 701                 if_printf(ifp, "could not create jumbo RX ring\n");
 702                 return error;
 703         }
 704
 705         /*
 706          * Create DMA stuffs for jumbo buffer block.
 707          */
 708         error = bge_dma_block_alloc(sc, BGE_JMEM,
 709                                     &sc->bge_cdata.bge_jumbo_tag,
 710                                     &sc->bge_cdata.bge_jumbo_map,
 711                                     (void **)&sc->bge_ldata.bge_jumbo_buf,
 712                                     &paddr);
 713         if (error) {
 714                 if_printf(ifp, "could not create jumbo buffer\n");
 715                 return error;
 716         }
 717
 718         SLIST_INIT(&sc->bge_jfree_listhead);
 719
 720         /*
 721          * Now divide it up into 9K pieces and save the addresses
 722          * in an array. Note that we play an evil trick here by using
 723          * the first few bytes in the buffer to hold the the address
 724          * of the softc structure for this interface. This is because
 725          * bge_jfree() needs it, but it is called by the mbuf management
 726          * code which will not pass it to us explicitly.
 727          */
 728         for (i = 0, ptr = sc->bge_ldata.bge_jumbo_buf; i < BGE_JSLOTS; i++) {
 729                 entry = &sc->bge_cdata.bge_jslots[i];
 730                 entry->bge_sc = sc;
 731                 entry->bge_buf = ptr;
 732                 entry->bge_paddr = paddr;
 733                 entry->bge_inuse = 0;
 734                 entry->bge_slot = i;
 735                 SLIST_INSERT_HEAD(&sc->bge_jfree_listhead, entry, jslot_link);
 736
 737                 ptr += BGE_JLEN;
 738                 paddr += BGE_JLEN;
 739         }
 740         return 0;
 741 }
 742
 743 static void
 744 bge_free_jumbo_mem(struct bge_softc *sc)
 745 {
 746         /* Destroy jumbo RX ring. */
 747         bge_dma_block_free(sc->bge_cdata.bge_rx_jumbo_ring_tag,
 748                            sc->bge_cdata.bge_rx_jumbo_ring_map,
 749                            sc->bge_ldata.bge_rx_jumbo_ring);
 750
 751         /* Destroy jumbo buffer block. */
 752         bge_dma_block_free(sc->bge_cdata.bge_jumbo_tag,
 753                            sc->bge_cdata.bge_jumbo_map,
 754                            sc->bge_ldata.bge_jumbo_buf);
 755 }
 756
 757 /*
 758  * Allocate a jumbo buffer.
 759  */
 760 static struct bge_jslot *
 761 bge_jalloc(struct bge_softc *sc)
 762 {
 763         struct bge_jslot *entry;
 764
 765         lwkt_serialize_enter(&sc->bge_jslot_serializer);
 766         entry = SLIST_FIRST(&sc->bge_jfree_listhead);
 767         if (entry) {
 768                 SLIST_REMOVE_HEAD(&sc->bge_jfree_listhead, jslot_link);
 769                 entry->bge_inuse = 1;
 770         } else {
 771                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 772         }
 773         lwkt_serialize_exit(&sc->bge_jslot_serializer);
 774         return(entry);
 775 }
 776
 777 /*
 778  * Adjust usage count on a jumbo buffer.
 779  */
 780 static void
 781 bge_jref(void *arg)
 782 {
 783         struct bge_jslot *entry = (struct bge_jslot *)arg;
 784         struct bge_softc *sc = entry->bge_sc;
 785
 786         if (sc == NULL)
 787                 panic("bge_jref: can't find softc pointer!");
 788
 789         if (&sc->bge_cdata.bge_jslots[entry->bge_slot] != entry) {
 790                 panic("bge_jref: asked to reference buffer "
 791                     "that we don't manage!");
 792         } else if (entry->bge_inuse == 0) {
 793                 panic("bge_jref: buffer already free!");
 794         } else {
 795                 atomic_add_int(&entry->bge_inuse, 1);
 796         }
 797 }
 798
 799 /*
 800  * Release a jumbo buffer.
 801  */
 802 static void
 803 bge_jfree(void *arg)
 804 {
 805         struct bge_jslot *entry = (struct bge_jslot *)arg;
 806         struct bge_softc *sc = entry->bge_sc;
 807
 808         if (sc == NULL)
 809                 panic("bge_jfree: can't find softc pointer!");
 810
 811         if (&sc->bge_cdata.bge_jslots[entry->bge_slot] != entry) {
 812                 panic("bge_jfree: asked to free buffer that we don't manage!");
 813         } else if (entry->bge_inuse == 0) {
 814                 panic("bge_jfree: buffer already free!");
 815         } else {
 816                 /*
 817                  * Possible MP race to 0, use the serializer.  The atomic insn
 818                  * is still needed for races against bge_jref().
 819                  */
 820                 lwkt_serialize_enter(&sc->bge_jslot_serializer);
 821                 atomic_subtract_int(&entry->bge_inuse, 1);
 822                 if (entry->bge_inuse == 0) {
 823                         SLIST_INSERT_HEAD(&sc->bge_jfree_listhead,
 824                                           entry, jslot_link);
 825                 }
 826                 lwkt_serialize_exit(&sc->bge_jslot_serializer);
 827         }
 828 }
 829
 830
 831 /*
 832  * Intialize a standard receive ring descriptor.
 833  */
 834 static int
 835 bge_newbuf_std(struct bge_softc *sc, int i, struct mbuf *m)
 836 {
 837         struct mbuf *m_new = NULL;
 838         struct bge_dmamap_arg ctx;
 839         bus_dma_segment_t seg;
 840         struct bge_rx_bd *r;
 841         int error;
 842
 843         if (m == NULL) {
 844                 m_new = m_getcl(MB_DONTWAIT, MT_DATA, M_PKTHDR);
 845                 if (m_new == NULL)
 846                         return ENOBUFS;
 847         } else {
 848                 m_new = m;
 849                 m_new->m_data = m_new->m_ext.ext_buf;
 850         }
 851         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
 852
 853         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0)
 854                 m_adj(m_new, ETHER_ALIGN);
 855
 856         ctx.bge_maxsegs = 1;
 857         ctx.bge_segs = &seg;
 858         error = bus_dmamap_load_mbuf(sc->bge_cdata.bge_mtag,
 859                                      sc->bge_cdata.bge_rx_std_dmamap[i],
 860                                      m_new, bge_dma_map_mbuf, &ctx,
 861                                      BUS_DMA_NOWAIT);
 862         if (error || ctx.bge_maxsegs == 0) {
 863                 if (!error) {
 864                         if_printf(&sc->arpcom.ac_if, "too many segments?!\n");
 865                         bus_dmamap_unload(sc->bge_cdata.bge_mtag,
 866                                           sc->bge_cdata.bge_rx_std_dmamap[i]);
 867                 }
 868                 if (m == NULL)
 869                         m_freem(m_new);
 870                 return ENOMEM;
 871         }
 872
 873         sc->bge_cdata.bge_rx_std_chain[i] = m_new;
 874
 875         r = &sc->bge_ldata.bge_rx_std_ring[i];
 876         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(ctx.bge_segs[0].ds_addr);
 877         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(ctx.bge_segs[0].ds_addr);
 878         r->bge_flags = BGE_RXBDFLAG_END;
 879         r->bge_len = m_new->m_len;
 880         r->bge_idx = i;
 881
 882         bus_dmamap_sync(sc->bge_cdata.bge_mtag,
 883                         sc->bge_cdata.bge_rx_std_dmamap[i],
 884                         BUS_DMASYNC_PREREAD);
 885         return 0;
 886 }
 887
 888 /*
 889  * Initialize a jumbo receive ring descriptor. This allocates
 890  * a jumbo buffer from the pool managed internally by the driver.
 891  */
 892 static int
 893 bge_newbuf_jumbo(struct bge_softc *sc, int i, struct mbuf *m)
 894 {
 895         struct mbuf *m_new = NULL;
 896         struct bge_jslot *buf;
 897         struct bge_rx_bd *r;
 898         bus_addr_t paddr;
 899
 900         if (m == NULL) {
 901                 /* Allocate the mbuf. */
 902                 MGETHDR(m_new, MB_DONTWAIT, MT_DATA);
 903                 if (m_new == NULL)
 904                         return(ENOBUFS);
 905
 906                 /* Allocate the jumbo buffer */
 907                 buf = bge_jalloc(sc);
 908                 if (buf == NULL) {
 909                         m_freem(m_new);
 910                         if_printf(&sc->arpcom.ac_if, "jumbo allocation failed "
 911                             "-- packet dropped!\n");
 912                         return ENOBUFS;
 913                 }
 914
 915                 /* Attach the buffer to the mbuf. */
 916                 m_new->m_ext.ext_arg = buf;
 917                 m_new->m_ext.ext_buf = buf->bge_buf;
 918                 m_new->m_ext.ext_free = bge_jfree;
 919                 m_new->m_ext.ext_ref = bge_jref;
 920                 m_new->m_ext.ext_size = BGE_JUMBO_FRAMELEN;
 921
 922                 m_new->m_flags |= M_EXT;
 923         } else {
 924                 KKASSERT(m->m_flags & M_EXT);
 925                 m_new = m;
 926                 buf = m_new->m_ext.ext_arg;
 927         }
 928         m_new->m_data = m_new->m_ext.ext_buf;
 929         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
 930
 931         paddr = buf->bge_paddr;
 932         if ((sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) == 0) {
 933                 m_adj(m_new, ETHER_ALIGN);
 934                 paddr += ETHER_ALIGN;
 935         }
 936
 937         /* Set up the descriptor. */
 938         sc->bge_cdata.bge_rx_jumbo_chain[i] = m_new;
 939
 940         r = &sc->bge_ldata.bge_rx_jumbo_ring[i];
 941         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(paddr);
 942         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(paddr);
 943         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 944         r->bge_len = m_new->m_len;
 945         r->bge_idx = i;
 946
 947         return 0;
 948 }
 949
 950 /*
 951  * The standard receive ring has 512 entries in it. At 2K per mbuf cluster,
 952  * that's 1MB or memory, which is a lot. For now, we fill only the first
 953  * 256 ring entries and hope that our CPU is fast enough to keep up with
 954  * the NIC.
 955  */
 956 static int
 957 bge_init_rx_ring_std(struct bge_softc *sc)
 958 {
 959         int i;
 960
 961         for (i = 0; i < BGE_SSLOTS; i++) {
 962                 if (bge_newbuf_std(sc, i, NULL) == ENOBUFS)
 963                         return(ENOBUFS);
 964         };
 965
 966         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
 967                         sc->bge_cdata.bge_rx_std_ring_map,
 968                         BUS_DMASYNC_PREWRITE);
 969
 970         sc->bge_std = i - 1;
 971         CSR_WRITE_4(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std);
 972
 973         return(0);
 974 }
 975
 976 static void
 977 bge_free_rx_ring_std(struct bge_softc *sc)
 978 {
 979         int i;
 980
 981         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 982                 if (sc->bge_cdata.bge_rx_std_chain[i] != NULL) {
 983                         bus_dmamap_unload(sc->bge_cdata.bge_mtag,
 984                                           sc->bge_cdata.bge_rx_std_dmamap[i]);
 985                         m_freem(sc->bge_cdata.bge_rx_std_chain[i]);
 986                         sc->bge_cdata.bge_rx_std_chain[i] = NULL;
 987                 }
 988                 bzero(&sc->bge_ldata.bge_rx_std_ring[i],
 989                     sizeof(struct bge_rx_bd));
 990         }
 991 }
 992
 993 static int
 994 bge_init_rx_ring_jumbo(struct bge_softc *sc)
 995 {
 996         int i;
 997         struct bge_rcb *rcb;
 998
 999         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1000                 if (bge_newbuf_jumbo(sc, i, NULL) == ENOBUFS)
1001                         return(ENOBUFS);
1002         };
1003
1004         bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
1005                         sc->bge_cdata.bge_rx_jumbo_ring_map,
1006                         BUS_DMASYNC_PREWRITE);
1007
1008         sc->bge_jumbo = i - 1;
1009
1010         rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
1011         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
1012         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1013
1014         CSR_WRITE_4(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo);
1015
1016         return(0);
1017 }
1018
1019 static void
1020 bge_free_rx_ring_jumbo(struct bge_softc *sc)
1021 {
1022         int i;
1023
1024         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1025                 if (sc->bge_cdata.bge_rx_jumbo_chain[i] != NULL) {
1026                         m_freem(sc->bge_cdata.bge_rx_jumbo_chain[i]);
1027                         sc->bge_cdata.bge_rx_jumbo_chain[i] = NULL;
1028                 }
1029                 bzero(&sc->bge_ldata.bge_rx_jumbo_ring[i],
1030                     sizeof(struct bge_rx_bd));
1031         }
1032 }
1033
1034 static void
1035 bge_free_tx_ring(struct bge_softc *sc)
1036 {
1037         int i;
1038
1039         for (i = 0; i < BGE_TX_RING_CNT; i++) {
1040                 if (sc->bge_cdata.bge_tx_chain[i] != NULL) {
1041                         bus_dmamap_unload(sc->bge_cdata.bge_mtag,
1042                                           sc->bge_cdata.bge_tx_dmamap[i]);
1043                         m_freem(sc->bge_cdata.bge_tx_chain[i]);
1044                         sc->bge_cdata.bge_tx_chain[i] = NULL;
1045                 }
1046                 bzero(&sc->bge_ldata.bge_tx_ring[i],
1047                     sizeof(struct bge_tx_bd));
1048         }
1049 }
1050
1051 static int
1052 bge_init_tx_ring(struct bge_softc *sc)
1053 {
1054         sc->bge_txcnt = 0;
1055         sc->bge_tx_saved_considx = 0;
1056         sc->bge_tx_prodidx = 0;
1057
1058         /* Initialize transmit producer index for host-memory send ring. */
1059         CSR_WRITE_4(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bge_tx_prodidx);
1060
1061         /* 5700 b2 errata */
1062         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1063                 CSR_WRITE_4(sc, BGE_MBX_TX_HOST_PROD0_LO, 0);
1064
1065         CSR_WRITE_4(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1066         /* 5700 b2 errata */
1067         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
1068                 CSR_WRITE_4(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
1069
1070         return(0);
1071 }
1072
1073 static void
1074 bge_setmulti(struct bge_softc *sc)
1075 {
1076         struct ifnet *ifp;
1077         struct ifmultiaddr *ifma;
1078         uint32_t hashes[4] = { 0, 0, 0, 0 };
1079         int h, i;
1080
1081         ifp = &sc->arpcom.ac_if;
1082
1083         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1084                 for (i = 0; i < 4; i++)
1085                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1086                 return;
1087         }
1088
1089         /* First, zot all the existing filters. */
1090         for (i = 0; i < 4; i++)
1091                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1092
1093         /* Now program new ones. */
1094         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1095                 if (ifma->ifma_addr->sa_family != AF_LINK)
1096                         continue;
1097                 h = ether_crc32_le(
1098                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1099                     ETHER_ADDR_LEN) & 0x7f;
1100                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1101         }
1102
1103         for (i = 0; i < 4; i++)
1104                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1105 }
1106
1107 /*
1108  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1109  * self-test results.
1110  */
1111 static int
1112 bge_chipinit(struct bge_softc *sc)
1113 {
1114         int i;
1115         uint32_t dma_rw_ctl;
1116
1117         /* Set endian type before we access any non-PCI registers. */
1118         pci_write_config(sc->bge_dev, BGE_PCI_MISC_CTL, BGE_INIT, 4);
1119
1120         /*
1121          * Check the 'ROM failed' bit on the RX CPU to see if
1122          * self-tests passed.
1123          */
1124         if (CSR_READ_4(sc, BGE_RXCPU_MODE) & BGE_RXCPUMODE_ROMFAIL) {
1125                 if_printf(&sc->arpcom.ac_if,
1126                           "RX CPU self-diagnostics failed!\n");
1127                 return(ENODEV);
1128         }
1129
1130         /* Clear the MAC control register */
1131         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
1132
1133         /*
1134          * Clear the MAC statistics block in the NIC's
1135          * internal memory.
1136          */
1137         for (i = BGE_STATS_BLOCK;
1138             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1139                 BGE_MEMWIN_WRITE(sc, i, 0);
1140
1141         for (i = BGE_STATUS_BLOCK;
1142             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1143                 BGE_MEMWIN_WRITE(sc, i, 0);
1144
1145         /* Set up the PCI DMA control register. */
1146         if (sc->bge_flags & BGE_FLAG_PCIE) {
1147                 /* PCI Express */
1148                 dma_rw_ctl = BGE_PCI_READ_CMD|BGE_PCI_WRITE_CMD |
1149                     (0xf << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1150                     (0x2 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1151         } else if (sc->bge_flags & BGE_FLAG_PCIX) {
1152                 /* PCI-X bus */
1153                 if (BGE_IS_5714_FAMILY(sc)) {
1154                         dma_rw_ctl = BGE_PCI_READ_CMD|BGE_PCI_WRITE_CMD;
1155                         dma_rw_ctl &= ~BGE_PCIDMARWCTL_ONEDMA_ATONCE; /* XXX */
1156                         /* XXX magic values, Broadcom-supplied Linux driver */
1157                         if (sc->bge_asicrev == BGE_ASICREV_BCM5780) {
1158                                 dma_rw_ctl |= (1 << 20) | (1 << 18) |
1159                                     BGE_PCIDMARWCTL_ONEDMA_ATONCE;
1160                         } else {
1161                                 dma_rw_ctl |= (1 << 20) | (1 << 18) | (1 << 15);
1162                         }
1163                 } else if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1164                         /*
1165                          * The 5704 uses a different encoding of read/write
1166                          * watermarks.
1167                          */
1168                         dma_rw_ctl = BGE_PCI_READ_CMD|BGE_PCI_WRITE_CMD |
1169                             (0x7 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1170                             (0x3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1171                 } else {
1172                         dma_rw_ctl = BGE_PCI_READ_CMD|BGE_PCI_WRITE_CMD |
1173                             (0x3 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1174                             (0x3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT) |
1175                             (0x0F);
1176                 }
1177
1178                 /*
1179                  * 5703 and 5704 need ONEDMA_AT_ONCE as a workaround
1180                  * for hardware bugs.
1181                  */
1182                 if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1183                     sc->bge_asicrev == BGE_ASICREV_BCM5704) {
1184                         uint32_t tmp;
1185
1186                         tmp = CSR_READ_4(sc, BGE_PCI_CLKCTL) & 0x1f;
1187                         if (tmp == 0x6 || tmp == 0x7)
1188                                 dma_rw_ctl |= BGE_PCIDMARWCTL_ONEDMA_ATONCE;
1189                 }
1190         } else {
1191                 /* Conventional PCI bus */
1192                 dma_rw_ctl = BGE_PCI_READ_CMD|BGE_PCI_WRITE_CMD |
1193                     (0x7 << BGE_PCIDMARWCTL_RD_WAT_SHIFT) |
1194                     (0x7 << BGE_PCIDMARWCTL_WR_WAT_SHIFT) |
1195                     (0x0F);
1196         }
1197
1198         if (sc->bge_asicrev == BGE_ASICREV_BCM5703 ||
1199             sc->bge_asicrev == BGE_ASICREV_BCM5704 ||
1200             sc->bge_asicrev == BGE_ASICREV_BCM5705)
1201                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_MINDMA;
1202         pci_write_config(sc->bge_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1203
1204         /*
1205          * Set up general mode register.
1206          */
1207         CSR_WRITE_4(sc, BGE_MODE_CTL, BGE_DMA_SWAP_OPTIONS|
1208             BGE_MODECTL_MAC_ATTN_INTR|BGE_MODECTL_HOST_SEND_BDS|
1209             BGE_MODECTL_TX_NO_PHDR_CSUM);
1210
1211         /*
1212          * Disable memory write invalidate.  Apparently it is not supported
1213          * properly by these devices.
1214          */
1215         PCI_CLRBIT(sc->bge_dev, BGE_PCI_CMD, PCIM_CMD_MWIEN, 4);
1216
1217         /* Set the timer prescaler (always 66Mhz) */
1218         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1219
1220         return(0);
1221 }
1222
1223 static int
1224 bge_blockinit(struct bge_softc *sc)
1225 {
1226         struct bge_rcb *rcb;
1227         bus_size_t vrcb;
1228         bge_hostaddr taddr;
1229         uint32_t val;
1230         int i;
1231
1232         /*
1233          * Initialize the memory window pointer register so that
1234          * we can access the first 32K of internal NIC RAM. This will
1235          * allow us to set up the TX send ring RCBs and the RX return
1236          * ring RCBs, plus other things which live in NIC memory.
1237          */
1238         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1239
1240         /* Note: the BCM5704 has a smaller mbuf space than other chips. */
1241
1242         if (!BGE_IS_5705_PLUS(sc)) {
1243                 /* Configure mbuf memory pool */
1244                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_BASEADDR, BGE_BUFFPOOL_1);
1245                 if (sc->bge_asicrev == BGE_ASICREV_BCM5704)
1246                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x10000);
1247                 else
1248                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_LEN, 0x18000);
1249
1250                 /* Configure DMA resource pool */
1251                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_BASEADDR,
1252                     BGE_DMA_DESCRIPTORS);
1253                 CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LEN, 0x2000);
1254         }
1255
1256         /* Configure mbuf pool watermarks */
1257         if (BGE_IS_5705_PLUS(sc)) {
1258                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1259                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1260         } else {
1261                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x50);
1262                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x20);
1263         }
1264         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1265
1266         /* Configure DMA resource watermarks */
1267         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1268         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1269
1270         /* Enable buffer manager */
1271         if (!BGE_IS_5705_PLUS(sc)) {
1272                 CSR_WRITE_4(sc, BGE_BMAN_MODE,
1273                     BGE_BMANMODE_ENABLE|BGE_BMANMODE_LOMBUF_ATTN);
1274
1275                 /* Poll for buffer manager start indication */
1276                 for (i = 0; i < BGE_TIMEOUT; i++) {
1277                         if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1278                                 break;
1279                         DELAY(10);
1280                 }
1281
1282                 if (i == BGE_TIMEOUT) {
1283                         if_printf(&sc->arpcom.ac_if,
1284                                   "buffer manager failed to start\n");
1285                         return(ENXIO);
1286                 }
1287         }
1288
1289         /* Enable flow-through queues */
1290         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1291         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1292
1293         /* Wait until queue initialization is complete */
1294         for (i = 0; i < BGE_TIMEOUT; i++) {
1295                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1296                         break;
1297                 DELAY(10);
1298         }
1299
1300         if (i == BGE_TIMEOUT) {
1301                 if_printf(&sc->arpcom.ac_if,
1302                           "flow-through queue init failed\n");
1303                 return(ENXIO);
1304         }
1305
1306         /* Initialize the standard RX ring control block */
1307         rcb = &sc->bge_ldata.bge_info.bge_std_rx_rcb;
1308         rcb->bge_hostaddr.bge_addr_lo =
1309             BGE_ADDR_LO(sc->bge_ldata.bge_rx_std_ring_paddr);
1310         rcb->bge_hostaddr.bge_addr_hi =
1311             BGE_ADDR_HI(sc->bge_ldata.bge_rx_std_ring_paddr);
1312         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
1313             sc->bge_cdata.bge_rx_std_ring_map, BUS_DMASYNC_PREREAD);
1314         if (BGE_IS_5705_PLUS(sc))
1315                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1316         else
1317                 rcb->bge_maxlen_flags =
1318                     BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN, 0);
1319         rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1320         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1321         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1322         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1323         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1324
1325         /*
1326          * Initialize the jumbo RX ring control block
1327          * We set the 'ring disabled' bit in the flags
1328          * field until we're actually ready to start
1329          * using this ring (i.e. once we set the MTU
1330          * high enough to require it).
1331          */
1332         if (BGE_IS_JUMBO_CAPABLE(sc)) {
1333                 rcb = &sc->bge_ldata.bge_info.bge_jumbo_rx_rcb;
1334
1335                 rcb->bge_hostaddr.bge_addr_lo =
1336                     BGE_ADDR_LO(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
1337                 rcb->bge_hostaddr.bge_addr_hi =
1338                     BGE_ADDR_HI(sc->bge_ldata.bge_rx_jumbo_ring_paddr);
1339                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
1340                     sc->bge_cdata.bge_rx_jumbo_ring_map,
1341                     BUS_DMASYNC_PREREAD);
1342                 rcb->bge_maxlen_flags =
1343                     BGE_RCB_MAXLEN_FLAGS(BGE_MAX_FRAMELEN,
1344                     BGE_RCB_FLAG_RING_DISABLED);
1345                 rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1346                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1347                     rcb->bge_hostaddr.bge_addr_hi);
1348                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1349                     rcb->bge_hostaddr.bge_addr_lo);
1350                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1351                     rcb->bge_maxlen_flags);
1352                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1353
1354                 /* Set up dummy disabled mini ring RCB */
1355                 rcb = &sc->bge_ldata.bge_info.bge_mini_rx_rcb;
1356                 rcb->bge_maxlen_flags =
1357                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED);
1358                 CSR_WRITE_4(sc, BGE_RX_MINI_RCB_MAXLEN_FLAGS,
1359                     rcb->bge_maxlen_flags);
1360         }
1361
1362         /*
1363          * Set the BD ring replentish thresholds. The recommended
1364          * values are 1/8th the number of descriptors allocated to
1365          * each ring.
1366          */
1367         if (BGE_IS_5705_PLUS(sc))
1368                 val = 8;
1369         else
1370                 val = BGE_STD_RX_RING_CNT / 8;
1371         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1372         CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH, BGE_JUMBO_RX_RING_CNT/8);
1373
1374         /*
1375          * Disable all unused send rings by setting the 'ring disabled'
1376          * bit in the flags field of all the TX send ring control blocks.
1377          * These are located in NIC memory.
1378          */
1379         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1380         for (i = 0; i < BGE_TX_RINGS_EXTSSRAM_MAX; i++) {
1381                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1382                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1383                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1384                 vrcb += sizeof(struct bge_rcb);
1385         }
1386
1387         /* Configure TX RCB 0 (we use only the first ring) */
1388         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1389         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_tx_ring_paddr);
1390         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1391         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1392         RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1393             BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1394         if (!BGE_IS_5705_PLUS(sc)) {
1395                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1396                     BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1397         }
1398
1399         /* Disable all unused RX return rings */
1400         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1401         for (i = 0; i < BGE_RX_RINGS_MAX; i++) {
1402                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1403                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1404                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1405                     BGE_RCB_MAXLEN_FLAGS(sc->bge_return_ring_cnt,
1406                     BGE_RCB_FLAG_RING_DISABLED));
1407                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1408                 CSR_WRITE_4(sc, BGE_MBX_RX_CONS0_LO +
1409                     (i * (sizeof(uint64_t))), 0);
1410                 vrcb += sizeof(struct bge_rcb);
1411         }
1412
1413         /* Initialize RX ring indexes */
1414         CSR_WRITE_4(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1415         CSR_WRITE_4(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1416         CSR_WRITE_4(sc, BGE_MBX_RX_MINI_PROD_LO, 0);
1417
1418         /*
1419          * Set up RX return ring 0
1420          * Note that the NIC address for RX return rings is 0x00000000.
1421          * The return rings live entirely within the host, so the
1422          * nicaddr field in the RCB isn't used.
1423          */
1424         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1425         BGE_HOSTADDR(taddr, sc->bge_ldata.bge_rx_return_ring_paddr);
1426         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1427         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1428         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0x00000000);
1429         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1430             BGE_RCB_MAXLEN_FLAGS(sc->bge_return_ring_cnt, 0));
1431
1432         /* Set random backoff seed for TX */
1433         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1434             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1435             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1436             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1437             BGE_TX_BACKOFF_SEED_MASK);
1438
1439         /* Set inter-packet gap */
1440         CSR_WRITE_4(sc, BGE_TX_LENGTHS, 0x2620);
1441
1442         /*
1443          * Specify which ring to use for packets that don't match
1444          * any RX rules.
1445          */
1446         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1447
1448         /*
1449          * Configure number of RX lists. One interrupt distribution
1450          * list, sixteen active lists, one bad frames class.
1451          */
1452         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1453
1454         /* Inialize RX list placement stats mask. */
1455         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1456         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1457
1458         /* Disable host coalescing until we get it set up */
1459         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1460
1461         /* Poll to make sure it's shut down. */
1462         for (i = 0; i < BGE_TIMEOUT; i++) {
1463                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1464                         break;
1465                 DELAY(10);
1466         }
1467
1468         if (i == BGE_TIMEOUT) {
1469                 if_printf(&sc->arpcom.ac_if,
1470                           "host coalescing engine failed to idle\n");
1471                 return(ENXIO);
1472         }
1473
1474         /* Set up host coalescing defaults */
1475         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bge_rx_coal_ticks);
1476         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bge_tx_coal_ticks);
1477         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bge_rx_max_coal_bds);
1478         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bge_tx_max_coal_bds);
1479         if (!BGE_IS_5705_PLUS(sc)) {
1480                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS_INT, 0);
1481                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS_INT, 0);
1482         }
1483         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 1);
1484         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 1);
1485
1486         /* Set up address of statistics block */
1487         if (!BGE_IS_5705_PLUS(sc)) {
1488                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_HI,
1489                     BGE_ADDR_HI(sc->bge_ldata.bge_stats_paddr));
1490                 CSR_WRITE_4(sc, BGE_HCC_STATS_ADDR_LO,
1491                     BGE_ADDR_LO(sc->bge_ldata.bge_stats_paddr));
1492
1493                 CSR_WRITE_4(sc, BGE_HCC_STATS_BASEADDR, BGE_STATS_BLOCK);
1494                 CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_BASEADDR, BGE_STATUS_BLOCK);
1495                 CSR_WRITE_4(sc, BGE_HCC_STATS_TICKS, sc->bge_stat_ticks);
1496         }
1497
1498         /* Set up address of status block */
1499         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1500             BGE_ADDR_HI(sc->bge_ldata.bge_status_block_paddr));
1501         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1502             BGE_ADDR_LO(sc->bge_ldata.bge_status_block_paddr));
1503         sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx = 0;
1504         sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx = 0;
1505
1506         /* Turn on host coalescing state machine */
1507         CSR_WRITE_4(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
1508
1509         /* Turn on RX BD completion state machine and enable attentions */
1510         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1511             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1512
1513         /* Turn on RX list placement state machine */
1514         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1515
1516         /* Turn on RX list selector state machine. */
1517         if (!BGE_IS_5705_PLUS(sc))
1518                 CSR_WRITE_4(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
1519
1520         /* Turn on DMA, clear stats */
1521         CSR_WRITE_4(sc, BGE_MAC_MODE, BGE_MACMODE_TXDMA_ENB|
1522             BGE_MACMODE_RXDMA_ENB|BGE_MACMODE_RX_STATS_CLEAR|
1523             BGE_MACMODE_TX_STATS_CLEAR|BGE_MACMODE_RX_STATS_ENB|
1524             BGE_MACMODE_TX_STATS_ENB|BGE_MACMODE_FRMHDR_DMA_ENB|
1525             ((sc->bge_flags & BGE_FLAG_TBI) ?
1526              BGE_PORTMODE_TBI : BGE_PORTMODE_MII));
1527
1528         /* Set misc. local control, enable interrupts on attentions */
1529         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1530
1531 #ifdef notdef
1532         /* Assert GPIO pins for PHY reset */
1533         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1534             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1535         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1536             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1537 #endif
1538
1539         /* Turn on DMA completion state machine */
1540         if (!BGE_IS_5705_PLUS(sc))
1541                 CSR_WRITE_4(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
1542
1543         /* Turn on write DMA state machine */
1544         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1545         if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
1546             sc->bge_asicrev == BGE_ASICREV_BCM5787)
1547                 val |= (1 << 29);       /* Enable host coalescing bug fix. */
1548         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1549         DELAY(40);
1550
1551         /* Turn on read DMA state machine */
1552         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1553         if (sc->bge_flags & BGE_FLAG_PCIE)
1554                 val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1555         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1556         DELAY(40);
1557
1558         /* Turn on RX data completion state machine */
1559         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1560
1561         /* Turn on RX BD initiator state machine */
1562         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1563
1564         /* Turn on RX data and RX BD initiator state machine */
1565         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1566
1567         /* Turn on Mbuf cluster free state machine */
1568         if (!BGE_IS_5705_PLUS(sc))
1569                 CSR_WRITE_4(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
1570
1571         /* Turn on send BD completion state machine */
1572         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1573
1574         /* Turn on send data completion state machine */
1575         CSR_WRITE_4(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
1576
1577         /* Turn on send data initiator state machine */
1578         CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1579
1580         /* Turn on send BD initiator state machine */
1581         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1582
1583         /* Turn on send BD selector state machine */
1584         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1585
1586         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1587         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1588             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1589
1590         /* ack/clear link change events */
1591         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1592             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1593             BGE_MACSTAT_LINK_CHANGED);
1594         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1595
1596         /* Enable PHY auto polling (for MII/GMII only) */
1597         if (sc->bge_flags & BGE_FLAG_TBI) {
1598                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1599         } else {
1600                 BGE_SETBIT(sc, BGE_MI_MODE, BGE_MIMODE_AUTOPOLL|10<<16);
1601                 if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
1602                     sc->bge_chipid != BGE_CHIPID_BCM5700_B2) {
1603                         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB,
1604                             BGE_EVTENB_MI_INTERRUPT);
1605                 }
1606         }
1607
1608         /*
1609          * Clear any pending link state attention.
1610          * Otherwise some link state change events may be lost until attention
1611          * is cleared by bge_intr() -> bge_softc.bge_link_upd() sequence.
1612          * It's not necessary on newer BCM chips - perhaps enabling link
1613          * state change attentions implies clearing pending attention.
1614          */
1615         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1616             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1617             BGE_MACSTAT_LINK_CHANGED);
1618
1619         /* Enable link state change attentions. */
1620         BGE_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1621
1622         return(0);
1623 }
1624
1625 /*
1626  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1627  * against our list and return its name if we find a match. Note
1628  * that since the Broadcom controller contains VPD support, we
1629  * can get the device name string from the controller itself instead
1630  * of the compiled-in string. This is a little slow, but it guarantees
1631  * we'll always announce the right product name.
1632  */
1633 static int
1634 bge_probe(device_t dev)
1635 {
1636         const struct bge_type *t;
1637         uint16_t product, vendor;
1638
1639         product = pci_get_device(dev);
1640         vendor = pci_get_vendor(dev);
1641
1642         for (t = bge_devs; t->bge_name != NULL; t++) {
1643                 if (vendor == t->bge_vid && product == t->bge_did)
1644                         break;
1645         }
1646         if (t->bge_name == NULL)
1647                 return(ENXIO);
1648
1649         device_set_desc(dev, t->bge_name);
1650         if (pci_get_subvendor(dev) == PCI_VENDOR_DELL) {
1651                 struct bge_softc *sc = device_get_softc(dev);
1652                 sc->bge_flags |= BGE_FLAG_NO_3LED;
1653         }
1654         return(0);
1655 }
1656
1657 static int
1658 bge_attach(device_t dev)
1659 {
1660         struct ifnet *ifp;
1661         struct bge_softc *sc;
1662         uint32_t hwcfg = 0;
1663         uint32_t mac_addr = 0;
1664         int error = 0, rid;
1665         uint8_t ether_addr[ETHER_ADDR_LEN];
1666
1667         sc = device_get_softc(dev);
1668         sc->bge_dev = dev;
1669         callout_init(&sc->bge_stat_timer);
1670         lwkt_serialize_init(&sc->bge_jslot_serializer);
1671
1672         /*
1673          * Map control/status registers.
1674          */
1675         pci_enable_busmaster(dev);
1676
1677         rid = BGE_PCI_BAR0;
1678         sc->bge_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1679             RF_ACTIVE);
1680
1681         if (sc->bge_res == NULL) {
1682                 device_printf(dev, "couldn't map memory\n");
1683                 return ENXIO;
1684         }
1685
1686         sc->bge_btag = rman_get_bustag(sc->bge_res);
1687         sc->bge_bhandle = rman_get_bushandle(sc->bge_res);
1688
1689         /* Save various chip information */
1690         sc->bge_chipid =
1691             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) &
1692             BGE_PCIMISCCTL_ASICREV;
1693         sc->bge_asicrev = BGE_ASICREV(sc->bge_chipid);
1694         sc->bge_chiprev = BGE_CHIPREV(sc->bge_chipid);
1695
1696         /* Save chipset family. */
1697         switch (sc->bge_asicrev) {
1698         case BGE_ASICREV_BCM5700:
1699         case BGE_ASICREV_BCM5701:
1700         case BGE_ASICREV_BCM5703:
1701         case BGE_ASICREV_BCM5704:
1702                 sc->bge_flags |= BGE_FLAG_5700_FAMILY | BGE_FLAG_JUMBO;
1703                 break;
1704
1705         case BGE_ASICREV_BCM5714_A0:
1706         case BGE_ASICREV_BCM5780:
1707         case BGE_ASICREV_BCM5714:
1708                 sc->bge_flags |= BGE_FLAG_5714_FAMILY;
1709                 /* Fall through */
1710
1711         case BGE_ASICREV_BCM5750:
1712         case BGE_ASICREV_BCM5752:
1713         case BGE_ASICREV_BCM5755:
1714         case BGE_ASICREV_BCM5787:
1715                 sc->bge_flags |= BGE_FLAG_575X_PLUS;
1716                 /* Fall through */
1717
1718         case BGE_ASICREV_BCM5705:
1719                 sc->bge_flags |= BGE_FLAG_5705_PLUS;
1720                 break;
1721         }
1722
1723         /*
1724          * Set various quirk flags.
1725          */
1726
1727         sc->bge_flags |= BGE_FLAG_ETH_WIRESPEED;
1728         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 ||
1729             (sc->bge_asicrev == BGE_ASICREV_BCM5705 &&
1730              (sc->bge_chipid != BGE_CHIPID_BCM5705_A0 &&
1731               sc->bge_chipid != BGE_CHIPID_BCM5705_A1)) ||
1732             sc->bge_asicrev == BGE_ASICREV_BCM5906)
1733                 sc->bge_flags &= ~BGE_FLAG_ETH_WIRESPEED;
1734
1735         if (sc->bge_chipid == BGE_CHIPID_BCM5701_A0 ||
1736             sc->bge_chipid == BGE_CHIPID_BCM5701_B0)
1737                 sc->bge_flags |= BGE_FLAG_CRC_BUG;
1738
1739         if (sc->bge_chiprev == BGE_CHIPREV_5703_AX ||
1740             sc->bge_chiprev == BGE_CHIPREV_5704_AX)
1741                 sc->bge_flags |= BGE_FLAG_ADC_BUG;
1742
1743         if (sc->bge_chipid == BGE_CHIPID_BCM5704_A0)
1744                 sc->bge_flags |= BGE_FLAG_5704_A0_BUG;
1745
1746         if (BGE_IS_5705_PLUS(sc)) {
1747                 if (sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
1748                     sc->bge_asicrev == BGE_ASICREV_BCM5787) {
1749                         uint32_t product = pci_get_device(dev);
1750
1751                         if (product != PCI_PRODUCT_BROADCOM_BCM5722 &&
1752                             product != PCI_PRODUCT_BROADCOM_BCM5756)
1753                                 sc->bge_flags |= BGE_FLAG_JITTER_BUG;
1754                         if (product == PCI_PRODUCT_BROADCOM_BCM5755M)
1755                                 sc->bge_flags |= BGE_FLAG_ADJUST_TRIM;
1756                 } else if (sc->bge_asicrev != BGE_ASICREV_BCM5906) {
1757                         sc->bge_flags |= BGE_FLAG_BER_BUG;
1758                 }
1759         }
1760
1761         /* Allocate interrupt */
1762         rid = 0;
1763
1764         sc->bge_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1765             RF_SHAREABLE | RF_ACTIVE);
1766
1767         if (sc->bge_irq == NULL) {
1768                 device_printf(dev, "couldn't map interrupt\n");
1769                 error = ENXIO;
1770                 goto fail;
1771         }
1772
1773         /*
1774          * Check if this is a PCI-X or PCI Express device.
1775          */
1776         if (BGE_IS_5705_PLUS(sc)) {
1777                 sc->bge_expr_ptr = pci_get_pciecap_ptr(dev);
1778
1779                 if (sc->bge_expr_ptr != 0) {
1780                         sc->bge_flags |= BGE_FLAG_PCIE;
1781                         bge_set_max_readrq(sc);
1782                 }
1783         } else {
1784                 /*
1785                  * Check if the device is in PCI-X Mode.
1786                  * (This bit is not valid on PCI Express controllers.)
1787                  */
1788                 if ((pci_read_config(sc->bge_dev, BGE_PCI_PCISTATE, 4) &
1789                     BGE_PCISTATE_PCI_BUSMODE) == 0)
1790                         sc->bge_flags |= BGE_FLAG_PCIX;
1791         }
1792
1793         device_printf(dev, "CHIP ID 0x%08x; "
1794                       "ASIC REV 0x%02x; CHIP REV 0x%02x; %s\n",
1795                       sc->bge_chipid, sc->bge_asicrev, sc->bge_chiprev,
1796                       (sc->bge_flags & BGE_FLAG_PCIX) ? "PCI-X"
1797                       : ((sc->bge_flags & BGE_FLAG_PCIE) ?
1798                         "PCI-E" : "PCI"));
1799
1800         ifp = &sc->arpcom.ac_if;
1801         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1802
1803         /* Try to reset the chip. */
1804         bge_reset(sc);
1805
1806         if (bge_chipinit(sc)) {
1807                 device_printf(dev, "chip initialization failed\n");
1808                 error = ENXIO;
1809                 goto fail;
1810         }
1811
1812         /*
1813          * Get station address from the EEPROM.
1814          */
1815         mac_addr = bge_readmem_ind(sc, 0x0c14);
1816         if ((mac_addr >> 16) == 0x484b) {
1817                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
1818                 ether_addr[1] = (uint8_t)mac_addr;
1819                 mac_addr = bge_readmem_ind(sc, 0x0c18);
1820                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
1821                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
1822                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
1823                 ether_addr[5] = (uint8_t)mac_addr;
1824         } else if (bge_read_eeprom(sc, ether_addr,
1825             BGE_EE_MAC_OFFSET + 2, ETHER_ADDR_LEN)) {
1826                 device_printf(dev, "failed to read station address\n");
1827                 error = ENXIO;
1828                 goto fail;
1829         }
1830
1831         /* 5705/5750 limits RX return ring to 512 entries. */
1832         if (BGE_IS_5705_PLUS(sc))
1833                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT_5705;
1834         else
1835                 sc->bge_return_ring_cnt = BGE_RETURN_RING_CNT;
1836
1837         error = bge_dma_alloc(sc);
1838         if (error)
1839                 goto fail;
1840
1841         /* Set default tuneable values. */
1842         sc->bge_stat_ticks = BGE_TICKS_PER_SEC;
1843         sc->bge_rx_coal_ticks = bge_rx_coal_ticks;
1844         sc->bge_tx_coal_ticks = bge_tx_coal_ticks;
1845         sc->bge_rx_max_coal_bds = bge_rx_max_coal_bds;
1846         sc->bge_tx_max_coal_bds = bge_tx_max_coal_bds;
1847
1848         /* Set up ifnet structure */
1849         ifp->if_softc = sc;
1850         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1851         ifp->if_ioctl = bge_ioctl;
1852         ifp->if_start = bge_start;
1853 #ifdef DEVICE_POLLING
1854         ifp->if_poll = bge_poll;
1855 #endif
1856         ifp->if_watchdog = bge_watchdog;
1857         ifp->if_init = bge_init;
1858         ifp->if_mtu = ETHERMTU;
1859         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1860         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
1861         ifq_set_ready(&ifp->if_snd);
1862
1863         /*
1864          * 5700 B0 chips do not support checksumming correctly due
1865          * to hardware bugs.
1866          */
1867         if (sc->bge_chipid != BGE_CHIPID_BCM5700_B0) {
1868                 ifp->if_capabilities |= IFCAP_HWCSUM;
1869                 ifp->if_hwassist = BGE_CSUM_FEATURES;
1870         }
1871         ifp->if_capenable = ifp->if_capabilities;
1872
1873         /*
1874          * Figure out what sort of media we have by checking the
1875          * hardware config word in the first 32k of NIC internal memory,
1876          * or fall back to examining the EEPROM if necessary.
1877          * Note: on some BCM5700 cards, this value appears to be unset.
1878          * If that's the case, we have to rely on identifying the NIC
1879          * by its PCI subsystem ID, as we do below for the SysKonnect
1880          * SK-9D41.
1881          */
1882         if (bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER)
1883                 hwcfg = bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
1884         else {
1885                 if (bge_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
1886                                     sizeof(hwcfg))) {
1887                         device_printf(dev, "failed to read EEPROM\n");
1888                         error = ENXIO;
1889                         goto fail;
1890                 }
1891                 hwcfg = ntohl(hwcfg);
1892         }
1893
1894         if ((hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
1895                 sc->bge_flags |= BGE_FLAG_TBI;
1896
1897         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
1898         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41)
1899                 sc->bge_flags |= BGE_FLAG_TBI;
1900
1901         if (sc->bge_flags & BGE_FLAG_TBI) {
1902                 ifmedia_init(&sc->bge_ifmedia, IFM_IMASK,
1903                     bge_ifmedia_upd, bge_ifmedia_sts);
1904                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
1905                 ifmedia_add(&sc->bge_ifmedia,
1906                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
1907                 ifmedia_add(&sc->bge_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
1908                 ifmedia_set(&sc->bge_ifmedia, IFM_ETHER|IFM_AUTO);
1909                 sc->bge_ifmedia.ifm_media = sc->bge_ifmedia.ifm_cur->ifm_media;
1910         } else {
1911                 /*
1912                  * Do transceiver setup.
1913                  */
1914                 if (mii_phy_probe(dev, &sc->bge_miibus,
1915                     bge_ifmedia_upd, bge_ifmedia_sts)) {
1916                         device_printf(dev, "MII without any PHY!\n");
1917                         error = ENXIO;
1918                         goto fail;
1919                 }
1920         }
1921
1922         /*
1923          * When using the BCM5701 in PCI-X mode, data corruption has
1924          * been observed in the first few bytes of some received packets.
1925          * Aligning the packet buffer in memory eliminates the corruption.
1926          * Unfortunately, this misaligns the packet payloads.  On platforms
1927          * which do not support unaligned accesses, we will realign the
1928          * payloads by copying the received packets.
1929          */
1930         if (sc->bge_asicrev == BGE_ASICREV_BCM5701 &&
1931             (sc->bge_flags & BGE_FLAG_PCIX))
1932                 sc->bge_flags |= BGE_FLAG_RX_ALIGNBUG;
1933
1934         if (sc->bge_asicrev == BGE_ASICREV_BCM5700 &&
1935             sc->bge_chipid != BGE_CHIPID_BCM5700_B2) {
1936                 sc->bge_link_upd = bge_bcm5700_link_upd;
1937                 sc->bge_link_chg = BGE_MACSTAT_MI_INTERRUPT;
1938         } else if (sc->bge_flags & BGE_FLAG_TBI) {
1939                 sc->bge_link_upd = bge_tbi_link_upd;
1940                 sc->bge_link_chg = BGE_MACSTAT_LINK_CHANGED;
1941         } else {
1942                 sc->bge_link_upd = bge_copper_link_upd;
1943                 sc->bge_link_chg = BGE_MACSTAT_LINK_CHANGED;
1944         }
1945
1946         /*
1947          * Create sysctl nodes.
1948          */
1949         sysctl_ctx_init(&sc->bge_sysctl_ctx);
1950         sc->bge_sysctl_tree = SYSCTL_ADD_NODE(&sc->bge_sysctl_ctx,
1951                                               SYSCTL_STATIC_CHILDREN(_hw),
1952                                               OID_AUTO,
1953                                               device_get_nameunit(dev),
1954                                               CTLFLAG_RD, 0, "");
1955         if (sc->bge_sysctl_tree == NULL) {
1956                 device_printf(dev, "can't add sysctl node\n");
1957                 error = ENXIO;
1958                 goto fail;
1959         }
1960
1961         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
1962                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
1963                         OID_AUTO, "rx_coal_ticks",
1964                         CTLTYPE_INT | CTLFLAG_RW,
1965                         sc, 0, bge_sysctl_rx_coal_ticks, "I",
1966                         "Receive coalescing ticks (usec).");
1967         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
1968                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
1969                         OID_AUTO, "tx_coal_ticks",
1970                         CTLTYPE_INT | CTLFLAG_RW,
1971                         sc, 0, bge_sysctl_tx_coal_ticks, "I",
1972                         "Transmit coalescing ticks (usec).");
1973         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
1974                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
1975                         OID_AUTO, "rx_max_coal_bds",
1976                         CTLTYPE_INT | CTLFLAG_RW,
1977                         sc, 0, bge_sysctl_rx_max_coal_bds, "I",
1978                         "Receive max coalesced BD count.");
1979         SYSCTL_ADD_PROC(&sc->bge_sysctl_ctx,
1980                         SYSCTL_CHILDREN(sc->bge_sysctl_tree),
1981                         OID_AUTO, "tx_max_coal_bds",
1982                         CTLTYPE_INT | CTLFLAG_RW,
1983                         sc, 0, bge_sysctl_tx_max_coal_bds, "I",
1984                         "Transmit max coalesced BD count.");
1985
1986         /*
1987          * Call MI attach routine.
1988          */
1989         ether_ifattach(ifp, ether_addr, NULL);
1990
1991         error = bus_setup_intr(dev, sc->bge_irq, INTR_MPSAFE,
1992                                bge_intr, sc, &sc->bge_intrhand,
1993                                ifp->if_serializer);
1994         if (error) {
1995                 ether_ifdetach(ifp);
1996                 device_printf(dev, "couldn't set up irq\n");
1997                 goto fail;
1998         }
1999
2000         ifp->if_cpuid = ithread_cpuid(rman_get_start(sc->bge_irq));
2001         KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus);
2002
2003         return(0);
2004 fail:
2005         bge_detach(dev);
2006         return(error);
2007 }
2008
2009 static int
2010 bge_detach(device_t dev)
2011 {
2012         struct bge_softc *sc = device_get_softc(dev);
2013
2014         if (device_is_attached(dev)) {
2015                 struct ifnet *ifp = &sc->arpcom.ac_if;
2016
2017                 lwkt_serialize_enter(ifp->if_serializer);
2018                 bge_stop(sc);
2019                 bge_reset(sc);
2020                 bus_teardown_intr(dev, sc->bge_irq, sc->bge_intrhand);
2021                 lwkt_serialize_exit(ifp->if_serializer);
2022
2023                 ether_ifdetach(ifp);
2024         }
2025
2026         if (sc->bge_flags & BGE_FLAG_TBI)
2027                 ifmedia_removeall(&sc->bge_ifmedia);
2028         if (sc->bge_miibus)
2029                 device_delete_child(dev, sc->bge_miibus);
2030         bus_generic_detach(dev);
2031
2032         if (sc->bge_irq != NULL)
2033                 bus_release_resource(dev, SYS_RES_IRQ, 0, sc->bge_irq);
2034
2035         if (sc->bge_res != NULL)
2036                 bus_release_resource(dev, SYS_RES_MEMORY,
2037                     BGE_PCI_BAR0, sc->bge_res);
2038
2039         if (sc->bge_sysctl_tree != NULL)
2040                 sysctl_ctx_free(&sc->bge_sysctl_ctx);
2041
2042         bge_dma_free(sc);
2043
2044         return 0;
2045 }
2046
2047 static void
2048 bge_reset(struct bge_softc *sc)
2049 {
2050         device_t dev;
2051         uint32_t cachesize, command, pcistate, reset;
2052         void (*write_op)(struct bge_softc *, uint32_t, uint32_t);
2053         int i, val = 0;
2054
2055         dev = sc->bge_dev;
2056
2057         if (BGE_IS_575X_PLUS(sc) && !BGE_IS_5714_FAMILY(sc)) {
2058                 if (sc->bge_flags & BGE_FLAG_PCIE)
2059                         write_op = bge_writemem_direct;
2060                 else
2061                         write_op = bge_writemem_ind;
2062         } else {
2063                 write_op = bge_writereg_ind;
2064         }
2065
2066         /* Save some important PCI state. */
2067         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2068         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2069         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2070
2071         pci_write_config(dev, BGE_PCI_MISC_CTL,
2072             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2073             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW, 4);
2074
2075         /* Disable fastboot on controllers that support it. */
2076         if (sc->bge_asicrev == BGE_ASICREV_BCM5752 ||
2077             sc->bge_asicrev == BGE_ASICREV_BCM5755 ||
2078             sc->bge_asicrev == BGE_ASICREV_BCM5787) {
2079                 if (bootverbose)
2080                         if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2081                 CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2082         }
2083
2084         /*
2085          * Write the magic number to SRAM at offset 0xB50.
2086          * When firmware finishes its initialization it will
2087          * write ~BGE_MAGIC_NUMBER to the same location.
2088          */
2089         bge_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2090
2091         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2092
2093         /* XXX: Broadcom Linux driver. */
2094         if (sc->bge_flags & BGE_FLAG_PCIE) {
2095                 if (CSR_READ_4(sc, 0x7e2c) == 0x60)     /* PCIE 1.0 */
2096                         CSR_WRITE_4(sc, 0x7e2c, 0x20);
2097                 if (sc->bge_chipid != BGE_CHIPID_BCM5750_A0) {
2098                         /* Prevent PCIE link training during global reset */
2099                         CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2100                         reset |= (1<<29);
2101                 }
2102         }
2103
2104         /*
2105          * Set GPHY Power Down Override to leave GPHY
2106          * powered up in D0 uninitialized.
2107          */
2108         if (BGE_IS_5705_PLUS(sc))
2109                 reset |= 0x04000000;
2110
2111         /* Issue global reset */
2112         write_op(sc, BGE_MISC_CFG, reset);
2113
2114         DELAY(1000);
2115
2116         /* XXX: Broadcom Linux driver. */
2117         if (sc->bge_flags & BGE_FLAG_PCIE) {
2118                 if (sc->bge_chipid == BGE_CHIPID_BCM5750_A0) {
2119                         uint32_t v;
2120
2121                         DELAY(500000); /* wait for link training to complete */
2122                         v = pci_read_config(dev, 0xc4, 4);
2123                         pci_write_config(dev, 0xc4, v | (1<<15), 4);
2124                 }
2125                 /*
2126                  * Set PCIE max payload size to 128 bytes and
2127                  * clear error status.
2128                  */
2129                 pci_write_config(dev, 0xd8, 0xf5000, 4);
2130         }
2131
2132         /* Reset some of the PCI state that got zapped by reset */
2133         pci_write_config(dev, BGE_PCI_MISC_CTL,
2134             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2135             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW, 4);
2136         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2137         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2138         write_op(sc, BGE_MISC_CFG, (65 << 1));
2139
2140         /* Enable memory arbiter. */
2141         if (BGE_IS_5714_FAMILY(sc)) {
2142                 uint32_t val;
2143
2144                 val = CSR_READ_4(sc, BGE_MARB_MODE);
2145                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE | val);
2146         } else {
2147                 CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2148         }
2149
2150         /*
2151          * Poll until we see the 1's complement of the magic number.
2152          * This indicates that the firmware initialization
2153          * is complete.
2154          */
2155         for (i = 0; i < BGE_TIMEOUT; i++) {
2156                 val = bge_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2157                 if (val == ~BGE_MAGIC_NUMBER)
2158                         break;
2159                 DELAY(10);
2160         }
2161
2162         if (i == BGE_TIMEOUT) {
2163                 if_printf(&sc->arpcom.ac_if, "firmware handshake timed out,"
2164                           "found 0x%08x\n", val);
2165                 return;
2166         }
2167
2168         /*
2169          * XXX Wait for the value of the PCISTATE register to
2170          * return to its original pre-reset state. This is a
2171          * fairly good indicator of reset completion. If we don't
2172          * wait for the reset to fully complete, trying to read
2173          * from the device's non-PCI registers may yield garbage
2174          * results.
2175          */
2176         for (i = 0; i < BGE_TIMEOUT; i++) {
2177                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2178                         break;
2179                 DELAY(10);
2180         }
2181
2182         if (sc->bge_flags & BGE_FLAG_PCIE) {
2183                 reset = bge_readmem_ind(sc, 0x7c00);
2184                 bge_writemem_ind(sc, 0x7c00, reset | (1 << 25));
2185         }
2186
2187         /* Fix up byte swapping */
2188         CSR_WRITE_4(sc, BGE_MODE_CTL, BGE_DMA_SWAP_OPTIONS |
2189             BGE_MODECTL_BYTESWAP_DATA);
2190
2191         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2192
2193         /*
2194          * The 5704 in TBI mode apparently needs some special
2195          * adjustment to insure the SERDES drive level is set
2196          * to 1.2V.
2197          */
2198         if (sc->bge_asicrev == BGE_ASICREV_BCM5704 &&
2199             (sc->bge_flags & BGE_FLAG_TBI)) {
2200                 uint32_t serdescfg;
2201
2202                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2203                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2204                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2205         }
2206
2207         /* XXX: Broadcom Linux driver. */
2208         if ((sc->bge_flags & BGE_FLAG_PCIE) &&
2209             sc->bge_chipid != BGE_CHIPID_BCM5750_A0) {
2210                 uint32_t v;
2211
2212                 v = CSR_READ_4(sc, 0x7c00);
2213                 CSR_WRITE_4(sc, 0x7c00, v | (1<<25));
2214         }
2215
2216         DELAY(10000);
2217 }
2218
2219 /*
2220  * Frame reception handling. This is called if there's a frame
2221  * on the receive return list.
2222  *
2223  * Note: we have to be able to handle two possibilities here:
2224  * 1) the frame is from the jumbo recieve ring
2225  * 2) the frame is from the standard receive ring
2226  */
2227
2228 static void
2229 bge_rxeof(struct bge_softc *sc)
2230 {
2231         struct ifnet *ifp;
2232         int stdcnt = 0, jumbocnt = 0;
2233         struct mbuf_chain chain[MAXCPU];
2234
2235         if (sc->bge_rx_saved_considx ==
2236             sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx)
2237                 return;
2238
2239         ether_input_chain_init(chain);
2240
2241         ifp = &sc->arpcom.ac_if;
2242
2243         bus_dmamap_sync(sc->bge_cdata.bge_rx_return_ring_tag,
2244                         sc->bge_cdata.bge_rx_return_ring_map,
2245                         BUS_DMASYNC_POSTREAD);
2246         bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
2247                         sc->bge_cdata.bge_rx_std_ring_map,
2248                         BUS_DMASYNC_POSTREAD);
2249         if (BGE_IS_JUMBO_CAPABLE(sc)) {
2250                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
2251                                 sc->bge_cdata.bge_rx_jumbo_ring_map,
2252                                 BUS_DMASYNC_POSTREAD);
2253         }
2254
2255         while (sc->bge_rx_saved_considx !=
2256                sc->bge_ldata.bge_status_block->bge_idx[0].bge_rx_prod_idx) {
2257                 struct bge_rx_bd        *cur_rx;
2258                 uint32_t                rxidx;
2259                 struct mbuf             *m = NULL;
2260                 uint16_t                vlan_tag = 0;
2261                 int                     have_tag = 0;
2262
2263                 cur_rx =
2264             &sc->bge_ldata.bge_rx_return_ring[sc->bge_rx_saved_considx];
2265
2266                 rxidx = cur_rx->bge_idx;
2267                 BGE_INC(sc->bge_rx_saved_considx, sc->bge_return_ring_cnt);
2268                 logif(rx_pkt);
2269
2270                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2271                         have_tag = 1;
2272                         vlan_tag = cur_rx->bge_vlan_tag;
2273                 }
2274
2275                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
2276                         BGE_INC(sc->bge_jumbo, BGE_JUMBO_RX_RING_CNT);
2277                         m = sc->bge_cdata.bge_rx_jumbo_chain[rxidx];
2278                         sc->bge_cdata.bge_rx_jumbo_chain[rxidx] = NULL;
2279                         jumbocnt++;
2280                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2281                                 ifp->if_ierrors++;
2282                                 bge_newbuf_jumbo(sc, sc->bge_jumbo, m);
2283                                 continue;
2284                         }
2285                         if (bge_newbuf_jumbo(sc,
2286                             sc->bge_jumbo, NULL) == ENOBUFS) {
2287                                 ifp->if_ierrors++;
2288                                 bge_newbuf_jumbo(sc, sc->bge_jumbo, m);
2289                                 continue;
2290                         }
2291                 } else {
2292                         BGE_INC(sc->bge_std, BGE_STD_RX_RING_CNT);
2293                         bus_dmamap_sync(sc->bge_cdata.bge_mtag,
2294                                         sc->bge_cdata.bge_rx_std_dmamap[rxidx],
2295                                         BUS_DMASYNC_POSTREAD);
2296                         bus_dmamap_unload(sc->bge_cdata.bge_mtag,
2297                                 sc->bge_cdata.bge_rx_std_dmamap[rxidx]);
2298                         m = sc->bge_cdata.bge_rx_std_chain[rxidx];
2299                         sc->bge_cdata.bge_rx_std_chain[rxidx] = NULL;
2300                         stdcnt++;
2301                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2302                                 ifp->if_ierrors++;
2303                                 bge_newbuf_std(sc, sc->bge_std, m);
2304                                 continue;
2305                         }
2306                         if (bge_newbuf_std(sc, sc->bge_std,
2307                             NULL) == ENOBUFS) {
2308                                 ifp->if_ierrors++;
2309                                 bge_newbuf_std(sc, sc->bge_std, m);
2310                                 continue;
2311                         }
2312                 }
2313
2314                 ifp->if_ipackets++;
2315 #ifndef __i386__
2316                 /*
2317                  * The i386 allows unaligned accesses, but for other
2318                  * platforms we must make sure the payload is aligned.
2319                  */
2320                 if (sc->bge_flags & BGE_FLAG_RX_ALIGNBUG) {
2321                         bcopy(m->m_data, m->m_data + ETHER_ALIGN,
2322                             cur_rx->bge_len);
2323                         m->m_data += ETHER_ALIGN;
2324                 }
2325 #endif
2326                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2327                 m->m_pkthdr.rcvif = ifp;
2328
2329                 if (ifp->if_capenable & IFCAP_RXCSUM) {
2330                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2331                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2332                                 if ((cur_rx->bge_ip_csum ^ 0xffff) == 0)
2333                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2334                         }
2335                         if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) &&
2336                             m->m_pkthdr.len >= BGE_MIN_FRAME) {
2337                                 m->m_pkthdr.csum_data =
2338                                         cur_rx->bge_tcp_udp_csum;
2339                                 m->m_pkthdr.csum_flags |=
2340                                         CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
2341                         }
2342                 }
2343
2344                 /*
2345                  * If we received a packet with a vlan tag, pass it
2346                  * to vlan_input() instead of ether_input().
2347                  */
2348                 if (have_tag) {
2349                         m->m_flags |= M_VLANTAG;
2350                         m->m_pkthdr.ether_vlantag = vlan_tag;
2351                         have_tag = vlan_tag = 0;
2352                 }
2353                 ether_input_chain(ifp, m, chain);
2354         }
2355
2356         ether_input_dispatch(chain);
2357
2358         if (stdcnt > 0) {
2359                 bus_dmamap_sync(sc->bge_cdata.bge_rx_std_ring_tag,
2360                                 sc->bge_cdata.bge_rx_std_ring_map,
2361                                 BUS_DMASYNC_PREWRITE);
2362         }
2363
2364         if (BGE_IS_JUMBO_CAPABLE(sc) && jumbocnt > 0) {
2365                 bus_dmamap_sync(sc->bge_cdata.bge_rx_jumbo_ring_tag,
2366                                 sc->bge_cdata.bge_rx_jumbo_ring_map,
2367                                 BUS_DMASYNC_PREWRITE);
2368         }
2369
2370         CSR_WRITE_4(sc, BGE_MBX_RX_CONS0_LO, sc->bge_rx_saved_considx);
2371         if (stdcnt)
2372                 CSR_WRITE_4(sc, BGE_MBX_RX_STD_PROD_LO, sc->bge_std);
2373         if (jumbocnt)
2374                 CSR_WRITE_4(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bge_jumbo);
2375 }
2376
2377 static void
2378 bge_txeof(struct bge_softc *sc)
2379 {
2380         struct bge_tx_bd *cur_tx = NULL;
2381         struct ifnet *ifp;
2382
2383         if (sc->bge_tx_saved_considx ==
2384             sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx)
2385                 return;
2386
2387         ifp = &sc->arpcom.ac_if;
2388
2389         bus_dmamap_sync(sc->bge_cdata.bge_tx_ring_tag,
2390                         sc->bge_cdata.bge_tx_ring_map,
2391                         BUS_DMASYNC_POSTREAD);
2392
2393         /*
2394          * Go through our tx ring and free mbufs for those
2395          * frames that have been sent.
2396          */
2397         while (sc->bge_tx_saved_considx !=
2398                sc->bge_ldata.bge_status_block->bge_idx[0].bge_tx_cons_idx) {
2399                 uint32_t idx = 0;
2400
2401                 idx = sc->bge_tx_saved_considx;
2402                 cur_tx = &sc->bge_ldata.bge_tx_ring[idx];
2403                 if (cur_tx->bge_flags & BGE_TXBDFLAG_END)
2404                         ifp->if_opackets++;
2405                 if (sc->bge_cdata.bge_tx_chain[idx] != NULL) {
2406                         bus_dmamap_sync(sc->bge_cdata.bge_mtag,
2407                                         sc->bge_cdata.bge_tx_dmamap[idx],
2408                                         BUS_DMASYNC_POSTWRITE);
2409                         bus_dmamap_unload(sc->bge_cdata.bge_mtag,
2410                             sc->bge_cdata.bge_tx_dmamap[idx]);
2411                         m_freem(sc->bge_cdata.bge_tx_chain[idx]);
2412                         sc->bge_cdata.bge_tx_chain[idx] = NULL;
2413                 }
2414                 sc->bge_txcnt--;
2415                 BGE_INC(sc->bge_tx_saved_considx, BGE_TX_RING_CNT);
2416                 logif(tx_pkt);
2417         }
2418
2419         if (cur_tx != NULL &&
2420             (BGE_TX_RING_CNT - sc->bge_txcnt) >=
2421             (BGE_NSEG_RSVD + BGE_NSEG_SPARE))
2422                 ifp->if_flags &= ~IFF_OACTIVE;
2423
2424         if (sc->bge_txcnt == 0)
2425                 ifp->if_timer = 0;
2426
2427         if (!ifq_is_empty(&ifp->if_snd))
2428                 if_devstart(ifp);
2429 }
2430
2431 #ifdef DEVICE_POLLING
2432
2433 static void
2434 bge_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2435 {
2436         struct bge_softc *sc = ifp->if_softc;
2437         uint32_t status;
2438
2439         switch(cmd) {
2440         case POLL_REGISTER:
2441                 bge_disable_intr(sc);
2442                 break;
2443         case POLL_DEREGISTER:
2444                 bge_enable_intr(sc);
2445                 break;
2446         case POLL_AND_CHECK_STATUS:
2447                 bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
2448                                 sc->bge_cdata.bge_status_map,
2449                                 BUS_DMASYNC_POSTREAD);
2450
2451                 /*
2452                  * Process link state changes.
2453                  */
2454                 status = CSR_READ_4(sc, BGE_MAC_STS);
2455                 if ((status & sc->bge_link_chg) || sc->bge_link_evt) {
2456                         sc->bge_link_evt = 0;
2457                         sc->bge_link_upd(sc, status);
2458                 }
2459                 /* fall through */
2460         case POLL_ONLY:
2461                 if (ifp->if_flags & IFF_RUNNING) {
2462                         bge_rxeof(sc);
2463                         bge_txeof(sc);
2464                 }
2465                 break;
2466         }
2467 }
2468
2469 #endif
2470
2471 static void
2472 bge_intr(void *xsc)
2473 {
2474         struct bge_softc *sc = xsc;
2475         struct ifnet *ifp = &sc->arpcom.ac_if;
2476         uint32_t status;
2477
2478         logif(intr);
2479
2480         /*
2481          * Ack the interrupt by writing something to BGE_MBX_IRQ0_LO.  Don't
2482          * disable interrupts by writing nonzero like we used to, since with
2483          * our current organization this just gives complications and
2484          * pessimizations for re-enabling interrupts.  We used to have races
2485          * instead of the necessary complications.  Disabling interrupts
2486          * would just reduce the chance of a status update while we are
2487          * running (by switching to the interrupt-mode coalescence
2488          * parameters), but this chance is already very low so it is more
2489          * efficient to get another interrupt than prevent it.
2490          *
2491          * We do the ack first to ensure another interrupt if there is a
2492          * status update after the ack.  We don't check for the status
2493          * changing later because it is more efficient to get another
2494          * interrupt than prevent it, not quite as above (not checking is
2495          * a smaller optimization than not toggling the interrupt enable,
2496          * since checking doesn't involve PCI accesses and toggling require
2497          * the status check).  So toggling would probably be a pessimization
2498          * even with MSI.  It would only be needed for using a task queue.
2499          */
2500         CSR_WRITE_4(sc, BGE_MBX_IRQ0_LO, 0);
2501
2502         bus_dmamap_sync(sc->bge_cdata.bge_status_tag,
2503                         sc->bge_cdata.bge_status_map,
2504                         BUS_DMASYNC_POSTREAD);
2505
2506         /*
2507          * Process link state changes.
2508          */
2509         status = CSR_READ_4(sc, BGE_MAC_STS);
2510         if ((status & sc->bge_link_chg) || sc->bge_link_evt) {
2511                 sc->bge_link_evt = 0;
2512                 sc->bge_link_upd(sc, status);
2513         }
2514
2515         if (ifp->if_flags & IFF_RUNNING) {
2516                 /* Check RX return ring producer/consumer */
2517                 bge_rxeof(sc);
2518
2519                 /* Check TX ring producer/consumer */
2520                 bge_txeof(sc);
2521         }
2522
2523         if (sc->bge_coal_chg)
2524                 bge_coal_change(sc);
2525 }
2526
2527 static void
2528 bge_tick(void *xsc)
2529 {
2530         struct bge_softc *sc = xsc;
2531         struct ifnet *ifp = &sc->arpcom.ac_if;
2532
2533         lwkt_serialize_enter(ifp->if_serializer);
2534
2535         if (BGE_IS_5705_PLUS(sc))
2536                 bge_stats_update_regs(sc);
2537         else
2538                 bge_stats_update(sc);
2539
2540         if (sc->bge_flags & BGE_FLAG_TBI) {
2541                 /*
2542                  * Since in TBI mode auto-polling can't be used we should poll
2543                  * link status manually. Here we register pending link event
2544                  * and trigger interrupt.
2545                  */
2546                 sc->bge_link_evt++;
2547                 BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
2548         } else if (!sc->bge_link) {
2549                 mii_tick(device_get_softc(sc->bge_miibus));
2550         }
2551
2552         callout_reset(&sc->bge_stat_timer, hz, bge_tick, sc);
2553
2554         lwkt_serialize_exit(ifp->if_serializer);
2555 }
2556
2557 static void
2558 bge_stats_update_regs(struct bge_softc *sc)
2559 {
2560         struct ifnet *ifp = &sc->arpcom.ac_if;
2561         struct bge_mac_stats_regs stats;
2562         uint32_t *s;
2563         int i;
2564
2565         s = (uint32_t *)&stats;
2566         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
2567                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
2568                 s++;
2569         }
2570
2571         ifp->if_collisions +=
2572            (stats.dot3StatsSingleCollisionFrames +
2573            stats.dot3StatsMultipleCollisionFrames +
2574            stats.dot3StatsExcessiveCollisions +
2575            stats.dot3StatsLateCollisions) -
2576            ifp->if_collisions;
2577 }
2578
2579 static void
2580 bge_stats_update(struct bge_softc *sc)
2581 {
2582         struct ifnet *ifp = &sc->arpcom.ac_if;
2583         bus_size_t stats;
2584
2585         stats = BGE_MEMWIN_START + BGE_STATS_BLOCK;
2586
2587 #define READ_STAT(sc, stats, stat)      \
2588         CSR_READ_4(sc, stats + offsetof(struct bge_stats, stat))
2589
2590         ifp->if_collisions +=
2591            (READ_STAT(sc, stats,
2592                 txstats.dot3StatsSingleCollisionFrames.bge_addr_lo) +
2593             READ_STAT(sc, stats,
2594                 txstats.dot3StatsMultipleCollisionFrames.bge_addr_lo) +
2595             READ_STAT(sc, stats,
2596                 txstats.dot3StatsExcessiveCollisions.bge_addr_lo) +
2597             READ_STAT(sc, stats,
2598                 txstats.dot3StatsLateCollisions.bge_addr_lo)) -
2599            ifp->if_collisions;
2600
2601 #undef READ_STAT
2602
2603 #ifdef notdef
2604         ifp->if_collisions +=
2605            (sc->bge_rdata->bge_info.bge_stats.dot3StatsSingleCollisionFrames +
2606            sc->bge_rdata->bge_info.bge_stats.dot3StatsMultipleCollisionFrames +
2607            sc->bge_rdata->bge_info.bge_stats.dot3StatsExcessiveCollisions +
2608            sc->bge_rdata->bge_info.bge_stats.dot3StatsLateCollisions) -
2609            ifp->if_collisions;
2610 #endif
2611 }
2612
2613 static __inline int
2614 bge_cksum_pad(struct mbuf *pkt)
2615 {
2616         struct mbuf *last = NULL;
2617         int padlen;
2618
2619         padlen = BGE_MIN_FRAME - pkt->m_pkthdr.len;
2620
2621         /* if there's only the packet-header and we can pad there, use it. */
2622         if (pkt->m_pkthdr.len == pkt->m_len &&
2623             M_TRAILINGSPACE(pkt) >= padlen) {
2624                 last = pkt;
2625         } else {
2626                 /*
2627                  * Walk packet chain to find last mbuf. We will either
2628                  * pad there, or append a new mbuf and pad it
2629                  * (thus perhaps avoiding the bcm5700 dma-min bug).
2630                  */
2631                 for (last = pkt; last->m_next != NULL; last = last->m_next)
2632                         ; /* EMPTY */
2633
2634                 /* `last' now points to last in chain. */
2635                 if (M_TRAILINGSPACE(last) < padlen) {
2636                         /* Allocate new empty mbuf, pad it.  Compact later. */
2637                         struct mbuf *n;
2638                         MGET(n, MB_DONTWAIT, MT_DATA);
2639                         if (n == NULL)
2640                                 return ENOBUFS;
2641                         n->m_len = 0;
2642                         last->m_next = n;
2643                         last = n;
2644                 }
2645         }
2646         KKASSERT(M_TRAILINGSPACE(last) >= padlen);
2647         KKASSERT(M_WRITABLE(last));
2648
2649         /* Now zero the pad area, to avoid the bge cksum-assist bug */
2650         bzero(mtod(last, char *) + last->m_len, padlen);
2651         last->m_len += padlen;
2652         pkt->m_pkthdr.len += padlen;
2653         return 0;
2654 }
2655
2656 /*
2657  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
2658  * pointers to descriptors.
2659  */
2660 static int
2661 bge_encap(struct bge_softc *sc, struct mbuf **m_head0, uint32_t *txidx)
2662 {
2663         struct bge_tx_bd *d = NULL;
2664         uint16_t csum_flags = 0;
2665         struct bge_dmamap_arg ctx;
2666         bus_dma_segment_t segs[BGE_NSEG_NEW];
2667         bus_dmamap_t map;
2668         int error, maxsegs, idx, i;
2669         struct mbuf *m_head = *m_head0;
2670
2671         if (m_head->m_pkthdr.csum_flags) {
2672                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
2673                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
2674                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2675                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
2676                 if (m_head->m_flags & M_LASTFRAG)
2677                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
2678                 else if (m_head->m_flags & M_FRAG)
2679                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
2680         }
2681
2682         idx = *txidx;
2683         map = sc->bge_cdata.bge_tx_dmamap[idx];
2684
2685         maxsegs = (BGE_TX_RING_CNT - sc->bge_txcnt) - BGE_NSEG_RSVD;
2686         KASSERT(maxsegs >= BGE_NSEG_SPARE,
2687                 ("not enough segments %d\n", maxsegs));
2688
2689         if (maxsegs > BGE_NSEG_NEW)
2690                 maxsegs = BGE_NSEG_NEW;
2691
2692         /*
2693          * Pad outbound frame to BGE_MIN_FRAME for an unusual reason.
2694          * The bge hardware will pad out Tx runts to BGE_MIN_FRAME,
2695          * but when such padded frames employ the bge IP/TCP checksum
2696          * offload, the hardware checksum assist gives incorrect results
2697          * (possibly from incorporating its own padding into the UDP/TCP
2698          * checksum; who knows).  If we pad such runts with zeros, the
2699          * onboard checksum comes out correct.
2700          */
2701         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
2702             m_head->m_pkthdr.len < BGE_MIN_FRAME) {
2703                 error = bge_cksum_pad(m_head);
2704                 if (error)
2705                         goto back;
2706         }
2707
2708         ctx.bge_segs = segs;
2709         ctx.bge_maxsegs = maxsegs;
2710         error = bus_dmamap_load_mbuf(sc->bge_cdata.bge_mtag, map, m_head,
2711                                      bge_dma_map_mbuf, &ctx, BUS_DMA_NOWAIT);
2712         if (error == EFBIG || ctx.bge_maxsegs == 0) {
2713                 struct mbuf *m_new;
2714
2715                 if (!error)
2716                         bus_dmamap_unload(sc->bge_cdata.bge_mtag, map);
2717
2718                 m_new = m_defrag(m_head, MB_DONTWAIT);
2719                 if (m_new == NULL) {
2720                         if_printf(&sc->arpcom.ac_if,
2721                                   "could not defrag TX mbuf\n");
2722                         error = ENOBUFS;
2723                         goto back;
2724                 } else {
2725                         m_head = m_new;
2726                         *m_head0 = m_head;
2727                 }
2728
2729                 ctx.bge_segs = segs;
2730                 ctx.bge_maxsegs = maxsegs;
2731                 error = bus_dmamap_load_mbuf(sc->bge_cdata.bge_mtag, map,
2732                                              m_head, bge_dma_map_mbuf, &ctx,
2733                                              BUS_DMA_NOWAIT);
2734                 if (error || ctx.bge_maxsegs == 0) {
2735                         if_printf(&sc->arpcom.ac_if,
2736                                   "could not defrag TX mbuf\n");
2737                         if (!error) {
2738                                 bus_dmamap_unload(sc->bge_cdata.bge_mtag, map);
2739                                 error = EFBIG;
2740                         }
2741                         goto back;
2742                 }
2743         } else if (error) {
2744                 if_printf(&sc->arpcom.ac_if, "could not map TX mbuf\n");
2745                 goto back;
2746         }
2747
2748         bus_dmamap_sync(sc->bge_cdata.bge_mtag, map, BUS_DMASYNC_PREWRITE);
2749
2750         for (i = 0; ; i++) {
2751                 d = &sc->bge_ldata.bge_tx_ring[idx];
2752
2753                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(ctx.bge_segs[i].ds_addr);
2754                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(ctx.bge_segs[i].ds_addr);
2755                 d->bge_len = segs[i].ds_len;
2756                 d->bge_flags = csum_flags;
2757
2758                 if (i == ctx.bge_maxsegs - 1)
2759                         break;
2760                 BGE_INC(idx, BGE_TX_RING_CNT);
2761         }
2762         /* Mark the last segment as end of packet... */
2763         d->bge_flags |= BGE_TXBDFLAG_END;
2764
2765         /* Set vlan tag to the first segment of the packet. */
2766         d = &sc->bge_ldata.bge_tx_ring[*txidx];
2767         if (m_head->m_flags & M_VLANTAG) {
2768                 d->bge_flags |= BGE_TXBDFLAG_VLAN_TAG;
2769                 d->bge_vlan_tag = m_head->m_pkthdr.ether_vlantag;
2770         } else {
2771                 d->bge_vlan_tag = 0;
2772         }
2773
2774         /*
2775          * Insure that the map for this transmission is placed at
2776          * the array index of the last descriptor in this chain.
2777          */
2778         sc->bge_cdata.bge_tx_dmamap[*txidx] = sc->bge_cdata.bge_tx_dmamap[idx];
2779         sc->bge_cdata.bge_tx_dmamap[idx] = map;
2780         sc->bge_cdata.bge_tx_chain[idx] = m_head;
2781         sc->bge_txcnt += ctx.bge_maxsegs;
2782
2783         BGE_INC(idx, BGE_TX_RING_CNT);
2784         *txidx = idx;
2785 back:
2786         if (error) {
2787                 m_freem(m_head);
2788                 *m_head0 = NULL;
2789         }
2790         return error;
2791 }
2792
2793 /*
2794  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
2795  * to the mbuf data regions directly in the transmit descriptors.
2796  */
2797 static void
2798 bge_start(struct ifnet *ifp)
2799 {
2800         struct bge_softc *sc = ifp->if_softc;
2801         struct mbuf *m_head = NULL;
2802         uint32_t prodidx;
2803         int need_trans;
2804
2805         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
2806                 return;
2807
2808         prodidx = sc->bge_tx_prodidx;
2809
2810         need_trans = 0;
2811         while (sc->bge_cdata.bge_tx_chain[prodidx] == NULL) {
2812                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
2813                 if (m_head == NULL)
2814                         break;
2815
2816                 /*
2817                  * XXX
2818                  * The code inside the if() block is never reached since we
2819                  * must mark CSUM_IP_FRAGS in our if_hwassist to start getting
2820                  * requests to checksum TCP/UDP in a fragmented packet.
2821                  *
2822                  * XXX
2823                  * safety overkill.  If this is a fragmented packet chain
2824                  * with delayed TCP/UDP checksums, then only encapsulate
2825                  * it if we have enough descriptors to handle the entire
2826                  * chain at once.
2827                  * (paranoia -- may not actually be needed)
2828                  */
2829                 if ((m_head->m_flags & M_FIRSTFRAG) &&
2830                     (m_head->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) {
2831                         if ((BGE_TX_RING_CNT - sc->bge_txcnt) <
2832                             m_head->m_pkthdr.csum_data + BGE_NSEG_RSVD) {
2833                                 ifp->if_flags |= IFF_OACTIVE;
2834                                 ifq_prepend(&ifp->if_snd, m_head);
2835                                 break;
2836                         }
2837                 }
2838
2839                 /*
2840                  * Sanity check: avoid coming within BGE_NSEG_RSVD
2841                  * descriptors of the end of the ring.  Also make
2842                  * sure there are BGE_NSEG_SPARE descriptors for
2843                  * jumbo buffers' defragmentation.
2844                  */
2845                 if ((BGE_TX_RING_CNT - sc->bge_txcnt) <
2846                     (BGE_NSEG_RSVD + BGE_NSEG_SPARE)) {
2847                         ifp->if_flags |= IFF_OACTIVE;
2848                         ifq_prepend(&ifp->if_snd, m_head);
2849                         break;
2850                 }
2851
2852                 /*
2853                  * Pack the data into the transmit ring. If we
2854                  * don't have room, set the OACTIVE flag and wait
2855                  * for the NIC to drain the ring.
2856                  */
2857                 if (bge_encap(sc, &m_head, &prodidx)) {
2858                         ifp->if_flags |= IFF_OACTIVE;
2859                         ifp->if_oerrors++;
2860                         break;
2861                 }
2862                 need_trans = 1;
2863
2864                 ETHER_BPF_MTAP(ifp, m_head);
2865         }
2866
2867         if (!need_trans)
2868                 return;
2869
2870         /* Transmit */
2871         CSR_WRITE_4(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
2872         /* 5700 b2 errata */
2873         if (sc->bge_chiprev == BGE_CHIPREV_5700_BX)
2874                 CSR_WRITE_4(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
2875
2876         sc->bge_tx_prodidx = prodidx;
2877
2878         /*
2879          * Set a timeout in case the chip goes out to lunch.
2880          */
2881         ifp->if_timer = 5;
2882 }
2883
2884 static void
2885 bge_init(void *xsc)
2886 {
2887         struct bge_softc *sc = xsc;
2888         struct ifnet *ifp = &sc->arpcom.ac_if;
2889         uint16_t *m;
2890
2891         ASSERT_SERIALIZED(ifp->if_serializer);
2892
2893         if (ifp->if_flags & IFF_RUNNING)
2894                 return;
2895
2896         /* Cancel pending I/O and flush buffers. */
2897         bge_stop(sc);
2898         bge_reset(sc);
2899         bge_chipinit(sc);
2900
2901         /*
2902          * Init the various state machines, ring
2903          * control blocks and firmware.
2904          */
2905         if (bge_blockinit(sc)) {
2906                 if_printf(ifp, "initialization failure\n");
2907                 return;
2908         }
2909
2910         /* Specify MTU. */
2911         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
2912             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
2913
2914         /* Load our MAC address. */
2915         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
2916         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
2917         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
2918
2919         /* Enable or disable promiscuous mode as needed. */
2920         bge_setpromisc(sc);
2921
2922         /* Program multicast filter. */
2923         bge_setmulti(sc);
2924
2925         /* Init RX ring. */
2926         bge_init_rx_ring_std(sc);
2927
2928         /*
2929          * Workaround for a bug in 5705 ASIC rev A0. Poll the NIC's
2930          * memory to insure that the chip has in fact read the first
2931          * entry of the ring.
2932          */
2933         if (sc->bge_chipid == BGE_CHIPID_BCM5705_A0) {
2934                 uint32_t                v, i;
2935                 for (i = 0; i < 10; i++) {
2936                         DELAY(20);
2937                         v = bge_readmem_ind(sc, BGE_STD_RX_RINGS + 8);
2938                         if (v == (MCLBYTES - ETHER_ALIGN))
2939                                 break;
2940                 }
2941                 if (i == 10)
2942                         if_printf(ifp, "5705 A0 chip failed to load RX ring\n");
2943         }
2944
2945         /* Init jumbo RX ring. */
2946         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN))
2947                 bge_init_rx_ring_jumbo(sc);
2948
2949         /* Init our RX return ring index */
2950         sc->bge_rx_saved_considx = 0;
2951
2952         /* Init TX ring. */
2953         bge_init_tx_ring(sc);
2954
2955         /* Turn on transmitter */
2956         BGE_SETBIT(sc, BGE_TX_MODE, BGE_TXMODE_ENABLE);
2957
2958         /* Turn on receiver */
2959         BGE_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
2960
2961         /* Tell firmware we're alive. */
2962         BGE_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
2963
2964         /* Enable host interrupts if polling(4) is not enabled. */
2965         BGE_SETBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA);
2966 #ifdef DEVICE_POLLING
2967         if (ifp->if_flags & IFF_POLLING)
2968                 bge_disable_intr(sc);
2969         else
2970 #endif
2971         bge_enable_intr(sc);
2972
2973         bge_ifmedia_upd(ifp);
2974
2975         ifp->if_flags |= IFF_RUNNING;
2976         ifp->if_flags &= ~IFF_OACTIVE;
2977
2978         callout_reset(&sc->bge_stat_timer, hz, bge_tick, sc);
2979 }
2980
2981 /*
2982  * Set media options.
2983  */
2984 static int
2985 bge_ifmedia_upd(struct ifnet *ifp)
2986 {
2987         struct bge_softc *sc = ifp->if_softc;
2988
2989         /* If this is a 1000baseX NIC, enable the TBI port. */
2990         if (sc->bge_flags & BGE_FLAG_TBI) {
2991                 struct ifmedia *ifm = &sc->bge_ifmedia;
2992
2993                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
2994                         return(EINVAL);
2995
2996                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
2997                 case IFM_AUTO:
2998                         /*
2999                          * The BCM5704 ASIC appears to have a special
3000                          * mechanism for programming the autoneg
3001                          * advertisement registers in TBI mode.
3002                          */
3003                         if (!bge_fake_autoneg &&
3004                             sc->bge_asicrev == BGE_ASICREV_BCM5704) {
3005                                 uint32_t sgdig;
3006
3007                                 CSR_WRITE_4(sc, BGE_TX_TBI_AUTONEG, 0);
3008                                 sgdig = CSR_READ_4(sc, BGE_SGDIG_CFG);
3009                                 sgdig |= BGE_SGDIGCFG_AUTO |
3010                                          BGE_SGDIGCFG_PAUSE_CAP |
3011                                          BGE_SGDIGCFG_ASYM_PAUSE;
3012                                 CSR_WRITE_4(sc, BGE_SGDIG_CFG,
3013                                             sgdig | BGE_SGDIGCFG_SEND);
3014                                 DELAY(5);
3015                                 CSR_WRITE_4(sc, BGE_SGDIG_CFG, sgdig);
3016                         }
3017                         break;
3018                 case IFM_1000_SX:
3019                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3020                                 BGE_CLRBIT(sc, BGE_MAC_MODE,
3021                                     BGE_MACMODE_HALF_DUPLEX);
3022                         } else {
3023                                 BGE_SETBIT(sc, BGE_MAC_MODE,
3024                                     BGE_MACMODE_HALF_DUPLEX);
3025                         }
3026                         break;
3027                 default:
3028                         return(EINVAL);
3029                 }
3030         } else {
3031                 struct mii_data *mii = device_get_softc(sc->bge_miibus);
3032
3033                 sc->bge_link_evt++;
3034                 sc->bge_link = 0;
3035                 if (mii->mii_instance) {
3036                         struct mii_softc *miisc;
3037
3038                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3039                                 mii_phy_reset(miisc);
3040                 }
3041                 mii_mediachg(mii);
3042         }
3043         return(0);
3044 }
3045
3046 /*
3047  * Report current media status.
3048  */
3049 static void
3050 bge_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3051 {
3052         struct bge_softc *sc = ifp->if_softc;
3053
3054         if (sc->bge_flags & BGE_FLAG_TBI) {
3055                 ifmr->ifm_status = IFM_AVALID;
3056                 ifmr->ifm_active = IFM_ETHER;
3057                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3058                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3059                         ifmr->ifm_status |= IFM_ACTIVE;
3060                 } else {
3061                         ifmr->ifm_active |= IFM_NONE;
3062                         return;
3063                 }
3064
3065                 ifmr->ifm_active |= IFM_1000_SX;
3066                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3067                         ifmr->ifm_active |= IFM_HDX;
3068                 else
3069                         ifmr->ifm_active |= IFM_FDX;
3070         } else {
3071                 struct mii_data *mii = device_get_softc(sc->bge_miibus);
3072
3073                 mii_pollstat(mii);
3074                 ifmr->ifm_active = mii->mii_media_active;
3075                 ifmr->ifm_status = mii->mii_media_status;
3076         }
3077 }
3078
3079 static int
3080 bge_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3081 {
3082         struct bge_softc *sc = ifp->if_softc;
3083         struct ifreq *ifr = (struct ifreq *)data;
3084         int mask, error = 0;
3085
3086         ASSERT_SERIALIZED(ifp->if_serializer);
3087
3088         switch (command) {
3089         case SIOCSIFMTU:
3090                 if ((!BGE_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3091                     (BGE_IS_JUMBO_CAPABLE(sc) &&
3092                      ifr->ifr_mtu > BGE_JUMBO_MTU)) {
3093                         error = EINVAL;
3094                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3095                         ifp->if_mtu = ifr->ifr_mtu;
3096                         ifp->if_flags &= ~IFF_RUNNING;
3097                         bge_init(sc);
3098                 }
3099                 break;
3100         case SIOCSIFFLAGS:
3101                 if (ifp->if_flags & IFF_UP) {
3102                         if (ifp->if_flags & IFF_RUNNING) {
3103                                 mask = ifp->if_flags ^ sc->bge_if_flags;
3104
3105                                 /*
3106                                  * If only the state of the PROMISC flag
3107                                  * changed, then just use the 'set promisc
3108                                  * mode' command instead of reinitializing
3109                                  * the entire NIC. Doing a full re-init
3110                                  * means reloading the firmware and waiting
3111                                  * for it to start up, which may take a
3112                                  * second or two.  Similarly for ALLMULTI.
3113                                  */
3114                                 if (mask & IFF_PROMISC)
3115                                         bge_setpromisc(sc);
3116                                 if (mask & IFF_ALLMULTI)
3117                                         bge_setmulti(sc);
3118                         } else {
3119                                 bge_init(sc);
3120                         }
3121                 } else {
3122                         if (ifp->if_flags & IFF_RUNNING)
3123                                 bge_stop(sc);
3124                 }
3125                 sc->bge_if_flags = ifp->if_flags;
3126                 break;
3127         case SIOCADDMULTI:
3128         case SIOCDELMULTI:
3129                 if (ifp->if_flags & IFF_RUNNING)
3130                         bge_setmulti(sc);
3131                 break;
3132         case SIOCSIFMEDIA:
3133         case SIOCGIFMEDIA:
3134                 if (sc->bge_flags & BGE_FLAG_TBI) {
3135                         error = ifmedia_ioctl(ifp, ifr,
3136                             &sc->bge_ifmedia, command);
3137                 } else {
3138                         struct mii_data *mii;
3139
3140                         mii = device_get_softc(sc->bge_miibus);
3141                         error = ifmedia_ioctl(ifp, ifr,
3142                                               &mii->mii_media, command);
3143                 }
3144                 break;
3145         case SIOCSIFCAP:
3146                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3147                 if (mask & IFCAP_HWCSUM) {
3148                         ifp->if_capenable ^= IFCAP_HWCSUM;
3149                         if (IFCAP_HWCSUM & ifp->if_capenable)
3150                                 ifp->if_hwassist = BGE_CSUM_FEATURES;
3151                         else
3152                                 ifp->if_hwassist = 0;
3153                 }
3154                 break;
3155         default:
3156                 error = ether_ioctl(ifp, command, data);
3157                 break;
3158         }
3159         return error;
3160 }
3161
3162 static void
3163 bge_watchdog(struct ifnet *ifp)
3164 {
3165         struct bge_softc *sc = ifp->if_softc;
3166
3167         if_printf(ifp, "watchdog timeout -- resetting\n");
3168
3169         ifp->if_flags &= ~IFF_RUNNING;
3170         bge_init(sc);
3171
3172         ifp->if_oerrors++;
3173
3174         if (!ifq_is_empty(&ifp->if_snd))
3175                 if_devstart(ifp);
3176 }
3177
3178 /*
3179  * Stop the adapter and free any mbufs allocated to the
3180  * RX and TX lists.
3181  */
3182 static void
3183 bge_stop(struct bge_softc *sc)
3184 {
3185         struct ifnet *ifp = &sc->arpcom.ac_if;
3186         struct ifmedia_entry *ifm;
3187         struct mii_data *mii = NULL;
3188         int mtmp, itmp;
3189
3190         ASSERT_SERIALIZED(ifp->if_serializer);
3191
3192         if ((sc->bge_flags & BGE_FLAG_TBI) == 0)
3193                 mii = device_get_softc(sc->bge_miibus);
3194
3195         callout_stop(&sc->bge_stat_timer);
3196
3197         /*
3198          * Disable all of the receiver blocks
3199          */
3200         BGE_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3201         BGE_CLRBIT(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3202         BGE_CLRBIT(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3203         if (!BGE_IS_5705_PLUS(sc))
3204                 BGE_CLRBIT(sc, BGE_RXLS_MODE, BGE_RXLSMODE_ENABLE);
3205         BGE_CLRBIT(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3206         BGE_CLRBIT(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3207         BGE_CLRBIT(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3208
3209         /*
3210          * Disable all of the transmit blocks
3211          */
3212         BGE_CLRBIT(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3213         BGE_CLRBIT(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3214         BGE_CLRBIT(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3215         BGE_CLRBIT(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3216         BGE_CLRBIT(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3217         if (!BGE_IS_5705_PLUS(sc))
3218                 BGE_CLRBIT(sc, BGE_DMAC_MODE, BGE_DMACMODE_ENABLE);
3219         BGE_CLRBIT(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3220
3221         /*
3222          * Shut down all of the memory managers and related
3223          * state machines.
3224          */
3225         BGE_CLRBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3226         BGE_CLRBIT(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3227         if (!BGE_IS_5705_PLUS(sc))
3228                 BGE_CLRBIT(sc, BGE_MBCF_MODE, BGE_MBCFMODE_ENABLE);
3229         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3230         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3231         if (!BGE_IS_5705_PLUS(sc)) {
3232                 BGE_CLRBIT(sc, BGE_BMAN_MODE, BGE_BMANMODE_ENABLE);
3233                 BGE_CLRBIT(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
3234         }
3235
3236         /* Disable host interrupts. */
3237         bge_disable_intr(sc);
3238
3239         /*
3240          * Tell firmware we're shutting down.
3241          */
3242         BGE_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3243
3244         /* Free the RX lists. */
3245         bge_free_rx_ring_std(sc);
3246
3247         /* Free jumbo RX list. */
3248         if (BGE_IS_JUMBO_CAPABLE(sc))
3249                 bge_free_rx_ring_jumbo(sc);
3250
3251         /* Free TX buffers. */
3252         bge_free_tx_ring(sc);
3253
3254         /*
3255          * Isolate/power down the PHY, but leave the media selection
3256          * unchanged so that things will be put back to normal when
3257          * we bring the interface back up.
3258          *
3259          * 'mii' may be NULL in the following cases:
3260          * - The device uses TBI.
3261          * - bge_stop() is called by bge_detach().
3262          */
3263         if (mii != NULL) {
3264                 itmp = ifp->if_flags;
3265                 ifp->if_flags |= IFF_UP;
3266                 ifm = mii->mii_media.ifm_cur;
3267                 mtmp = ifm->ifm_media;
3268                 ifm->ifm_media = IFM_ETHER|IFM_NONE;
3269                 mii_mediachg(mii);
3270                 ifm->ifm_media = mtmp;
3271                 ifp->if_flags = itmp;
3272         }
3273
3274         sc->bge_link = 0;
3275         sc->bge_coal_chg = 0;
3276
3277         sc->bge_tx_saved_considx = BGE_TXCONS_UNSET;
3278
3279         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
3280         ifp->if_timer = 0;
3281 }
3282
3283 /*
3284  * Stop all chip I/O so that the kernel's probe routines don't
3285  * get confused by errant DMAs when rebooting.
3286  */
3287 static void
3288 bge_shutdown(device_t dev)
3289 {
3290         struct bge_softc *sc = device_get_softc(dev);
3291         struct ifnet *ifp = &sc->arpcom.ac_if;
3292
3293         lwkt_serialize_enter(ifp->if_serializer);
3294         bge_stop(sc);
3295         bge_reset(sc);
3296         lwkt_serialize_exit(ifp->if_serializer);
3297 }
3298
3299 static int
3300 bge_suspend(device_t dev)
3301 {
3302         struct bge_softc *sc = device_get_softc(dev);
3303         struct ifnet *ifp = &sc->arpcom.ac_if;
3304
3305         lwkt_serialize_enter(ifp->if_serializer);
3306         bge_stop(sc);
3307         lwkt_serialize_exit(ifp->if_serializer);
3308
3309         return 0;
3310 }
3311
3312 static int
3313 bge_resume(device_t dev)
3314 {
3315         struct bge_softc *sc = device_get_softc(dev);
3316         struct ifnet *ifp = &sc->arpcom.ac_if;
3317
3318         lwkt_serialize_enter(ifp->if_serializer);
3319
3320         if (ifp->if_flags & IFF_UP) {
3321                 bge_init(sc);
3322
3323                 if (!ifq_is_empty(&ifp->if_snd))
3324                         if_devstart(ifp);
3325         }
3326
3327         lwkt_serialize_exit(ifp->if_serializer);
3328
3329         return 0;
3330 }
3331
3332 static void
3333 bge_setpromisc(struct bge_softc *sc)
3334 {
3335         struct ifnet *ifp = &sc->arpcom.ac_if;
3336
3337         if (ifp->if_flags & IFF_PROMISC)
3338                 BGE_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3339         else
3340                 BGE_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3341 }
3342
3343 static void
3344 bge_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
3345 {
3346         struct bge_dmamap_arg *ctx = arg;
3347
3348         if (error)
3349                 return;
3350
3351         KASSERT(nsegs == 1 && ctx->bge_maxsegs == 1,
3352                 ("only one segment is allowed\n"));
3353
3354         ctx->bge_segs[0] = *segs;
3355 }
3356
3357 static void
3358 bge_dma_map_mbuf(void *arg, bus_dma_segment_t *segs, int nsegs,
3359                  bus_size_t mapsz __unused, int error)
3360 {
3361         struct bge_dmamap_arg *ctx = arg;
3362         int i;
3363
3364         if (error)
3365                 return;
3366
3367         if (nsegs > ctx->bge_maxsegs) {
3368                 ctx->bge_maxsegs = 0;
3369                 return;
3370         }
3371
3372         ctx->bge_maxsegs = nsegs;
3373         for (i = 0; i < nsegs; ++i)
3374                 ctx->bge_segs[i] = segs[i];
3375 }
3376
3377 static void
3378 bge_dma_free(struct bge_softc *sc)
3379 {
3380         int i;
3381
3382         /* Destroy RX/TX mbuf DMA stuffs. */
3383         if (sc->bge_cdata.bge_mtag != NULL) {
3384                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3385                         if (sc->bge_cdata.bge_rx_std_dmamap[i]) {
3386                                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag,
3387                                     sc->bge_cdata.bge_rx_std_dmamap[i]);
3388                         }
3389                 }
3390
3391                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
3392                         if (sc->bge_cdata.bge_tx_dmamap[i]) {
3393                                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag,
3394                                     sc->bge_cdata.bge_tx_dmamap[i]);
3395                         }
3396                 }
3397                 bus_dma_tag_destroy(sc->bge_cdata.bge_mtag);
3398         }
3399
3400         /* Destroy standard RX ring */
3401         bge_dma_block_free(sc->bge_cdata.bge_rx_std_ring_tag,
3402                            sc->bge_cdata.bge_rx_std_ring_map,
3403                            sc->bge_ldata.bge_rx_std_ring);
3404
3405         if (BGE_IS_JUMBO_CAPABLE(sc))
3406                 bge_free_jumbo_mem(sc);
3407
3408         /* Destroy RX return ring */
3409         bge_dma_block_free(sc->bge_cdata.bge_rx_return_ring_tag,
3410                            sc->bge_cdata.bge_rx_return_ring_map,
3411                            sc->bge_ldata.bge_rx_return_ring);
3412
3413         /* Destroy TX ring */
3414         bge_dma_block_free(sc->bge_cdata.bge_tx_ring_tag,
3415                            sc->bge_cdata.bge_tx_ring_map,
3416                            sc->bge_ldata.bge_tx_ring);
3417
3418         /* Destroy status block */
3419         bge_dma_block_free(sc->bge_cdata.bge_status_tag,
3420                            sc->bge_cdata.bge_status_map,
3421                            sc->bge_ldata.bge_status_block);
3422
3423         /* Destroy statistics block */
3424         bge_dma_block_free(sc->bge_cdata.bge_stats_tag,
3425                            sc->bge_cdata.bge_stats_map,
3426                            sc->bge_ldata.bge_stats);
3427
3428         /* Destroy the parent tag */
3429         if (sc->bge_cdata.bge_parent_tag != NULL)
3430                 bus_dma_tag_destroy(sc->bge_cdata.bge_parent_tag);
3431 }
3432
3433 static int
3434 bge_dma_alloc(struct bge_softc *sc)
3435 {
3436         struct ifnet *ifp = &sc->arpcom.ac_if;
3437         int nseg, i, error;
3438
3439         /*
3440          * Allocate the parent bus DMA tag appropriate for PCI.
3441          */
3442         error = bus_dma_tag_create(NULL, 1, 0,
3443                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3444                                    NULL, NULL,
3445                                    MAXBSIZE, BGE_NSEG_NEW,
3446                                    BUS_SPACE_MAXSIZE_32BIT,
3447                                    0, &sc->bge_cdata.bge_parent_tag);
3448         if (error) {
3449                 if_printf(ifp, "could not allocate parent dma tag\n");
3450                 return error;
3451         }
3452
3453         /*
3454          * Create DMA tag for mbufs.
3455          */
3456         nseg = BGE_NSEG_NEW;
3457         error = bus_dma_tag_create(sc->bge_cdata.bge_parent_tag, 1, 0,
3458                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3459                                    NULL, NULL,
3460                                    MCLBYTES * nseg, nseg, MCLBYTES,
3461                                    BUS_DMA_ALLOCNOW, &sc->bge_cdata.bge_mtag);
3462         if (error) {
3463                 if_printf(ifp, "could not allocate mbuf dma tag\n");
3464                 return error;
3465         }
3466
3467         /*
3468          * Create DMA maps for TX/RX mbufs.
3469          */
3470         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3471                 error = bus_dmamap_create(sc->bge_cdata.bge_mtag, 0,
3472                                           &sc->bge_cdata.bge_rx_std_dmamap[i]);
3473                 if (error) {
3474                         int j;
3475
3476                         for (j = 0; j < i; ++j) {
3477                                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag,
3478                                         sc->bge_cdata.bge_rx_std_dmamap[j]);
3479                         }
3480                         bus_dma_tag_destroy(sc->bge_cdata.bge_mtag);
3481                         sc->bge_cdata.bge_mtag = NULL;
3482
3483                         if_printf(ifp, "could not create DMA map for RX\n");
3484                         return error;
3485                 }
3486         }
3487
3488         for (i = 0; i < BGE_TX_RING_CNT; i++) {
3489                 error = bus_dmamap_create(sc->bge_cdata.bge_mtag, 0,
3490                                           &sc->bge_cdata.bge_tx_dmamap[i]);
3491                 if (error) {
3492                         int j;
3493
3494                         for (j = 0; j < BGE_STD_RX_RING_CNT; ++j) {
3495                                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag,
3496                                         sc->bge_cdata.bge_rx_std_dmamap[j]);
3497                         }
3498                         for (j = 0; j < i; ++j) {
3499                                 bus_dmamap_destroy(sc->bge_cdata.bge_mtag,
3500                                         sc->bge_cdata.bge_tx_dmamap[j]);
3501                         }
3502                         bus_dma_tag_destroy(sc->bge_cdata.bge_mtag);
3503                         sc->bge_cdata.bge_mtag = NULL;
3504
3505                         if_printf(ifp, "could not create DMA map for TX\n");
3506                         return error;
3507                 }
3508         }
3509
3510         /*
3511          * Create DMA stuffs for standard RX ring.
3512          */
3513         error = bge_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
3514                                     &sc->bge_cdata.bge_rx_std_ring_tag,
3515                                     &sc->bge_cdata.bge_rx_std_ring_map,
3516                                     (void **)&sc->bge_ldata.bge_rx_std_ring,
3517                                     &sc->bge_ldata.bge_rx_std_ring_paddr);
3518         if (error) {
3519                 if_printf(ifp, "could not create std RX ring\n");
3520                 return error;
3521         }
3522
3523         /*
3524          * Create jumbo buffer pool.
3525          */
3526         if (BGE_IS_JUMBO_CAPABLE(sc)) {
3527                 error = bge_alloc_jumbo_mem(sc);
3528                 if (error) {
3529                         if_printf(ifp, "could not create jumbo buffer pool\n");
3530                         return error;
3531                 }
3532         }
3533
3534         /*
3535          * Create DMA stuffs for RX return ring.
3536          */
3537         error = bge_dma_block_alloc(sc, BGE_RX_RTN_RING_SZ(sc),
3538                                     &sc->bge_cdata.bge_rx_return_ring_tag,
3539                                     &sc->bge_cdata.bge_rx_return_ring_map,
3540                                     (void **)&sc->bge_ldata.bge_rx_return_ring,
3541                                     &sc->bge_ldata.bge_rx_return_ring_paddr);
3542         if (error) {
3543                 if_printf(ifp, "could not create RX ret ring\n");
3544                 return error;
3545         }
3546
3547         /*
3548          * Create DMA stuffs for TX ring.
3549          */
3550         error = bge_dma_block_alloc(sc, BGE_TX_RING_SZ,
3551                                     &sc->bge_cdata.bge_tx_ring_tag,
3552                                     &sc->bge_cdata.bge_tx_ring_map,
3553                                     (void **)&sc->bge_ldata.bge_tx_ring,
3554                                     &sc->bge_ldata.bge_tx_ring_paddr);
3555         if (error) {
3556                 if_printf(ifp, "could not create TX ring\n");
3557                 return error;
3558         }
3559
3560         /*
3561          * Create DMA stuffs for status block.
3562          */
3563         error = bge_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
3564                                     &sc->bge_cdata.bge_status_tag,
3565                                     &sc->bge_cdata.bge_status_map,
3566                                     (void **)&sc->bge_ldata.bge_status_block,
3567                                     &sc->bge_ldata.bge_status_block_paddr);
3568         if (error) {
3569                 if_printf(ifp, "could not create status block\n");
3570                 return error;
3571         }
3572
3573         /*
3574          * Create DMA stuffs for statistics block.
3575          */
3576         error = bge_dma_block_alloc(sc, BGE_STATS_SZ,
3577                                     &sc->bge_cdata.bge_stats_tag,
3578                                     &sc->bge_cdata.bge_stats_map,
3579                                     (void **)&sc->bge_ldata.bge_stats,
3580                                     &sc->bge_ldata.bge_stats_paddr);
3581         if (error) {
3582                 if_printf(ifp, "could not create stats block\n");
3583                 return error;
3584         }
3585         return 0;
3586 }
3587
3588 static int
3589 bge_dma_block_alloc(struct bge_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
3590                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
3591 {
3592         struct ifnet *ifp = &sc->arpcom.ac_if;
3593         struct bge_dmamap_arg ctx;
3594         bus_dma_segment_t seg;
3595         int error;
3596
3597         /*
3598          * Create DMA tag
3599          */
3600         error = bus_dma_tag_create(sc->bge_cdata.bge_parent_tag, PAGE_SIZE, 0,
3601                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3602                                    NULL, NULL, size, 1, size, 0, tag);
3603         if (error) {
3604                 if_printf(ifp, "could not allocate dma tag\n");
3605                 return error;
3606         }
3607
3608         /*
3609          * Allocate DMA'able memory
3610          */
3611         error = bus_dmamem_alloc(*tag, addr, BUS_DMA_WAITOK | BUS_DMA_ZERO,
3612                                  map);
3613         if (error) {
3614                 if_printf(ifp, "could not allocate dma memory\n");
3615                 bus_dma_tag_destroy(*tag);
3616                 *tag = NULL;
3617                 return error;
3618         }
3619
3620         /*
3621          * Load the DMA'able memory
3622          */
3623         ctx.bge_maxsegs = 1;
3624         ctx.bge_segs = &seg;
3625         error = bus_dmamap_load(*tag, *map, *addr, size, bge_dma_map_addr, &ctx,
3626                                 BUS_DMA_WAITOK);
3627         if (error) {
3628                 if_printf(ifp, "could not load dma memory\n");
3629                 bus_dmamem_free(*tag, *addr, *map);
3630                 bus_dma_tag_destroy(*tag);
3631                 *tag = NULL;
3632                 return error;
3633         }
3634         *paddr = ctx.bge_segs[0].ds_addr;
3635
3636         return 0;
3637 }
3638
3639 static void
3640 bge_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
3641 {
3642         if (tag != NULL) {
3643                 bus_dmamap_unload(tag, map);
3644                 bus_dmamem_free(tag, addr, map);
3645                 bus_dma_tag_destroy(tag);
3646         }
3647 }
3648
3649 /*
3650  * Grrr. The link status word in the status block does
3651  * not work correctly on the BCM5700 rev AX and BX chips,
3652  * according to all available information. Hence, we have
3653  * to enable MII interrupts in order to properly obtain
3654  * async link changes. Unfortunately, this also means that
3655  * we have to read the MAC status register to detect link
3656  * changes, thereby adding an additional register access to
3657  * the interrupt handler.
3658  *
3659  * XXX: perhaps link state detection procedure used for
3660  * BGE_CHIPID_BCM5700_B2 can be used for others BCM5700 revisions.
3661  */
3662 static void
3663 bge_bcm5700_link_upd(struct bge_softc *sc, uint32_t status __unused)
3664 {
3665         struct ifnet *ifp = &sc->arpcom.ac_if;
3666         struct mii_data *mii = device_get_softc(sc->bge_miibus);
3667
3668         mii_pollstat(mii);
3669
3670         if (!sc->bge_link &&
3671             (mii->mii_media_status & IFM_ACTIVE) &&
3672             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3673                 sc->bge_link++;
3674                 if (bootverbose)
3675                         if_printf(ifp, "link UP\n");
3676         } else if (sc->bge_link &&
3677             (!(mii->mii_media_status & IFM_ACTIVE) ||
3678             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3679                 sc->bge_link = 0;
3680                 if (bootverbose)
3681                         if_printf(ifp, "link DOWN\n");
3682         }
3683
3684         /* Clear the interrupt. */
3685         CSR_WRITE_4(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_MI_INTERRUPT);
3686         bge_miibus_readreg(sc->bge_dev, 1, BRGPHY_MII_ISR);
3687         bge_miibus_writereg(sc->bge_dev, 1, BRGPHY_MII_IMR, BRGPHY_INTRS);
3688 }
3689
3690 static void
3691 bge_tbi_link_upd(struct bge_softc *sc, uint32_t status)
3692 {
3693         struct ifnet *ifp = &sc->arpcom.ac_if;
3694
3695 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
3696
3697         /*
3698          * Sometimes PCS encoding errors are detected in
3699          * TBI mode (on fiber NICs), and for some reason
3700          * the chip will signal them as link changes.
3701          * If we get a link change event, but the 'PCS
3702          * encoding error' bit in the MAC status register
3703          * is set, don't bother doing a link check.
3704          * This avoids spurious "gigabit link up" messages
3705          * that sometimes appear on fiber NICs during
3706          * periods of heavy traffic.
3707          */
3708         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
3709                 if (!sc->bge_link) {
3710                         sc->bge_link++;
3711                         if (sc->bge_asicrev == BGE_ASICREV_BCM5704) {
3712                                 BGE_CLRBIT(sc, BGE_MAC_MODE,
3713                                     BGE_MACMODE_TBI_SEND_CFGS);
3714                         }
3715                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
3716
3717                         if (bootverbose)
3718                                 if_printf(ifp, "link UP\n");
3719
3720                         ifp->if_link_state = LINK_STATE_UP;
3721                         if_link_state_change(ifp);
3722                 }
3723         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
3724                 if (sc->bge_link) {
3725                         sc->bge_link = 0;
3726
3727                         if (bootverbose)
3728                                 if_printf(ifp, "link DOWN\n");
3729
3730                         ifp->if_link_state = LINK_STATE_DOWN;
3731                         if_link_state_change(ifp);
3732                 }
3733         }
3734
3735 #undef PCS_ENCODE_ERR
3736
3737         /* Clear the attention. */
3738         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3739             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3740             BGE_MACSTAT_LINK_CHANGED);
3741 }
3742
3743 static void
3744 bge_copper_link_upd(struct bge_softc *sc, uint32_t status __unused)
3745 {
3746         /*
3747          * Check that the AUTOPOLL bit is set before
3748          * processing the event as a real link change.
3749          * Turning AUTOPOLL on and off in the MII read/write
3750          * functions will often trigger a link status
3751          * interrupt for no reason.
3752          */
3753         if (CSR_READ_4(sc, BGE_MI_MODE) & BGE_MIMODE_AUTOPOLL) {
3754                 struct ifnet *ifp = &sc->arpcom.ac_if;
3755                 struct mii_data *mii = device_get_softc(sc->bge_miibus);
3756
3757                 mii_pollstat(mii);
3758
3759                 if (!sc->bge_link &&
3760                     (mii->mii_media_status & IFM_ACTIVE) &&
3761                     IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3762                         sc->bge_link++;
3763                         if (bootverbose)
3764                                 if_printf(ifp, "link UP\n");
3765                 } else if (sc->bge_link &&
3766                     (!(mii->mii_media_status & IFM_ACTIVE) ||
3767                     IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3768                         sc->bge_link = 0;
3769                         if (bootverbose)
3770                                 if_printf(ifp, "link DOWN\n");
3771                 }
3772         }
3773
3774         /* Clear the attention. */
3775         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3776             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3777             BGE_MACSTAT_LINK_CHANGED);
3778 }
3779
3780 static int
3781 bge_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
3782 {
3783         struct bge_softc *sc = arg1;
3784
3785         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
3786                                    &sc->bge_rx_coal_ticks,
3787                                    BGE_RX_COAL_TICKS_CHG);
3788 }
3789
3790 static int
3791 bge_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
3792 {
3793         struct bge_softc *sc = arg1;
3794
3795         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
3796                                    &sc->bge_tx_coal_ticks,
3797                                    BGE_TX_COAL_TICKS_CHG);
3798 }
3799
3800 static int
3801 bge_sysctl_rx_max_coal_bds(SYSCTL_HANDLER_ARGS)
3802 {
3803         struct bge_softc *sc = arg1;
3804
3805         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
3806                                    &sc->bge_rx_max_coal_bds,
3807                                    BGE_RX_MAX_COAL_BDS_CHG);
3808 }
3809
3810 static int
3811 bge_sysctl_tx_max_coal_bds(SYSCTL_HANDLER_ARGS)
3812 {
3813         struct bge_softc *sc = arg1;
3814
3815         return bge_sysctl_coal_chg(oidp, arg1, arg2, req,
3816                                    &sc->bge_tx_max_coal_bds,
3817                                    BGE_TX_MAX_COAL_BDS_CHG);
3818 }
3819
3820 static int
3821 bge_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
3822                     uint32_t coal_chg_mask)
3823 {
3824         struct bge_softc *sc = arg1;
3825         struct ifnet *ifp = &sc->arpcom.ac_if;
3826         int error = 0, v;
3827
3828         lwkt_serialize_enter(ifp->if_serializer);
3829
3830         v = *coal;
3831         error = sysctl_handle_int(oidp, &v, 0, req);
3832         if (!error && req->newptr != NULL) {
3833                 if (v < 0) {
3834                         error = EINVAL;
3835                 } else {
3836                         *coal = v;
3837                         sc->bge_coal_chg |= coal_chg_mask;
3838                 }
3839         }
3840
3841         lwkt_serialize_exit(ifp->if_serializer);
3842         return error;
3843 }
3844
3845 static void
3846 bge_coal_change(struct bge_softc *sc)
3847 {
3848         struct ifnet *ifp = &sc->arpcom.ac_if;
3849         uint32_t val;
3850
3851         ASSERT_SERIALIZED(ifp->if_serializer);
3852
3853         if (sc->bge_coal_chg & BGE_RX_COAL_TICKS_CHG) {
3854                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
3855                             sc->bge_rx_coal_ticks);
3856                 DELAY(10);
3857                 val = CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS);
3858
3859                 if (bootverbose) {
3860                         if_printf(ifp, "rx_coal_ticks -> %u\n",
3861                                   sc->bge_rx_coal_ticks);
3862                 }
3863         }
3864
3865         if (sc->bge_coal_chg & BGE_TX_COAL_TICKS_CHG) {
3866                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
3867                             sc->bge_tx_coal_ticks);
3868                 DELAY(10);
3869                 val = CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS);
3870
3871                 if (bootverbose) {
3872                         if_printf(ifp, "tx_coal_ticks -> %u\n",
3873                                   sc->bge_tx_coal_ticks);
3874                 }
3875         }
3876
3877         if (sc->bge_coal_chg & BGE_RX_MAX_COAL_BDS_CHG) {
3878                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
3879                             sc->bge_rx_max_coal_bds);
3880                 DELAY(10);
3881                 val = CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS);
3882
3883                 if (bootverbose) {
3884                         if_printf(ifp, "rx_max_coal_bds -> %u\n",
3885                                   sc->bge_rx_max_coal_bds);
3886                 }
3887         }
3888
3889         if (sc->bge_coal_chg & BGE_TX_MAX_COAL_BDS_CHG) {
3890                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS,
3891                             sc->bge_tx_max_coal_bds);
3892                 DELAY(10);
3893                 val = CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS);
3894
3895                 if (bootverbose) {
3896                         if_printf(ifp, "tx_max_coal_bds -> %u\n",
3897                                   sc->bge_tx_max_coal_bds);
3898                 }
3899         }
3900
3901         sc->bge_coal_chg = 0;
3902 }
3903
3904 static void
3905 bge_enable_intr(struct bge_softc *sc)
3906 {
3907         struct ifnet *ifp = &sc->arpcom.ac_if;
3908
3909         lwkt_serialize_handler_enable(ifp->if_serializer);
3910
3911         /*
3912          * Enable interrupt.
3913          */
3914         CSR_WRITE_4(sc, BGE_MBX_IRQ0_LO, 0);
3915
3916         /*
3917          * Unmask the interrupt when we stop polling.
3918          */
3919         BGE_CLRBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_MASK_PCI_INTR);
3920
3921         /*
3922          * Trigger another interrupt, since above writing
3923          * to interrupt mailbox0 may acknowledge pending
3924          * interrupt.
3925          */
3926         BGE_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
3927 }
3928
3929 static void
3930 bge_disable_intr(struct bge_softc *sc)
3931 {
3932         struct ifnet *ifp = &sc->arpcom.ac_if;
3933
3934         /*
3935          * Mask the interrupt when we start polling.
3936          */
3937         BGE_SETBIT(sc, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_MASK_PCI_INTR);
3938
3939         /*
3940          * Acknowledge possible asserted interrupt.
3941          */
3942         CSR_WRITE_4(sc, BGE_MBX_IRQ0_LO, 1);
3943
3944         lwkt_serialize_handler_disable(ifp->if_serializer);
3945 }