sys/dev/netif/bnx/if_bnx.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  */
  35
  36
  37 #include "opt_polling.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/kernel.h>
  43 #include <sys/interrupt.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/queue.h>
  47 #include <sys/rman.h>
  48 #include <sys/serialize.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <net/bpf.h>
  54 #include <net/ethernet.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <net/if_dl.h>
  58 #include <net/if_media.h>
  59 #include <net/if_types.h>
  60 #include <net/ifq_var.h>
  61 #include <net/vlan/if_vlan_var.h>
  62 #include <net/vlan/if_vlan_ether.h>
  63
  64 #include <dev/netif/mii_layer/mii.h>
  65 #include <dev/netif/mii_layer/miivar.h>
  66 #include <dev/netif/mii_layer/brgphyreg.h>
  67
  68 #include <bus/pci/pcidevs.h>
  69 #include <bus/pci/pcireg.h>
  70 #include <bus/pci/pcivar.h>
  71
  72 #include <dev/netif/bge/if_bgereg.h>
  73 #include <dev/netif/bnx/if_bnxvar.h>
  74
  75 /* "device miibus" required.  See GENERIC if you get errors here. */
  76 #include "miibus_if.h"
  77
  78 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP)
  79
  80 static const struct bnx_type {
  81         uint16_t                bnx_vid;
  82         uint16_t                bnx_did;
  83         char                    *bnx_name;
  84 } bnx_devs[] = {
  85         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
  86                 "Broadcom BCM5717 Gigabit Ethernet" },
  87         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
  88                 "Broadcom BCM5718 Gigabit Ethernet" },
  89         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
  90                 "Broadcom BCM5719 Gigabit Ethernet" },
  91         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
  92                 "Broadcom BCM5720 Gigabit Ethernet" },
  93
  94         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
  95                 "Broadcom BCM57761 Gigabit Ethernet" },
  96         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
  97                 "Broadcom BCM57781 Gigabit Ethernet" },
  98         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
  99                 "Broadcom BCM57791 Fast Ethernet" },
 100         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
 101                 "Broadcom BCM57765 Gigabit Ethernet" },
 102         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
 103                 "Broadcom BCM57785 Gigabit Ethernet" },
 104         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
 105                 "Broadcom BCM57795 Fast Ethernet" },
 106
 107         { 0, 0, NULL }
 108 };
 109
 110 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
 111 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
 112
 113 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
 114
 115 static int      bnx_probe(device_t);
 116 static int      bnx_attach(device_t);
 117 static int      bnx_detach(device_t);
 118 static void     bnx_shutdown(device_t);
 119 static int      bnx_suspend(device_t);
 120 static int      bnx_resume(device_t);
 121 static int      bnx_miibus_readreg(device_t, int, int);
 122 static int      bnx_miibus_writereg(device_t, int, int, int);
 123 static void     bnx_miibus_statchg(device_t);
 124
 125 #ifdef DEVICE_POLLING
 126 static void     bnx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count);
 127 #endif
 128 static void     bnx_intr_legacy(void *);
 129 static void     bnx_msi(void *);
 130 static void     bnx_msi_oneshot(void *);
 131 static void     bnx_intr(struct bnx_softc *);
 132 static void     bnx_enable_intr(struct bnx_softc *);
 133 static void     bnx_disable_intr(struct bnx_softc *);
 134 static void     bnx_txeof(struct bnx_softc *, uint16_t);
 135 static void     bnx_rxeof(struct bnx_softc *, uint16_t);
 136
 137 static void     bnx_start(struct ifnet *);
 138 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 139 static void     bnx_init(void *);
 140 static void     bnx_stop(struct bnx_softc *);
 141 static void     bnx_watchdog(struct ifnet *);
 142 static int      bnx_ifmedia_upd(struct ifnet *);
 143 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 144 static void     bnx_tick(void *);
 145
 146 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
 147 static void     bnx_free_jumbo_mem(struct bnx_softc *);
 148 static struct bnx_jslot
 149                 *bnx_jalloc(struct bnx_softc *);
 150 static void     bnx_jfree(void *);
 151 static void     bnx_jref(void *);
 152 static int      bnx_newbuf_std(struct bnx_softc *, int, int);
 153 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
 154 static void     bnx_setup_rxdesc_std(struct bnx_softc *, int);
 155 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
 156 static int      bnx_init_rx_ring_std(struct bnx_softc *);
 157 static void     bnx_free_rx_ring_std(struct bnx_softc *);
 158 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
 159 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
 160 static void     bnx_free_tx_ring(struct bnx_softc *);
 161 static int      bnx_init_tx_ring(struct bnx_softc *);
 162 static int      bnx_dma_alloc(struct bnx_softc *);
 163 static void     bnx_dma_free(struct bnx_softc *);
 164 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
 165                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
 166 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 167 static struct mbuf *
 168                 bnx_defrag_shortdma(struct mbuf *);
 169 static int      bnx_encap(struct bnx_softc *, struct mbuf **, uint32_t *);
 170
 171 static void     bnx_reset(struct bnx_softc *);
 172 static int      bnx_chipinit(struct bnx_softc *);
 173 static int      bnx_blockinit(struct bnx_softc *);
 174 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
 175 static void     bnx_enable_msi(struct bnx_softc *sc);
 176 static void     bnx_setmulti(struct bnx_softc *);
 177 static void     bnx_setpromisc(struct bnx_softc *);
 178 static void     bnx_stats_update_regs(struct bnx_softc *);
 179 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
 180
 181 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
 182 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
 183 #ifdef notdef
 184 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
 185 #endif
 186 static void     bnx_writereg_ind(struct bnx_softc *, uint32_t, uint32_t);
 187 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
 188 static void     bnx_writembx(struct bnx_softc *, int, int);
 189 static uint8_t  bnx_nvram_getbyte(struct bnx_softc *, int, uint8_t *);
 190 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
 191 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
 192 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
 193
 194 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
 195 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
 196 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
 197 static void     bnx_link_poll(struct bnx_softc *);
 198
 199 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
 200 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
 201 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
 202 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
 203
 204 static void     bnx_coal_change(struct bnx_softc *);
 205 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 206 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 207 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
 208 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
 209 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 210 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 211 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
 212                     int, int, uint32_t);
 213
 214 static int      bnx_msi_enable = 1;
 215 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
 216
 217 static device_method_t bnx_methods[] = {
 218         /* Device interface */
 219         DEVMETHOD(device_probe,         bnx_probe),
 220         DEVMETHOD(device_attach,        bnx_attach),
 221         DEVMETHOD(device_detach,        bnx_detach),
 222         DEVMETHOD(device_shutdown,      bnx_shutdown),
 223         DEVMETHOD(device_suspend,       bnx_suspend),
 224         DEVMETHOD(device_resume,        bnx_resume),
 225
 226         /* bus interface */
 227         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 228         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 229
 230         /* MII interface */
 231         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
 232         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
 233         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
 234
 235         { 0, 0 }
 236 };
 237
 238 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
 239 static devclass_t bnx_devclass;
 240
 241 DECLARE_DUMMY_MODULE(if_bnx);
 242 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
 243 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
 244
 245 static uint32_t
 246 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
 247 {
 248         device_t dev = sc->bnx_dev;
 249         uint32_t val;
 250
 251         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
 252             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 253                 return 0;
 254
 255         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 256         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 257         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 258         return (val);
 259 }
 260
 261 static void
 262 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
 263 {
 264         device_t dev = sc->bnx_dev;
 265
 266         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
 267             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
 268                 return;
 269
 270         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 271         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 272         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 273 }
 274
 275 #ifdef notdef
 276 static uint32_t
 277 bnx_readreg_ind(struct bnx_softc *sc, uin32_t off)
 278 {
 279         device_t dev = sc->bnx_dev;
 280
 281         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 282         return(pci_read_config(dev, BGE_PCI_REG_DATA, 4));
 283 }
 284 #endif
 285
 286 static void
 287 bnx_writereg_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
 288 {
 289         device_t dev = sc->bnx_dev;
 290
 291         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
 292         pci_write_config(dev, BGE_PCI_REG_DATA, val, 4);
 293 }
 294
 295 static void
 296 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
 297 {
 298         CSR_WRITE_4(sc, off, val);
 299 }
 300
 301 static void
 302 bnx_writembx(struct bnx_softc *sc, int off, int val)
 303 {
 304         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906)
 305                 off += BGE_LPMBX_IRQ0_HI - BGE_MBX_IRQ0_HI;
 306
 307         CSR_WRITE_4(sc, off, val);
 308 }
 309
 310 static uint8_t
 311 bnx_nvram_getbyte(struct bnx_softc *sc, int addr, uint8_t *dest)
 312 {
 313         uint32_t access, byte = 0;
 314         int i;
 315
 316         /* Lock. */
 317         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
 318         for (i = 0; i < 8000; i++) {
 319                 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1)
 320                         break;
 321                 DELAY(20);
 322         }
 323         if (i == 8000)
 324                 return (1);
 325
 326         /* Enable access. */
 327         access = CSR_READ_4(sc, BGE_NVRAM_ACCESS);
 328         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access | BGE_NVRAMACC_ENABLE);
 329
 330         CSR_WRITE_4(sc, BGE_NVRAM_ADDR, addr & 0xfffffffc);
 331         CSR_WRITE_4(sc, BGE_NVRAM_CMD, BGE_NVRAM_READCMD);
 332         for (i = 0; i < BNX_TIMEOUT * 10; i++) {
 333                 DELAY(10);
 334                 if (CSR_READ_4(sc, BGE_NVRAM_CMD) & BGE_NVRAMCMD_DONE) {
 335                         DELAY(10);
 336                         break;
 337                 }
 338         }
 339
 340         if (i == BNX_TIMEOUT * 10) {
 341                 if_printf(&sc->arpcom.ac_if, "nvram read timed out\n");
 342                 return (1);
 343         }
 344
 345         /* Get result. */
 346         byte = CSR_READ_4(sc, BGE_NVRAM_RDDATA);
 347
 348         *dest = (bswap32(byte) >> ((addr % 4) * 8)) & 0xFF;
 349
 350         /* Disable access. */
 351         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access);
 352
 353         /* Unlock. */
 354         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_CLR1);
 355         CSR_READ_4(sc, BGE_NVRAM_SWARB);
 356
 357         return (0);
 358 }
 359
 360 /*
 361  * Read a sequence of bytes from NVRAM.
 362  */
 363 static int
 364 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
 365 {
 366         int err = 0, i;
 367         uint8_t byte = 0;
 368
 369         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
 370                 return (1);
 371
 372         for (i = 0; i < cnt; i++) {
 373                 err = bnx_nvram_getbyte(sc, off + i, &byte);
 374                 if (err)
 375                         break;
 376                 *(dest + i) = byte;
 377         }
 378
 379         return (err ? 1 : 0);
 380 }
 381
 382 /*
 383  * Read a byte of data stored in the EEPROM at address 'addr.' The
 384  * BCM570x supports both the traditional bitbang interface and an
 385  * auto access interface for reading the EEPROM. We use the auto
 386  * access method.
 387  */
 388 static uint8_t
 389 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
 390 {
 391         int i;
 392         uint32_t byte = 0;
 393
 394         /*
 395          * Enable use of auto EEPROM access so we can avoid
 396          * having to use the bitbang method.
 397          */
 398         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 399
 400         /* Reset the EEPROM, load the clock period. */
 401         CSR_WRITE_4(sc, BGE_EE_ADDR,
 402             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 403         DELAY(20);
 404
 405         /* Issue the read EEPROM command. */
 406         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 407
 408         /* Wait for completion */
 409         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
 410                 DELAY(10);
 411                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 412                         break;
 413         }
 414
 415         if (i == BNX_TIMEOUT) {
 416                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 417                 return(1);
 418         }
 419
 420         /* Get result. */
 421         byte = CSR_READ_4(sc, BGE_EE_DATA);
 422
 423         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 424
 425         return(0);
 426 }
 427
 428 /*
 429  * Read a sequence of bytes from the EEPROM.
 430  */
 431 static int
 432 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
 433 {
 434         size_t i;
 435         int err;
 436         uint8_t byte;
 437
 438         for (byte = 0, err = 0, i = 0; i < len; i++) {
 439                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
 440                 if (err)
 441                         break;
 442                 *(dest + i) = byte;
 443         }
 444
 445         return(err ? 1 : 0);
 446 }
 447
 448 static int
 449 bnx_miibus_readreg(device_t dev, int phy, int reg)
 450 {
 451         struct bnx_softc *sc = device_get_softc(dev);
 452         uint32_t val;
 453         int i;
 454
 455         KASSERT(phy == sc->bnx_phyno,
 456             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 457
 458         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 459         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 460                 CSR_WRITE_4(sc, BGE_MI_MODE,
 461                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 462                 DELAY(80);
 463         }
 464
 465         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
 466             BGE_MIPHY(phy) | BGE_MIREG(reg));
 467
 468         /* Poll for the PHY register access to complete. */
 469         for (i = 0; i < BNX_TIMEOUT; i++) {
 470                 DELAY(10);
 471                 val = CSR_READ_4(sc, BGE_MI_COMM);
 472                 if ((val & BGE_MICOMM_BUSY) == 0) {
 473                         DELAY(5);
 474                         val = CSR_READ_4(sc, BGE_MI_COMM);
 475                         break;
 476                 }
 477         }
 478         if (i == BNX_TIMEOUT) {
 479                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
 480                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 481                 val = 0;
 482         }
 483
 484         /* Restore the autopoll bit if necessary. */
 485         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 486                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 487                 DELAY(80);
 488         }
 489
 490         if (val & BGE_MICOMM_READFAIL)
 491                 return 0;
 492
 493         return (val & 0xFFFF);
 494 }
 495
 496 static int
 497 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
 498 {
 499         struct bnx_softc *sc = device_get_softc(dev);
 500         int i;
 501
 502         KASSERT(phy == sc->bnx_phyno,
 503             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 504
 505         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
 506             (reg == BRGPHY_MII_1000CTL || reg == BRGPHY_MII_AUXCTL))
 507                return 0;
 508
 509         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 510         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 511                 CSR_WRITE_4(sc, BGE_MI_MODE,
 512                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 513                 DELAY(80);
 514         }
 515
 516         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
 517             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
 518
 519         for (i = 0; i < BNX_TIMEOUT; i++) {
 520                 DELAY(10);
 521                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 522                         DELAY(5);
 523                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 524                         break;
 525                 }
 526         }
 527         if (i == BNX_TIMEOUT) {
 528                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 529                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
 530         }
 531
 532         /* Restore the autopoll bit if necessary. */
 533         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 534                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 535                 DELAY(80);
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void
 542 bnx_miibus_statchg(device_t dev)
 543 {
 544         struct bnx_softc *sc;
 545         struct mii_data *mii;
 546
 547         sc = device_get_softc(dev);
 548         mii = device_get_softc(sc->bnx_miibus);
 549
 550         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 551             (IFM_ACTIVE | IFM_AVALID)) {
 552                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 553                 case IFM_10_T:
 554                 case IFM_100_TX:
 555                         sc->bnx_link = 1;
 556                         break;
 557                 case IFM_1000_T:
 558                 case IFM_1000_SX:
 559                 case IFM_2500_SX:
 560                         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
 561                                 sc->bnx_link = 1;
 562                         else
 563                                 sc->bnx_link = 0;
 564                         break;
 565                 default:
 566                         sc->bnx_link = 0;
 567                         break;
 568                 }
 569         } else {
 570                 sc->bnx_link = 0;
 571         }
 572         if (sc->bnx_link == 0)
 573                 return;
 574
 575         BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
 576         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
 577             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
 578                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
 579         } else {
 580                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
 581         }
 582
 583         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
 584                 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 585         } else {
 586                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 587         }
 588 }
 589
 590 /*
 591  * Memory management for jumbo frames.
 592  */
 593 static int
 594 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
 595 {
 596         struct ifnet *ifp = &sc->arpcom.ac_if;
 597         struct bnx_jslot *entry;
 598         uint8_t *ptr;
 599         bus_addr_t paddr;
 600         int i, error;
 601
 602         /*
 603          * Create tag for jumbo mbufs.
 604          * This is really a bit of a kludge. We allocate a special
 605          * jumbo buffer pool which (thanks to the way our DMA
 606          * memory allocation works) will consist of contiguous
 607          * pages. This means that even though a jumbo buffer might
 608          * be larger than a page size, we don't really need to
 609          * map it into more than one DMA segment. However, the
 610          * default mbuf tag will result in multi-segment mappings,
 611          * so we have to create a special jumbo mbuf tag that
 612          * lets us get away with mapping the jumbo buffers as
 613          * a single segment. I think eventually the driver should
 614          * be changed so that it uses ordinary mbufs and cluster
 615          * buffers, i.e. jumbo frames can span multiple DMA
 616          * descriptors. But that's a project for another day.
 617          */
 618
 619         /*
 620          * Create DMA stuffs for jumbo RX ring.
 621          */
 622         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 623                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 624                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 625                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
 626                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
 627         if (error) {
 628                 if_printf(ifp, "could not create jumbo RX ring\n");
 629                 return error;
 630         }
 631
 632         /*
 633          * Create DMA stuffs for jumbo buffer block.
 634          */
 635         error = bnx_dma_block_alloc(sc, BNX_JMEM,
 636                                     &sc->bnx_cdata.bnx_jumbo_tag,
 637                                     &sc->bnx_cdata.bnx_jumbo_map,
 638                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
 639                                     &paddr);
 640         if (error) {
 641                 if_printf(ifp, "could not create jumbo buffer\n");
 642                 return error;
 643         }
 644
 645         SLIST_INIT(&sc->bnx_jfree_listhead);
 646
 647         /*
 648          * Now divide it up into 9K pieces and save the addresses
 649          * in an array. Note that we play an evil trick here by using
 650          * the first few bytes in the buffer to hold the the address
 651          * of the softc structure for this interface. This is because
 652          * bnx_jfree() needs it, but it is called by the mbuf management
 653          * code which will not pass it to us explicitly.
 654          */
 655         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
 656                 entry = &sc->bnx_cdata.bnx_jslots[i];
 657                 entry->bnx_sc = sc;
 658                 entry->bnx_buf = ptr;
 659                 entry->bnx_paddr = paddr;
 660                 entry->bnx_inuse = 0;
 661                 entry->bnx_slot = i;
 662                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
 663
 664                 ptr += BNX_JLEN;
 665                 paddr += BNX_JLEN;
 666         }
 667         return 0;
 668 }
 669
 670 static void
 671 bnx_free_jumbo_mem(struct bnx_softc *sc)
 672 {
 673         /* Destroy jumbo RX ring. */
 674         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 675                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 676                            sc->bnx_ldata.bnx_rx_jumbo_ring);
 677
 678         /* Destroy jumbo buffer block. */
 679         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
 680                            sc->bnx_cdata.bnx_jumbo_map,
 681                            sc->bnx_ldata.bnx_jumbo_buf);
 682 }
 683
 684 /*
 685  * Allocate a jumbo buffer.
 686  */
 687 static struct bnx_jslot *
 688 bnx_jalloc(struct bnx_softc *sc)
 689 {
 690         struct bnx_jslot *entry;
 691
 692         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 693         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
 694         if (entry) {
 695                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
 696                 entry->bnx_inuse = 1;
 697         } else {
 698                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 699         }
 700         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 701         return(entry);
 702 }
 703
 704 /*
 705  * Adjust usage count on a jumbo buffer.
 706  */
 707 static void
 708 bnx_jref(void *arg)
 709 {
 710         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 711         struct bnx_softc *sc = entry->bnx_sc;
 712
 713         if (sc == NULL)
 714                 panic("bnx_jref: can't find softc pointer!");
 715
 716         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 717                 panic("bnx_jref: asked to reference buffer "
 718                     "that we don't manage!");
 719         } else if (entry->bnx_inuse == 0) {
 720                 panic("bnx_jref: buffer already free!");
 721         } else {
 722                 atomic_add_int(&entry->bnx_inuse, 1);
 723         }
 724 }
 725
 726 /*
 727  * Release a jumbo buffer.
 728  */
 729 static void
 730 bnx_jfree(void *arg)
 731 {
 732         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 733         struct bnx_softc *sc = entry->bnx_sc;
 734
 735         if (sc == NULL)
 736                 panic("bnx_jfree: can't find softc pointer!");
 737
 738         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 739                 panic("bnx_jfree: asked to free buffer that we don't manage!");
 740         } else if (entry->bnx_inuse == 0) {
 741                 panic("bnx_jfree: buffer already free!");
 742         } else {
 743                 /*
 744                  * Possible MP race to 0, use the serializer.  The atomic insn
 745                  * is still needed for races against bnx_jref().
 746                  */
 747                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 748                 atomic_subtract_int(&entry->bnx_inuse, 1);
 749                 if (entry->bnx_inuse == 0) {
 750                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead,
 751                                           entry, jslot_link);
 752                 }
 753                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 754         }
 755 }
 756
 757
 758 /*
 759  * Intialize a standard receive ring descriptor.
 760  */
 761 static int
 762 bnx_newbuf_std(struct bnx_softc *sc, int i, int init)
 763 {
 764         struct mbuf *m_new = NULL;
 765         bus_dma_segment_t seg;
 766         bus_dmamap_t map;
 767         int error, nsegs;
 768
 769         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
 770         if (m_new == NULL)
 771                 return ENOBUFS;
 772         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
 773         m_adj(m_new, ETHER_ALIGN);
 774
 775         error = bus_dmamap_load_mbuf_segment(sc->bnx_cdata.bnx_rx_mtag,
 776                         sc->bnx_cdata.bnx_rx_tmpmap, m_new,
 777                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
 778         if (error) {
 779                 m_freem(m_new);
 780                 return error;
 781         }
 782
 783         if (!init) {
 784                 bus_dmamap_sync(sc->bnx_cdata.bnx_rx_mtag,
 785                                 sc->bnx_cdata.bnx_rx_std_dmamap[i],
 786                                 BUS_DMASYNC_POSTREAD);
 787                 bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
 788                         sc->bnx_cdata.bnx_rx_std_dmamap[i]);
 789         }
 790
 791         map = sc->bnx_cdata.bnx_rx_tmpmap;
 792         sc->bnx_cdata.bnx_rx_tmpmap = sc->bnx_cdata.bnx_rx_std_dmamap[i];
 793         sc->bnx_cdata.bnx_rx_std_dmamap[i] = map;
 794
 795         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_mbuf = m_new;
 796         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_paddr = seg.ds_addr;
 797
 798         bnx_setup_rxdesc_std(sc, i);
 799         return 0;
 800 }
 801
 802 static void
 803 bnx_setup_rxdesc_std(struct bnx_softc *sc, int i)
 804 {
 805         struct bnx_rxchain *rc;
 806         struct bge_rx_bd *r;
 807
 808         rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
 809         r = &sc->bnx_ldata.bnx_rx_std_ring[i];
 810
 811         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
 812         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
 813         r->bge_len = rc->bnx_mbuf->m_len;
 814         r->bge_idx = i;
 815         r->bge_flags = BGE_RXBDFLAG_END;
 816 }
 817
 818 /*
 819  * Initialize a jumbo receive ring descriptor. This allocates
 820  * a jumbo buffer from the pool managed internally by the driver.
 821  */
 822 static int
 823 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
 824 {
 825         struct mbuf *m_new = NULL;
 826         struct bnx_jslot *buf;
 827         bus_addr_t paddr;
 828
 829         /* Allocate the mbuf. */
 830         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
 831         if (m_new == NULL)
 832                 return ENOBUFS;
 833
 834         /* Allocate the jumbo buffer */
 835         buf = bnx_jalloc(sc);
 836         if (buf == NULL) {
 837                 m_freem(m_new);
 838                 return ENOBUFS;
 839         }
 840
 841         /* Attach the buffer to the mbuf. */
 842         m_new->m_ext.ext_arg = buf;
 843         m_new->m_ext.ext_buf = buf->bnx_buf;
 844         m_new->m_ext.ext_free = bnx_jfree;
 845         m_new->m_ext.ext_ref = bnx_jref;
 846         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
 847
 848         m_new->m_flags |= M_EXT;
 849
 850         m_new->m_data = m_new->m_ext.ext_buf;
 851         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
 852
 853         paddr = buf->bnx_paddr;
 854         m_adj(m_new, ETHER_ALIGN);
 855         paddr += ETHER_ALIGN;
 856
 857         /* Save necessary information */
 858         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_mbuf = m_new;
 859         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_paddr = paddr;
 860
 861         /* Set up the descriptor. */
 862         bnx_setup_rxdesc_jumbo(sc, i);
 863         return 0;
 864 }
 865
 866 static void
 867 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
 868 {
 869         struct bge_rx_bd *r;
 870         struct bnx_rxchain *rc;
 871
 872         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
 873         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 874
 875         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
 876         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
 877         r->bge_len = rc->bnx_mbuf->m_len;
 878         r->bge_idx = i;
 879         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 880 }
 881
 882 static int
 883 bnx_init_rx_ring_std(struct bnx_softc *sc)
 884 {
 885         int i, error;
 886
 887         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 888                 error = bnx_newbuf_std(sc, i, 1);
 889                 if (error)
 890                         return error;
 891         };
 892
 893         sc->bnx_std = BGE_STD_RX_RING_CNT - 1;
 894         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
 895
 896         return(0);
 897 }
 898
 899 static void
 900 bnx_free_rx_ring_std(struct bnx_softc *sc)
 901 {
 902         int i;
 903
 904         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 905                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
 906
 907                 if (rc->bnx_mbuf != NULL) {
 908                         bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
 909                                           sc->bnx_cdata.bnx_rx_std_dmamap[i]);
 910                         m_freem(rc->bnx_mbuf);
 911                         rc->bnx_mbuf = NULL;
 912                 }
 913                 bzero(&sc->bnx_ldata.bnx_rx_std_ring[i],
 914                     sizeof(struct bge_rx_bd));
 915         }
 916 }
 917
 918 static int
 919 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
 920 {
 921         struct bge_rcb *rcb;
 922         int i, error;
 923
 924         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 925                 error = bnx_newbuf_jumbo(sc, i, 1);
 926                 if (error)
 927                         return error;
 928         };
 929
 930         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
 931
 932         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
 933         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
 934         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
 935
 936         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
 937
 938         return(0);
 939 }
 940
 941 static void
 942 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
 943 {
 944         int i;
 945
 946         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 947                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 948
 949                 if (rc->bnx_mbuf != NULL) {
 950                         m_freem(rc->bnx_mbuf);
 951                         rc->bnx_mbuf = NULL;
 952                 }
 953                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
 954                     sizeof(struct bge_rx_bd));
 955         }
 956 }
 957
 958 static void
 959 bnx_free_tx_ring(struct bnx_softc *sc)
 960 {
 961         int i;
 962
 963         for (i = 0; i < BGE_TX_RING_CNT; i++) {
 964                 if (sc->bnx_cdata.bnx_tx_chain[i] != NULL) {
 965                         bus_dmamap_unload(sc->bnx_cdata.bnx_tx_mtag,
 966                                           sc->bnx_cdata.bnx_tx_dmamap[i]);
 967                         m_freem(sc->bnx_cdata.bnx_tx_chain[i]);
 968                         sc->bnx_cdata.bnx_tx_chain[i] = NULL;
 969                 }
 970                 bzero(&sc->bnx_ldata.bnx_tx_ring[i],
 971                     sizeof(struct bge_tx_bd));
 972         }
 973 }
 974
 975 static int
 976 bnx_init_tx_ring(struct bnx_softc *sc)
 977 {
 978         sc->bnx_txcnt = 0;
 979         sc->bnx_tx_saved_considx = 0;
 980         sc->bnx_tx_prodidx = 0;
 981
 982         /* Initialize transmit producer index for host-memory send ring. */
 983         bnx_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bnx_tx_prodidx);
 984         bnx_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
 985
 986         return(0);
 987 }
 988
 989 static void
 990 bnx_setmulti(struct bnx_softc *sc)
 991 {
 992         struct ifnet *ifp;
 993         struct ifmultiaddr *ifma;
 994         uint32_t hashes[4] = { 0, 0, 0, 0 };
 995         int h, i;
 996
 997         ifp = &sc->arpcom.ac_if;
 998
 999         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1000                 for (i = 0; i < 4; i++)
1001                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1002                 return;
1003         }
1004
1005         /* First, zot all the existing filters. */
1006         for (i = 0; i < 4; i++)
1007                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1008
1009         /* Now program new ones. */
1010         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1011                 if (ifma->ifma_addr->sa_family != AF_LINK)
1012                         continue;
1013                 h = ether_crc32_le(
1014                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1015                     ETHER_ADDR_LEN) & 0x7f;
1016                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1017         }
1018
1019         for (i = 0; i < 4; i++)
1020                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1021 }
1022
1023 /*
1024  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1025  * self-test results.
1026  */
1027 static int
1028 bnx_chipinit(struct bnx_softc *sc)
1029 {
1030         uint32_t dma_rw_ctl, mode_ctl;
1031         int i;
1032
1033         /* Set endian type before we access any non-PCI registers. */
1034         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
1035             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
1036
1037         /* Clear the MAC control register */
1038         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
1039
1040         /*
1041          * Clear the MAC statistics block in the NIC's
1042          * internal memory.
1043          */
1044         for (i = BGE_STATS_BLOCK;
1045             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1046                 BNX_MEMWIN_WRITE(sc, i, 0);
1047
1048         for (i = BGE_STATUS_BLOCK;
1049             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1050                 BNX_MEMWIN_WRITE(sc, i, 0);
1051
1052         /* Set up the PCI DMA control register. */
1053         dma_rw_ctl = BGE_PCI_READ_CMD | BGE_PCI_WRITE_CMD |
1054             (0x3 << BGE_PCIDMARWCTL_WR_WAT_SHIFT);
1055
1056         if (BNX_IS_5717_PLUS(sc)) {
1057                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1058                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1059                         dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1060                 /*
1061                  * Enable HW workaround for controllers that misinterpret
1062                  * a status tag update and leave interrupts permanently
1063                  * disabled.
1064                  */
1065                 if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1066                     sc->bnx_asicrev != BGE_ASICREV_BCM57765)
1067                         dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1068         }
1069         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1070
1071         /*
1072          * Set up general mode register.
1073          */
1074         mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
1075             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1076         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1077
1078         /*
1079          * Disable memory write invalidate.  Apparently it is not supported
1080          * properly by these devices.  Also ensure that INTx isn't disabled,
1081          * as these chips need it even when using MSI.
1082          */
1083         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1084             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1085
1086         /* Set the timer prescaler (always 66Mhz) */
1087         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1088
1089         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
1090                 DELAY(40);      /* XXX */
1091
1092                 /* Put PHY into ready state */
1093                 BNX_CLRBIT(sc, BGE_MISC_CFG, BGE_MISCCFG_EPHY_IDDQ);
1094                 CSR_READ_4(sc, BGE_MISC_CFG); /* Flush */
1095                 DELAY(40);
1096         }
1097
1098         return(0);
1099 }
1100
1101 static int
1102 bnx_blockinit(struct bnx_softc *sc)
1103 {
1104         struct bge_rcb *rcb;
1105         bus_size_t vrcb;
1106         bge_hostaddr taddr;
1107         uint32_t val;
1108         int i, limit;
1109
1110         /*
1111          * Initialize the memory window pointer register so that
1112          * we can access the first 32K of internal NIC RAM. This will
1113          * allow us to set up the TX send ring RCBs and the RX return
1114          * ring RCBs, plus other things which live in NIC memory.
1115          */
1116         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1117
1118         /* Configure mbuf pool watermarks */
1119         if (BNX_IS_5717_PLUS(sc)) {
1120                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1121                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1122                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1123                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1124                 } else {
1125                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1126                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1127                 }
1128         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
1129                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1130                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x04);
1131                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x10);
1132         } else {
1133                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1134                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1135                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1136         }
1137
1138         /* Configure DMA resource watermarks */
1139         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1140         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1141
1142         /* Enable buffer manager */
1143         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1144         /*
1145          * Change the arbitration algorithm of TXMBUF read request to
1146          * round-robin instead of priority based for BCM5719.  When
1147          * TXFIFO is almost empty, RDMA will hold its request until
1148          * TXFIFO is not almost empty.
1149          */
1150         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1151                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1152         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1153
1154         /* Poll for buffer manager start indication */
1155         for (i = 0; i < BNX_TIMEOUT; i++) {
1156                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1157                         break;
1158                 DELAY(10);
1159         }
1160
1161         if (i == BNX_TIMEOUT) {
1162                 if_printf(&sc->arpcom.ac_if,
1163                           "buffer manager failed to start\n");
1164                 return(ENXIO);
1165         }
1166
1167         /* Enable flow-through queues */
1168         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1169         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1170
1171         /* Wait until queue initialization is complete */
1172         for (i = 0; i < BNX_TIMEOUT; i++) {
1173                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1174                         break;
1175                 DELAY(10);
1176         }
1177
1178         if (i == BNX_TIMEOUT) {
1179                 if_printf(&sc->arpcom.ac_if,
1180                           "flow-through queue init failed\n");
1181                 return(ENXIO);
1182         }
1183
1184         /*
1185          * Summary of rings supported by the controller:
1186          *
1187          * Standard Receive Producer Ring
1188          * - This ring is used to feed receive buffers for "standard"
1189          *   sized frames (typically 1536 bytes) to the controller.
1190          *
1191          * Jumbo Receive Producer Ring
1192          * - This ring is used to feed receive buffers for jumbo sized
1193          *   frames (i.e. anything bigger than the "standard" frames)
1194          *   to the controller.
1195          *
1196          * Mini Receive Producer Ring
1197          * - This ring is used to feed receive buffers for "mini"
1198          *   sized frames to the controller.
1199          * - This feature required external memory for the controller
1200          *   but was never used in a production system.  Should always
1201          *   be disabled.
1202          *
1203          * Receive Return Ring
1204          * - After the controller has placed an incoming frame into a
1205          *   receive buffer that buffer is moved into a receive return
1206          *   ring.  The driver is then responsible to passing the
1207          *   buffer up to the stack.  Many versions of the controller
1208          *   support multiple RR rings.
1209          *
1210          * Send Ring
1211          * - This ring is used for outgoing frames.  Many versions of
1212          *   the controller support multiple send rings.
1213          */
1214
1215         /* Initialize the standard receive producer ring control block. */
1216         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1217         rcb->bge_hostaddr.bge_addr_lo =
1218             BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1219         rcb->bge_hostaddr.bge_addr_hi =
1220             BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1221         if (BNX_IS_5717_PLUS(sc)) {
1222                 /*
1223                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1224                  * Bits 15-2 : Maximum RX frame size
1225                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1226                  * Bit 0     : Reserved
1227                  */
1228                 rcb->bge_maxlen_flags =
1229                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1230         } else {
1231                 /*
1232                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1233                  * Bits 15-2 : Reserved (should be 0)
1234                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1235                  * Bit 0     : Reserved
1236                  */
1237                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1238         }
1239         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1240             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1241             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1242                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1243         else
1244                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1245         /* Write the standard receive producer ring control block. */
1246         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1247         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1248         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1249         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1250         /* Reset the standard receive producer ring producer index. */
1251         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1252
1253         /*
1254          * Initialize the jumbo RX producer ring control
1255          * block.  We set the 'ring disabled' bit in the
1256          * flags field until we're actually ready to start
1257          * using this ring (i.e. once we set the MTU
1258          * high enough to require it).
1259          */
1260         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1261                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1262                 /* Get the jumbo receive producer ring RCB parameters. */
1263                 rcb->bge_hostaddr.bge_addr_lo =
1264                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1265                 rcb->bge_hostaddr.bge_addr_hi =
1266                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1267                 rcb->bge_maxlen_flags =
1268                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1269                     BGE_RCB_FLAG_RING_DISABLED);
1270                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1271                     sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1272                     sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1273                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1274                 else
1275                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1276                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1277                     rcb->bge_hostaddr.bge_addr_hi);
1278                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1279                     rcb->bge_hostaddr.bge_addr_lo);
1280                 /* Program the jumbo receive producer ring RCB parameters. */
1281                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1282                     rcb->bge_maxlen_flags);
1283                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1284                 /* Reset the jumbo receive producer ring producer index. */
1285                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1286         }
1287
1288         /* Choose de-pipeline mode for BCM5906 A0, A1 and A2. */
1289         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
1290             (sc->bnx_chipid == BGE_CHIPID_BCM5906_A0 ||
1291              sc->bnx_chipid == BGE_CHIPID_BCM5906_A1 ||
1292              sc->bnx_chipid == BGE_CHIPID_BCM5906_A2)) {
1293                 CSR_WRITE_4(sc, BGE_ISO_PKT_TX,
1294                     (CSR_READ_4(sc, BGE_ISO_PKT_TX) & ~3) | 2);
1295         }
1296
1297         /*
1298          * The BD ring replenish thresholds control how often the
1299          * hardware fetches new BD's from the producer rings in host
1300          * memory.  Setting the value too low on a busy system can
1301          * starve the hardware and recue the throughpout.
1302          *
1303          * Set the BD ring replentish thresholds. The recommended
1304          * values are 1/8th the number of descriptors allocated to
1305          * each ring.
1306          */
1307         val = 8;
1308         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1309         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1310                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1311                     BGE_JUMBO_RX_RING_CNT/8);
1312         }
1313         if (BNX_IS_5717_PLUS(sc)) {
1314                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1315                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1316         }
1317
1318         /*
1319          * Disable all send rings by setting the 'ring disabled' bit
1320          * in the flags field of all the TX send ring control blocks,
1321          * located in NIC memory.
1322          */
1323         limit = 1;
1324         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1325         for (i = 0; i < limit; i++) {
1326                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1327                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1328                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1329                 vrcb += sizeof(struct bge_rcb);
1330         }
1331
1332         /* Configure send ring RCB 0 (we use only the first ring) */
1333         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1334         BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_tx_ring_paddr);
1335         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1336         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1337         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1338             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1339             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1340                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
1341         } else {
1342                 RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1343                     BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1344         }
1345         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1346             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1347
1348         /*
1349          * Disable all receive return rings by setting the
1350          * 'ring disabled' bit in the flags field of all the receive
1351          * return ring control blocks, located in NIC memory.
1352          */
1353         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1354             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1355             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1356                 /* Should be 17, use 16 until we get an SRAM map. */
1357                 limit = 16;
1358         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM57765) {
1359                 limit = 4;
1360         } else {
1361                 limit = 1;
1362         }
1363         /* Disable all receive return rings. */
1364         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1365         for (i = 0; i < limit; i++) {
1366                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1367                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1368                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1369                     BGE_RCB_FLAG_RING_DISABLED);
1370                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1371                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1372                     (i * (sizeof(uint64_t))), 0);
1373                 vrcb += sizeof(struct bge_rcb);
1374         }
1375
1376         /*
1377          * Set up receive return ring 0.  Note that the NIC address
1378          * for RX return rings is 0x0.  The return rings live entirely
1379          * within the host, so the nicaddr field in the RCB isn't used.
1380          */
1381         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1382         BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_rx_return_ring_paddr);
1383         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1384         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1385         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1386         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1387             BGE_RCB_MAXLEN_FLAGS(sc->bnx_return_ring_cnt, 0));
1388
1389         /* Set random backoff seed for TX */
1390         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1391             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1392             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1393             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1394             BGE_TX_BACKOFF_SEED_MASK);
1395
1396         /* Set inter-packet gap */
1397         val = 0x2620;
1398         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1399                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1400                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1401         }
1402         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1403
1404         /*
1405          * Specify which ring to use for packets that don't match
1406          * any RX rules.
1407          */
1408         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1409
1410         /*
1411          * Configure number of RX lists. One interrupt distribution
1412          * list, sixteen active lists, one bad frames class.
1413          */
1414         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1415
1416         /* Inialize RX list placement stats mask. */
1417         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1418         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1419
1420         /* Disable host coalescing until we get it set up */
1421         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1422
1423         /* Poll to make sure it's shut down. */
1424         for (i = 0; i < BNX_TIMEOUT; i++) {
1425                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1426                         break;
1427                 DELAY(10);
1428         }
1429
1430         if (i == BNX_TIMEOUT) {
1431                 if_printf(&sc->arpcom.ac_if,
1432                           "host coalescing engine failed to idle\n");
1433                 return(ENXIO);
1434         }
1435
1436         /* Set up host coalescing defaults */
1437         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bnx_rx_coal_ticks);
1438         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bnx_tx_coal_ticks);
1439         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bnx_rx_coal_bds);
1440         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bnx_tx_coal_bds);
1441         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, sc->bnx_rx_coal_bds_int);
1442         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, sc->bnx_tx_coal_bds_int);
1443
1444         /* Set up address of status block */
1445         bzero(sc->bnx_ldata.bnx_status_block, BGE_STATUS_BLK_SZ);
1446         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1447             BGE_ADDR_HI(sc->bnx_ldata.bnx_status_block_paddr));
1448         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1449             BGE_ADDR_LO(sc->bnx_ldata.bnx_status_block_paddr));
1450
1451         /* Set up status block partail update size. */
1452         val = BGE_STATBLKSZ_32BYTE;
1453 #if 0
1454         /*
1455          * Does not seem to have visible effect in both
1456          * bulk data (1472B UDP datagram) and tiny data
1457          * (18B UDP datagram) TX tests.
1458          */
1459         val |= BGE_HCCMODE_CLRTICK_TX;
1460 #endif
1461         /* Turn on host coalescing state machine */
1462         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1463
1464         /* Turn on RX BD completion state machine and enable attentions */
1465         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1466             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1467
1468         /* Turn on RX list placement state machine */
1469         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1470
1471         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1472             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1473             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1474             BGE_MACMODE_FRMHDR_DMA_ENB;
1475
1476         if (sc->bnx_flags & BNX_FLAG_TBI)
1477                 val |= BGE_PORTMODE_TBI;
1478         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1479                 val |= BGE_PORTMODE_GMII;
1480         else
1481                 val |= BGE_PORTMODE_MII;
1482
1483         /* Turn on DMA, clear stats */
1484         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1485
1486         /* Set misc. local control, enable interrupts on attentions */
1487         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1488
1489 #ifdef notdef
1490         /* Assert GPIO pins for PHY reset */
1491         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1492             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1493         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1494             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1495 #endif
1496
1497         /* Turn on write DMA state machine */
1498         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1499         /* Enable host coalescing bug fix. */
1500         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1501         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1502                 /* Request larger DMA burst size to get better performance. */
1503                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1504         }
1505         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1506         DELAY(40);
1507
1508         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761 ||
1509             sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1510             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1511             sc->bnx_asicrev == BGE_ASICREV_BCM57780 ||
1512             BNX_IS_5717_PLUS(sc)) {
1513                 uint32_t dmactl;
1514
1515                 dmactl = CSR_READ_4(sc, BGE_RDMA_RSRVCTRL);
1516                 /*
1517                  * Adjust tx margin to prevent TX data corruption and
1518                  * fix internal FIFO overflow.
1519                  */
1520                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1521                     sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1522                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1523                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1524                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1525                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1526                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1527                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1528                 }
1529                 /*
1530                  * Enable fix for read DMA FIFO overruns.
1531                  * The fix is to limit the number of RX BDs
1532                  * the hardware would fetch at a fime.
1533                  */
1534                 CSR_WRITE_4(sc, BGE_RDMA_RSRVCTRL,
1535                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1536         }
1537
1538         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1539                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1540                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1541                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1542                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1543         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1544                 /*
1545                  * Allow 4KB burst length reads for non-LSO frames.
1546                  * Enable 512B burst length reads for buffer descriptors.
1547                  */
1548                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1549                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1550                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1551                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1552         }
1553
1554         /* Turn on read DMA state machine */
1555         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1556         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1557                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1558         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1559             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1560             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1561                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1562                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1563                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1564         }
1565         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1566                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1567                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1568                 /*
1569                  * Allow multiple outstanding read requests from
1570                  * non-LSO read DMA engine.
1571                  */
1572                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1573         }
1574         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1575         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1576         DELAY(40);
1577
1578         /* Turn on RX data completion state machine */
1579         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1580
1581         /* Turn on RX BD initiator state machine */
1582         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1583
1584         /* Turn on RX data and RX BD initiator state machine */
1585         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1586
1587         /* Turn on send BD completion state machine */
1588         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1589
1590         /* Turn on send data completion state machine */
1591         val = BGE_SDCMODE_ENABLE;
1592         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1593                 val |= BGE_SDCMODE_CDELAY;
1594         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1595
1596         /* Turn on send data initiator state machine */
1597         CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1598
1599         /* Turn on send BD initiator state machine */
1600         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1601
1602         /* Turn on send BD selector state machine */
1603         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1604
1605         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1606         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1607             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1608
1609         /* ack/clear link change events */
1610         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1611             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1612             BGE_MACSTAT_LINK_CHANGED);
1613         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1614
1615         /*
1616          * Enable attention when the link has changed state for
1617          * devices that use auto polling.
1618          */
1619         if (sc->bnx_flags & BNX_FLAG_TBI) {
1620                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1621         } else {
1622                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1623                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1624                         DELAY(80);
1625                 }
1626         }
1627
1628         /*
1629          * Clear any pending link state attention.
1630          * Otherwise some link state change events may be lost until attention
1631          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1632          * It's not necessary on newer BCM chips - perhaps enabling link
1633          * state change attentions implies clearing pending attention.
1634          */
1635         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1636             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1637             BGE_MACSTAT_LINK_CHANGED);
1638
1639         /* Enable link state change attentions. */
1640         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1641
1642         return(0);
1643 }
1644
1645 /*
1646  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1647  * against our list and return its name if we find a match. Note
1648  * that since the Broadcom controller contains VPD support, we
1649  * can get the device name string from the controller itself instead
1650  * of the compiled-in string. This is a little slow, but it guarantees
1651  * we'll always announce the right product name.
1652  */
1653 static int
1654 bnx_probe(device_t dev)
1655 {
1656         const struct bnx_type *t;
1657         uint16_t product, vendor;
1658
1659         if (!pci_is_pcie(dev))
1660                 return ENXIO;
1661
1662         product = pci_get_device(dev);
1663         vendor = pci_get_vendor(dev);
1664
1665         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1666                 if (vendor == t->bnx_vid && product == t->bnx_did)
1667                         break;
1668         }
1669         if (t->bnx_name == NULL)
1670                 return ENXIO;
1671
1672         device_set_desc(dev, t->bnx_name);
1673         return 0;
1674 }
1675
1676 static int
1677 bnx_attach(device_t dev)
1678 {
1679         struct ifnet *ifp;
1680         struct bnx_softc *sc;
1681         uint32_t hwcfg = 0, misccfg;
1682         int error = 0, rid, capmask;
1683         uint8_t ether_addr[ETHER_ADDR_LEN];
1684         uint16_t product, vendor;
1685         driver_intr_t *intr_func;
1686         uintptr_t mii_priv = 0;
1687         u_int intr_flags;
1688
1689         sc = device_get_softc(dev);
1690         sc->bnx_dev = dev;
1691         callout_init(&sc->bnx_stat_timer);
1692         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1693
1694         product = pci_get_device(dev);
1695         vendor = pci_get_vendor(dev);
1696
1697 #ifndef BURN_BRIDGES
1698         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1699                 uint32_t irq, mem;
1700
1701                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1702                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1703
1704                 device_printf(dev, "chip is in D%d power mode "
1705                     "-- setting to D0\n", pci_get_powerstate(dev));
1706
1707                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1708
1709                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1710                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1711         }
1712 #endif  /* !BURN_BRIDGE */
1713
1714         /*
1715          * Map control/status registers.
1716          */
1717         pci_enable_busmaster(dev);
1718
1719         rid = BGE_PCI_BAR0;
1720         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1721             RF_ACTIVE);
1722
1723         if (sc->bnx_res == NULL) {
1724                 device_printf(dev, "couldn't map memory\n");
1725                 return ENXIO;
1726         }
1727
1728         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1729         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1730
1731         /* Save various chip information */
1732         sc->bnx_chipid =
1733             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1734             BGE_PCIMISCCTL_ASICREV_SHIFT;
1735         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1736                 /* All chips having dedicated ASICREV register have CPMU */
1737                 sc->bnx_flags |= BNX_FLAG_CPMU;
1738
1739                 switch (product) {
1740                 case PCI_PRODUCT_BROADCOM_BCM5717:
1741                 case PCI_PRODUCT_BROADCOM_BCM5718:
1742                 case PCI_PRODUCT_BROADCOM_BCM5719:
1743                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1744                         sc->bnx_chipid = pci_read_config(dev,
1745                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1746                         break;
1747
1748                 case PCI_PRODUCT_BROADCOM_BCM57761:
1749                 case PCI_PRODUCT_BROADCOM_BCM57765:
1750                 case PCI_PRODUCT_BROADCOM_BCM57781:
1751                 case PCI_PRODUCT_BROADCOM_BCM57785:
1752                 case PCI_PRODUCT_BROADCOM_BCM57791:
1753                 case PCI_PRODUCT_BROADCOM_BCM57795:
1754                         sc->bnx_chipid = pci_read_config(dev,
1755                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1756                         break;
1757
1758                 default:
1759                         sc->bnx_chipid = pci_read_config(dev,
1760                             BGE_PCI_PRODID_ASICREV, 4);
1761                         break;
1762                 }
1763         }
1764         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1765         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1766
1767         switch (sc->bnx_asicrev) {
1768         case BGE_ASICREV_BCM5717:
1769         case BGE_ASICREV_BCM5719:
1770         case BGE_ASICREV_BCM5720:
1771         case BGE_ASICREV_BCM57765:
1772                 sc->bnx_flags |= BNX_FLAG_5717_PLUS;
1773                 break;
1774         }
1775         sc->bnx_flags |= BNX_FLAG_SHORTDMA;
1776
1777         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906)
1778                 sc->bnx_flags |= BNX_FLAG_NO_EEPROM;
1779
1780         misccfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID_MASK;
1781
1782         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
1783         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1784             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1785                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
1786         else
1787                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
1788         device_printf(dev, "CHIP ID 0x%08x; "
1789                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
1790                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
1791
1792         /*
1793          * Set various PHY quirk flags.
1794          */
1795
1796         capmask = MII_CAPMASK_DEFAULT;
1797         if ((sc->bnx_asicrev == BGE_ASICREV_BCM5703 &&
1798              (misccfg == 0x4000 || misccfg == 0x8000)) ||
1799             (sc->bnx_asicrev == BGE_ASICREV_BCM5705 &&
1800              vendor == PCI_VENDOR_BROADCOM &&
1801              (product == PCI_PRODUCT_BROADCOM_BCM5901 ||
1802               product == PCI_PRODUCT_BROADCOM_BCM5901A2 ||
1803               product == PCI_PRODUCT_BROADCOM_BCM5705F)) ||
1804             (vendor == PCI_VENDOR_BROADCOM &&
1805              (product == PCI_PRODUCT_BROADCOM_BCM5751F ||
1806               product == PCI_PRODUCT_BROADCOM_BCM5753F ||
1807               product == PCI_PRODUCT_BROADCOM_BCM5787F)) ||
1808             product == PCI_PRODUCT_BROADCOM_BCM57790 ||
1809             sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
1810                 /* 10/100 only */
1811                 capmask &= ~BMSR_EXTSTAT;
1812         }
1813
1814         mii_priv |= BRGPHY_FLAG_WIRESPEED;
1815
1816         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906 &&
1817             sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1818             sc->bnx_asicrev != BGE_ASICREV_BCM5719 &&
1819             sc->bnx_asicrev != BGE_ASICREV_BCM5720 &&
1820             sc->bnx_asicrev != BGE_ASICREV_BCM5785 &&
1821             sc->bnx_asicrev != BGE_ASICREV_BCM57765 &&
1822             sc->bnx_asicrev != BGE_ASICREV_BCM57780) {
1823                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5755 ||
1824                     sc->bnx_asicrev == BGE_ASICREV_BCM5761 ||
1825                     sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1826                     sc->bnx_asicrev == BGE_ASICREV_BCM5787) {
1827                         if (product != PCI_PRODUCT_BROADCOM_BCM5722 &&
1828                             product != PCI_PRODUCT_BROADCOM_BCM5756)
1829                                 mii_priv |= BRGPHY_FLAG_JITTER_BUG;
1830                         if (product == PCI_PRODUCT_BROADCOM_BCM5755M)
1831                                 mii_priv |= BRGPHY_FLAG_ADJUST_TRIM;
1832                 } else {
1833                         mii_priv |= BRGPHY_FLAG_BER_BUG;
1834                 }
1835         }
1836
1837         /*
1838          * Allocate interrupt
1839          */
1840         sc->bnx_irq_type = pci_alloc_1intr(dev, bnx_msi_enable, &sc->bnx_irq_rid,
1841             &intr_flags);
1842
1843         sc->bnx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bnx_irq_rid,
1844             intr_flags);
1845         if (sc->bnx_irq == NULL) {
1846                 device_printf(dev, "couldn't map interrupt\n");
1847                 error = ENXIO;
1848                 goto fail;
1849         }
1850
1851         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
1852                 sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
1853                 bnx_enable_msi(sc);
1854         }
1855
1856         /* Initialize if_name earlier, so if_printf could be used */
1857         ifp = &sc->arpcom.ac_if;
1858         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1859
1860         /* Try to reset the chip. */
1861         bnx_reset(sc);
1862
1863         if (bnx_chipinit(sc)) {
1864                 device_printf(dev, "chip initialization failed\n");
1865                 error = ENXIO;
1866                 goto fail;
1867         }
1868
1869         /*
1870          * Get station address
1871          */
1872         error = bnx_get_eaddr(sc, ether_addr);
1873         if (error) {
1874                 device_printf(dev, "failed to read station address\n");
1875                 goto fail;
1876         }
1877
1878         if (BNX_IS_5717_PLUS(sc)) {
1879                 sc->bnx_return_ring_cnt = BGE_RETURN_RING_CNT;
1880         } else {
1881                 /* 5705/5750 limits RX return ring to 512 entries. */
1882                 sc->bnx_return_ring_cnt = BGE_RETURN_RING_CNT_5705;
1883         }
1884
1885         error = bnx_dma_alloc(sc);
1886         if (error)
1887                 goto fail;
1888
1889         /* Set default tuneable values. */
1890         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
1891         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
1892         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
1893         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
1894         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_DEF;
1895         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_DEF;
1896
1897         /* Set up ifnet structure */
1898         ifp->if_softc = sc;
1899         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1900         ifp->if_ioctl = bnx_ioctl;
1901         ifp->if_start = bnx_start;
1902 #ifdef DEVICE_POLLING
1903         ifp->if_poll = bnx_poll;
1904 #endif
1905         ifp->if_watchdog = bnx_watchdog;
1906         ifp->if_init = bnx_init;
1907         ifp->if_mtu = ETHERMTU;
1908         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1909         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
1910         ifq_set_ready(&ifp->if_snd);
1911
1912         ifp->if_capabilities |= IFCAP_HWCSUM;
1913         ifp->if_hwassist = BNX_CSUM_FEATURES;
1914         ifp->if_capenable = ifp->if_capabilities;
1915
1916         /*
1917          * Figure out what sort of media we have by checking the
1918          * hardware config word in the first 32k of NIC internal memory,
1919          * or fall back to examining the EEPROM if necessary.
1920          * Note: on some BCM5700 cards, this value appears to be unset.
1921          * If that's the case, we have to rely on identifying the NIC
1922          * by its PCI subsystem ID, as we do below for the SysKonnect
1923          * SK-9D41.
1924          */
1925         if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) {
1926                 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
1927         } else {
1928                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
1929                                     sizeof(hwcfg))) {
1930                         device_printf(dev, "failed to read EEPROM\n");
1931                         error = ENXIO;
1932                         goto fail;
1933                 }
1934                 hwcfg = ntohl(hwcfg);
1935         }
1936
1937         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
1938         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
1939             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
1940                 sc->bnx_flags |= BNX_FLAG_TBI;
1941
1942         /* Setup MI MODE */
1943         if (sc->bnx_flags & BNX_FLAG_CPMU)
1944                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
1945         else
1946                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
1947
1948         /* Setup link status update stuffs */
1949         if (sc->bnx_flags & BNX_FLAG_TBI) {
1950                 sc->bnx_link_upd = bnx_tbi_link_upd;
1951                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1952         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1953                 sc->bnx_link_upd = bnx_autopoll_link_upd;
1954                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1955         } else {
1956                 sc->bnx_link_upd = bnx_copper_link_upd;
1957                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1958         }
1959
1960         /* Set default PHY address */
1961         sc->bnx_phyno = 1;
1962
1963         /*
1964          * PHY address mapping for various devices.
1965          *
1966          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
1967          * ---------+-------+-------+-------+-------+
1968          * BCM57XX  |   1   |   X   |   X   |   X   |
1969          * BCM5704  |   1   |   X   |   1   |   X   |
1970          * BCM5717  |   1   |   8   |   2   |   9   |
1971          * BCM5719  |   1   |   8   |   2   |   9   |
1972          * BCM5720  |   1   |   8   |   2   |   9   |
1973          *
1974          * Other addresses may respond but they are not
1975          * IEEE compliant PHYs and should be ignored.
1976          */
1977         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1978             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1979             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1980                 int f;
1981
1982                 f = pci_get_function(dev);
1983                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
1984                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
1985                             BGE_SGDIGSTS_IS_SERDES)
1986                                 sc->bnx_phyno = f + 8;
1987                         else
1988                                 sc->bnx_phyno = f + 1;
1989                 } else {
1990                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
1991                             BGE_CPMU_PHY_STRAP_IS_SERDES)
1992                                 sc->bnx_phyno = f + 8;
1993                         else
1994                                 sc->bnx_phyno = f + 1;
1995                 }
1996         }
1997
1998         if (sc->bnx_flags & BNX_FLAG_TBI) {
1999                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
2000                     bnx_ifmedia_upd, bnx_ifmedia_sts);
2001                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2002                 ifmedia_add(&sc->bnx_ifmedia,
2003                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2004                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2005                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2006                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2007         } else {
2008                 struct mii_probe_args mii_args;
2009
2010                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2011                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2012                 mii_args.mii_capmask = capmask;
2013                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2014                 mii_args.mii_priv = mii_priv;
2015
2016                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2017                 if (error) {
2018                         device_printf(dev, "MII without any PHY!\n");
2019                         goto fail;
2020                 }
2021         }
2022
2023         /*
2024          * Create sysctl nodes.
2025          */
2026         sysctl_ctx_init(&sc->bnx_sysctl_ctx);
2027         sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx,
2028                                               SYSCTL_STATIC_CHILDREN(_hw),
2029                                               OID_AUTO,
2030                                               device_get_nameunit(dev),
2031                                               CTLFLAG_RD, 0, "");
2032         if (sc->bnx_sysctl_tree == NULL) {
2033                 device_printf(dev, "can't add sysctl node\n");
2034                 error = ENXIO;
2035                 goto fail;
2036         }
2037
2038         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2039                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2040                         OID_AUTO, "rx_coal_ticks",
2041                         CTLTYPE_INT | CTLFLAG_RW,
2042                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2043                         "Receive coalescing ticks (usec).");
2044         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2045                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2046                         OID_AUTO, "tx_coal_ticks",
2047                         CTLTYPE_INT | CTLFLAG_RW,
2048                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2049                         "Transmit coalescing ticks (usec).");
2050         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2051                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2052                         OID_AUTO, "rx_coal_bds",
2053                         CTLTYPE_INT | CTLFLAG_RW,
2054                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2055                         "Receive max coalesced BD count.");
2056         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2057                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2058                         OID_AUTO, "tx_coal_bds",
2059                         CTLTYPE_INT | CTLFLAG_RW,
2060                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2061                         "Transmit max coalesced BD count.");
2062         /*
2063          * A common design characteristic for many Broadcom
2064          * client controllers is that they only support a
2065          * single outstanding DMA read operation on the PCIe
2066          * bus. This means that it will take twice as long to
2067          * fetch a TX frame that is split into header and
2068          * payload buffers as it does to fetch a single,
2069          * contiguous TX frame (2 reads vs. 1 read). For these
2070          * controllers, coalescing buffers to reduce the number
2071          * of memory reads is effective way to get maximum
2072          * performance(about 940Mbps).  Without collapsing TX
2073          * buffers the maximum TCP bulk transfer performance
2074          * is about 850Mbps. However forcing coalescing mbufs
2075          * consumes a lot of CPU cycles, so leave it off by
2076          * default.
2077          */
2078         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2079             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2080             "force_defrag", CTLFLAG_RW, &sc->bnx_force_defrag, 0,
2081             "Force defragment on TX path");
2082
2083         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2084             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2085             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2086             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2087             "Receive max coalesced BD count during interrupt.");
2088         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2089             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2090             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2091             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2092             "Transmit max coalesced BD count during interrupt.");
2093
2094         /*
2095          * Call MI attach routine.
2096          */
2097         ether_ifattach(ifp, ether_addr, NULL);
2098
2099         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
2100                 if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
2101                         intr_func = bnx_msi_oneshot;
2102                         if (bootverbose)
2103                                 device_printf(dev, "oneshot MSI\n");
2104                 } else {
2105                         intr_func = bnx_msi;
2106                 }
2107         } else {
2108                 intr_func = bnx_intr_legacy;
2109         }
2110         error = bus_setup_intr(dev, sc->bnx_irq, INTR_MPSAFE, intr_func, sc,
2111             &sc->bnx_intrhand, ifp->if_serializer);
2112         if (error) {
2113                 ether_ifdetach(ifp);
2114                 device_printf(dev, "couldn't set up irq\n");
2115                 goto fail;
2116         }
2117
2118         ifp->if_cpuid = rman_get_cpuid(sc->bnx_irq);
2119         KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus);
2120
2121         return(0);
2122 fail:
2123         bnx_detach(dev);
2124         return(error);
2125 }
2126
2127 static int
2128 bnx_detach(device_t dev)
2129 {
2130         struct bnx_softc *sc = device_get_softc(dev);
2131
2132         if (device_is_attached(dev)) {
2133                 struct ifnet *ifp = &sc->arpcom.ac_if;
2134
2135                 lwkt_serialize_enter(ifp->if_serializer);
2136                 bnx_stop(sc);
2137                 bnx_reset(sc);
2138                 bus_teardown_intr(dev, sc->bnx_irq, sc->bnx_intrhand);
2139                 lwkt_serialize_exit(ifp->if_serializer);
2140
2141                 ether_ifdetach(ifp);
2142         }
2143
2144         if (sc->bnx_flags & BNX_FLAG_TBI)
2145                 ifmedia_removeall(&sc->bnx_ifmedia);
2146         if (sc->bnx_miibus)
2147                 device_delete_child(dev, sc->bnx_miibus);
2148         bus_generic_detach(dev);
2149
2150         if (sc->bnx_irq != NULL) {
2151                 bus_release_resource(dev, SYS_RES_IRQ, sc->bnx_irq_rid,
2152                     sc->bnx_irq);
2153         }
2154         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI)
2155                 pci_release_msi(dev);
2156
2157         if (sc->bnx_res != NULL) {
2158                 bus_release_resource(dev, SYS_RES_MEMORY,
2159                     BGE_PCI_BAR0, sc->bnx_res);
2160         }
2161
2162         if (sc->bnx_sysctl_tree != NULL)
2163                 sysctl_ctx_free(&sc->bnx_sysctl_ctx);
2164
2165         bnx_dma_free(sc);
2166
2167         return 0;
2168 }
2169
2170 static void
2171 bnx_reset(struct bnx_softc *sc)
2172 {
2173         device_t dev;
2174         uint32_t cachesize, command, pcistate, reset;
2175         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2176         int i, val = 0;
2177         uint16_t devctl;
2178
2179         dev = sc->bnx_dev;
2180
2181         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
2182                 write_op = bnx_writemem_direct;
2183         else
2184                 write_op = bnx_writereg_ind;
2185
2186         /* Save some important PCI state. */
2187         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2188         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2189         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2190
2191         pci_write_config(dev, BGE_PCI_MISC_CTL,
2192             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2193             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2194             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2195
2196         /* Disable fastboot on controllers that support it. */
2197         if (bootverbose)
2198                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2199         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2200
2201         /*
2202          * Write the magic number to SRAM at offset 0xB50.
2203          * When firmware finishes its initialization it will
2204          * write ~BGE_MAGIC_NUMBER to the same location.
2205          */
2206         bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2207
2208         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2209
2210         /* XXX: Broadcom Linux driver. */
2211         /* Force PCI-E 1.0a mode */
2212         if (sc->bnx_asicrev != BGE_ASICREV_BCM5785 &&
2213             !BNX_IS_5717_PLUS(sc) &&
2214             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2215             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2216              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2217                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2218                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2219         }
2220         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2221                 /* Prevent PCIE link training during global reset */
2222                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2223                 reset |= (1<<29);
2224         }
2225
2226         /*
2227          * Set GPHY Power Down Override to leave GPHY
2228          * powered up in D0 uninitialized.
2229          */
2230         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2231                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2232
2233         /* Issue global reset */
2234         write_op(sc, BGE_MISC_CFG, reset);
2235
2236         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
2237                 uint32_t status, ctrl;
2238
2239                 status = CSR_READ_4(sc, BGE_VCPU_STATUS);
2240                 CSR_WRITE_4(sc, BGE_VCPU_STATUS,
2241                     status | BGE_VCPU_STATUS_DRV_RESET);
2242                 ctrl = CSR_READ_4(sc, BGE_VCPU_EXT_CTRL);
2243                 CSR_WRITE_4(sc, BGE_VCPU_EXT_CTRL,
2244                     ctrl & ~BGE_VCPU_EXT_CTRL_HALT_CPU);
2245         }
2246
2247         DELAY(1000);
2248
2249         /* XXX: Broadcom Linux driver. */
2250         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2251                 uint32_t v;
2252
2253                 DELAY(500000); /* wait for link training to complete */
2254                 v = pci_read_config(dev, 0xc4, 4);
2255                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2256         }
2257
2258         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2259
2260         /* Disable no snoop and disable relaxed ordering. */
2261         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2262
2263         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2264         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2265                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2266                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2267         }
2268
2269         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2270             devctl, 2);
2271
2272         /* Clear error status. */
2273         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2274             PCIEM_DEVSTS_CORR_ERR |
2275             PCIEM_DEVSTS_NFATAL_ERR |
2276             PCIEM_DEVSTS_FATAL_ERR |
2277             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2278
2279         /* Reset some of the PCI state that got zapped by reset */
2280         pci_write_config(dev, BGE_PCI_MISC_CTL,
2281             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2282             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2283             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2284         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2285         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2286         write_op(sc, BGE_MISC_CFG, (65 << 1));
2287
2288         /* Enable memory arbiter */
2289         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2290
2291         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
2292                 for (i = 0; i < BNX_TIMEOUT; i++) {
2293                         val = CSR_READ_4(sc, BGE_VCPU_STATUS);
2294                         if (val & BGE_VCPU_STATUS_INIT_DONE)
2295                                 break;
2296                         DELAY(100);
2297                 }
2298                 if (i == BNX_TIMEOUT) {
2299                         if_printf(&sc->arpcom.ac_if, "reset timed out\n");
2300                         return;
2301                 }
2302         } else {
2303                 /*
2304                  * Poll until we see the 1's complement of the magic number.
2305                  * This indicates that the firmware initialization
2306                  * is complete.
2307                  */
2308                 for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2309                         val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2310                         if (val == ~BGE_MAGIC_NUMBER)
2311                                 break;
2312                         DELAY(10);
2313                 }
2314                 if (i == BNX_FIRMWARE_TIMEOUT) {
2315                         if_printf(&sc->arpcom.ac_if, "firmware handshake "
2316                                   "timed out, found 0x%08x\n", val);
2317                 }
2318
2319                 /* BCM57765 A0 needs additional time before accessing. */
2320                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2321                         DELAY(10 * 1000);
2322         }
2323
2324         /*
2325          * XXX Wait for the value of the PCISTATE register to
2326          * return to its original pre-reset state. This is a
2327          * fairly good indicator of reset completion. If we don't
2328          * wait for the reset to fully complete, trying to read
2329          * from the device's non-PCI registers may yield garbage
2330          * results.
2331          */
2332         for (i = 0; i < BNX_TIMEOUT; i++) {
2333                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2334                         break;
2335                 DELAY(10);
2336         }
2337
2338         /* Fix up byte swapping */
2339         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2340
2341         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2342
2343         /*
2344          * The 5704 in TBI mode apparently needs some special
2345          * adjustment to insure the SERDES drive level is set
2346          * to 1.2V.
2347          */
2348         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2349             (sc->bnx_flags & BNX_FLAG_TBI)) {
2350                 uint32_t serdescfg;
2351
2352                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2353                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2354                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2355         }
2356
2357         /* XXX: Broadcom Linux driver. */
2358         if (!BNX_IS_5717_PLUS(sc) &&
2359             sc->bnx_chipid != BGE_CHIPID_BCM5750_A0 &&
2360             sc->bnx_asicrev != BGE_ASICREV_BCM5785) {
2361                 uint32_t v;
2362
2363                 /* Enable Data FIFO protection. */
2364                 v = CSR_READ_4(sc, 0x7c00);
2365                 CSR_WRITE_4(sc, 0x7c00, v | (1<<25));
2366         }
2367
2368         DELAY(10000);
2369
2370         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2371                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2372                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2373         }
2374 }
2375
2376 /*
2377  * Frame reception handling. This is called if there's a frame
2378  * on the receive return list.
2379  *
2380  * Note: we have to be able to handle two possibilities here:
2381  * 1) the frame is from the jumbo recieve ring
2382  * 2) the frame is from the standard receive ring
2383  */
2384
2385 static void
2386 bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod)
2387 {
2388         struct ifnet *ifp;
2389         int stdcnt = 0, jumbocnt = 0;
2390
2391         ifp = &sc->arpcom.ac_if;
2392
2393         while (sc->bnx_rx_saved_considx != rx_prod) {
2394                 struct bge_rx_bd        *cur_rx;
2395                 uint32_t                rxidx;
2396                 struct mbuf             *m = NULL;
2397                 uint16_t                vlan_tag = 0;
2398                 int                     have_tag = 0;
2399
2400                 cur_rx =
2401             &sc->bnx_ldata.bnx_rx_return_ring[sc->bnx_rx_saved_considx];
2402
2403                 rxidx = cur_rx->bge_idx;
2404                 BNX_INC(sc->bnx_rx_saved_considx, sc->bnx_return_ring_cnt);
2405
2406                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2407                         have_tag = 1;
2408                         vlan_tag = cur_rx->bge_vlan_tag;
2409                 }
2410
2411                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
2412                         BNX_INC(sc->bnx_jumbo, BGE_JUMBO_RX_RING_CNT);
2413                         jumbocnt++;
2414
2415                         if (rxidx != sc->bnx_jumbo) {
2416                                 ifp->if_ierrors++;
2417                                 if_printf(ifp, "sw jumbo index(%d) "
2418                                     "and hw jumbo index(%d) mismatch, drop!\n",
2419                                     sc->bnx_jumbo, rxidx);
2420                                 bnx_setup_rxdesc_jumbo(sc, rxidx);
2421                                 continue;
2422                         }
2423
2424                         m = sc->bnx_cdata.bnx_rx_jumbo_chain[rxidx].bnx_mbuf;
2425                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2426                                 ifp->if_ierrors++;
2427                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2428                                 continue;
2429                         }
2430                         if (bnx_newbuf_jumbo(sc, sc->bnx_jumbo, 0)) {
2431                                 ifp->if_ierrors++;
2432                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2433                                 continue;
2434                         }
2435                 } else {
2436                         BNX_INC(sc->bnx_std, BGE_STD_RX_RING_CNT);
2437                         stdcnt++;
2438
2439                         if (rxidx != sc->bnx_std) {
2440                                 ifp->if_ierrors++;
2441                                 if_printf(ifp, "sw std index(%d) "
2442                                     "and hw std index(%d) mismatch, drop!\n",
2443                                     sc->bnx_std, rxidx);
2444                                 bnx_setup_rxdesc_std(sc, rxidx);
2445                                 continue;
2446                         }
2447
2448                         m = sc->bnx_cdata.bnx_rx_std_chain[rxidx].bnx_mbuf;
2449                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2450                                 ifp->if_ierrors++;
2451                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2452                                 continue;
2453                         }
2454                         if (bnx_newbuf_std(sc, sc->bnx_std, 0)) {
2455                                 ifp->if_ierrors++;
2456                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2457                                 continue;
2458                         }
2459                 }
2460
2461                 ifp->if_ipackets++;
2462                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2463                 m->m_pkthdr.rcvif = ifp;
2464
2465                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2466                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2467                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2468                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2469                                 if ((cur_rx->bge_error_flag &
2470                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2471                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2472                         }
2473                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2474                                 m->m_pkthdr.csum_data =
2475                                     cur_rx->bge_tcp_udp_csum;
2476                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2477                                     CSUM_PSEUDO_HDR;
2478                         }
2479                 }
2480
2481                 /*
2482                  * If we received a packet with a vlan tag, pass it
2483                  * to vlan_input() instead of ether_input().
2484                  */
2485                 if (have_tag) {
2486                         m->m_flags |= M_VLANTAG;
2487                         m->m_pkthdr.ether_vlantag = vlan_tag;
2488                         have_tag = vlan_tag = 0;
2489                 }
2490                 ifp->if_input(ifp, m);
2491         }
2492
2493         bnx_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bnx_rx_saved_considx);
2494         if (stdcnt)
2495                 bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
2496         if (jumbocnt)
2497                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
2498 }
2499
2500 static void
2501 bnx_txeof(struct bnx_softc *sc, uint16_t tx_cons)
2502 {
2503         struct bge_tx_bd *cur_tx = NULL;
2504         struct ifnet *ifp;
2505
2506         ifp = &sc->arpcom.ac_if;
2507
2508         /*
2509          * Go through our tx ring and free mbufs for those
2510          * frames that have been sent.
2511          */
2512         while (sc->bnx_tx_saved_considx != tx_cons) {
2513                 uint32_t idx = 0;
2514
2515                 idx = sc->bnx_tx_saved_considx;
2516                 cur_tx = &sc->bnx_ldata.bnx_tx_ring[idx];
2517                 if (cur_tx->bge_flags & BGE_TXBDFLAG_END)
2518                         ifp->if_opackets++;
2519                 if (sc->bnx_cdata.bnx_tx_chain[idx] != NULL) {
2520                         bus_dmamap_unload(sc->bnx_cdata.bnx_tx_mtag,
2521                             sc->bnx_cdata.bnx_tx_dmamap[idx]);
2522                         m_freem(sc->bnx_cdata.bnx_tx_chain[idx]);
2523                         sc->bnx_cdata.bnx_tx_chain[idx] = NULL;
2524                 }
2525                 sc->bnx_txcnt--;
2526                 BNX_INC(sc->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2527         }
2528
2529         if (cur_tx != NULL &&
2530             (BGE_TX_RING_CNT - sc->bnx_txcnt) >=
2531             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2532                 ifp->if_flags &= ~IFF_OACTIVE;
2533
2534         if (sc->bnx_txcnt == 0)
2535                 ifp->if_timer = 0;
2536
2537         if (!ifq_is_empty(&ifp->if_snd))
2538                 if_devstart(ifp);
2539 }
2540
2541 #ifdef DEVICE_POLLING
2542
2543 static void
2544 bnx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2545 {
2546         struct bnx_softc *sc = ifp->if_softc;
2547         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2548         uint16_t rx_prod, tx_cons;
2549
2550         switch(cmd) {
2551         case POLL_REGISTER:
2552                 bnx_disable_intr(sc);
2553                 break;
2554         case POLL_DEREGISTER:
2555                 bnx_enable_intr(sc);
2556                 break;
2557         case POLL_AND_CHECK_STATUS:
2558                 /*
2559                  * Process link state changes.
2560                  */
2561                 bnx_link_poll(sc);
2562                 /* Fall through */
2563         case POLL_ONLY:
2564                 sc->bnx_status_tag = sblk->bge_status_tag;
2565                 /*
2566                  * Use a load fence to ensure that status_tag
2567                  * is saved  before rx_prod and tx_cons.
2568                  */
2569                 cpu_lfence();
2570
2571                 rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2572                 tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2573                 if (ifp->if_flags & IFF_RUNNING) {
2574                         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2575                         if (sc->bnx_rx_saved_considx != rx_prod)
2576                                 bnx_rxeof(sc, rx_prod);
2577
2578                         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2579                         if (sc->bnx_tx_saved_considx != tx_cons)
2580                                 bnx_txeof(sc, tx_cons);
2581                 }
2582                 break;
2583         }
2584 }
2585
2586 #endif
2587
2588 static void
2589 bnx_intr_legacy(void *xsc)
2590 {
2591         struct bnx_softc *sc = xsc;
2592         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2593
2594         if (sc->bnx_status_tag == sblk->bge_status_tag) {
2595                 uint32_t val;
2596
2597                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
2598                 if (val & BGE_PCISTAT_INTR_NOTACT)
2599                         return;
2600         }
2601
2602         /*
2603          * NOTE:
2604          * Interrupt will have to be disabled if tagged status
2605          * is used, else interrupt will always be asserted on
2606          * certain chips (at least on BCM5750 AX/BX).
2607          */
2608         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2609
2610         bnx_intr(sc);
2611 }
2612
2613 static void
2614 bnx_msi(void *xsc)
2615 {
2616         struct bnx_softc *sc = xsc;
2617
2618         /* Disable interrupt first */
2619         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2620         bnx_intr(sc);
2621 }
2622
2623 static void
2624 bnx_msi_oneshot(void *xsc)
2625 {
2626         bnx_intr(xsc);
2627 }
2628
2629 static void
2630 bnx_intr(struct bnx_softc *sc)
2631 {
2632         struct ifnet *ifp = &sc->arpcom.ac_if;
2633         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2634         uint16_t rx_prod, tx_cons;
2635         uint32_t status;
2636
2637         sc->bnx_status_tag = sblk->bge_status_tag;
2638         /*
2639          * Use a load fence to ensure that status_tag is saved
2640          * before rx_prod, tx_cons and status.
2641          */
2642         cpu_lfence();
2643
2644         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2645         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2646         status = sblk->bge_status;
2647
2648         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
2649                 bnx_link_poll(sc);
2650
2651         if (ifp->if_flags & IFF_RUNNING) {
2652                 if (sc->bnx_rx_saved_considx != rx_prod)
2653                         bnx_rxeof(sc, rx_prod);
2654
2655                 if (sc->bnx_tx_saved_considx != tx_cons)
2656                         bnx_txeof(sc, tx_cons);
2657         }
2658
2659         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
2660
2661         if (sc->bnx_coal_chg)
2662                 bnx_coal_change(sc);
2663 }
2664
2665 static void
2666 bnx_tick(void *xsc)
2667 {
2668         struct bnx_softc *sc = xsc;
2669         struct ifnet *ifp = &sc->arpcom.ac_if;
2670
2671         lwkt_serialize_enter(ifp->if_serializer);
2672
2673         bnx_stats_update_regs(sc);
2674
2675         if (sc->bnx_flags & BNX_FLAG_TBI) {
2676                 /*
2677                  * Since in TBI mode auto-polling can't be used we should poll
2678                  * link status manually. Here we register pending link event
2679                  * and trigger interrupt.
2680                  */
2681                 sc->bnx_link_evt++;
2682                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
2683         } else if (!sc->bnx_link) {
2684                 mii_tick(device_get_softc(sc->bnx_miibus));
2685         }
2686
2687         callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
2688
2689         lwkt_serialize_exit(ifp->if_serializer);
2690 }
2691
2692 static void
2693 bnx_stats_update_regs(struct bnx_softc *sc)
2694 {
2695         struct ifnet *ifp = &sc->arpcom.ac_if;
2696         struct bge_mac_stats_regs stats;
2697         uint32_t *s;
2698         int i;
2699
2700         s = (uint32_t *)&stats;
2701         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
2702                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
2703                 s++;
2704         }
2705
2706         ifp->if_collisions +=
2707            (stats.dot3StatsSingleCollisionFrames +
2708            stats.dot3StatsMultipleCollisionFrames +
2709            stats.dot3StatsExcessiveCollisions +
2710            stats.dot3StatsLateCollisions) -
2711            ifp->if_collisions;
2712 }
2713
2714 /*
2715  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
2716  * pointers to descriptors.
2717  */
2718 static int
2719 bnx_encap(struct bnx_softc *sc, struct mbuf **m_head0, uint32_t *txidx)
2720 {
2721         struct bge_tx_bd *d = NULL;
2722         uint16_t csum_flags = 0;
2723         bus_dma_segment_t segs[BNX_NSEG_NEW];
2724         bus_dmamap_t map;
2725         int error, maxsegs, nsegs, idx, i;
2726         struct mbuf *m_head = *m_head0, *m_new;
2727
2728         if (m_head->m_pkthdr.csum_flags) {
2729                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
2730                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
2731                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2732                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
2733                 if (m_head->m_flags & M_LASTFRAG)
2734                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
2735                 else if (m_head->m_flags & M_FRAG)
2736                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
2737         }
2738
2739         idx = *txidx;
2740         map = sc->bnx_cdata.bnx_tx_dmamap[idx];
2741
2742         maxsegs = (BGE_TX_RING_CNT - sc->bnx_txcnt) - BNX_NSEG_RSVD;
2743         KASSERT(maxsegs >= BNX_NSEG_SPARE,
2744                 ("not enough segments %d", maxsegs));
2745
2746         if (maxsegs > BNX_NSEG_NEW)
2747                 maxsegs = BNX_NSEG_NEW;
2748
2749         /*
2750          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
2751          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
2752          * but when such padded frames employ the bge IP/TCP checksum
2753          * offload, the hardware checksum assist gives incorrect results
2754          * (possibly from incorporating its own padding into the UDP/TCP
2755          * checksum; who knows).  If we pad such runts with zeros, the
2756          * onboard checksum comes out correct.
2757          */
2758         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
2759             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
2760                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
2761                 if (error)
2762                         goto back;
2763         }
2764
2765         if ((sc->bnx_flags & BNX_FLAG_SHORTDMA) && m_head->m_next != NULL) {
2766                 m_new = bnx_defrag_shortdma(m_head);
2767                 if (m_new == NULL) {
2768                         error = ENOBUFS;
2769                         goto back;
2770                 }
2771                 *m_head0 = m_head = m_new;
2772         }
2773         if (sc->bnx_force_defrag && m_head->m_next != NULL) {
2774                 /*
2775                  * Forcefully defragment mbuf chain to overcome hardware
2776                  * limitation which only support a single outstanding
2777                  * DMA read operation.  If it fails, keep moving on using
2778                  * the original mbuf chain.
2779                  */
2780                 m_new = m_defrag(m_head, MB_DONTWAIT);
2781                 if (m_new != NULL)
2782                         *m_head0 = m_head = m_new;
2783         }
2784
2785         error = bus_dmamap_load_mbuf_defrag(sc->bnx_cdata.bnx_tx_mtag, map,
2786                         m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
2787         if (error)
2788                 goto back;
2789
2790         m_head = *m_head0;
2791         bus_dmamap_sync(sc->bnx_cdata.bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
2792
2793         for (i = 0; ; i++) {
2794                 d = &sc->bnx_ldata.bnx_tx_ring[idx];
2795
2796                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
2797                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
2798                 d->bge_len = segs[i].ds_len;
2799                 d->bge_flags = csum_flags;
2800
2801                 if (i == nsegs - 1)
2802                         break;
2803                 BNX_INC(idx, BGE_TX_RING_CNT);
2804         }
2805         /* Mark the last segment as end of packet... */
2806         d->bge_flags |= BGE_TXBDFLAG_END;
2807
2808         /* Set vlan tag to the first segment of the packet. */
2809         d = &sc->bnx_ldata.bnx_tx_ring[*txidx];
2810         if (m_head->m_flags & M_VLANTAG) {
2811                 d->bge_flags |= BGE_TXBDFLAG_VLAN_TAG;
2812                 d->bge_vlan_tag = m_head->m_pkthdr.ether_vlantag;
2813         } else {
2814                 d->bge_vlan_tag = 0;
2815         }
2816
2817         /*
2818          * Insure that the map for this transmission is placed at
2819          * the array index of the last descriptor in this chain.
2820          */
2821         sc->bnx_cdata.bnx_tx_dmamap[*txidx] = sc->bnx_cdata.bnx_tx_dmamap[idx];
2822         sc->bnx_cdata.bnx_tx_dmamap[idx] = map;
2823         sc->bnx_cdata.bnx_tx_chain[idx] = m_head;
2824         sc->bnx_txcnt += nsegs;
2825
2826         BNX_INC(idx, BGE_TX_RING_CNT);
2827         *txidx = idx;
2828 back:
2829         if (error) {
2830                 m_freem(*m_head0);
2831                 *m_head0 = NULL;
2832         }
2833         return error;
2834 }
2835
2836 /*
2837  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
2838  * to the mbuf data regions directly in the transmit descriptors.
2839  */
2840 static void
2841 bnx_start(struct ifnet *ifp)
2842 {
2843         struct bnx_softc *sc = ifp->if_softc;
2844         struct mbuf *m_head = NULL;
2845         uint32_t prodidx;
2846         int need_trans;
2847
2848         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
2849                 return;
2850
2851         prodidx = sc->bnx_tx_prodidx;
2852
2853         need_trans = 0;
2854         while (sc->bnx_cdata.bnx_tx_chain[prodidx] == NULL) {
2855                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
2856                 if (m_head == NULL)
2857                         break;
2858
2859                 /*
2860                  * XXX
2861                  * The code inside the if() block is never reached since we
2862                  * must mark CSUM_IP_FRAGS in our if_hwassist to start getting
2863                  * requests to checksum TCP/UDP in a fragmented packet.
2864                  *
2865                  * XXX
2866                  * safety overkill.  If this is a fragmented packet chain
2867                  * with delayed TCP/UDP checksums, then only encapsulate
2868                  * it if we have enough descriptors to handle the entire
2869                  * chain at once.
2870                  * (paranoia -- may not actually be needed)
2871                  */
2872                 if ((m_head->m_flags & M_FIRSTFRAG) &&
2873                     (m_head->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) {
2874                         if ((BGE_TX_RING_CNT - sc->bnx_txcnt) <
2875                             m_head->m_pkthdr.csum_data + BNX_NSEG_RSVD) {
2876                                 ifp->if_flags |= IFF_OACTIVE;
2877                                 ifq_prepend(&ifp->if_snd, m_head);
2878                                 break;
2879                         }
2880                 }
2881
2882                 /*
2883                  * Sanity check: avoid coming within BGE_NSEG_RSVD
2884                  * descriptors of the end of the ring.  Also make
2885                  * sure there are BGE_NSEG_SPARE descriptors for
2886                  * jumbo buffers' defragmentation.
2887                  */
2888                 if ((BGE_TX_RING_CNT - sc->bnx_txcnt) <
2889                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
2890                         ifp->if_flags |= IFF_OACTIVE;
2891                         ifq_prepend(&ifp->if_snd, m_head);
2892                         break;
2893                 }
2894
2895                 /*
2896                  * Pack the data into the transmit ring. If we
2897                  * don't have room, set the OACTIVE flag and wait
2898                  * for the NIC to drain the ring.
2899                  */
2900                 if (bnx_encap(sc, &m_head, &prodidx)) {
2901                         ifp->if_flags |= IFF_OACTIVE;
2902                         ifp->if_oerrors++;
2903                         break;
2904                 }
2905                 need_trans = 1;
2906
2907                 ETHER_BPF_MTAP(ifp, m_head);
2908         }
2909
2910         if (!need_trans)
2911                 return;
2912
2913         /* Transmit */
2914         bnx_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
2915
2916         sc->bnx_tx_prodidx = prodidx;
2917
2918         /*
2919          * Set a timeout in case the chip goes out to lunch.
2920          */
2921         ifp->if_timer = 5;
2922 }
2923
2924 static void
2925 bnx_init(void *xsc)
2926 {
2927         struct bnx_softc *sc = xsc;
2928         struct ifnet *ifp = &sc->arpcom.ac_if;
2929         uint16_t *m;
2930         uint32_t mode;
2931
2932         ASSERT_SERIALIZED(ifp->if_serializer);
2933
2934         /* Cancel pending I/O and flush buffers. */
2935         bnx_stop(sc);
2936         bnx_reset(sc);
2937         bnx_chipinit(sc);
2938
2939         /*
2940          * Init the various state machines, ring
2941          * control blocks and firmware.
2942          */
2943         if (bnx_blockinit(sc)) {
2944                 if_printf(ifp, "initialization failure\n");
2945                 bnx_stop(sc);
2946                 return;
2947         }
2948
2949         /* Specify MTU. */
2950         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
2951             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
2952
2953         /* Load our MAC address. */
2954         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
2955         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
2956         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
2957
2958         /* Enable or disable promiscuous mode as needed. */
2959         bnx_setpromisc(sc);
2960
2961         /* Program multicast filter. */
2962         bnx_setmulti(sc);
2963
2964         /* Init RX ring. */
2965         if (bnx_init_rx_ring_std(sc)) {
2966                 if_printf(ifp, "RX ring initialization failed\n");
2967                 bnx_stop(sc);
2968                 return;
2969         }
2970
2971         /* Init jumbo RX ring. */
2972         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
2973                 if (bnx_init_rx_ring_jumbo(sc)) {
2974                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
2975                         bnx_stop(sc);
2976                         return;
2977                 }
2978         }
2979
2980         /* Init our RX return ring index */
2981         sc->bnx_rx_saved_considx = 0;
2982
2983         /* Init TX ring. */
2984         bnx_init_tx_ring(sc);
2985
2986         /* Enable TX MAC state machine lockup fix. */
2987         mode = CSR_READ_4(sc, BGE_TX_MODE);
2988         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
2989         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2990                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
2991                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
2992                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
2993         }
2994         /* Turn on transmitter */
2995         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
2996
2997         /* Turn on receiver */
2998         BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
2999
3000         /*
3001          * Set the number of good frames to receive after RX MBUF
3002          * Low Watermark has been reached.  After the RX MAC receives
3003          * this number of frames, it will drop subsequent incoming
3004          * frames until the MBUF High Watermark is reached.
3005          */
3006         if (sc->bnx_asicrev == BGE_ASICREV_BCM57765)
3007                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3008         else
3009                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3010
3011         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
3012                 if (bootverbose) {
3013                         if_printf(ifp, "MSI_MODE: %#x\n",
3014                             CSR_READ_4(sc, BGE_MSI_MODE));
3015                 }
3016         }
3017
3018         /* Tell firmware we're alive. */
3019         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3020
3021         /* Enable host interrupts if polling(4) is not enabled. */
3022         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3023 #ifdef DEVICE_POLLING
3024         if (ifp->if_flags & IFF_POLLING)
3025                 bnx_disable_intr(sc);
3026         else
3027 #endif
3028         bnx_enable_intr(sc);
3029
3030         bnx_ifmedia_upd(ifp);
3031
3032         ifp->if_flags |= IFF_RUNNING;
3033         ifp->if_flags &= ~IFF_OACTIVE;
3034
3035         callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
3036 }
3037
3038 /*
3039  * Set media options.
3040  */
3041 static int
3042 bnx_ifmedia_upd(struct ifnet *ifp)
3043 {
3044         struct bnx_softc *sc = ifp->if_softc;
3045
3046         /* If this is a 1000baseX NIC, enable the TBI port. */
3047         if (sc->bnx_flags & BNX_FLAG_TBI) {
3048                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3049
3050                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3051                         return(EINVAL);
3052
3053                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3054                 case IFM_AUTO:
3055                         break;
3056
3057                 case IFM_1000_SX:
3058                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3059                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3060                                     BGE_MACMODE_HALF_DUPLEX);
3061                         } else {
3062                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3063                                     BGE_MACMODE_HALF_DUPLEX);
3064                         }
3065                         break;
3066                 default:
3067                         return(EINVAL);
3068                 }
3069         } else {
3070                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3071
3072                 sc->bnx_link_evt++;
3073                 sc->bnx_link = 0;
3074                 if (mii->mii_instance) {
3075                         struct mii_softc *miisc;
3076
3077                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3078                                 mii_phy_reset(miisc);
3079                 }
3080                 mii_mediachg(mii);
3081
3082                 /*
3083                  * Force an interrupt so that we will call bnx_link_upd
3084                  * if needed and clear any pending link state attention.
3085                  * Without this we are not getting any further interrupts
3086                  * for link state changes and thus will not UP the link and
3087                  * not be able to send in bnx_start.  The only way to get
3088                  * things working was to receive a packet and get an RX
3089                  * intr.
3090                  *
3091                  * bnx_tick should help for fiber cards and we might not
3092                  * need to do this here if BNX_FLAG_TBI is set but as
3093                  * we poll for fiber anyway it should not harm.
3094                  */
3095                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3096         }
3097         return(0);
3098 }
3099
3100 /*
3101  * Report current media status.
3102  */
3103 static void
3104 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3105 {
3106         struct bnx_softc *sc = ifp->if_softc;
3107
3108         if (sc->bnx_flags & BNX_FLAG_TBI) {
3109                 ifmr->ifm_status = IFM_AVALID;
3110                 ifmr->ifm_active = IFM_ETHER;
3111                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3112                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3113                         ifmr->ifm_status |= IFM_ACTIVE;
3114                 } else {
3115                         ifmr->ifm_active |= IFM_NONE;
3116                         return;
3117                 }
3118
3119                 ifmr->ifm_active |= IFM_1000_SX;
3120                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3121                         ifmr->ifm_active |= IFM_HDX;
3122                 else
3123                         ifmr->ifm_active |= IFM_FDX;
3124         } else {
3125                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3126
3127                 mii_pollstat(mii);
3128                 ifmr->ifm_active = mii->mii_media_active;
3129                 ifmr->ifm_status = mii->mii_media_status;
3130         }
3131 }
3132
3133 static int
3134 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3135 {
3136         struct bnx_softc *sc = ifp->if_softc;
3137         struct ifreq *ifr = (struct ifreq *)data;
3138         int mask, error = 0;
3139
3140         ASSERT_SERIALIZED(ifp->if_serializer);
3141
3142         switch (command) {
3143         case SIOCSIFMTU:
3144                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3145                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3146                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3147                         error = EINVAL;
3148                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3149                         ifp->if_mtu = ifr->ifr_mtu;
3150                         if (ifp->if_flags & IFF_RUNNING)
3151                                 bnx_init(sc);
3152                 }
3153                 break;
3154         case SIOCSIFFLAGS:
3155                 if (ifp->if_flags & IFF_UP) {
3156                         if (ifp->if_flags & IFF_RUNNING) {
3157                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3158
3159                                 /*
3160                                  * If only the state of the PROMISC flag
3161                                  * changed, then just use the 'set promisc
3162                                  * mode' command instead of reinitializing
3163                                  * the entire NIC. Doing a full re-init
3164                                  * means reloading the firmware and waiting
3165                                  * for it to start up, which may take a
3166                                  * second or two.  Similarly for ALLMULTI.
3167                                  */
3168                                 if (mask & IFF_PROMISC)
3169                                         bnx_setpromisc(sc);
3170                                 if (mask & IFF_ALLMULTI)
3171                                         bnx_setmulti(sc);
3172                         } else {
3173                                 bnx_init(sc);
3174                         }
3175                 } else if (ifp->if_flags & IFF_RUNNING) {
3176                         bnx_stop(sc);
3177                 }
3178                 sc->bnx_if_flags = ifp->if_flags;
3179                 break;
3180         case SIOCADDMULTI:
3181         case SIOCDELMULTI:
3182                 if (ifp->if_flags & IFF_RUNNING)
3183                         bnx_setmulti(sc);
3184                 break;
3185         case SIOCSIFMEDIA:
3186         case SIOCGIFMEDIA:
3187                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3188                         error = ifmedia_ioctl(ifp, ifr,
3189                             &sc->bnx_ifmedia, command);
3190                 } else {
3191                         struct mii_data *mii;
3192
3193                         mii = device_get_softc(sc->bnx_miibus);
3194                         error = ifmedia_ioctl(ifp, ifr,
3195                                               &mii->mii_media, command);
3196                 }
3197                 break;
3198         case SIOCSIFCAP:
3199                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3200                 if (mask & IFCAP_HWCSUM) {
3201                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3202                         if (IFCAP_HWCSUM & ifp->if_capenable)
3203                                 ifp->if_hwassist = BNX_CSUM_FEATURES;
3204                         else
3205                                 ifp->if_hwassist = 0;
3206                 }
3207                 break;
3208         default:
3209                 error = ether_ioctl(ifp, command, data);
3210                 break;
3211         }
3212         return error;
3213 }
3214
3215 static void
3216 bnx_watchdog(struct ifnet *ifp)
3217 {
3218         struct bnx_softc *sc = ifp->if_softc;
3219
3220         if_printf(ifp, "watchdog timeout -- resetting\n");
3221
3222         bnx_init(sc);
3223
3224         ifp->if_oerrors++;
3225
3226         if (!ifq_is_empty(&ifp->if_snd))
3227                 if_devstart(ifp);
3228 }
3229
3230 /*
3231  * Stop the adapter and free any mbufs allocated to the
3232  * RX and TX lists.
3233  */
3234 static void
3235 bnx_stop(struct bnx_softc *sc)
3236 {
3237         struct ifnet *ifp = &sc->arpcom.ac_if;
3238
3239         ASSERT_SERIALIZED(ifp->if_serializer);
3240
3241         callout_stop(&sc->bnx_stat_timer);
3242
3243         /*
3244          * Disable all of the receiver blocks
3245          */
3246         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3247         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3248         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3249         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3250         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3251         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3252
3253         /*
3254          * Disable all of the transmit blocks
3255          */
3256         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3257         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3258         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3259         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3260         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3261         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3262
3263         /*
3264          * Shut down all of the memory managers and related
3265          * state machines.
3266          */
3267         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3268         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3269         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3270         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3271
3272         /* Disable host interrupts. */
3273         bnx_disable_intr(sc);
3274
3275         /*
3276          * Tell firmware we're shutting down.
3277          */
3278         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3279
3280         /* Free the RX lists. */
3281         bnx_free_rx_ring_std(sc);
3282
3283         /* Free jumbo RX list. */
3284         if (BNX_IS_JUMBO_CAPABLE(sc))
3285                 bnx_free_rx_ring_jumbo(sc);
3286
3287         /* Free TX buffers. */
3288         bnx_free_tx_ring(sc);
3289
3290         sc->bnx_status_tag = 0;
3291         sc->bnx_link = 0;
3292         sc->bnx_coal_chg = 0;
3293
3294         sc->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
3295
3296         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
3297         ifp->if_timer = 0;
3298 }
3299
3300 /*
3301  * Stop all chip I/O so that the kernel's probe routines don't
3302  * get confused by errant DMAs when rebooting.
3303  */
3304 static void
3305 bnx_shutdown(device_t dev)
3306 {
3307         struct bnx_softc *sc = device_get_softc(dev);
3308         struct ifnet *ifp = &sc->arpcom.ac_if;
3309
3310         lwkt_serialize_enter(ifp->if_serializer);
3311         bnx_stop(sc);
3312         bnx_reset(sc);
3313         lwkt_serialize_exit(ifp->if_serializer);
3314 }
3315
3316 static int
3317 bnx_suspend(device_t dev)
3318 {
3319         struct bnx_softc *sc = device_get_softc(dev);
3320         struct ifnet *ifp = &sc->arpcom.ac_if;
3321
3322         lwkt_serialize_enter(ifp->if_serializer);
3323         bnx_stop(sc);
3324         lwkt_serialize_exit(ifp->if_serializer);
3325
3326         return 0;
3327 }
3328
3329 static int
3330 bnx_resume(device_t dev)
3331 {
3332         struct bnx_softc *sc = device_get_softc(dev);
3333         struct ifnet *ifp = &sc->arpcom.ac_if;
3334
3335         lwkt_serialize_enter(ifp->if_serializer);
3336
3337         if (ifp->if_flags & IFF_UP) {
3338                 bnx_init(sc);
3339
3340                 if (!ifq_is_empty(&ifp->if_snd))
3341                         if_devstart(ifp);
3342         }
3343
3344         lwkt_serialize_exit(ifp->if_serializer);
3345
3346         return 0;
3347 }
3348
3349 static void
3350 bnx_setpromisc(struct bnx_softc *sc)
3351 {
3352         struct ifnet *ifp = &sc->arpcom.ac_if;
3353
3354         if (ifp->if_flags & IFF_PROMISC)
3355                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3356         else
3357                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3358 }
3359
3360 static void
3361 bnx_dma_free(struct bnx_softc *sc)
3362 {
3363         int i;
3364
3365         /* Destroy RX mbuf DMA stuffs. */
3366         if (sc->bnx_cdata.bnx_rx_mtag != NULL) {
3367                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3368                         bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3369                             sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3370                 }
3371                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3372                                    sc->bnx_cdata.bnx_rx_tmpmap);
3373                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3374         }
3375
3376         /* Destroy TX mbuf DMA stuffs. */
3377         if (sc->bnx_cdata.bnx_tx_mtag != NULL) {
3378                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
3379                         bus_dmamap_destroy(sc->bnx_cdata.bnx_tx_mtag,
3380                             sc->bnx_cdata.bnx_tx_dmamap[i]);
3381                 }
3382                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_tx_mtag);
3383         }
3384
3385         /* Destroy standard RX ring */
3386         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_std_ring_tag,
3387                            sc->bnx_cdata.bnx_rx_std_ring_map,
3388                            sc->bnx_ldata.bnx_rx_std_ring);
3389
3390         if (BNX_IS_JUMBO_CAPABLE(sc))
3391                 bnx_free_jumbo_mem(sc);
3392
3393         /* Destroy RX return ring */
3394         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_return_ring_tag,
3395                            sc->bnx_cdata.bnx_rx_return_ring_map,
3396                            sc->bnx_ldata.bnx_rx_return_ring);
3397
3398         /* Destroy TX ring */
3399         bnx_dma_block_free(sc->bnx_cdata.bnx_tx_ring_tag,
3400                            sc->bnx_cdata.bnx_tx_ring_map,
3401                            sc->bnx_ldata.bnx_tx_ring);
3402
3403         /* Destroy status block */
3404         bnx_dma_block_free(sc->bnx_cdata.bnx_status_tag,
3405                            sc->bnx_cdata.bnx_status_map,
3406                            sc->bnx_ldata.bnx_status_block);
3407
3408         /* Destroy the parent tag */
3409         if (sc->bnx_cdata.bnx_parent_tag != NULL)
3410                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
3411 }
3412
3413 static int
3414 bnx_dma_alloc(struct bnx_softc *sc)
3415 {
3416         struct ifnet *ifp = &sc->arpcom.ac_if;
3417         int i, error;
3418
3419         /*
3420          * Allocate the parent bus DMA tag appropriate for PCI.
3421          *
3422          * All of the NetExtreme/NetLink controllers have 4GB boundary
3423          * DMA bug.
3424          * Whenever an address crosses a multiple of the 4GB boundary
3425          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
3426          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
3427          * state machine will lockup and cause the device to hang.
3428          */
3429         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
3430                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3431                                    NULL, NULL,
3432                                    BUS_SPACE_MAXSIZE_32BIT, 0,
3433                                    BUS_SPACE_MAXSIZE_32BIT,
3434                                    0, &sc->bnx_cdata.bnx_parent_tag);
3435         if (error) {
3436                 if_printf(ifp, "could not allocate parent dma tag\n");
3437                 return error;
3438         }
3439
3440         /*
3441          * Create DMA tag and maps for RX mbufs.
3442          */
3443         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3444                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3445                                    NULL, NULL, MCLBYTES, 1, MCLBYTES,
3446                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK,
3447                                    &sc->bnx_cdata.bnx_rx_mtag);
3448         if (error) {
3449                 if_printf(ifp, "could not allocate RX mbuf dma tag\n");
3450                 return error;
3451         }
3452
3453         error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3454                                   BUS_DMA_WAITOK, &sc->bnx_cdata.bnx_rx_tmpmap);
3455         if (error) {
3456                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3457                 sc->bnx_cdata.bnx_rx_mtag = NULL;
3458                 return error;
3459         }
3460
3461         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3462                 error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3463                                           BUS_DMA_WAITOK,
3464                                           &sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3465                 if (error) {
3466                         int j;
3467
3468                         for (j = 0; j < i; ++j) {
3469                                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3470                                         sc->bnx_cdata.bnx_rx_std_dmamap[j]);
3471                         }
3472                         bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3473                         sc->bnx_cdata.bnx_rx_mtag = NULL;
3474
3475                         if_printf(ifp, "could not create DMA map for RX\n");
3476                         return error;
3477                 }
3478         }
3479
3480         /*
3481          * Create DMA tag and maps for TX mbufs.
3482          */
3483         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3484                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3485                                    NULL, NULL,
3486                                    BNX_JUMBO_FRAMELEN, BNX_NSEG_NEW, MCLBYTES,
3487                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK |
3488                                    BUS_DMA_ONEBPAGE,
3489                                    &sc->bnx_cdata.bnx_tx_mtag);
3490         if (error) {
3491                 if_printf(ifp, "could not allocate TX mbuf dma tag\n");
3492                 return error;
3493         }
3494
3495         for (i = 0; i < BGE_TX_RING_CNT; i++) {
3496                 error = bus_dmamap_create(sc->bnx_cdata.bnx_tx_mtag,
3497                                           BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
3498                                           &sc->bnx_cdata.bnx_tx_dmamap[i]);
3499                 if (error) {
3500                         int j;
3501
3502                         for (j = 0; j < i; ++j) {
3503                                 bus_dmamap_destroy(sc->bnx_cdata.bnx_tx_mtag,
3504                                         sc->bnx_cdata.bnx_tx_dmamap[j]);
3505                         }
3506                         bus_dma_tag_destroy(sc->bnx_cdata.bnx_tx_mtag);
3507                         sc->bnx_cdata.bnx_tx_mtag = NULL;
3508
3509                         if_printf(ifp, "could not create DMA map for TX\n");
3510                         return error;
3511                 }
3512         }
3513
3514         /*
3515          * Create DMA stuffs for standard RX ring.
3516          */
3517         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
3518                                     &sc->bnx_cdata.bnx_rx_std_ring_tag,
3519                                     &sc->bnx_cdata.bnx_rx_std_ring_map,
3520                                     (void *)&sc->bnx_ldata.bnx_rx_std_ring,
3521                                     &sc->bnx_ldata.bnx_rx_std_ring_paddr);
3522         if (error) {
3523                 if_printf(ifp, "could not create std RX ring\n");
3524                 return error;
3525         }
3526
3527         /*
3528          * Create jumbo buffer pool.
3529          */
3530         if (BNX_IS_JUMBO_CAPABLE(sc)) {
3531                 error = bnx_alloc_jumbo_mem(sc);
3532                 if (error) {
3533                         if_printf(ifp, "could not create jumbo buffer pool\n");
3534                         return error;
3535                 }
3536         }
3537
3538         /*
3539          * Create DMA stuffs for RX return ring.
3540          */
3541         error = bnx_dma_block_alloc(sc,
3542             BGE_RX_RTN_RING_SZ(sc->bnx_return_ring_cnt),
3543             &sc->bnx_cdata.bnx_rx_return_ring_tag,
3544             &sc->bnx_cdata.bnx_rx_return_ring_map,
3545             (void *)&sc->bnx_ldata.bnx_rx_return_ring,
3546             &sc->bnx_ldata.bnx_rx_return_ring_paddr);
3547         if (error) {
3548                 if_printf(ifp, "could not create RX ret ring\n");
3549                 return error;
3550         }
3551
3552         /*
3553          * Create DMA stuffs for TX ring.
3554          */
3555         error = bnx_dma_block_alloc(sc, BGE_TX_RING_SZ,
3556                                     &sc->bnx_cdata.bnx_tx_ring_tag,
3557                                     &sc->bnx_cdata.bnx_tx_ring_map,
3558                                     (void *)&sc->bnx_ldata.bnx_tx_ring,
3559                                     &sc->bnx_ldata.bnx_tx_ring_paddr);
3560         if (error) {
3561                 if_printf(ifp, "could not create TX ring\n");
3562                 return error;
3563         }
3564
3565         /*
3566          * Create DMA stuffs for status block.
3567          */
3568         error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
3569                                     &sc->bnx_cdata.bnx_status_tag,
3570                                     &sc->bnx_cdata.bnx_status_map,
3571                                     (void *)&sc->bnx_ldata.bnx_status_block,
3572                                     &sc->bnx_ldata.bnx_status_block_paddr);
3573         if (error) {
3574                 if_printf(ifp, "could not create status block\n");
3575                 return error;
3576         }
3577
3578         return 0;
3579 }
3580
3581 static int
3582 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
3583                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
3584 {
3585         bus_dmamem_t dmem;
3586         int error;
3587
3588         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
3589                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3590                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
3591         if (error)
3592                 return error;
3593
3594         *tag = dmem.dmem_tag;
3595         *map = dmem.dmem_map;
3596         *addr = dmem.dmem_addr;
3597         *paddr = dmem.dmem_busaddr;
3598
3599         return 0;
3600 }
3601
3602 static void
3603 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
3604 {
3605         if (tag != NULL) {
3606                 bus_dmamap_unload(tag, map);
3607                 bus_dmamem_free(tag, addr, map);
3608                 bus_dma_tag_destroy(tag);
3609         }
3610 }
3611
3612 static void
3613 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
3614 {
3615         struct ifnet *ifp = &sc->arpcom.ac_if;
3616
3617 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
3618
3619         /*
3620          * Sometimes PCS encoding errors are detected in
3621          * TBI mode (on fiber NICs), and for some reason
3622          * the chip will signal them as link changes.
3623          * If we get a link change event, but the 'PCS
3624          * encoding error' bit in the MAC status register
3625          * is set, don't bother doing a link check.
3626          * This avoids spurious "gigabit link up" messages
3627          * that sometimes appear on fiber NICs during
3628          * periods of heavy traffic.
3629          */
3630         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
3631                 if (!sc->bnx_link) {
3632                         sc->bnx_link++;
3633                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
3634                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3635                                     BGE_MACMODE_TBI_SEND_CFGS);
3636                         }
3637                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
3638
3639                         if (bootverbose)
3640                                 if_printf(ifp, "link UP\n");
3641
3642                         ifp->if_link_state = LINK_STATE_UP;
3643                         if_link_state_change(ifp);
3644                 }
3645         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
3646                 if (sc->bnx_link) {
3647                         sc->bnx_link = 0;
3648
3649                         if (bootverbose)
3650                                 if_printf(ifp, "link DOWN\n");
3651
3652                         ifp->if_link_state = LINK_STATE_DOWN;
3653                         if_link_state_change(ifp);
3654                 }
3655         }
3656
3657 #undef PCS_ENCODE_ERR
3658
3659         /* Clear the attention. */
3660         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3661             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3662             BGE_MACSTAT_LINK_CHANGED);
3663 }
3664
3665 static void
3666 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3667 {
3668         struct ifnet *ifp = &sc->arpcom.ac_if;
3669         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3670
3671         mii_pollstat(mii);
3672         bnx_miibus_statchg(sc->bnx_dev);
3673
3674         if (bootverbose) {
3675                 if (sc->bnx_link)
3676                         if_printf(ifp, "link UP\n");
3677                 else
3678                         if_printf(ifp, "link DOWN\n");
3679         }
3680
3681         /* Clear the attention. */
3682         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3683             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3684             BGE_MACSTAT_LINK_CHANGED);
3685 }
3686
3687 static void
3688 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3689 {
3690         struct ifnet *ifp = &sc->arpcom.ac_if;
3691         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3692
3693         mii_pollstat(mii);
3694
3695         if (!sc->bnx_link &&
3696             (mii->mii_media_status & IFM_ACTIVE) &&
3697             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3698                 sc->bnx_link++;
3699                 if (bootverbose)
3700                         if_printf(ifp, "link UP\n");
3701         } else if (sc->bnx_link &&
3702             (!(mii->mii_media_status & IFM_ACTIVE) ||
3703             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3704                 sc->bnx_link = 0;
3705                 if (bootverbose)
3706                         if_printf(ifp, "link DOWN\n");
3707         }
3708
3709         /* Clear the attention. */
3710         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3711             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3712             BGE_MACSTAT_LINK_CHANGED);
3713 }
3714
3715 static int
3716 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
3717 {
3718         struct bnx_softc *sc = arg1;
3719
3720         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3721             &sc->bnx_rx_coal_ticks,
3722             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
3723             BNX_RX_COAL_TICKS_CHG);
3724 }
3725
3726 static int
3727 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
3728 {
3729         struct bnx_softc *sc = arg1;
3730
3731         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3732             &sc->bnx_tx_coal_ticks,
3733             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
3734             BNX_TX_COAL_TICKS_CHG);
3735 }
3736
3737 static int
3738 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
3739 {
3740         struct bnx_softc *sc = arg1;
3741
3742         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3743             &sc->bnx_rx_coal_bds,
3744             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3745             BNX_RX_COAL_BDS_CHG);
3746 }
3747
3748 static int
3749 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
3750 {
3751         struct bnx_softc *sc = arg1;
3752
3753         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3754             &sc->bnx_tx_coal_bds,
3755             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3756             BNX_TX_COAL_BDS_CHG);
3757 }
3758
3759 static int
3760 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3761 {
3762         struct bnx_softc *sc = arg1;
3763
3764         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3765             &sc->bnx_rx_coal_bds_int,
3766             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3767             BNX_RX_COAL_BDS_INT_CHG);
3768 }
3769
3770 static int
3771 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3772 {
3773         struct bnx_softc *sc = arg1;
3774
3775         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3776             &sc->bnx_tx_coal_bds_int,
3777             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3778             BNX_TX_COAL_BDS_INT_CHG);
3779 }
3780
3781 static int
3782 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
3783     int coal_min, int coal_max, uint32_t coal_chg_mask)
3784 {
3785         struct bnx_softc *sc = arg1;
3786         struct ifnet *ifp = &sc->arpcom.ac_if;
3787         int error = 0, v;
3788
3789         lwkt_serialize_enter(ifp->if_serializer);
3790
3791         v = *coal;
3792         error = sysctl_handle_int(oidp, &v, 0, req);
3793         if (!error && req->newptr != NULL) {
3794                 if (v < coal_min || v > coal_max) {
3795                         error = EINVAL;
3796                 } else {
3797                         *coal = v;
3798                         sc->bnx_coal_chg |= coal_chg_mask;
3799                 }
3800         }
3801
3802         lwkt_serialize_exit(ifp->if_serializer);
3803         return error;
3804 }
3805
3806 static void
3807 bnx_coal_change(struct bnx_softc *sc)
3808 {
3809         struct ifnet *ifp = &sc->arpcom.ac_if;
3810         uint32_t val;
3811
3812         ASSERT_SERIALIZED(ifp->if_serializer);
3813
3814         if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
3815                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
3816                             sc->bnx_rx_coal_ticks);
3817                 DELAY(10);
3818                 val = CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS);
3819
3820                 if (bootverbose) {
3821                         if_printf(ifp, "rx_coal_ticks -> %u\n",
3822                                   sc->bnx_rx_coal_ticks);
3823                 }
3824         }
3825
3826         if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) {
3827                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
3828                             sc->bnx_tx_coal_ticks);
3829                 DELAY(10);
3830                 val = CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS);
3831
3832                 if (bootverbose) {
3833                         if_printf(ifp, "tx_coal_ticks -> %u\n",
3834                                   sc->bnx_tx_coal_ticks);
3835                 }
3836         }
3837
3838         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) {
3839                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
3840                             sc->bnx_rx_coal_bds);
3841                 DELAY(10);
3842                 val = CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS);
3843
3844                 if (bootverbose) {
3845                         if_printf(ifp, "rx_coal_bds -> %u\n",
3846                                   sc->bnx_rx_coal_bds);
3847                 }
3848         }
3849
3850         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) {
3851                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS,
3852                             sc->bnx_tx_coal_bds);
3853                 DELAY(10);
3854                 val = CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS);
3855
3856                 if (bootverbose) {
3857                         if_printf(ifp, "tx_max_coal_bds -> %u\n",
3858                                   sc->bnx_tx_coal_bds);
3859                 }
3860         }
3861
3862         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) {
3863                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
3864                     sc->bnx_rx_coal_bds_int);
3865                 DELAY(10);
3866                 val = CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT);
3867
3868                 if (bootverbose) {
3869                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
3870                             sc->bnx_rx_coal_bds_int);
3871                 }
3872         }
3873
3874         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) {
3875                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
3876                     sc->bnx_tx_coal_bds_int);
3877                 DELAY(10);
3878                 val = CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT);
3879
3880                 if (bootverbose) {
3881                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
3882                             sc->bnx_tx_coal_bds_int);
3883                 }
3884         }
3885
3886         sc->bnx_coal_chg = 0;
3887 }
3888
3889 static void
3890 bnx_enable_intr(struct bnx_softc *sc)
3891 {
3892         struct ifnet *ifp = &sc->arpcom.ac_if;
3893
3894         lwkt_serialize_handler_enable(ifp->if_serializer);
3895
3896         /*
3897          * Enable interrupt.
3898          */
3899         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
3900         if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
3901                 /* XXX Linux driver */
3902                 bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
3903         }
3904
3905         /*
3906          * Unmask the interrupt when we stop polling.
3907          */
3908         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
3909             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
3910
3911         /*
3912          * Trigger another interrupt, since above writing
3913          * to interrupt mailbox0 may acknowledge pending
3914          * interrupt.
3915          */
3916         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
3917 }
3918
3919 static void
3920 bnx_disable_intr(struct bnx_softc *sc)
3921 {
3922         struct ifnet *ifp = &sc->arpcom.ac_if;
3923
3924         /*
3925          * Mask the interrupt when we start polling.
3926          */
3927         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
3928             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
3929
3930         /*
3931          * Acknowledge possible asserted interrupt.
3932          */
3933         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3934
3935         lwkt_serialize_handler_disable(ifp->if_serializer);
3936 }
3937
3938 static int
3939 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[])
3940 {
3941         uint32_t mac_addr;
3942         int ret = 1;
3943
3944         mac_addr = bnx_readmem_ind(sc, 0x0c14);
3945         if ((mac_addr >> 16) == 0x484b) {
3946                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
3947                 ether_addr[1] = (uint8_t)mac_addr;
3948                 mac_addr = bnx_readmem_ind(sc, 0x0c18);
3949                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
3950                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
3951                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
3952                 ether_addr[5] = (uint8_t)mac_addr;
3953                 ret = 0;
3954         }
3955         return ret;
3956 }
3957
3958 static int
3959 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[])
3960 {
3961         int mac_offset = BGE_EE_MAC_OFFSET;
3962
3963         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906)
3964                 mac_offset = BGE_EE_MAC_OFFSET_5906;
3965
3966         return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN);
3967 }
3968
3969 static int
3970 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[])
3971 {
3972         if (sc->bnx_flags & BNX_FLAG_NO_EEPROM)
3973                 return 1;
3974
3975         return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
3976                                ETHER_ADDR_LEN);
3977 }
3978
3979 static int
3980 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[])
3981 {
3982         static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = {
3983                 /* NOTE: Order is critical */
3984                 bnx_get_eaddr_mem,
3985                 bnx_get_eaddr_nvram,
3986                 bnx_get_eaddr_eeprom,
3987                 NULL
3988         };
3989         const bnx_eaddr_fcn_t *func;
3990
3991         for (func = bnx_eaddr_funcs; *func != NULL; ++func) {
3992                 if ((*func)(sc, eaddr) == 0)
3993                         break;
3994         }
3995         return (*func == NULL ? ENXIO : 0);
3996 }
3997
3998 /*
3999  * NOTE: 'm' is not freed upon failure
4000  */
4001 struct mbuf *
4002 bnx_defrag_shortdma(struct mbuf *m)
4003 {
4004         struct mbuf *n;
4005         int found;
4006
4007         /*
4008          * If device receive two back-to-back send BDs with less than
4009          * or equal to 8 total bytes then the device may hang.  The two
4010          * back-to-back send BDs must in the same frame for this failure
4011          * to occur.  Scan mbuf chains and see whether two back-to-back
4012          * send BDs are there.  If this is the case, allocate new mbuf
4013          * and copy the frame to workaround the silicon bug.
4014          */
4015         for (n = m, found = 0; n != NULL; n = n->m_next) {
4016                 if (n->m_len < 8) {
4017                         found++;
4018                         if (found > 1)
4019                                 break;
4020                         continue;
4021                 }
4022                 found = 0;
4023         }
4024
4025         if (found > 1)
4026                 n = m_defrag(m, MB_DONTWAIT);
4027         else
4028                 n = m;
4029         return n;
4030 }
4031
4032 static void
4033 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit)
4034 {
4035         int i;
4036
4037         BNX_CLRBIT(sc, reg, bit);
4038         for (i = 0; i < BNX_TIMEOUT; i++) {
4039                 if ((CSR_READ_4(sc, reg) & bit) == 0)
4040                         return;
4041                 DELAY(100);
4042         }
4043 }
4044
4045 static void
4046 bnx_link_poll(struct bnx_softc *sc)
4047 {
4048         uint32_t status;
4049
4050         status = CSR_READ_4(sc, BGE_MAC_STS);
4051         if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) {
4052                 sc->bnx_link_evt = 0;
4053                 sc->bnx_link_upd(sc, status);
4054         }
4055 }
4056
4057 static void
4058 bnx_enable_msi(struct bnx_softc *sc)
4059 {
4060         uint32_t msi_mode;
4061
4062         msi_mode = CSR_READ_4(sc, BGE_MSI_MODE);
4063         msi_mode |= BGE_MSIMODE_ENABLE;
4064         if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
4065                 /*
4066                  * NOTE:
4067                  * 5718-PG105-R says that "one shot" mode
4068                  * does not work if MSI is used, however,
4069                  * it obviously works.
4070                  */
4071                 msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE;
4072         }
4073         CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode);
4074 }
4075
4076 static uint32_t
4077 bnx_dma_swap_options(struct bnx_softc *sc)
4078 {
4079         uint32_t dma_options;
4080
4081         dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
4082             BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
4083 #if BYTE_ORDER == BIG_ENDIAN
4084         dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
4085 #endif
4086         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
4087                 dma_options |= BGE_MODECTL_BYTESWAP_B2HRX_DATA |
4088                     BGE_MODECTL_WORDSWAP_B2HRX_DATA | BGE_MODECTL_B2HRX_ENABLE |
4089                     BGE_MODECTL_HTX2B_ENABLE;
4090         }
4091         return dma_options;
4092 }