sys/dev/netif/bnx/if_bnx.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  */
  35
  36 #include "opt_bnx.h"
  37 #include "opt_ifpoll.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/kernel.h>
  43 #include <sys/interrupt.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/queue.h>
  47 #include <sys/rman.h>
  48 #include <sys/serialize.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <netinet/ip.h>
  54 #include <netinet/tcp.h>
  55
  56 #include <net/bpf.h>
  57 #include <net/ethernet.h>
  58 #include <net/if.h>
  59 #include <net/if_arp.h>
  60 #include <net/if_dl.h>
  61 #include <net/if_media.h>
  62 #include <net/if_poll.h>
  63 #include <net/if_types.h>
  64 #include <net/ifq_var.h>
  65 #include <net/vlan/if_vlan_var.h>
  66 #include <net/vlan/if_vlan_ether.h>
  67
  68 #include <dev/netif/mii_layer/mii.h>
  69 #include <dev/netif/mii_layer/miivar.h>
  70 #include <dev/netif/mii_layer/brgphyreg.h>
  71
  72 #include <bus/pci/pcidevs.h>
  73 #include <bus/pci/pcireg.h>
  74 #include <bus/pci/pcivar.h>
  75
  76 #include <dev/netif/bge/if_bgereg.h>
  77 #include <dev/netif/bnx/if_bnxvar.h>
  78
  79 /* "device miibus" required.  See GENERIC if you get errors here. */
  80 #include "miibus_if.h"
  81
  82 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
  83
  84 #define BNX_INTR_CKINTVL        ((10 * hz) / 1000)      /* 10ms */
  85
  86 static const struct bnx_type {
  87         uint16_t                bnx_vid;
  88         uint16_t                bnx_did;
  89         char                    *bnx_name;
  90 } bnx_devs[] = {
  91         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
  92                 "Broadcom BCM5717 Gigabit Ethernet" },
  93         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C,
  94                 "Broadcom BCM5717C Gigabit Ethernet" },
  95         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
  96                 "Broadcom BCM5718 Gigabit Ethernet" },
  97         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
  98                 "Broadcom BCM5719 Gigabit Ethernet" },
  99         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
 100                 "Broadcom BCM5720 Gigabit Ethernet" },
 101
 102         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725,
 103                 "Broadcom BCM5725 Gigabit Ethernet" },
 104         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727,
 105                 "Broadcom BCM5727 Gigabit Ethernet" },
 106         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762,
 107                 "Broadcom BCM5762 Gigabit Ethernet" },
 108
 109         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
 110                 "Broadcom BCM57761 Gigabit Ethernet" },
 111         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762,
 112                 "Broadcom BCM57762 Gigabit Ethernet" },
 113         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
 114                 "Broadcom BCM57765 Gigabit Ethernet" },
 115         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766,
 116                 "Broadcom BCM57766 Gigabit Ethernet" },
 117         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
 118                 "Broadcom BCM57781 Gigabit Ethernet" },
 119         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782,
 120                 "Broadcom BCM57782 Gigabit Ethernet" },
 121         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
 122                 "Broadcom BCM57785 Gigabit Ethernet" },
 123         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786,
 124                 "Broadcom BCM57786 Gigabit Ethernet" },
 125         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
 126                 "Broadcom BCM57791 Fast Ethernet" },
 127         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
 128                 "Broadcom BCM57795 Fast Ethernet" },
 129
 130         { 0, 0, NULL }
 131 };
 132
 133 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
 134 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
 135 #define BNX_IS_57765_PLUS(sc)           ((sc)->bnx_flags & BNX_FLAG_57765_PLUS)
 136 #define BNX_IS_57765_FAMILY(sc)  \
 137         ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY)
 138
 139 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
 140
 141 static int      bnx_probe(device_t);
 142 static int      bnx_attach(device_t);
 143 static int      bnx_detach(device_t);
 144 static void     bnx_shutdown(device_t);
 145 static int      bnx_suspend(device_t);
 146 static int      bnx_resume(device_t);
 147 static int      bnx_miibus_readreg(device_t, int, int);
 148 static int      bnx_miibus_writereg(device_t, int, int, int);
 149 static void     bnx_miibus_statchg(device_t);
 150
 151 #ifdef IFPOLL_ENABLE
 152 static void     bnx_npoll(struct ifnet *, struct ifpoll_info *);
 153 static void     bnx_npoll_compat(struct ifnet *, void *, int);
 154 #endif
 155 static void     bnx_intr_legacy(void *);
 156 static void     bnx_msi(void *);
 157 static void     bnx_msi_oneshot(void *);
 158 static void     bnx_intr(struct bnx_softc *);
 159 static void     bnx_enable_intr(struct bnx_softc *);
 160 static void     bnx_disable_intr(struct bnx_softc *);
 161 static void     bnx_txeof(struct bnx_tx_ring *, uint16_t);
 162 static void     bnx_rxeof(struct bnx_softc *, uint16_t, int);
 163
 164 static void     bnx_start(struct ifnet *, struct ifaltq_subque *);
 165 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 166 static void     bnx_init(void *);
 167 static void     bnx_stop(struct bnx_softc *);
 168 static void     bnx_watchdog(struct ifnet *);
 169 static int      bnx_ifmedia_upd(struct ifnet *);
 170 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 171 static void     bnx_tick(void *);
 172
 173 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
 174 static void     bnx_free_jumbo_mem(struct bnx_softc *);
 175 static struct bnx_jslot
 176                 *bnx_jalloc(struct bnx_softc *);
 177 static void     bnx_jfree(void *);
 178 static void     bnx_jref(void *);
 179 static int      bnx_newbuf_std(struct bnx_softc *, int, int);
 180 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
 181 static void     bnx_setup_rxdesc_std(struct bnx_softc *, int);
 182 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
 183 static int      bnx_init_rx_ring_std(struct bnx_softc *);
 184 static void     bnx_free_rx_ring_std(struct bnx_softc *);
 185 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
 186 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
 187 static void     bnx_free_tx_ring(struct bnx_tx_ring *);
 188 static int      bnx_init_tx_ring(struct bnx_tx_ring *);
 189 static int      bnx_create_tx_ring(struct bnx_tx_ring *);
 190 static void     bnx_destroy_tx_ring(struct bnx_tx_ring *);
 191 static int      bnx_dma_alloc(struct bnx_softc *);
 192 static void     bnx_dma_free(struct bnx_softc *);
 193 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
 194                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
 195 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 196 static struct mbuf *
 197                 bnx_defrag_shortdma(struct mbuf *);
 198 static int      bnx_encap(struct bnx_tx_ring *, struct mbuf **,
 199                     uint32_t *, int *);
 200 static int      bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
 201                     uint16_t *, uint16_t *);
 202
 203 static void     bnx_reset(struct bnx_softc *);
 204 static int      bnx_chipinit(struct bnx_softc *);
 205 static int      bnx_blockinit(struct bnx_softc *);
 206 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
 207 static void     bnx_enable_msi(struct bnx_softc *sc);
 208 static void     bnx_setmulti(struct bnx_softc *);
 209 static void     bnx_setpromisc(struct bnx_softc *);
 210 static void     bnx_stats_update_regs(struct bnx_softc *);
 211 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
 212
 213 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
 214 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
 215 #ifdef notdef
 216 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
 217 #endif
 218 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
 219 static void     bnx_writembx(struct bnx_softc *, int, int);
 220 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
 221 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
 222 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
 223
 224 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
 225 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
 226 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
 227 static void     bnx_link_poll(struct bnx_softc *);
 228
 229 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
 230 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
 231 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
 232 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
 233
 234 static void     bnx_coal_change(struct bnx_softc *);
 235 static int      bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS);
 236 static int      bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS);
 237 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 238 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 239 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
 240 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
 241 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 242 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 243 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
 244                     int, int, uint32_t);
 245
 246 static int      bnx_msi_enable = 1;
 247 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
 248
 249 static device_method_t bnx_methods[] = {
 250         /* Device interface */
 251         DEVMETHOD(device_probe,         bnx_probe),
 252         DEVMETHOD(device_attach,        bnx_attach),
 253         DEVMETHOD(device_detach,        bnx_detach),
 254         DEVMETHOD(device_shutdown,      bnx_shutdown),
 255         DEVMETHOD(device_suspend,       bnx_suspend),
 256         DEVMETHOD(device_resume,        bnx_resume),
 257
 258         /* bus interface */
 259         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 260         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 261
 262         /* MII interface */
 263         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
 264         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
 265         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
 266
 267         DEVMETHOD_END
 268 };
 269
 270 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
 271 static devclass_t bnx_devclass;
 272
 273 DECLARE_DUMMY_MODULE(if_bnx);
 274 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
 275 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
 276
 277 static uint32_t
 278 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
 279 {
 280         device_t dev = sc->bnx_dev;
 281         uint32_t val;
 282
 283         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 284         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 285         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 286         return (val);
 287 }
 288
 289 static void
 290 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
 291 {
 292         device_t dev = sc->bnx_dev;
 293
 294         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 295         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 296         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 297 }
 298
 299 static void
 300 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
 301 {
 302         CSR_WRITE_4(sc, off, val);
 303 }
 304
 305 static void
 306 bnx_writembx(struct bnx_softc *sc, int off, int val)
 307 {
 308         CSR_WRITE_4(sc, off, val);
 309 }
 310
 311 /*
 312  * Read a sequence of bytes from NVRAM.
 313  */
 314 static int
 315 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
 316 {
 317         return (1);
 318 }
 319
 320 /*
 321  * Read a byte of data stored in the EEPROM at address 'addr.' The
 322  * BCM570x supports both the traditional bitbang interface and an
 323  * auto access interface for reading the EEPROM. We use the auto
 324  * access method.
 325  */
 326 static uint8_t
 327 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
 328 {
 329         int i;
 330         uint32_t byte = 0;
 331
 332         /*
 333          * Enable use of auto EEPROM access so we can avoid
 334          * having to use the bitbang method.
 335          */
 336         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 337
 338         /* Reset the EEPROM, load the clock period. */
 339         CSR_WRITE_4(sc, BGE_EE_ADDR,
 340             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 341         DELAY(20);
 342
 343         /* Issue the read EEPROM command. */
 344         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 345
 346         /* Wait for completion */
 347         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
 348                 DELAY(10);
 349                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 350                         break;
 351         }
 352
 353         if (i == BNX_TIMEOUT) {
 354                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 355                 return(1);
 356         }
 357
 358         /* Get result. */
 359         byte = CSR_READ_4(sc, BGE_EE_DATA);
 360
 361         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 362
 363         return(0);
 364 }
 365
 366 /*
 367  * Read a sequence of bytes from the EEPROM.
 368  */
 369 static int
 370 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
 371 {
 372         size_t i;
 373         int err;
 374         uint8_t byte;
 375
 376         for (byte = 0, err = 0, i = 0; i < len; i++) {
 377                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
 378                 if (err)
 379                         break;
 380                 *(dest + i) = byte;
 381         }
 382
 383         return(err ? 1 : 0);
 384 }
 385
 386 static int
 387 bnx_miibus_readreg(device_t dev, int phy, int reg)
 388 {
 389         struct bnx_softc *sc = device_get_softc(dev);
 390         uint32_t val;
 391         int i;
 392
 393         KASSERT(phy == sc->bnx_phyno,
 394             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 395
 396         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 397         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 398                 CSR_WRITE_4(sc, BGE_MI_MODE,
 399                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 400                 DELAY(80);
 401         }
 402
 403         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
 404             BGE_MIPHY(phy) | BGE_MIREG(reg));
 405
 406         /* Poll for the PHY register access to complete. */
 407         for (i = 0; i < BNX_TIMEOUT; i++) {
 408                 DELAY(10);
 409                 val = CSR_READ_4(sc, BGE_MI_COMM);
 410                 if ((val & BGE_MICOMM_BUSY) == 0) {
 411                         DELAY(5);
 412                         val = CSR_READ_4(sc, BGE_MI_COMM);
 413                         break;
 414                 }
 415         }
 416         if (i == BNX_TIMEOUT) {
 417                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
 418                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 419                 val = 0;
 420         }
 421
 422         /* Restore the autopoll bit if necessary. */
 423         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 424                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 425                 DELAY(80);
 426         }
 427
 428         if (val & BGE_MICOMM_READFAIL)
 429                 return 0;
 430
 431         return (val & 0xFFFF);
 432 }
 433
 434 static int
 435 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
 436 {
 437         struct bnx_softc *sc = device_get_softc(dev);
 438         int i;
 439
 440         KASSERT(phy == sc->bnx_phyno,
 441             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 442
 443         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 444         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 445                 CSR_WRITE_4(sc, BGE_MI_MODE,
 446                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 447                 DELAY(80);
 448         }
 449
 450         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
 451             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
 452
 453         for (i = 0; i < BNX_TIMEOUT; i++) {
 454                 DELAY(10);
 455                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 456                         DELAY(5);
 457                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 458                         break;
 459                 }
 460         }
 461         if (i == BNX_TIMEOUT) {
 462                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 463                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
 464         }
 465
 466         /* Restore the autopoll bit if necessary. */
 467         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 468                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 469                 DELAY(80);
 470         }
 471
 472         return 0;
 473 }
 474
 475 static void
 476 bnx_miibus_statchg(device_t dev)
 477 {
 478         struct bnx_softc *sc;
 479         struct mii_data *mii;
 480
 481         sc = device_get_softc(dev);
 482         mii = device_get_softc(sc->bnx_miibus);
 483
 484         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 485             (IFM_ACTIVE | IFM_AVALID)) {
 486                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 487                 case IFM_10_T:
 488                 case IFM_100_TX:
 489                         sc->bnx_link = 1;
 490                         break;
 491                 case IFM_1000_T:
 492                 case IFM_1000_SX:
 493                 case IFM_2500_SX:
 494                         sc->bnx_link = 1;
 495                         break;
 496                 default:
 497                         sc->bnx_link = 0;
 498                         break;
 499                 }
 500         } else {
 501                 sc->bnx_link = 0;
 502         }
 503         if (sc->bnx_link == 0)
 504                 return;
 505
 506         BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
 507         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
 508             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
 509                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
 510         } else {
 511                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
 512         }
 513
 514         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
 515                 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 516         } else {
 517                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 518         }
 519 }
 520
 521 /*
 522  * Memory management for jumbo frames.
 523  */
 524 static int
 525 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
 526 {
 527         struct ifnet *ifp = &sc->arpcom.ac_if;
 528         struct bnx_jslot *entry;
 529         uint8_t *ptr;
 530         bus_addr_t paddr;
 531         int i, error;
 532
 533         /*
 534          * Create tag for jumbo mbufs.
 535          * This is really a bit of a kludge. We allocate a special
 536          * jumbo buffer pool which (thanks to the way our DMA
 537          * memory allocation works) will consist of contiguous
 538          * pages. This means that even though a jumbo buffer might
 539          * be larger than a page size, we don't really need to
 540          * map it into more than one DMA segment. However, the
 541          * default mbuf tag will result in multi-segment mappings,
 542          * so we have to create a special jumbo mbuf tag that
 543          * lets us get away with mapping the jumbo buffers as
 544          * a single segment. I think eventually the driver should
 545          * be changed so that it uses ordinary mbufs and cluster
 546          * buffers, i.e. jumbo frames can span multiple DMA
 547          * descriptors. But that's a project for another day.
 548          */
 549
 550         /*
 551          * Create DMA stuffs for jumbo RX ring.
 552          */
 553         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 554                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 555                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 556                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
 557                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
 558         if (error) {
 559                 if_printf(ifp, "could not create jumbo RX ring\n");
 560                 return error;
 561         }
 562
 563         /*
 564          * Create DMA stuffs for jumbo buffer block.
 565          */
 566         error = bnx_dma_block_alloc(sc, BNX_JMEM,
 567                                     &sc->bnx_cdata.bnx_jumbo_tag,
 568                                     &sc->bnx_cdata.bnx_jumbo_map,
 569                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
 570                                     &paddr);
 571         if (error) {
 572                 if_printf(ifp, "could not create jumbo buffer\n");
 573                 return error;
 574         }
 575
 576         SLIST_INIT(&sc->bnx_jfree_listhead);
 577
 578         /*
 579          * Now divide it up into 9K pieces and save the addresses
 580          * in an array. Note that we play an evil trick here by using
 581          * the first few bytes in the buffer to hold the the address
 582          * of the softc structure for this interface. This is because
 583          * bnx_jfree() needs it, but it is called by the mbuf management
 584          * code which will not pass it to us explicitly.
 585          */
 586         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
 587                 entry = &sc->bnx_cdata.bnx_jslots[i];
 588                 entry->bnx_sc = sc;
 589                 entry->bnx_buf = ptr;
 590                 entry->bnx_paddr = paddr;
 591                 entry->bnx_inuse = 0;
 592                 entry->bnx_slot = i;
 593                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
 594
 595                 ptr += BNX_JLEN;
 596                 paddr += BNX_JLEN;
 597         }
 598         return 0;
 599 }
 600
 601 static void
 602 bnx_free_jumbo_mem(struct bnx_softc *sc)
 603 {
 604         /* Destroy jumbo RX ring. */
 605         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 606                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 607                            sc->bnx_ldata.bnx_rx_jumbo_ring);
 608
 609         /* Destroy jumbo buffer block. */
 610         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
 611                            sc->bnx_cdata.bnx_jumbo_map,
 612                            sc->bnx_ldata.bnx_jumbo_buf);
 613 }
 614
 615 /*
 616  * Allocate a jumbo buffer.
 617  */
 618 static struct bnx_jslot *
 619 bnx_jalloc(struct bnx_softc *sc)
 620 {
 621         struct bnx_jslot *entry;
 622
 623         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 624         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
 625         if (entry) {
 626                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
 627                 entry->bnx_inuse = 1;
 628         } else {
 629                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 630         }
 631         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 632         return(entry);
 633 }
 634
 635 /*
 636  * Adjust usage count on a jumbo buffer.
 637  */
 638 static void
 639 bnx_jref(void *arg)
 640 {
 641         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 642         struct bnx_softc *sc = entry->bnx_sc;
 643
 644         if (sc == NULL)
 645                 panic("bnx_jref: can't find softc pointer!");
 646
 647         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 648                 panic("bnx_jref: asked to reference buffer "
 649                     "that we don't manage!");
 650         } else if (entry->bnx_inuse == 0) {
 651                 panic("bnx_jref: buffer already free!");
 652         } else {
 653                 atomic_add_int(&entry->bnx_inuse, 1);
 654         }
 655 }
 656
 657 /*
 658  * Release a jumbo buffer.
 659  */
 660 static void
 661 bnx_jfree(void *arg)
 662 {
 663         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 664         struct bnx_softc *sc = entry->bnx_sc;
 665
 666         if (sc == NULL)
 667                 panic("bnx_jfree: can't find softc pointer!");
 668
 669         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 670                 panic("bnx_jfree: asked to free buffer that we don't manage!");
 671         } else if (entry->bnx_inuse == 0) {
 672                 panic("bnx_jfree: buffer already free!");
 673         } else {
 674                 /*
 675                  * Possible MP race to 0, use the serializer.  The atomic insn
 676                  * is still needed for races against bnx_jref().
 677                  */
 678                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 679                 atomic_subtract_int(&entry->bnx_inuse, 1);
 680                 if (entry->bnx_inuse == 0) {
 681                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead,
 682                                           entry, jslot_link);
 683                 }
 684                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 685         }
 686 }
 687
 688
 689 /*
 690  * Intialize a standard receive ring descriptor.
 691  */
 692 static int
 693 bnx_newbuf_std(struct bnx_softc *sc, int i, int init)
 694 {
 695         struct mbuf *m_new = NULL;
 696         bus_dma_segment_t seg;
 697         bus_dmamap_t map;
 698         int error, nsegs;
 699
 700         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
 701         if (m_new == NULL)
 702                 return ENOBUFS;
 703         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
 704         m_adj(m_new, ETHER_ALIGN);
 705
 706         error = bus_dmamap_load_mbuf_segment(sc->bnx_cdata.bnx_rx_mtag,
 707                         sc->bnx_cdata.bnx_rx_tmpmap, m_new,
 708                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
 709         if (error) {
 710                 m_freem(m_new);
 711                 return error;
 712         }
 713
 714         if (!init) {
 715                 bus_dmamap_sync(sc->bnx_cdata.bnx_rx_mtag,
 716                                 sc->bnx_cdata.bnx_rx_std_dmamap[i],
 717                                 BUS_DMASYNC_POSTREAD);
 718                 bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
 719                         sc->bnx_cdata.bnx_rx_std_dmamap[i]);
 720         }
 721
 722         map = sc->bnx_cdata.bnx_rx_tmpmap;
 723         sc->bnx_cdata.bnx_rx_tmpmap = sc->bnx_cdata.bnx_rx_std_dmamap[i];
 724         sc->bnx_cdata.bnx_rx_std_dmamap[i] = map;
 725
 726         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_mbuf = m_new;
 727         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_paddr = seg.ds_addr;
 728
 729         bnx_setup_rxdesc_std(sc, i);
 730         return 0;
 731 }
 732
 733 static void
 734 bnx_setup_rxdesc_std(struct bnx_softc *sc, int i)
 735 {
 736         struct bnx_rxchain *rc;
 737         struct bge_rx_bd *r;
 738
 739         rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
 740         r = &sc->bnx_ldata.bnx_rx_std_ring[i];
 741
 742         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
 743         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
 744         r->bge_len = rc->bnx_mbuf->m_len;
 745         r->bge_idx = i;
 746         r->bge_flags = BGE_RXBDFLAG_END;
 747 }
 748
 749 /*
 750  * Initialize a jumbo receive ring descriptor. This allocates
 751  * a jumbo buffer from the pool managed internally by the driver.
 752  */
 753 static int
 754 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
 755 {
 756         struct mbuf *m_new = NULL;
 757         struct bnx_jslot *buf;
 758         bus_addr_t paddr;
 759
 760         /* Allocate the mbuf. */
 761         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
 762         if (m_new == NULL)
 763                 return ENOBUFS;
 764
 765         /* Allocate the jumbo buffer */
 766         buf = bnx_jalloc(sc);
 767         if (buf == NULL) {
 768                 m_freem(m_new);
 769                 return ENOBUFS;
 770         }
 771
 772         /* Attach the buffer to the mbuf. */
 773         m_new->m_ext.ext_arg = buf;
 774         m_new->m_ext.ext_buf = buf->bnx_buf;
 775         m_new->m_ext.ext_free = bnx_jfree;
 776         m_new->m_ext.ext_ref = bnx_jref;
 777         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
 778
 779         m_new->m_flags |= M_EXT;
 780
 781         m_new->m_data = m_new->m_ext.ext_buf;
 782         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
 783
 784         paddr = buf->bnx_paddr;
 785         m_adj(m_new, ETHER_ALIGN);
 786         paddr += ETHER_ALIGN;
 787
 788         /* Save necessary information */
 789         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_mbuf = m_new;
 790         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_paddr = paddr;
 791
 792         /* Set up the descriptor. */
 793         bnx_setup_rxdesc_jumbo(sc, i);
 794         return 0;
 795 }
 796
 797 static void
 798 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
 799 {
 800         struct bge_rx_bd *r;
 801         struct bnx_rxchain *rc;
 802
 803         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
 804         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 805
 806         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
 807         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
 808         r->bge_len = rc->bnx_mbuf->m_len;
 809         r->bge_idx = i;
 810         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 811 }
 812
 813 static int
 814 bnx_init_rx_ring_std(struct bnx_softc *sc)
 815 {
 816         int i, error;
 817
 818         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 819                 error = bnx_newbuf_std(sc, i, 1);
 820                 if (error)
 821                         return error;
 822         }
 823
 824         sc->bnx_std = BGE_STD_RX_RING_CNT - 1;
 825         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
 826
 827         return(0);
 828 }
 829
 830 static void
 831 bnx_free_rx_ring_std(struct bnx_softc *sc)
 832 {
 833         int i;
 834
 835         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 836                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
 837
 838                 if (rc->bnx_mbuf != NULL) {
 839                         bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
 840                                           sc->bnx_cdata.bnx_rx_std_dmamap[i]);
 841                         m_freem(rc->bnx_mbuf);
 842                         rc->bnx_mbuf = NULL;
 843                 }
 844                 bzero(&sc->bnx_ldata.bnx_rx_std_ring[i],
 845                     sizeof(struct bge_rx_bd));
 846         }
 847 }
 848
 849 static int
 850 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
 851 {
 852         struct bge_rcb *rcb;
 853         int i, error;
 854
 855         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 856                 error = bnx_newbuf_jumbo(sc, i, 1);
 857                 if (error)
 858                         return error;
 859         }
 860
 861         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
 862
 863         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
 864         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
 865         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
 866
 867         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
 868
 869         return(0);
 870 }
 871
 872 static void
 873 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
 874 {
 875         int i;
 876
 877         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 878                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 879
 880                 if (rc->bnx_mbuf != NULL) {
 881                         m_freem(rc->bnx_mbuf);
 882                         rc->bnx_mbuf = NULL;
 883                 }
 884                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
 885                     sizeof(struct bge_rx_bd));
 886         }
 887 }
 888
 889 static void
 890 bnx_free_tx_ring(struct bnx_tx_ring *txr)
 891 {
 892         int i;
 893
 894         for (i = 0; i < BGE_TX_RING_CNT; i++) {
 895                 if (txr->bnx_tx_chain[i] != NULL) {
 896                         bus_dmamap_unload(txr->bnx_tx_mtag,
 897                             txr->bnx_tx_dmamap[i]);
 898                         m_freem(txr->bnx_tx_chain[i]);
 899                         txr->bnx_tx_chain[i] = NULL;
 900                 }
 901                 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd));
 902         }
 903         txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
 904 }
 905
 906 static int
 907 bnx_init_tx_ring(struct bnx_tx_ring *txr)
 908 {
 909         txr->bnx_txcnt = 0;
 910         txr->bnx_tx_saved_considx = 0;
 911         txr->bnx_tx_prodidx = 0;
 912
 913         /* Initialize transmit producer index for host-memory send ring. */
 914         bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO,
 915             txr->bnx_tx_prodidx);
 916
 917         return(0);
 918 }
 919
 920 static void
 921 bnx_setmulti(struct bnx_softc *sc)
 922 {
 923         struct ifnet *ifp;
 924         struct ifmultiaddr *ifma;
 925         uint32_t hashes[4] = { 0, 0, 0, 0 };
 926         int h, i;
 927
 928         ifp = &sc->arpcom.ac_if;
 929
 930         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
 931                 for (i = 0; i < 4; i++)
 932                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
 933                 return;
 934         }
 935
 936         /* First, zot all the existing filters. */
 937         for (i = 0; i < 4; i++)
 938                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
 939
 940         /* Now program new ones. */
 941         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 942                 if (ifma->ifma_addr->sa_family != AF_LINK)
 943                         continue;
 944                 h = ether_crc32_le(
 945                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
 946                     ETHER_ADDR_LEN) & 0x7f;
 947                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
 948         }
 949
 950         for (i = 0; i < 4; i++)
 951                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
 952 }
 953
 954 /*
 955  * Do endian, PCI and DMA initialization. Also check the on-board ROM
 956  * self-test results.
 957  */
 958 static int
 959 bnx_chipinit(struct bnx_softc *sc)
 960 {
 961         uint32_t dma_rw_ctl, mode_ctl;
 962         int i;
 963
 964         /* Set endian type before we access any non-PCI registers. */
 965         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
 966             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
 967
 968         /* Clear the MAC control register */
 969         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
 970
 971         /*
 972          * Clear the MAC statistics block in the NIC's
 973          * internal memory.
 974          */
 975         for (i = BGE_STATS_BLOCK;
 976             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
 977                 BNX_MEMWIN_WRITE(sc, i, 0);
 978
 979         for (i = BGE_STATUS_BLOCK;
 980             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
 981                 BNX_MEMWIN_WRITE(sc, i, 0);
 982
 983         if (BNX_IS_57765_FAMILY(sc)) {
 984                 uint32_t val;
 985
 986                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) {
 987                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
 988                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
 989
 990                         /* Access the lower 1K of PL PCI-E block registers. */
 991                         CSR_WRITE_4(sc, BGE_MODE_CTL,
 992                             val | BGE_MODECTL_PCIE_PL_SEL);
 993
 994                         val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5);
 995                         val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ;
 996                         CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val);
 997
 998                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
 999                 }
1000                 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) {
1001                         /* Fix transmit hangs */
1002                         val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL);
1003                         val |= BGE_CPMU_PADRNG_CTL_RDIV2;
1004                         CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val);
1005
1006                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1007                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1008
1009                         /* Access the lower 1K of DL PCI-E block registers. */
1010                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1011                             val | BGE_MODECTL_PCIE_DL_SEL);
1012
1013                         val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX);
1014                         val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK;
1015                         val |= BGE_PCIE_DL_LO_FTSMAX_VAL;
1016                         CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val);
1017
1018                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1019                 }
1020
1021                 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK);
1022                 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK;
1023                 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25;
1024                 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val);
1025         }
1026
1027         /*
1028          * Set up the PCI DMA control register.
1029          */
1030         dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4);
1031         /*
1032          * Disable 32bytes cache alignment for DMA write to host memory
1033          *
1034          * NOTE:
1035          * 64bytes cache alignment for DMA write to host memory is still
1036          * enabled.
1037          */
1038         dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1039         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1040                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1041         /*
1042          * Enable HW workaround for controllers that misinterpret
1043          * a status tag update and leave interrupts permanently
1044          * disabled.
1045          */
1046         if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1047             sc->bnx_asicrev != BGE_ASICREV_BCM5762 &&
1048             !BNX_IS_57765_FAMILY(sc))
1049                 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1050         if (bootverbose) {
1051                 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n",
1052                     dma_rw_ctl);
1053         }
1054         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1055
1056         /*
1057          * Set up general mode register.
1058          */
1059         mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
1060             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1061         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1062
1063         /*
1064          * Disable memory write invalidate.  Apparently it is not supported
1065          * properly by these devices.  Also ensure that INTx isn't disabled,
1066          * as these chips need it even when using MSI.
1067          */
1068         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1069             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1070
1071         /* Set the timer prescaler (always 66Mhz) */
1072         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1073
1074         return(0);
1075 }
1076
1077 static int
1078 bnx_blockinit(struct bnx_softc *sc)
1079 {
1080         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
1081         struct bge_rcb *rcb;
1082         bus_size_t vrcb;
1083         bge_hostaddr taddr;
1084         uint32_t val;
1085         int i, limit;
1086
1087         /*
1088          * Initialize the memory window pointer register so that
1089          * we can access the first 32K of internal NIC RAM. This will
1090          * allow us to set up the TX send ring RCBs and the RX return
1091          * ring RCBs, plus other things which live in NIC memory.
1092          */
1093         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1094
1095         /* Configure mbuf pool watermarks */
1096         if (BNX_IS_57765_PLUS(sc)) {
1097                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1098                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1099                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1100                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1101                 } else {
1102                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1103                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1104                 }
1105         } else {
1106                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1107                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1108                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1109         }
1110
1111         /* Configure DMA resource watermarks */
1112         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1113         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1114
1115         /* Enable buffer manager */
1116         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1117         /*
1118          * Change the arbitration algorithm of TXMBUF read request to
1119          * round-robin instead of priority based for BCM5719.  When
1120          * TXFIFO is almost empty, RDMA will hold its request until
1121          * TXFIFO is not almost empty.
1122          */
1123         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1124                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1125         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1126             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 ||
1127             sc->bnx_chipid == BGE_CHIPID_BCM5720_A0)
1128                 val |= BGE_BMANMODE_LOMBUF_ATTN;
1129         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1130
1131         /* Poll for buffer manager start indication */
1132         for (i = 0; i < BNX_TIMEOUT; i++) {
1133                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1134                         break;
1135                 DELAY(10);
1136         }
1137
1138         if (i == BNX_TIMEOUT) {
1139                 if_printf(&sc->arpcom.ac_if,
1140                           "buffer manager failed to start\n");
1141                 return(ENXIO);
1142         }
1143
1144         /* Enable flow-through queues */
1145         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1146         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1147
1148         /* Wait until queue initialization is complete */
1149         for (i = 0; i < BNX_TIMEOUT; i++) {
1150                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1151                         break;
1152                 DELAY(10);
1153         }
1154
1155         if (i == BNX_TIMEOUT) {
1156                 if_printf(&sc->arpcom.ac_if,
1157                           "flow-through queue init failed\n");
1158                 return(ENXIO);
1159         }
1160
1161         /*
1162          * Summary of rings supported by the controller:
1163          *
1164          * Standard Receive Producer Ring
1165          * - This ring is used to feed receive buffers for "standard"
1166          *   sized frames (typically 1536 bytes) to the controller.
1167          *
1168          * Jumbo Receive Producer Ring
1169          * - This ring is used to feed receive buffers for jumbo sized
1170          *   frames (i.e. anything bigger than the "standard" frames)
1171          *   to the controller.
1172          *
1173          * Mini Receive Producer Ring
1174          * - This ring is used to feed receive buffers for "mini"
1175          *   sized frames to the controller.
1176          * - This feature required external memory for the controller
1177          *   but was never used in a production system.  Should always
1178          *   be disabled.
1179          *
1180          * Receive Return Ring
1181          * - After the controller has placed an incoming frame into a
1182          *   receive buffer that buffer is moved into a receive return
1183          *   ring.  The driver is then responsible to passing the
1184          *   buffer up to the stack.  Many versions of the controller
1185          *   support multiple RR rings.
1186          *
1187          * Send Ring
1188          * - This ring is used for outgoing frames.  Many versions of
1189          *   the controller support multiple send rings.
1190          */
1191
1192         /* Initialize the standard receive producer ring control block. */
1193         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1194         rcb->bge_hostaddr.bge_addr_lo =
1195             BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1196         rcb->bge_hostaddr.bge_addr_hi =
1197             BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1198         if (BNX_IS_57765_PLUS(sc)) {
1199                 /*
1200                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1201                  * Bits 15-2 : Maximum RX frame size
1202                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1203                  * Bit 0     : Reserved
1204                  */
1205                 rcb->bge_maxlen_flags =
1206                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1207         } else {
1208                 /*
1209                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1210                  * Bits 15-2 : Reserved (should be 0)
1211                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1212                  * Bit 0     : Reserved
1213                  */
1214                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1215         }
1216         if (BNX_IS_5717_PLUS(sc))
1217                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1218         else
1219                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1220         /* Write the standard receive producer ring control block. */
1221         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1222         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1223         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1224         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1225         /* Reset the standard receive producer ring producer index. */
1226         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1227
1228         /*
1229          * Initialize the jumbo RX producer ring control
1230          * block.  We set the 'ring disabled' bit in the
1231          * flags field until we're actually ready to start
1232          * using this ring (i.e. once we set the MTU
1233          * high enough to require it).
1234          */
1235         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1236                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1237                 /* Get the jumbo receive producer ring RCB parameters. */
1238                 rcb->bge_hostaddr.bge_addr_lo =
1239                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1240                 rcb->bge_hostaddr.bge_addr_hi =
1241                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1242                 rcb->bge_maxlen_flags =
1243                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1244                     BGE_RCB_FLAG_RING_DISABLED);
1245                 if (BNX_IS_5717_PLUS(sc))
1246                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1247                 else
1248                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1249                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1250                     rcb->bge_hostaddr.bge_addr_hi);
1251                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1252                     rcb->bge_hostaddr.bge_addr_lo);
1253                 /* Program the jumbo receive producer ring RCB parameters. */
1254                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1255                     rcb->bge_maxlen_flags);
1256                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1257                 /* Reset the jumbo receive producer ring producer index. */
1258                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1259         }
1260
1261         /*
1262          * The BD ring replenish thresholds control how often the
1263          * hardware fetches new BD's from the producer rings in host
1264          * memory.  Setting the value too low on a busy system can
1265          * starve the hardware and recue the throughpout.
1266          *
1267          * Set the BD ring replentish thresholds. The recommended
1268          * values are 1/8th the number of descriptors allocated to
1269          * each ring.
1270          */
1271         val = 8;
1272         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1273         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1274                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1275                     BGE_JUMBO_RX_RING_CNT/8);
1276         }
1277         if (BNX_IS_57765_PLUS(sc)) {
1278                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1279                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1280         }
1281
1282         /*
1283          * Disable all send rings by setting the 'ring disabled' bit
1284          * in the flags field of all the TX send ring control blocks,
1285          * located in NIC memory.
1286          */
1287         if (BNX_IS_5717_PLUS(sc))
1288                 limit = 4;
1289         else if (BNX_IS_57765_FAMILY(sc) ||
1290             sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1291                 limit = 2;
1292         else
1293                 limit = 1;
1294         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1295         for (i = 0; i < limit; i++) {
1296                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1297                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1298                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1299                 vrcb += sizeof(struct bge_rcb);
1300         }
1301
1302         /* Configure send ring RCB 0 (we use only the first ring) */
1303         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1304         BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr);
1305         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1306         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1307         if (BNX_IS_5717_PLUS(sc)) {
1308                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
1309         } else {
1310                 RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1311                     BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1312         }
1313         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1314             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1315
1316         /*
1317          * Disable all receive return rings by setting the
1318          * 'ring disabled' bit in the flags field of all the receive
1319          * return ring control blocks, located in NIC memory.
1320          */
1321         if (BNX_IS_5717_PLUS(sc)) {
1322                 /* Should be 17, use 16 until we get an SRAM map. */
1323                 limit = 16;
1324         } else if (BNX_IS_57765_FAMILY(sc) ||
1325             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1326                 limit = 4;
1327         } else {
1328                 limit = 1;
1329         }
1330         /* Disable all receive return rings. */
1331         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1332         for (i = 0; i < limit; i++) {
1333                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1334                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1335                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1336                     BGE_RCB_FLAG_RING_DISABLED);
1337                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1338                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1339                     (i * (sizeof(uint64_t))), 0);
1340                 vrcb += sizeof(struct bge_rcb);
1341         }
1342
1343         /*
1344          * Set up receive return ring 0.  Note that the NIC address
1345          * for RX return rings is 0x0.  The return rings live entirely
1346          * within the host, so the nicaddr field in the RCB isn't used.
1347          */
1348         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1349         BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_rx_return_ring_paddr);
1350         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1351         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1352         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1353         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1354             BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0));
1355
1356         /* Set random backoff seed for TX */
1357         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1358             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1359             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1360             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1361             BGE_TX_BACKOFF_SEED_MASK);
1362
1363         /* Set inter-packet gap */
1364         val = 0x2620;
1365         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1366             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1367                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1368                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1369         }
1370         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1371
1372         /*
1373          * Specify which ring to use for packets that don't match
1374          * any RX rules.
1375          */
1376         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1377
1378         /*
1379          * Configure number of RX lists. One interrupt distribution
1380          * list, sixteen active lists, one bad frames class.
1381          */
1382         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1383
1384         /* Inialize RX list placement stats mask. */
1385         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1386         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1387
1388         /* Disable host coalescing until we get it set up */
1389         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1390
1391         /* Poll to make sure it's shut down. */
1392         for (i = 0; i < BNX_TIMEOUT; i++) {
1393                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1394                         break;
1395                 DELAY(10);
1396         }
1397
1398         if (i == BNX_TIMEOUT) {
1399                 if_printf(&sc->arpcom.ac_if,
1400                           "host coalescing engine failed to idle\n");
1401                 return(ENXIO);
1402         }
1403
1404         /* Set up host coalescing defaults */
1405         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bnx_rx_coal_ticks);
1406         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bnx_tx_coal_ticks);
1407         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bnx_rx_coal_bds);
1408         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bnx_tx_coal_bds);
1409         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, sc->bnx_rx_coal_bds_int);
1410         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, sc->bnx_tx_coal_bds_int);
1411
1412         /* Set up address of status block */
1413         bzero(sc->bnx_ldata.bnx_status_block, BGE_STATUS_BLK_SZ);
1414         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1415             BGE_ADDR_HI(sc->bnx_ldata.bnx_status_block_paddr));
1416         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1417             BGE_ADDR_LO(sc->bnx_ldata.bnx_status_block_paddr));
1418
1419         /* Set up status block partail update size. */
1420         val = BGE_STATBLKSZ_32BYTE;
1421 #if 0
1422         /*
1423          * Does not seem to have visible effect in both
1424          * bulk data (1472B UDP datagram) and tiny data
1425          * (18B UDP datagram) TX tests.
1426          */
1427         val |= BGE_HCCMODE_CLRTICK_TX;
1428 #endif
1429         /* Turn on host coalescing state machine */
1430         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1431
1432         /* Turn on RX BD completion state machine and enable attentions */
1433         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1434             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1435
1436         /* Turn on RX list placement state machine */
1437         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1438
1439         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1440             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1441             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1442             BGE_MACMODE_FRMHDR_DMA_ENB;
1443
1444         if (sc->bnx_flags & BNX_FLAG_TBI)
1445                 val |= BGE_PORTMODE_TBI;
1446         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1447                 val |= BGE_PORTMODE_GMII;
1448         else
1449                 val |= BGE_PORTMODE_MII;
1450
1451         /* Turn on DMA, clear stats */
1452         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1453
1454         /* Set misc. local control, enable interrupts on attentions */
1455         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1456
1457 #ifdef notdef
1458         /* Assert GPIO pins for PHY reset */
1459         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1460             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1461         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1462             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1463 #endif
1464
1465         /* Turn on write DMA state machine */
1466         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1467         /* Enable host coalescing bug fix. */
1468         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1469         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1470                 /* Request larger DMA burst size to get better performance. */
1471                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1472         }
1473         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1474         DELAY(40);
1475
1476         if (BNX_IS_57765_PLUS(sc)) {
1477                 uint32_t dmactl, dmactl_reg;
1478
1479                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1480                         dmactl_reg = BGE_RDMA_RSRVCTRL2;
1481                 else
1482                         dmactl_reg = BGE_RDMA_RSRVCTRL;
1483
1484                 dmactl = CSR_READ_4(sc, dmactl_reg);
1485                 /*
1486                  * Adjust tx margin to prevent TX data corruption and
1487                  * fix internal FIFO overflow.
1488                  */
1489                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1490                     sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1491                     sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1492                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1493                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1494                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1495                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1496                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1497                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1498                 }
1499                 /*
1500                  * Enable fix for read DMA FIFO overruns.
1501                  * The fix is to limit the number of RX BDs
1502                  * the hardware would fetch at a fime.
1503                  */
1504                 CSR_WRITE_4(sc, dmactl_reg,
1505                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1506         }
1507
1508         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1509                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1510                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1511                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1512                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1513         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1514             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1515                 uint32_t ctrl_reg;
1516
1517                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1518                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2;
1519                 else
1520                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL;
1521
1522                 /*
1523                  * Allow 4KB burst length reads for non-LSO frames.
1524                  * Enable 512B burst length reads for buffer descriptors.
1525                  */
1526                 CSR_WRITE_4(sc, ctrl_reg,
1527                     CSR_READ_4(sc, ctrl_reg) |
1528                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1529                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1530         }
1531
1532         /* Turn on read DMA state machine */
1533         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1534         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1535                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1536         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1537             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1538             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1539                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1540                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1541                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1542         }
1543         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1544             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1545                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1546                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1547                 /*
1548                  * Allow multiple outstanding read requests from
1549                  * non-LSO read DMA engine.
1550                  */
1551                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1552         }
1553         if (sc->bnx_asicrev == BGE_ASICREV_BCM57766)
1554                 val |= BGE_RDMAMODE_JMB_2K_MMRR;
1555         if (sc->bnx_flags & BNX_FLAG_TSO)
1556                 val |= BGE_RDMAMODE_TSO4_ENABLE;
1557         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1558         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1559         DELAY(40);
1560
1561         /* Turn on RX data completion state machine */
1562         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1563
1564         /* Turn on RX BD initiator state machine */
1565         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1566
1567         /* Turn on RX data and RX BD initiator state machine */
1568         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1569
1570         /* Turn on send BD completion state machine */
1571         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1572
1573         /* Turn on send data completion state machine */
1574         val = BGE_SDCMODE_ENABLE;
1575         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1576                 val |= BGE_SDCMODE_CDELAY;
1577         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1578
1579         /* Turn on send data initiator state machine */
1580         if (sc->bnx_flags & BNX_FLAG_TSO) {
1581                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
1582                     BGE_SDIMODE_HW_LSO_PRE_DMA);
1583         } else {
1584                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1585         }
1586
1587         /* Turn on send BD initiator state machine */
1588         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1589
1590         /* Turn on send BD selector state machine */
1591         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1592
1593         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1594         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1595             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1596
1597         /* ack/clear link change events */
1598         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1599             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1600             BGE_MACSTAT_LINK_CHANGED);
1601         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1602
1603         /*
1604          * Enable attention when the link has changed state for
1605          * devices that use auto polling.
1606          */
1607         if (sc->bnx_flags & BNX_FLAG_TBI) {
1608                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1609         } else {
1610                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1611                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1612                         DELAY(80);
1613                 }
1614         }
1615
1616         /*
1617          * Clear any pending link state attention.
1618          * Otherwise some link state change events may be lost until attention
1619          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1620          * It's not necessary on newer BCM chips - perhaps enabling link
1621          * state change attentions implies clearing pending attention.
1622          */
1623         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1624             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1625             BGE_MACSTAT_LINK_CHANGED);
1626
1627         /* Enable link state change attentions. */
1628         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1629
1630         return(0);
1631 }
1632
1633 /*
1634  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1635  * against our list and return its name if we find a match. Note
1636  * that since the Broadcom controller contains VPD support, we
1637  * can get the device name string from the controller itself instead
1638  * of the compiled-in string. This is a little slow, but it guarantees
1639  * we'll always announce the right product name.
1640  */
1641 static int
1642 bnx_probe(device_t dev)
1643 {
1644         const struct bnx_type *t;
1645         uint16_t product, vendor;
1646
1647         if (!pci_is_pcie(dev))
1648                 return ENXIO;
1649
1650         product = pci_get_device(dev);
1651         vendor = pci_get_vendor(dev);
1652
1653         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1654                 if (vendor == t->bnx_vid && product == t->bnx_did)
1655                         break;
1656         }
1657         if (t->bnx_name == NULL)
1658                 return ENXIO;
1659
1660         device_set_desc(dev, t->bnx_name);
1661         return 0;
1662 }
1663
1664 static int
1665 bnx_attach(device_t dev)
1666 {
1667         struct ifnet *ifp;
1668         struct bnx_softc *sc;
1669         uint32_t hwcfg = 0;
1670         int error = 0, rid, capmask;
1671         uint8_t ether_addr[ETHER_ADDR_LEN];
1672         uint16_t product;
1673         driver_intr_t *intr_func;
1674         uintptr_t mii_priv = 0;
1675         u_int intr_flags;
1676 #ifdef BNX_TSO_DEBUG
1677         char desc[32];
1678         int i;
1679 #endif
1680
1681         sc = device_get_softc(dev);
1682         sc->bnx_dev = dev;
1683         callout_init_mp(&sc->bnx_stat_timer);
1684         callout_init_mp(&sc->bnx_intr_timer);
1685         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1686
1687         product = pci_get_device(dev);
1688
1689 #ifndef BURN_BRIDGES
1690         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1691                 uint32_t irq, mem;
1692
1693                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1694                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1695
1696                 device_printf(dev, "chip is in D%d power mode "
1697                     "-- setting to D0\n", pci_get_powerstate(dev));
1698
1699                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1700
1701                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1702                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1703         }
1704 #endif  /* !BURN_BRIDGE */
1705
1706         /*
1707          * Map control/status registers.
1708          */
1709         pci_enable_busmaster(dev);
1710
1711         rid = BGE_PCI_BAR0;
1712         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1713             RF_ACTIVE);
1714
1715         if (sc->bnx_res == NULL) {
1716                 device_printf(dev, "couldn't map memory\n");
1717                 return ENXIO;
1718         }
1719
1720         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1721         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1722
1723         /* Save various chip information */
1724         sc->bnx_chipid =
1725             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1726             BGE_PCIMISCCTL_ASICREV_SHIFT;
1727         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1728                 /* All chips having dedicated ASICREV register have CPMU */
1729                 sc->bnx_flags |= BNX_FLAG_CPMU;
1730
1731                 switch (product) {
1732                 case PCI_PRODUCT_BROADCOM_BCM5717:
1733                 case PCI_PRODUCT_BROADCOM_BCM5717C:
1734                 case PCI_PRODUCT_BROADCOM_BCM5718:
1735                 case PCI_PRODUCT_BROADCOM_BCM5719:
1736                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1737                 case PCI_PRODUCT_BROADCOM_BCM5725:
1738                 case PCI_PRODUCT_BROADCOM_BCM5727:
1739                 case PCI_PRODUCT_BROADCOM_BCM5762:
1740                         sc->bnx_chipid = pci_read_config(dev,
1741                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1742                         break;
1743
1744                 case PCI_PRODUCT_BROADCOM_BCM57761:
1745                 case PCI_PRODUCT_BROADCOM_BCM57762:
1746                 case PCI_PRODUCT_BROADCOM_BCM57765:
1747                 case PCI_PRODUCT_BROADCOM_BCM57766:
1748                 case PCI_PRODUCT_BROADCOM_BCM57781:
1749                 case PCI_PRODUCT_BROADCOM_BCM57782:
1750                 case PCI_PRODUCT_BROADCOM_BCM57785:
1751                 case PCI_PRODUCT_BROADCOM_BCM57786:
1752                 case PCI_PRODUCT_BROADCOM_BCM57791:
1753                 case PCI_PRODUCT_BROADCOM_BCM57795:
1754                         sc->bnx_chipid = pci_read_config(dev,
1755                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1756                         break;
1757
1758                 default:
1759                         sc->bnx_chipid = pci_read_config(dev,
1760                             BGE_PCI_PRODID_ASICREV, 4);
1761                         break;
1762                 }
1763         }
1764         if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0)
1765                 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0;
1766
1767         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1768         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1769
1770         switch (sc->bnx_asicrev) {
1771         case BGE_ASICREV_BCM5717:
1772         case BGE_ASICREV_BCM5719:
1773         case BGE_ASICREV_BCM5720:
1774                 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS;
1775                 break;
1776
1777         case BGE_ASICREV_BCM5762:
1778                 sc->bnx_flags |= BNX_FLAG_57765_PLUS;
1779                 break;
1780
1781         case BGE_ASICREV_BCM57765:
1782         case BGE_ASICREV_BCM57766:
1783                 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS;
1784                 break;
1785         }
1786
1787         sc->bnx_flags |= BNX_FLAG_TSO;
1788         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 &&
1789             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0)
1790                 sc->bnx_flags &= ~BNX_FLAG_TSO;
1791
1792         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1793             BNX_IS_57765_FAMILY(sc)) {
1794                 /*
1795                  * All BCM57785 and BCM5718 families chips have a bug that
1796                  * under certain situation interrupt will not be enabled
1797                  * even if status tag is written to BGE_MBX_IRQ0_LO mailbox.
1798                  *
1799                  * While BCM5719 and BCM5720 have a hardware workaround
1800                  * which could fix the above bug.
1801                  * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in
1802                  * bnx_chipinit().
1803                  *
1804                  * For the rest of the chips in these two families, we will
1805                  * have to poll the status block at high rate (10ms currently)
1806                  * to check whether the interrupt is hosed or not.
1807                  * See bnx_intr_check() for details.
1808                  */
1809                 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
1810         }
1811
1812         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
1813         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1814             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1815                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
1816         else
1817                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
1818         device_printf(dev, "CHIP ID 0x%08x; "
1819                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
1820                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
1821
1822         /*
1823          * Set various PHY quirk flags.
1824          */
1825
1826         capmask = MII_CAPMASK_DEFAULT;
1827         if (product == PCI_PRODUCT_BROADCOM_BCM57791 ||
1828             product == PCI_PRODUCT_BROADCOM_BCM57795) {
1829                 /* 10/100 only */
1830                 capmask &= ~BMSR_EXTSTAT;
1831         }
1832
1833         mii_priv |= BRGPHY_FLAG_WIRESPEED;
1834         if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0)
1835                 mii_priv |= BRGPHY_FLAG_5762_A0;
1836
1837         /* Initialize if_name earlier, so if_printf could be used */
1838         ifp = &sc->arpcom.ac_if;
1839         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1840
1841         /* Try to reset the chip. */
1842         bnx_reset(sc);
1843
1844         if (bnx_chipinit(sc)) {
1845                 device_printf(dev, "chip initialization failed\n");
1846                 error = ENXIO;
1847                 goto fail;
1848         }
1849
1850         /*
1851          * Get station address
1852          */
1853         error = bnx_get_eaddr(sc, ether_addr);
1854         if (error) {
1855                 device_printf(dev, "failed to read station address\n");
1856                 goto fail;
1857         }
1858
1859         /* XXX */
1860         sc->bnx_tx_ringcnt = 1;
1861
1862         error = bnx_dma_alloc(sc);
1863         if (error)
1864                 goto fail;
1865
1866         /*
1867          * Allocate interrupt
1868          */
1869         sc->bnx_irq_type = pci_alloc_1intr(dev, bnx_msi_enable, &sc->bnx_irq_rid,
1870             &intr_flags);
1871
1872         sc->bnx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bnx_irq_rid,
1873             intr_flags);
1874         if (sc->bnx_irq == NULL) {
1875                 device_printf(dev, "couldn't map interrupt\n");
1876                 error = ENXIO;
1877                 goto fail;
1878         }
1879
1880         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
1881                 sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
1882                 bnx_enable_msi(sc);
1883         }
1884
1885         /* Set default tuneable values. */
1886         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
1887         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
1888         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
1889         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
1890         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF;
1891         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF;
1892
1893         /* Set up ifnet structure */
1894         ifp->if_softc = sc;
1895         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1896         ifp->if_ioctl = bnx_ioctl;
1897         ifp->if_start = bnx_start;
1898 #ifdef IFPOLL_ENABLE
1899         ifp->if_npoll = bnx_npoll;
1900 #endif
1901         ifp->if_watchdog = bnx_watchdog;
1902         ifp->if_init = bnx_init;
1903         ifp->if_mtu = ETHERMTU;
1904         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1905         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
1906         ifq_set_ready(&ifp->if_snd);
1907
1908         ifp->if_capabilities |= IFCAP_HWCSUM;
1909         ifp->if_hwassist = BNX_CSUM_FEATURES;
1910         if (sc->bnx_flags & BNX_FLAG_TSO) {
1911                 ifp->if_capabilities |= IFCAP_TSO;
1912                 ifp->if_hwassist |= CSUM_TSO;
1913         }
1914         ifp->if_capenable = ifp->if_capabilities;
1915
1916         /*
1917          * Figure out what sort of media we have by checking the
1918          * hardware config word in the first 32k of NIC internal memory,
1919          * or fall back to examining the EEPROM if necessary.
1920          * Note: on some BCM5700 cards, this value appears to be unset.
1921          * If that's the case, we have to rely on identifying the NIC
1922          * by its PCI subsystem ID, as we do below for the SysKonnect
1923          * SK-9D41.
1924          */
1925         if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) {
1926                 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
1927         } else {
1928                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
1929                                     sizeof(hwcfg))) {
1930                         device_printf(dev, "failed to read EEPROM\n");
1931                         error = ENXIO;
1932                         goto fail;
1933                 }
1934                 hwcfg = ntohl(hwcfg);
1935         }
1936
1937         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
1938         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
1939             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
1940                 sc->bnx_flags |= BNX_FLAG_TBI;
1941
1942         /* Setup MI MODE */
1943         if (sc->bnx_flags & BNX_FLAG_CPMU)
1944                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
1945         else
1946                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
1947
1948         /* Setup link status update stuffs */
1949         if (sc->bnx_flags & BNX_FLAG_TBI) {
1950                 sc->bnx_link_upd = bnx_tbi_link_upd;
1951                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1952         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1953                 sc->bnx_link_upd = bnx_autopoll_link_upd;
1954                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1955         } else {
1956                 sc->bnx_link_upd = bnx_copper_link_upd;
1957                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
1958         }
1959
1960         /* Set default PHY address */
1961         sc->bnx_phyno = 1;
1962
1963         /*
1964          * PHY address mapping for various devices.
1965          *
1966          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
1967          * ---------+-------+-------+-------+-------+
1968          * BCM57XX  |   1   |   X   |   X   |   X   |
1969          * BCM5704  |   1   |   X   |   1   |   X   |
1970          * BCM5717  |   1   |   8   |   2   |   9   |
1971          * BCM5719  |   1   |   8   |   2   |   9   |
1972          * BCM5720  |   1   |   8   |   2   |   9   |
1973          *
1974          * Other addresses may respond but they are not
1975          * IEEE compliant PHYs and should be ignored.
1976          */
1977         if (BNX_IS_5717_PLUS(sc)) {
1978                 int f;
1979
1980                 f = pci_get_function(dev);
1981                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
1982                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
1983                             BGE_SGDIGSTS_IS_SERDES)
1984                                 sc->bnx_phyno = f + 8;
1985                         else
1986                                 sc->bnx_phyno = f + 1;
1987                 } else {
1988                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
1989                             BGE_CPMU_PHY_STRAP_IS_SERDES)
1990                                 sc->bnx_phyno = f + 8;
1991                         else
1992                                 sc->bnx_phyno = f + 1;
1993                 }
1994         }
1995
1996         if (sc->bnx_flags & BNX_FLAG_TBI) {
1997                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
1998                     bnx_ifmedia_upd, bnx_ifmedia_sts);
1999                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2000                 ifmedia_add(&sc->bnx_ifmedia,
2001                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2002                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2003                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2004                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2005         } else {
2006                 struct mii_probe_args mii_args;
2007
2008                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2009                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2010                 mii_args.mii_capmask = capmask;
2011                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2012                 mii_args.mii_priv = mii_priv;
2013
2014                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2015                 if (error) {
2016                         device_printf(dev, "MII without any PHY!\n");
2017                         goto fail;
2018                 }
2019         }
2020
2021         /*
2022          * Create sysctl nodes.
2023          */
2024         sysctl_ctx_init(&sc->bnx_sysctl_ctx);
2025         sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx,
2026                                               SYSCTL_STATIC_CHILDREN(_hw),
2027                                               OID_AUTO,
2028                                               device_get_nameunit(dev),
2029                                               CTLFLAG_RD, 0, "");
2030         if (sc->bnx_sysctl_tree == NULL) {
2031                 device_printf(dev, "can't add sysctl node\n");
2032                 error = ENXIO;
2033                 goto fail;
2034         }
2035
2036         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2037                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2038                         OID_AUTO, "rx_coal_ticks",
2039                         CTLTYPE_INT | CTLFLAG_RW,
2040                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2041                         "Receive coalescing ticks (usec).");
2042         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2043                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2044                         OID_AUTO, "tx_coal_ticks",
2045                         CTLTYPE_INT | CTLFLAG_RW,
2046                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2047                         "Transmit coalescing ticks (usec).");
2048         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2049                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2050                         OID_AUTO, "rx_coal_bds",
2051                         CTLTYPE_INT | CTLFLAG_RW,
2052                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2053                         "Receive max coalesced BD count.");
2054         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2055                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2056                         OID_AUTO, "tx_coal_bds",
2057                         CTLTYPE_INT | CTLFLAG_RW,
2058                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2059                         "Transmit max coalesced BD count.");
2060         /*
2061          * A common design characteristic for many Broadcom
2062          * client controllers is that they only support a
2063          * single outstanding DMA read operation on the PCIe
2064          * bus. This means that it will take twice as long to
2065          * fetch a TX frame that is split into header and
2066          * payload buffers as it does to fetch a single,
2067          * contiguous TX frame (2 reads vs. 1 read). For these
2068          * controllers, coalescing buffers to reduce the number
2069          * of memory reads is effective way to get maximum
2070          * performance(about 940Mbps).  Without collapsing TX
2071          * buffers the maximum TCP bulk transfer performance
2072          * is about 850Mbps. However forcing coalescing mbufs
2073          * consumes a lot of CPU cycles, so leave it off by
2074          * default.
2075          */
2076         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2077             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2078             "force_defrag", CTLTYPE_INT | CTLFLAG_RW,
2079             sc, 0, bnx_sysctl_force_defrag, "I",
2080             "Force defragment on TX path");
2081
2082         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2083             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2084             "tx_wreg", CTLTYPE_INT | CTLFLAG_RW,
2085             sc, 0, bnx_sysctl_tx_wreg, "I",
2086             "# of segments before writing to hardware register");
2087
2088         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2089             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2090             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2091             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2092             "Receive max coalesced BD count during interrupt.");
2093         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2094             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2095             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2096             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2097             "Transmit max coalesced BD count during interrupt.");
2098
2099 #ifdef BNX_TSO_DEBUG
2100         for (i = 0; i < BNX_TSO_NSTATS; ++i) {
2101                 ksnprintf(desc, sizeof(desc), "tso%d", i + 1);
2102                 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2103                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2104                     desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], "");
2105         }
2106 #endif
2107
2108         /*
2109          * Call MI attach routine.
2110          */
2111         ether_ifattach(ifp, ether_addr, NULL);
2112
2113         ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
2114
2115 #ifdef IFPOLL_ENABLE
2116         ifpoll_compat_setup(&sc->bnx_npoll,
2117             &sc->bnx_sysctl_ctx, sc->bnx_sysctl_tree,
2118             device_get_unit(dev), ifp->if_serializer);
2119 #endif
2120
2121         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
2122                 if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
2123                         intr_func = bnx_msi_oneshot;
2124                         if (bootverbose)
2125                                 device_printf(dev, "oneshot MSI\n");
2126                 } else {
2127                         intr_func = bnx_msi;
2128                 }
2129         } else {
2130                 intr_func = bnx_intr_legacy;
2131         }
2132         error = bus_setup_intr(dev, sc->bnx_irq, INTR_MPSAFE, intr_func, sc,
2133             &sc->bnx_intrhand, ifp->if_serializer);
2134         if (error) {
2135                 ether_ifdetach(ifp);
2136                 device_printf(dev, "couldn't set up irq\n");
2137                 goto fail;
2138         }
2139
2140         sc->bnx_intr_cpuid = rman_get_cpuid(sc->bnx_irq);
2141         sc->bnx_stat_cpuid = sc->bnx_intr_cpuid;
2142
2143         return(0);
2144 fail:
2145         bnx_detach(dev);
2146         return(error);
2147 }
2148
2149 static int
2150 bnx_detach(device_t dev)
2151 {
2152         struct bnx_softc *sc = device_get_softc(dev);
2153
2154         if (device_is_attached(dev)) {
2155                 struct ifnet *ifp = &sc->arpcom.ac_if;
2156
2157                 lwkt_serialize_enter(ifp->if_serializer);
2158                 bnx_stop(sc);
2159                 bnx_reset(sc);
2160                 bus_teardown_intr(dev, sc->bnx_irq, sc->bnx_intrhand);
2161                 lwkt_serialize_exit(ifp->if_serializer);
2162
2163                 ether_ifdetach(ifp);
2164         }
2165
2166         if (sc->bnx_flags & BNX_FLAG_TBI)
2167                 ifmedia_removeall(&sc->bnx_ifmedia);
2168         if (sc->bnx_miibus)
2169                 device_delete_child(dev, sc->bnx_miibus);
2170         bus_generic_detach(dev);
2171
2172         if (sc->bnx_irq != NULL) {
2173                 bus_release_resource(dev, SYS_RES_IRQ, sc->bnx_irq_rid,
2174                     sc->bnx_irq);
2175         }
2176         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI)
2177                 pci_release_msi(dev);
2178
2179         if (sc->bnx_res != NULL) {
2180                 bus_release_resource(dev, SYS_RES_MEMORY,
2181                     BGE_PCI_BAR0, sc->bnx_res);
2182         }
2183
2184         if (sc->bnx_sysctl_tree != NULL)
2185                 sysctl_ctx_free(&sc->bnx_sysctl_ctx);
2186
2187         bnx_dma_free(sc);
2188
2189         return 0;
2190 }
2191
2192 static void
2193 bnx_reset(struct bnx_softc *sc)
2194 {
2195         device_t dev;
2196         uint32_t cachesize, command, pcistate, reset;
2197         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2198         int i, val = 0;
2199         uint16_t devctl;
2200
2201         dev = sc->bnx_dev;
2202
2203         write_op = bnx_writemem_direct;
2204
2205         /* Save some important PCI state. */
2206         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2207         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2208         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2209
2210         pci_write_config(dev, BGE_PCI_MISC_CTL,
2211             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2212             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2213             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2214
2215         /* Disable fastboot on controllers that support it. */
2216         if (bootverbose)
2217                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2218         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2219
2220         /*
2221          * Write the magic number to SRAM at offset 0xB50.
2222          * When firmware finishes its initialization it will
2223          * write ~BGE_MAGIC_NUMBER to the same location.
2224          */
2225         bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2226
2227         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2228
2229         /* XXX: Broadcom Linux driver. */
2230         /* Force PCI-E 1.0a mode */
2231         if (!BNX_IS_57765_PLUS(sc) &&
2232             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2233             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2234              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2235                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2236                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2237         }
2238         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2239                 /* Prevent PCIE link training during global reset */
2240                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2241                 reset |= (1<<29);
2242         }
2243
2244         /*
2245          * Set GPHY Power Down Override to leave GPHY
2246          * powered up in D0 uninitialized.
2247          */
2248         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2249                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2250
2251         /* Issue global reset */
2252         write_op(sc, BGE_MISC_CFG, reset);
2253
2254         DELAY(1000);
2255
2256         /* XXX: Broadcom Linux driver. */
2257         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2258                 uint32_t v;
2259
2260                 DELAY(500000); /* wait for link training to complete */
2261                 v = pci_read_config(dev, 0xc4, 4);
2262                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2263         }
2264
2265         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2266
2267         /* Disable no snoop and disable relaxed ordering. */
2268         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2269
2270         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2271         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2272                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2273                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2274         }
2275
2276         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2277             devctl, 2);
2278
2279         /* Clear error status. */
2280         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2281             PCIEM_DEVSTS_CORR_ERR |
2282             PCIEM_DEVSTS_NFATAL_ERR |
2283             PCIEM_DEVSTS_FATAL_ERR |
2284             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2285
2286         /* Reset some of the PCI state that got zapped by reset */
2287         pci_write_config(dev, BGE_PCI_MISC_CTL,
2288             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2289             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2290             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2291         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2292         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2293         write_op(sc, BGE_MISC_CFG, (65 << 1));
2294
2295         /* Enable memory arbiter */
2296         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2297
2298         /*
2299          * Poll until we see the 1's complement of the magic number.
2300          * This indicates that the firmware initialization is complete.
2301          */
2302         for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2303                 val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2304                 if (val == ~BGE_MAGIC_NUMBER)
2305                         break;
2306                 DELAY(10);
2307         }
2308         if (i == BNX_FIRMWARE_TIMEOUT) {
2309                 if_printf(&sc->arpcom.ac_if, "firmware handshake "
2310                           "timed out, found 0x%08x\n", val);
2311         }
2312
2313         /* BCM57765 A0 needs additional time before accessing. */
2314         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2315                 DELAY(10 * 1000);
2316
2317         /*
2318          * XXX Wait for the value of the PCISTATE register to
2319          * return to its original pre-reset state. This is a
2320          * fairly good indicator of reset completion. If we don't
2321          * wait for the reset to fully complete, trying to read
2322          * from the device's non-PCI registers may yield garbage
2323          * results.
2324          */
2325         for (i = 0; i < BNX_TIMEOUT; i++) {
2326                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2327                         break;
2328                 DELAY(10);
2329         }
2330
2331         /* Fix up byte swapping */
2332         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2333
2334         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2335
2336         /*
2337          * The 5704 in TBI mode apparently needs some special
2338          * adjustment to insure the SERDES drive level is set
2339          * to 1.2V.
2340          */
2341         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2342             (sc->bnx_flags & BNX_FLAG_TBI)) {
2343                 uint32_t serdescfg;
2344
2345                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2346                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2347                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2348         }
2349
2350         CSR_WRITE_4(sc, BGE_MI_MODE,
2351             sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
2352         DELAY(80);
2353
2354         /* XXX: Broadcom Linux driver. */
2355         if (!BNX_IS_57765_PLUS(sc)) {
2356                 uint32_t v;
2357
2358                 /* Enable Data FIFO protection. */
2359                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
2360                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
2361         }
2362
2363         DELAY(10000);
2364
2365         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2366                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2367                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2368         }
2369 }
2370
2371 /*
2372  * Frame reception handling. This is called if there's a frame
2373  * on the receive return list.
2374  *
2375  * Note: we have to be able to handle two possibilities here:
2376  * 1) the frame is from the jumbo recieve ring
2377  * 2) the frame is from the standard receive ring
2378  */
2379
2380 static void
2381 bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod, int count)
2382 {
2383         struct ifnet *ifp;
2384         int stdcnt = 0, jumbocnt = 0;
2385
2386         ifp = &sc->arpcom.ac_if;
2387
2388         while (sc->bnx_rx_saved_considx != rx_prod && count != 0) {
2389                 struct bge_rx_bd        *cur_rx;
2390                 uint32_t                rxidx;
2391                 struct mbuf             *m = NULL;
2392                 uint16_t                vlan_tag = 0;
2393                 int                     have_tag = 0;
2394
2395                 --count;
2396
2397                 cur_rx =
2398             &sc->bnx_ldata.bnx_rx_return_ring[sc->bnx_rx_saved_considx];
2399
2400                 rxidx = cur_rx->bge_idx;
2401                 BNX_INC(sc->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
2402
2403                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2404                         have_tag = 1;
2405                         vlan_tag = cur_rx->bge_vlan_tag;
2406                 }
2407
2408                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
2409                         BNX_INC(sc->bnx_jumbo, BGE_JUMBO_RX_RING_CNT);
2410                         jumbocnt++;
2411
2412                         if (rxidx != sc->bnx_jumbo) {
2413                                 IFNET_STAT_INC(ifp, ierrors, 1);
2414                                 if_printf(ifp, "sw jumbo index(%d) "
2415                                     "and hw jumbo index(%d) mismatch, drop!\n",
2416                                     sc->bnx_jumbo, rxidx);
2417                                 bnx_setup_rxdesc_jumbo(sc, rxidx);
2418                                 continue;
2419                         }
2420
2421                         m = sc->bnx_cdata.bnx_rx_jumbo_chain[rxidx].bnx_mbuf;
2422                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2423                                 IFNET_STAT_INC(ifp, ierrors, 1);
2424                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2425                                 continue;
2426                         }
2427                         if (bnx_newbuf_jumbo(sc, sc->bnx_jumbo, 0)) {
2428                                 IFNET_STAT_INC(ifp, ierrors, 1);
2429                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2430                                 continue;
2431                         }
2432                 } else {
2433                         BNX_INC(sc->bnx_std, BGE_STD_RX_RING_CNT);
2434                         stdcnt++;
2435
2436                         if (rxidx != sc->bnx_std) {
2437                                 IFNET_STAT_INC(ifp, ierrors, 1);
2438                                 if_printf(ifp, "sw std index(%d) "
2439                                     "and hw std index(%d) mismatch, drop!\n",
2440                                     sc->bnx_std, rxidx);
2441                                 bnx_setup_rxdesc_std(sc, rxidx);
2442                                 continue;
2443                         }
2444
2445                         m = sc->bnx_cdata.bnx_rx_std_chain[rxidx].bnx_mbuf;
2446                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2447                                 IFNET_STAT_INC(ifp, ierrors, 1);
2448                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2449                                 continue;
2450                         }
2451                         if (bnx_newbuf_std(sc, sc->bnx_std, 0)) {
2452                                 IFNET_STAT_INC(ifp, ierrors, 1);
2453                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2454                                 continue;
2455                         }
2456                 }
2457
2458                 IFNET_STAT_INC(ifp, ipackets, 1);
2459                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2460                 m->m_pkthdr.rcvif = ifp;
2461
2462                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2463                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2464                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2465                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2466                                 if ((cur_rx->bge_error_flag &
2467                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2468                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2469                         }
2470                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2471                                 m->m_pkthdr.csum_data =
2472                                     cur_rx->bge_tcp_udp_csum;
2473                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2474                                     CSUM_PSEUDO_HDR;
2475                         }
2476                 }
2477
2478                 /*
2479                  * If we received a packet with a vlan tag, pass it
2480                  * to vlan_input() instead of ether_input().
2481                  */
2482                 if (have_tag) {
2483                         m->m_flags |= M_VLANTAG;
2484                         m->m_pkthdr.ether_vlantag = vlan_tag;
2485                 }
2486                 ifp->if_input(ifp, m);
2487         }
2488
2489         bnx_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bnx_rx_saved_considx);
2490         if (stdcnt)
2491                 bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
2492         if (jumbocnt)
2493                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
2494 }
2495
2496 static void
2497 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
2498 {
2499         struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if;
2500
2501         /*
2502          * Go through our tx ring and free mbufs for those
2503          * frames that have been sent.
2504          */
2505         while (txr->bnx_tx_saved_considx != tx_cons) {
2506                 uint32_t idx = 0;
2507
2508                 idx = txr->bnx_tx_saved_considx;
2509                 if (txr->bnx_tx_chain[idx] != NULL) {
2510                         IFNET_STAT_INC(ifp, opackets, 1);
2511                         bus_dmamap_unload(txr->bnx_tx_mtag,
2512                             txr->bnx_tx_dmamap[idx]);
2513                         m_freem(txr->bnx_tx_chain[idx]);
2514                         txr->bnx_tx_chain[idx] = NULL;
2515                 }
2516                 txr->bnx_txcnt--;
2517                 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2518         }
2519
2520         if ((BGE_TX_RING_CNT - txr->bnx_txcnt) >=
2521             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2522                 ifq_clr_oactive(&ifp->if_snd);
2523
2524         if (txr->bnx_txcnt == 0)
2525                 ifp->if_timer = 0;
2526
2527         if (!ifq_is_empty(&ifp->if_snd))
2528                 if_devstart(ifp);
2529 }
2530
2531 #ifdef IFPOLL_ENABLE
2532
2533 static void
2534 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
2535 {
2536         struct bnx_softc *sc = ifp->if_softc;
2537
2538         ASSERT_SERIALIZED(ifp->if_serializer);
2539
2540         if (info != NULL) {
2541                 int cpuid = sc->bnx_npoll.ifpc_cpuid;
2542
2543                 info->ifpi_rx[cpuid].poll_func = bnx_npoll_compat;
2544                 info->ifpi_rx[cpuid].arg = NULL;
2545                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
2546
2547                 if (ifp->if_flags & IFF_RUNNING)
2548                         bnx_disable_intr(sc);
2549                 ifq_set_cpuid(&ifp->if_snd, cpuid);
2550         } else {
2551                 if (ifp->if_flags & IFF_RUNNING)
2552                         bnx_enable_intr(sc);
2553                 ifq_set_cpuid(&ifp->if_snd, sc->bnx_intr_cpuid);
2554         }
2555 }
2556
2557 static void
2558 bnx_npoll_compat(struct ifnet *ifp, void *arg __unused, int cycle)
2559 {
2560         struct bnx_softc *sc = ifp->if_softc;
2561         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
2562         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2563         uint16_t rx_prod, tx_cons;
2564
2565         ASSERT_SERIALIZED(ifp->if_serializer);
2566
2567         if (sc->bnx_npoll.ifpc_stcount-- == 0) {
2568                 sc->bnx_npoll.ifpc_stcount = sc->bnx_npoll.ifpc_stfrac;
2569                 /*
2570                  * Process link state changes.
2571                  */
2572                 bnx_link_poll(sc);
2573         }
2574
2575         sc->bnx_status_tag = sblk->bge_status_tag;
2576
2577         /*
2578          * Use a load fence to ensure that status_tag is saved
2579          * before rx_prod and tx_cons.
2580          */
2581         cpu_lfence();
2582
2583         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2584         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2585
2586         if (sc->bnx_rx_saved_considx != rx_prod)
2587                 bnx_rxeof(sc, rx_prod, cycle);
2588
2589         if (txr->bnx_tx_saved_considx != tx_cons)
2590                 bnx_txeof(txr, tx_cons);
2591
2592         if (sc->bnx_coal_chg)
2593                 bnx_coal_change(sc);
2594 }
2595
2596 #endif  /* IFPOLL_ENABLE */
2597
2598 static void
2599 bnx_intr_legacy(void *xsc)
2600 {
2601         struct bnx_softc *sc = xsc;
2602         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2603
2604         if (sc->bnx_status_tag == sblk->bge_status_tag) {
2605                 uint32_t val;
2606
2607                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
2608                 if (val & BGE_PCISTAT_INTR_NOTACT)
2609                         return;
2610         }
2611
2612         /*
2613          * NOTE:
2614          * Interrupt will have to be disabled if tagged status
2615          * is used, else interrupt will always be asserted on
2616          * certain chips (at least on BCM5750 AX/BX).
2617          */
2618         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2619
2620         bnx_intr(sc);
2621 }
2622
2623 static void
2624 bnx_msi(void *xsc)
2625 {
2626         struct bnx_softc *sc = xsc;
2627
2628         /* Disable interrupt first */
2629         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2630         bnx_intr(sc);
2631 }
2632
2633 static void
2634 bnx_msi_oneshot(void *xsc)
2635 {
2636         bnx_intr(xsc);
2637 }
2638
2639 static void
2640 bnx_intr(struct bnx_softc *sc)
2641 {
2642         struct ifnet *ifp = &sc->arpcom.ac_if;
2643         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2644         uint16_t rx_prod, tx_cons;
2645         uint32_t status;
2646
2647         sc->bnx_status_tag = sblk->bge_status_tag;
2648         /*
2649          * Use a load fence to ensure that status_tag is saved
2650          * before rx_prod, tx_cons and status.
2651          */
2652         cpu_lfence();
2653
2654         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2655         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2656         status = sblk->bge_status;
2657
2658         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
2659                 bnx_link_poll(sc);
2660
2661         if (ifp->if_flags & IFF_RUNNING) {
2662                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
2663
2664                 if (sc->bnx_rx_saved_considx != rx_prod)
2665                         bnx_rxeof(sc, rx_prod, -1);
2666
2667                 if (txr->bnx_tx_saved_considx != tx_cons)
2668                         bnx_txeof(txr, tx_cons);
2669         }
2670
2671         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
2672
2673         if (sc->bnx_coal_chg)
2674                 bnx_coal_change(sc);
2675 }
2676
2677 static void
2678 bnx_tick(void *xsc)
2679 {
2680         struct bnx_softc *sc = xsc;
2681         struct ifnet *ifp = &sc->arpcom.ac_if;
2682
2683         lwkt_serialize_enter(ifp->if_serializer);
2684
2685         KKASSERT(mycpuid == sc->bnx_stat_cpuid);
2686
2687         bnx_stats_update_regs(sc);
2688
2689         if (sc->bnx_flags & BNX_FLAG_TBI) {
2690                 /*
2691                  * Since in TBI mode auto-polling can't be used we should poll
2692                  * link status manually. Here we register pending link event
2693                  * and trigger interrupt.
2694                  */
2695                 sc->bnx_link_evt++;
2696                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
2697         } else if (!sc->bnx_link) {
2698                 mii_tick(device_get_softc(sc->bnx_miibus));
2699         }
2700
2701         callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
2702
2703         lwkt_serialize_exit(ifp->if_serializer);
2704 }
2705
2706 static void
2707 bnx_stats_update_regs(struct bnx_softc *sc)
2708 {
2709         struct ifnet *ifp = &sc->arpcom.ac_if;
2710         struct bge_mac_stats_regs stats;
2711         uint32_t *s;
2712         int i;
2713
2714         s = (uint32_t *)&stats;
2715         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
2716                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
2717                 s++;
2718         }
2719
2720         IFNET_STAT_SET(ifp, collisions,
2721            (stats.dot3StatsSingleCollisionFrames +
2722            stats.dot3StatsMultipleCollisionFrames +
2723            stats.dot3StatsExcessiveCollisions +
2724            stats.dot3StatsLateCollisions));
2725 }
2726
2727 /*
2728  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
2729  * pointers to descriptors.
2730  */
2731 static int
2732 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
2733     int *segs_used)
2734 {
2735         struct bge_tx_bd *d = NULL;
2736         uint16_t csum_flags = 0, vlan_tag = 0, mss = 0;
2737         bus_dma_segment_t segs[BNX_NSEG_NEW];
2738         bus_dmamap_t map;
2739         int error, maxsegs, nsegs, idx, i;
2740         struct mbuf *m_head = *m_head0, *m_new;
2741
2742         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2743 #ifdef BNX_TSO_DEBUG
2744                 int tso_nsegs;
2745 #endif
2746
2747                 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags);
2748                 if (error)
2749                         return error;
2750                 m_head = *m_head0;
2751
2752 #ifdef BNX_TSO_DEBUG
2753                 tso_nsegs = (m_head->m_pkthdr.len /
2754                     m_head->m_pkthdr.tso_segsz) - 1;
2755                 if (tso_nsegs > (BNX_TSO_NSTATS - 1))
2756                         tso_nsegs = BNX_TSO_NSTATS - 1;
2757                 else if (tso_nsegs < 0)
2758                         tso_nsegs = 0;
2759                 txr->sc->bnx_tsosegs[tso_nsegs]++;
2760 #endif
2761         } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) {
2762                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
2763                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
2764                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2765                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
2766                 if (m_head->m_flags & M_LASTFRAG)
2767                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
2768                 else if (m_head->m_flags & M_FRAG)
2769                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
2770         }
2771         if (m_head->m_flags & M_VLANTAG) {
2772                 csum_flags |= BGE_TXBDFLAG_VLAN_TAG;
2773                 vlan_tag = m_head->m_pkthdr.ether_vlantag;
2774         }
2775
2776         idx = *txidx;
2777         map = txr->bnx_tx_dmamap[idx];
2778
2779         maxsegs = (BGE_TX_RING_CNT - txr->bnx_txcnt) - BNX_NSEG_RSVD;
2780         KASSERT(maxsegs >= BNX_NSEG_SPARE,
2781                 ("not enough segments %d", maxsegs));
2782
2783         if (maxsegs > BNX_NSEG_NEW)
2784                 maxsegs = BNX_NSEG_NEW;
2785
2786         /*
2787          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
2788          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
2789          * but when such padded frames employ the bge IP/TCP checksum
2790          * offload, the hardware checksum assist gives incorrect results
2791          * (possibly from incorporating its own padding into the UDP/TCP
2792          * checksum; who knows).  If we pad such runts with zeros, the
2793          * onboard checksum comes out correct.
2794          */
2795         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
2796             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
2797                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
2798                 if (error)
2799                         goto back;
2800         }
2801
2802         if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) &&
2803             m_head->m_next != NULL) {
2804                 m_new = bnx_defrag_shortdma(m_head);
2805                 if (m_new == NULL) {
2806                         error = ENOBUFS;
2807                         goto back;
2808                 }
2809                 *m_head0 = m_head = m_new;
2810         }
2811         if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 &&
2812             (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) &&
2813             m_head->m_next != NULL) {
2814                 /*
2815                  * Forcefully defragment mbuf chain to overcome hardware
2816                  * limitation which only support a single outstanding
2817                  * DMA read operation.  If it fails, keep moving on using
2818                  * the original mbuf chain.
2819                  */
2820                 m_new = m_defrag(m_head, MB_DONTWAIT);
2821                 if (m_new != NULL)
2822                         *m_head0 = m_head = m_new;
2823         }
2824
2825         error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map,
2826             m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
2827         if (error)
2828                 goto back;
2829         *segs_used += nsegs;
2830
2831         m_head = *m_head0;
2832         bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
2833
2834         for (i = 0; ; i++) {
2835                 d = &txr->bnx_tx_ring[idx];
2836
2837                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
2838                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
2839                 d->bge_len = segs[i].ds_len;
2840                 d->bge_flags = csum_flags;
2841                 d->bge_vlan_tag = vlan_tag;
2842                 d->bge_mss = mss;
2843
2844                 if (i == nsegs - 1)
2845                         break;
2846                 BNX_INC(idx, BGE_TX_RING_CNT);
2847         }
2848         /* Mark the last segment as end of packet... */
2849         d->bge_flags |= BGE_TXBDFLAG_END;
2850
2851         /*
2852          * Insure that the map for this transmission is placed at
2853          * the array index of the last descriptor in this chain.
2854          */
2855         txr->bnx_tx_dmamap[*txidx] = txr->bnx_tx_dmamap[idx];
2856         txr->bnx_tx_dmamap[idx] = map;
2857         txr->bnx_tx_chain[idx] = m_head;
2858         txr->bnx_txcnt += nsegs;
2859
2860         BNX_INC(idx, BGE_TX_RING_CNT);
2861         *txidx = idx;
2862 back:
2863         if (error) {
2864                 m_freem(*m_head0);
2865                 *m_head0 = NULL;
2866         }
2867         return error;
2868 }
2869
2870 /*
2871  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
2872  * to the mbuf data regions directly in the transmit descriptors.
2873  */
2874 static void
2875 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2876 {
2877         struct bnx_softc *sc = ifp->if_softc;
2878         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
2879         struct mbuf *m_head = NULL;
2880         uint32_t prodidx;
2881         int nsegs = 0;
2882
2883         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
2884
2885         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
2886                 return;
2887
2888         prodidx = txr->bnx_tx_prodidx;
2889
2890         while (txr->bnx_tx_chain[prodidx] == NULL) {
2891                 /*
2892                  * Sanity check: avoid coming within BGE_NSEG_RSVD
2893                  * descriptors of the end of the ring.  Also make
2894                  * sure there are BGE_NSEG_SPARE descriptors for
2895                  * jumbo buffers' or TSO segments' defragmentation.
2896                  */
2897                 if ((BGE_TX_RING_CNT - txr->bnx_txcnt) <
2898                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
2899                         ifq_set_oactive(&ifp->if_snd);
2900                         break;
2901                 }
2902
2903                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
2904                 if (m_head == NULL)
2905                         break;
2906
2907                 /*
2908                  * Pack the data into the transmit ring. If we
2909                  * don't have room, set the OACTIVE flag and wait
2910                  * for the NIC to drain the ring.
2911                  */
2912                 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) {
2913                         ifq_set_oactive(&ifp->if_snd);
2914                         IFNET_STAT_INC(ifp, oerrors, 1);
2915                         break;
2916                 }
2917
2918                 if (nsegs >= txr->bnx_tx_wreg) {
2919                         /* Transmit */
2920                         bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO,
2921                             prodidx);
2922                         nsegs = 0;
2923                 }
2924
2925                 ETHER_BPF_MTAP(ifp, m_head);
2926
2927                 /*
2928                  * Set a timeout in case the chip goes out to lunch.
2929                  */
2930                 ifp->if_timer = 5;
2931         }
2932
2933         if (nsegs > 0) {
2934                 /* Transmit */
2935                 bnx_writembx(txr->bnx_sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
2936         }
2937         txr->bnx_tx_prodidx = prodidx;
2938 }
2939
2940 static void
2941 bnx_init(void *xsc)
2942 {
2943         struct bnx_softc *sc = xsc;
2944         struct ifnet *ifp = &sc->arpcom.ac_if;
2945         uint16_t *m;
2946         uint32_t mode;
2947         int i;
2948
2949         ASSERT_SERIALIZED(ifp->if_serializer);
2950
2951         /* Cancel pending I/O and flush buffers. */
2952         bnx_stop(sc);
2953         bnx_reset(sc);
2954         bnx_chipinit(sc);
2955
2956         /*
2957          * Init the various state machines, ring
2958          * control blocks and firmware.
2959          */
2960         if (bnx_blockinit(sc)) {
2961                 if_printf(ifp, "initialization failure\n");
2962                 bnx_stop(sc);
2963                 return;
2964         }
2965
2966         /* Specify MTU. */
2967         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
2968             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
2969
2970         /* Load our MAC address. */
2971         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
2972         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
2973         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
2974
2975         /* Enable or disable promiscuous mode as needed. */
2976         bnx_setpromisc(sc);
2977
2978         /* Program multicast filter. */
2979         bnx_setmulti(sc);
2980
2981         /* Init RX ring. */
2982         if (bnx_init_rx_ring_std(sc)) {
2983                 if_printf(ifp, "RX ring initialization failed\n");
2984                 bnx_stop(sc);
2985                 return;
2986         }
2987
2988         /* Init jumbo RX ring. */
2989         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
2990                 if (bnx_init_rx_ring_jumbo(sc)) {
2991                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
2992                         bnx_stop(sc);
2993                         return;
2994                 }
2995         }
2996
2997         /* Init our RX return ring index */
2998         sc->bnx_rx_saved_considx = 0;
2999
3000         /* Init TX ring. */
3001         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3002                 bnx_init_tx_ring(&sc->bnx_tx_ring[i]);
3003
3004         /* Enable TX MAC state machine lockup fix. */
3005         mode = CSR_READ_4(sc, BGE_TX_MODE);
3006         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3007         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
3008             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
3009                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3010                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
3011                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3012         }
3013         /* Turn on transmitter */
3014         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3015
3016         /* Turn on receiver */
3017         BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3018
3019         /*
3020          * Set the number of good frames to receive after RX MBUF
3021          * Low Watermark has been reached.  After the RX MAC receives
3022          * this number of frames, it will drop subsequent incoming
3023          * frames until the MBUF High Watermark is reached.
3024          */
3025         if (BNX_IS_57765_FAMILY(sc))
3026                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3027         else
3028                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3029
3030         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
3031                 if (bootverbose) {
3032                         if_printf(ifp, "MSI_MODE: %#x\n",
3033                             CSR_READ_4(sc, BGE_MSI_MODE));
3034                 }
3035         }
3036
3037         /* Tell firmware we're alive. */
3038         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3039
3040         /* Enable host interrupts if polling(4) is not enabled. */
3041         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3042 #ifdef IFPOLL_ENABLE
3043         if (ifp->if_flags & IFF_NPOLLING)
3044                 bnx_disable_intr(sc);
3045         else
3046 #endif
3047         bnx_enable_intr(sc);
3048
3049         bnx_ifmedia_upd(ifp);
3050
3051         ifp->if_flags |= IFF_RUNNING;
3052         ifq_clr_oactive(&ifp->if_snd);
3053
3054         callout_reset_bycpu(&sc->bnx_stat_timer, hz, bnx_tick, sc,
3055             sc->bnx_stat_cpuid);
3056 }
3057
3058 /*
3059  * Set media options.
3060  */
3061 static int
3062 bnx_ifmedia_upd(struct ifnet *ifp)
3063 {
3064         struct bnx_softc *sc = ifp->if_softc;
3065
3066         /* If this is a 1000baseX NIC, enable the TBI port. */
3067         if (sc->bnx_flags & BNX_FLAG_TBI) {
3068                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3069
3070                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3071                         return(EINVAL);
3072
3073                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3074                 case IFM_AUTO:
3075                         break;
3076
3077                 case IFM_1000_SX:
3078                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3079                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3080                                     BGE_MACMODE_HALF_DUPLEX);
3081                         } else {
3082                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3083                                     BGE_MACMODE_HALF_DUPLEX);
3084                         }
3085                         break;
3086                 default:
3087                         return(EINVAL);
3088                 }
3089         } else {
3090                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3091
3092                 sc->bnx_link_evt++;
3093                 sc->bnx_link = 0;
3094                 if (mii->mii_instance) {
3095                         struct mii_softc *miisc;
3096
3097                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3098                                 mii_phy_reset(miisc);
3099                 }
3100                 mii_mediachg(mii);
3101
3102                 /*
3103                  * Force an interrupt so that we will call bnx_link_upd
3104                  * if needed and clear any pending link state attention.
3105                  * Without this we are not getting any further interrupts
3106                  * for link state changes and thus will not UP the link and
3107                  * not be able to send in bnx_start.  The only way to get
3108                  * things working was to receive a packet and get an RX
3109                  * intr.
3110                  *
3111                  * bnx_tick should help for fiber cards and we might not
3112                  * need to do this here if BNX_FLAG_TBI is set but as
3113                  * we poll for fiber anyway it should not harm.
3114                  */
3115                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3116         }
3117         return(0);
3118 }
3119
3120 /*
3121  * Report current media status.
3122  */
3123 static void
3124 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3125 {
3126         struct bnx_softc *sc = ifp->if_softc;
3127
3128         if (sc->bnx_flags & BNX_FLAG_TBI) {
3129                 ifmr->ifm_status = IFM_AVALID;
3130                 ifmr->ifm_active = IFM_ETHER;
3131                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3132                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3133                         ifmr->ifm_status |= IFM_ACTIVE;
3134                 } else {
3135                         ifmr->ifm_active |= IFM_NONE;
3136                         return;
3137                 }
3138
3139                 ifmr->ifm_active |= IFM_1000_SX;
3140                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3141                         ifmr->ifm_active |= IFM_HDX;
3142                 else
3143                         ifmr->ifm_active |= IFM_FDX;
3144         } else {
3145                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3146
3147                 mii_pollstat(mii);
3148                 ifmr->ifm_active = mii->mii_media_active;
3149                 ifmr->ifm_status = mii->mii_media_status;
3150         }
3151 }
3152
3153 static int
3154 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3155 {
3156         struct bnx_softc *sc = ifp->if_softc;
3157         struct ifreq *ifr = (struct ifreq *)data;
3158         int mask, error = 0;
3159
3160         ASSERT_SERIALIZED(ifp->if_serializer);
3161
3162         switch (command) {
3163         case SIOCSIFMTU:
3164                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3165                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3166                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3167                         error = EINVAL;
3168                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3169                         ifp->if_mtu = ifr->ifr_mtu;
3170                         if (ifp->if_flags & IFF_RUNNING)
3171                                 bnx_init(sc);
3172                 }
3173                 break;
3174         case SIOCSIFFLAGS:
3175                 if (ifp->if_flags & IFF_UP) {
3176                         if (ifp->if_flags & IFF_RUNNING) {
3177                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3178
3179                                 /*
3180                                  * If only the state of the PROMISC flag
3181                                  * changed, then just use the 'set promisc
3182                                  * mode' command instead of reinitializing
3183                                  * the entire NIC. Doing a full re-init
3184                                  * means reloading the firmware and waiting
3185                                  * for it to start up, which may take a
3186                                  * second or two.  Similarly for ALLMULTI.
3187                                  */
3188                                 if (mask & IFF_PROMISC)
3189                                         bnx_setpromisc(sc);
3190                                 if (mask & IFF_ALLMULTI)
3191                                         bnx_setmulti(sc);
3192                         } else {
3193                                 bnx_init(sc);
3194                         }
3195                 } else if (ifp->if_flags & IFF_RUNNING) {
3196                         bnx_stop(sc);
3197                 }
3198                 sc->bnx_if_flags = ifp->if_flags;
3199                 break;
3200         case SIOCADDMULTI:
3201         case SIOCDELMULTI:
3202                 if (ifp->if_flags & IFF_RUNNING)
3203                         bnx_setmulti(sc);
3204                 break;
3205         case SIOCSIFMEDIA:
3206         case SIOCGIFMEDIA:
3207                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3208                         error = ifmedia_ioctl(ifp, ifr,
3209                             &sc->bnx_ifmedia, command);
3210                 } else {
3211                         struct mii_data *mii;
3212
3213                         mii = device_get_softc(sc->bnx_miibus);
3214                         error = ifmedia_ioctl(ifp, ifr,
3215                                               &mii->mii_media, command);
3216                 }
3217                 break;
3218         case SIOCSIFCAP:
3219                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3220                 if (mask & IFCAP_HWCSUM) {
3221                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3222                         if (ifp->if_capenable & IFCAP_TXCSUM)
3223                                 ifp->if_hwassist |= BNX_CSUM_FEATURES;
3224                         else
3225                                 ifp->if_hwassist &= ~BNX_CSUM_FEATURES;
3226                 }
3227                 if (mask & IFCAP_TSO) {
3228                         ifp->if_capenable ^= (mask & IFCAP_TSO);
3229                         if (ifp->if_capenable & IFCAP_TSO)
3230                                 ifp->if_hwassist |= CSUM_TSO;
3231                         else
3232                                 ifp->if_hwassist &= ~CSUM_TSO;
3233                 }
3234                 break;
3235         default:
3236                 error = ether_ioctl(ifp, command, data);
3237                 break;
3238         }
3239         return error;
3240 }
3241
3242 static void
3243 bnx_watchdog(struct ifnet *ifp)
3244 {
3245         struct bnx_softc *sc = ifp->if_softc;
3246
3247         if_printf(ifp, "watchdog timeout -- resetting\n");
3248
3249         bnx_init(sc);
3250
3251         IFNET_STAT_INC(ifp, oerrors, 1);
3252
3253         if (!ifq_is_empty(&ifp->if_snd))
3254                 if_devstart(ifp);
3255 }
3256
3257 /*
3258  * Stop the adapter and free any mbufs allocated to the
3259  * RX and TX lists.
3260  */
3261 static void
3262 bnx_stop(struct bnx_softc *sc)
3263 {
3264         struct ifnet *ifp = &sc->arpcom.ac_if;
3265         int i;
3266
3267         ASSERT_SERIALIZED(ifp->if_serializer);
3268
3269         callout_stop(&sc->bnx_stat_timer);
3270
3271         /*
3272          * Disable all of the receiver blocks
3273          */
3274         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3275         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3276         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3277         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3278         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3279         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3280
3281         /*
3282          * Disable all of the transmit blocks
3283          */
3284         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3285         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3286         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3287         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3288         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3289         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3290
3291         /*
3292          * Shut down all of the memory managers and related
3293          * state machines.
3294          */
3295         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3296         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3297         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3298         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3299
3300         /* Disable host interrupts. */
3301         bnx_disable_intr(sc);
3302
3303         /*
3304          * Tell firmware we're shutting down.
3305          */
3306         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3307
3308         /* Free the RX lists. */
3309         bnx_free_rx_ring_std(sc);
3310
3311         /* Free jumbo RX list. */
3312         if (BNX_IS_JUMBO_CAPABLE(sc))
3313                 bnx_free_rx_ring_jumbo(sc);
3314
3315         /* Free TX buffers. */
3316         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3317                 bnx_free_tx_ring(&sc->bnx_tx_ring[i]);
3318
3319         sc->bnx_status_tag = 0;
3320         sc->bnx_link = 0;
3321         sc->bnx_coal_chg = 0;
3322
3323         ifp->if_flags &= ~IFF_RUNNING;
3324         ifq_clr_oactive(&ifp->if_snd);
3325         ifp->if_timer = 0;
3326 }
3327
3328 /*
3329  * Stop all chip I/O so that the kernel's probe routines don't
3330  * get confused by errant DMAs when rebooting.
3331  */
3332 static void
3333 bnx_shutdown(device_t dev)
3334 {
3335         struct bnx_softc *sc = device_get_softc(dev);
3336         struct ifnet *ifp = &sc->arpcom.ac_if;
3337
3338         lwkt_serialize_enter(ifp->if_serializer);
3339         bnx_stop(sc);
3340         bnx_reset(sc);
3341         lwkt_serialize_exit(ifp->if_serializer);
3342 }
3343
3344 static int
3345 bnx_suspend(device_t dev)
3346 {
3347         struct bnx_softc *sc = device_get_softc(dev);
3348         struct ifnet *ifp = &sc->arpcom.ac_if;
3349
3350         lwkt_serialize_enter(ifp->if_serializer);
3351         bnx_stop(sc);
3352         lwkt_serialize_exit(ifp->if_serializer);
3353
3354         return 0;
3355 }
3356
3357 static int
3358 bnx_resume(device_t dev)
3359 {
3360         struct bnx_softc *sc = device_get_softc(dev);
3361         struct ifnet *ifp = &sc->arpcom.ac_if;
3362
3363         lwkt_serialize_enter(ifp->if_serializer);
3364
3365         if (ifp->if_flags & IFF_UP) {
3366                 bnx_init(sc);
3367
3368                 if (!ifq_is_empty(&ifp->if_snd))
3369                         if_devstart(ifp);
3370         }
3371
3372         lwkt_serialize_exit(ifp->if_serializer);
3373
3374         return 0;
3375 }
3376
3377 static void
3378 bnx_setpromisc(struct bnx_softc *sc)
3379 {
3380         struct ifnet *ifp = &sc->arpcom.ac_if;
3381
3382         if (ifp->if_flags & IFF_PROMISC)
3383                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3384         else
3385                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3386 }
3387
3388 static void
3389 bnx_dma_free(struct bnx_softc *sc)
3390 {
3391         int i;
3392
3393         /* Destroy RX mbuf DMA stuffs. */
3394         if (sc->bnx_cdata.bnx_rx_mtag != NULL) {
3395                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3396                         bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3397                             sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3398                 }
3399                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3400                                    sc->bnx_cdata.bnx_rx_tmpmap);
3401                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3402         }
3403
3404         /* Destroy TX rings */
3405         if (sc->bnx_tx_ring != NULL) {
3406                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3407                         bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]);
3408                 kfree(sc->bnx_tx_ring, M_DEVBUF);
3409         }
3410
3411         /* Destroy standard RX ring */
3412         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_std_ring_tag,
3413                            sc->bnx_cdata.bnx_rx_std_ring_map,
3414                            sc->bnx_ldata.bnx_rx_std_ring);
3415
3416         if (BNX_IS_JUMBO_CAPABLE(sc))
3417                 bnx_free_jumbo_mem(sc);
3418
3419         /* Destroy RX return ring */
3420         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_return_ring_tag,
3421                            sc->bnx_cdata.bnx_rx_return_ring_map,
3422                            sc->bnx_ldata.bnx_rx_return_ring);
3423
3424         /* Destroy status block */
3425         bnx_dma_block_free(sc->bnx_cdata.bnx_status_tag,
3426                            sc->bnx_cdata.bnx_status_map,
3427                            sc->bnx_ldata.bnx_status_block);
3428
3429         /* Destroy the parent tag */
3430         if (sc->bnx_cdata.bnx_parent_tag != NULL)
3431                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
3432 }
3433
3434 static int
3435 bnx_dma_alloc(struct bnx_softc *sc)
3436 {
3437         struct ifnet *ifp = &sc->arpcom.ac_if;
3438         int i, error;
3439
3440         /*
3441          * Allocate the parent bus DMA tag appropriate for PCI.
3442          *
3443          * All of the NetExtreme/NetLink controllers have 4GB boundary
3444          * DMA bug.
3445          * Whenever an address crosses a multiple of the 4GB boundary
3446          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
3447          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
3448          * state machine will lockup and cause the device to hang.
3449          */
3450         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
3451                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3452                                    NULL, NULL,
3453                                    BUS_SPACE_MAXSIZE_32BIT, 0,
3454                                    BUS_SPACE_MAXSIZE_32BIT,
3455                                    0, &sc->bnx_cdata.bnx_parent_tag);
3456         if (error) {
3457                 if_printf(ifp, "could not allocate parent dma tag\n");
3458                 return error;
3459         }
3460
3461         /*
3462          * Create DMA tag and maps for RX mbufs.
3463          */
3464         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3465                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3466                                    NULL, NULL, MCLBYTES, 1, MCLBYTES,
3467                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK,
3468                                    &sc->bnx_cdata.bnx_rx_mtag);
3469         if (error) {
3470                 if_printf(ifp, "could not allocate RX mbuf dma tag\n");
3471                 return error;
3472         }
3473
3474         error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3475                                   BUS_DMA_WAITOK, &sc->bnx_cdata.bnx_rx_tmpmap);
3476         if (error) {
3477                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3478                 sc->bnx_cdata.bnx_rx_mtag = NULL;
3479                 return error;
3480         }
3481
3482         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3483                 error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3484                                           BUS_DMA_WAITOK,
3485                                           &sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3486                 if (error) {
3487                         int j;
3488
3489                         for (j = 0; j < i; ++j) {
3490                                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3491                                         sc->bnx_cdata.bnx_rx_std_dmamap[j]);
3492                         }
3493                         bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3494                         sc->bnx_cdata.bnx_rx_mtag = NULL;
3495
3496                         if_printf(ifp, "could not create DMA map for RX\n");
3497                         return error;
3498                 }
3499         }
3500
3501         /*
3502          * Create DMA stuffs for standard RX ring.
3503          */
3504         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
3505                                     &sc->bnx_cdata.bnx_rx_std_ring_tag,
3506                                     &sc->bnx_cdata.bnx_rx_std_ring_map,
3507                                     (void *)&sc->bnx_ldata.bnx_rx_std_ring,
3508                                     &sc->bnx_ldata.bnx_rx_std_ring_paddr);
3509         if (error) {
3510                 if_printf(ifp, "could not create std RX ring\n");
3511                 return error;
3512         }
3513
3514         /*
3515          * Create jumbo buffer pool.
3516          */
3517         if (BNX_IS_JUMBO_CAPABLE(sc)) {
3518                 error = bnx_alloc_jumbo_mem(sc);
3519                 if (error) {
3520                         if_printf(ifp, "could not create jumbo buffer pool\n");
3521                         return error;
3522                 }
3523         }
3524
3525         /*
3526          * Create DMA stuffs for RX return ring.
3527          */
3528         error = bnx_dma_block_alloc(sc,
3529             BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT),
3530             &sc->bnx_cdata.bnx_rx_return_ring_tag,
3531             &sc->bnx_cdata.bnx_rx_return_ring_map,
3532             (void *)&sc->bnx_ldata.bnx_rx_return_ring,
3533             &sc->bnx_ldata.bnx_rx_return_ring_paddr);
3534         if (error) {
3535                 if_printf(ifp, "could not create RX ret ring\n");
3536                 return error;
3537         }
3538
3539         /*
3540          * Create DMA stuffs for status block.
3541          */
3542         error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
3543                                     &sc->bnx_cdata.bnx_status_tag,
3544                                     &sc->bnx_cdata.bnx_status_map,
3545                                     (void *)&sc->bnx_ldata.bnx_status_block,
3546                                     &sc->bnx_ldata.bnx_status_block_paddr);
3547         if (error) {
3548                 if_printf(ifp, "could not create status block\n");
3549                 return error;
3550         }
3551
3552         sc->bnx_tx_ring = kmalloc_cachealign(
3553             sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF,
3554             M_WAITOK | M_ZERO);
3555         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3556                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3557
3558                 txr->bnx_sc = sc;
3559                 error = bnx_create_tx_ring(txr);
3560                 if (error) {
3561                         device_printf(sc->bnx_dev,
3562                             "can't create %dth tx ring\n", i);
3563                         return error;
3564                 }
3565         }
3566
3567         return 0;
3568 }
3569
3570 static int
3571 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
3572                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
3573 {
3574         bus_dmamem_t dmem;
3575         int error;
3576
3577         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
3578                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3579                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
3580         if (error)
3581                 return error;
3582
3583         *tag = dmem.dmem_tag;
3584         *map = dmem.dmem_map;
3585         *addr = dmem.dmem_addr;
3586         *paddr = dmem.dmem_busaddr;
3587
3588         return 0;
3589 }
3590
3591 static void
3592 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
3593 {
3594         if (tag != NULL) {
3595                 bus_dmamap_unload(tag, map);
3596                 bus_dmamem_free(tag, addr, map);
3597                 bus_dma_tag_destroy(tag);
3598         }
3599 }
3600
3601 static void
3602 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
3603 {
3604         struct ifnet *ifp = &sc->arpcom.ac_if;
3605
3606 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
3607
3608         /*
3609          * Sometimes PCS encoding errors are detected in
3610          * TBI mode (on fiber NICs), and for some reason
3611          * the chip will signal them as link changes.
3612          * If we get a link change event, but the 'PCS
3613          * encoding error' bit in the MAC status register
3614          * is set, don't bother doing a link check.
3615          * This avoids spurious "gigabit link up" messages
3616          * that sometimes appear on fiber NICs during
3617          * periods of heavy traffic.
3618          */
3619         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
3620                 if (!sc->bnx_link) {
3621                         sc->bnx_link++;
3622                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
3623                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3624                                     BGE_MACMODE_TBI_SEND_CFGS);
3625                         }
3626                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
3627
3628                         if (bootverbose)
3629                                 if_printf(ifp, "link UP\n");
3630
3631                         ifp->if_link_state = LINK_STATE_UP;
3632                         if_link_state_change(ifp);
3633                 }
3634         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
3635                 if (sc->bnx_link) {
3636                         sc->bnx_link = 0;
3637
3638                         if (bootverbose)
3639                                 if_printf(ifp, "link DOWN\n");
3640
3641                         ifp->if_link_state = LINK_STATE_DOWN;
3642                         if_link_state_change(ifp);
3643                 }
3644         }
3645
3646 #undef PCS_ENCODE_ERR
3647
3648         /* Clear the attention. */
3649         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3650             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3651             BGE_MACSTAT_LINK_CHANGED);
3652 }
3653
3654 static void
3655 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3656 {
3657         struct ifnet *ifp = &sc->arpcom.ac_if;
3658         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3659
3660         mii_pollstat(mii);
3661         bnx_miibus_statchg(sc->bnx_dev);
3662
3663         if (bootverbose) {
3664                 if (sc->bnx_link)
3665                         if_printf(ifp, "link UP\n");
3666                 else
3667                         if_printf(ifp, "link DOWN\n");
3668         }
3669
3670         /* Clear the attention. */
3671         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3672             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3673             BGE_MACSTAT_LINK_CHANGED);
3674 }
3675
3676 static void
3677 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3678 {
3679         struct ifnet *ifp = &sc->arpcom.ac_if;
3680         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3681
3682         mii_pollstat(mii);
3683
3684         if (!sc->bnx_link &&
3685             (mii->mii_media_status & IFM_ACTIVE) &&
3686             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3687                 sc->bnx_link++;
3688                 if (bootverbose)
3689                         if_printf(ifp, "link UP\n");
3690         } else if (sc->bnx_link &&
3691             (!(mii->mii_media_status & IFM_ACTIVE) ||
3692             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3693                 sc->bnx_link = 0;
3694                 if (bootverbose)
3695                         if_printf(ifp, "link DOWN\n");
3696         }
3697
3698         /* Clear the attention. */
3699         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3700             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3701             BGE_MACSTAT_LINK_CHANGED);
3702 }
3703
3704 static int
3705 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
3706 {
3707         struct bnx_softc *sc = arg1;
3708
3709         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3710             &sc->bnx_rx_coal_ticks,
3711             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
3712             BNX_RX_COAL_TICKS_CHG);
3713 }
3714
3715 static int
3716 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
3717 {
3718         struct bnx_softc *sc = arg1;
3719
3720         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3721             &sc->bnx_tx_coal_ticks,
3722             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
3723             BNX_TX_COAL_TICKS_CHG);
3724 }
3725
3726 static int
3727 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
3728 {
3729         struct bnx_softc *sc = arg1;
3730
3731         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3732             &sc->bnx_rx_coal_bds,
3733             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3734             BNX_RX_COAL_BDS_CHG);
3735 }
3736
3737 static int
3738 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
3739 {
3740         struct bnx_softc *sc = arg1;
3741
3742         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3743             &sc->bnx_tx_coal_bds,
3744             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3745             BNX_TX_COAL_BDS_CHG);
3746 }
3747
3748 static int
3749 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3750 {
3751         struct bnx_softc *sc = arg1;
3752
3753         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3754             &sc->bnx_rx_coal_bds_int,
3755             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3756             BNX_RX_COAL_BDS_INT_CHG);
3757 }
3758
3759 static int
3760 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3761 {
3762         struct bnx_softc *sc = arg1;
3763
3764         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3765             &sc->bnx_tx_coal_bds_int,
3766             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3767             BNX_TX_COAL_BDS_INT_CHG);
3768 }
3769
3770 static int
3771 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
3772     int coal_min, int coal_max, uint32_t coal_chg_mask)
3773 {
3774         struct bnx_softc *sc = arg1;
3775         struct ifnet *ifp = &sc->arpcom.ac_if;
3776         int error = 0, v;
3777
3778         lwkt_serialize_enter(ifp->if_serializer);
3779
3780         v = *coal;
3781         error = sysctl_handle_int(oidp, &v, 0, req);
3782         if (!error && req->newptr != NULL) {
3783                 if (v < coal_min || v > coal_max) {
3784                         error = EINVAL;
3785                 } else {
3786                         *coal = v;
3787                         sc->bnx_coal_chg |= coal_chg_mask;
3788                 }
3789         }
3790
3791         lwkt_serialize_exit(ifp->if_serializer);
3792         return error;
3793 }
3794
3795 static void
3796 bnx_coal_change(struct bnx_softc *sc)
3797 {
3798         struct ifnet *ifp = &sc->arpcom.ac_if;
3799
3800         ASSERT_SERIALIZED(ifp->if_serializer);
3801
3802         if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
3803                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
3804                             sc->bnx_rx_coal_ticks);
3805                 DELAY(10);
3806                 CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS);
3807
3808                 if (bootverbose) {
3809                         if_printf(ifp, "rx_coal_ticks -> %u\n",
3810                                   sc->bnx_rx_coal_ticks);
3811                 }
3812         }
3813
3814         if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) {
3815                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
3816                             sc->bnx_tx_coal_ticks);
3817                 DELAY(10);
3818                 CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS);
3819
3820                 if (bootverbose) {
3821                         if_printf(ifp, "tx_coal_ticks -> %u\n",
3822                                   sc->bnx_tx_coal_ticks);
3823                 }
3824         }
3825
3826         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) {
3827                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
3828                             sc->bnx_rx_coal_bds);
3829                 DELAY(10);
3830                 CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS);
3831
3832                 if (bootverbose) {
3833                         if_printf(ifp, "rx_coal_bds -> %u\n",
3834                                   sc->bnx_rx_coal_bds);
3835                 }
3836         }
3837
3838         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) {
3839                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS,
3840                             sc->bnx_tx_coal_bds);
3841                 DELAY(10);
3842                 CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS);
3843
3844                 if (bootverbose) {
3845                         if_printf(ifp, "tx_coal_bds -> %u\n",
3846                                   sc->bnx_tx_coal_bds);
3847                 }
3848         }
3849
3850         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) {
3851                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
3852                     sc->bnx_rx_coal_bds_int);
3853                 DELAY(10);
3854                 CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT);
3855
3856                 if (bootverbose) {
3857                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
3858                             sc->bnx_rx_coal_bds_int);
3859                 }
3860         }
3861
3862         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) {
3863                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
3864                     sc->bnx_tx_coal_bds_int);
3865                 DELAY(10);
3866                 CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT);
3867
3868                 if (bootverbose) {
3869                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
3870                             sc->bnx_tx_coal_bds_int);
3871                 }
3872         }
3873
3874         sc->bnx_coal_chg = 0;
3875 }
3876
3877 static void
3878 bnx_intr_check(void *xsc)
3879 {
3880         struct bnx_softc *sc = xsc;
3881         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0]; /* XXX */
3882         struct ifnet *ifp = &sc->arpcom.ac_if;
3883         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
3884
3885         lwkt_serialize_enter(ifp->if_serializer);
3886
3887         KKASSERT(mycpuid == sc->bnx_intr_cpuid);
3888
3889         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
3890                 lwkt_serialize_exit(ifp->if_serializer);
3891                 return;
3892         }
3893
3894         if (sblk->bge_idx[0].bge_rx_prod_idx != sc->bnx_rx_saved_considx ||
3895             sblk->bge_idx[0].bge_tx_cons_idx != txr->bnx_tx_saved_considx) {
3896                 if (sc->bnx_rx_check_considx == sc->bnx_rx_saved_considx &&
3897                     sc->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
3898                         if (!sc->bnx_intr_maylose) {
3899                                 sc->bnx_intr_maylose = TRUE;
3900                                 goto done;
3901                         }
3902                         if (bootverbose)
3903                                 if_printf(ifp, "lost interrupt\n");
3904                         bnx_msi(sc);
3905                 }
3906         }
3907         sc->bnx_intr_maylose = FALSE;
3908         sc->bnx_rx_check_considx = sc->bnx_rx_saved_considx;
3909         sc->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
3910
3911 done:
3912         callout_reset(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
3913             bnx_intr_check, sc);
3914         lwkt_serialize_exit(ifp->if_serializer);
3915 }
3916
3917 static void
3918 bnx_enable_intr(struct bnx_softc *sc)
3919 {
3920         struct ifnet *ifp = &sc->arpcom.ac_if;
3921
3922         lwkt_serialize_handler_enable(ifp->if_serializer);
3923
3924         /*
3925          * Enable interrupt.
3926          */
3927         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
3928         if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
3929                 /* XXX Linux driver */
3930                 bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
3931         }
3932
3933         /*
3934          * Unmask the interrupt when we stop polling.
3935          */
3936         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
3937             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
3938
3939         /*
3940          * Trigger another interrupt, since above writing
3941          * to interrupt mailbox0 may acknowledge pending
3942          * interrupt.
3943          */
3944         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
3945
3946         if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) {
3947                 sc->bnx_intr_maylose = FALSE;
3948                 sc->bnx_rx_check_considx = 0;
3949                 sc->bnx_tx_check_considx = 0;
3950
3951                 if (bootverbose)
3952                         if_printf(ifp, "status tag bug workaround\n");
3953
3954                 /* 10ms check interval */
3955                 callout_reset_bycpu(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
3956                     bnx_intr_check, sc, sc->bnx_intr_cpuid);
3957         }
3958 }
3959
3960 static void
3961 bnx_disable_intr(struct bnx_softc *sc)
3962 {
3963         struct ifnet *ifp = &sc->arpcom.ac_if;
3964
3965         /*
3966          * Mask the interrupt when we start polling.
3967          */
3968         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
3969             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
3970
3971         /*
3972          * Acknowledge possible asserted interrupt.
3973          */
3974         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3975
3976         callout_stop(&sc->bnx_intr_timer);
3977         sc->bnx_intr_maylose = FALSE;
3978         sc->bnx_rx_check_considx = 0;
3979         sc->bnx_tx_check_considx = 0;
3980
3981         sc->bnx_npoll.ifpc_stcount = 0;
3982
3983         lwkt_serialize_handler_disable(ifp->if_serializer);
3984 }
3985
3986 static int
3987 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[])
3988 {
3989         uint32_t mac_addr;
3990         int ret = 1;
3991
3992         mac_addr = bnx_readmem_ind(sc, 0x0c14);
3993         if ((mac_addr >> 16) == 0x484b) {
3994                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
3995                 ether_addr[1] = (uint8_t)mac_addr;
3996                 mac_addr = bnx_readmem_ind(sc, 0x0c18);
3997                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
3998                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
3999                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
4000                 ether_addr[5] = (uint8_t)mac_addr;
4001                 ret = 0;
4002         }
4003         return ret;
4004 }
4005
4006 static int
4007 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[])
4008 {
4009         int mac_offset = BGE_EE_MAC_OFFSET;
4010
4011         if (BNX_IS_5717_PLUS(sc)) {
4012                 int f;
4013
4014                 f = pci_get_function(sc->bnx_dev);
4015                 if (f & 1)
4016                         mac_offset = BGE_EE_MAC_OFFSET_5717;
4017                 if (f > 1)
4018                         mac_offset += BGE_EE_MAC_OFFSET_5717_OFF;
4019         }
4020
4021         return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN);
4022 }
4023
4024 static int
4025 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[])
4026 {
4027         if (sc->bnx_flags & BNX_FLAG_NO_EEPROM)
4028                 return 1;
4029
4030         return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
4031                                ETHER_ADDR_LEN);
4032 }
4033
4034 static int
4035 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[])
4036 {
4037         static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = {
4038                 /* NOTE: Order is critical */
4039                 bnx_get_eaddr_mem,
4040                 bnx_get_eaddr_nvram,
4041                 bnx_get_eaddr_eeprom,
4042                 NULL
4043         };
4044         const bnx_eaddr_fcn_t *func;
4045
4046         for (func = bnx_eaddr_funcs; *func != NULL; ++func) {
4047                 if ((*func)(sc, eaddr) == 0)
4048                         break;
4049         }
4050         return (*func == NULL ? ENXIO : 0);
4051 }
4052
4053 /*
4054  * NOTE: 'm' is not freed upon failure
4055  */
4056 struct mbuf *
4057 bnx_defrag_shortdma(struct mbuf *m)
4058 {
4059         struct mbuf *n;
4060         int found;
4061
4062         /*
4063          * If device receive two back-to-back send BDs with less than
4064          * or equal to 8 total bytes then the device may hang.  The two
4065          * back-to-back send BDs must in the same frame for this failure
4066          * to occur.  Scan mbuf chains and see whether two back-to-back
4067          * send BDs are there.  If this is the case, allocate new mbuf
4068          * and copy the frame to workaround the silicon bug.
4069          */
4070         for (n = m, found = 0; n != NULL; n = n->m_next) {
4071                 if (n->m_len < 8) {
4072                         found++;
4073                         if (found > 1)
4074                                 break;
4075                         continue;
4076                 }
4077                 found = 0;
4078         }
4079
4080         if (found > 1)
4081                 n = m_defrag(m, MB_DONTWAIT);
4082         else
4083                 n = m;
4084         return n;
4085 }
4086
4087 static void
4088 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit)
4089 {
4090         int i;
4091
4092         BNX_CLRBIT(sc, reg, bit);
4093         for (i = 0; i < BNX_TIMEOUT; i++) {
4094                 if ((CSR_READ_4(sc, reg) & bit) == 0)
4095                         return;
4096                 DELAY(100);
4097         }
4098 }
4099
4100 static void
4101 bnx_link_poll(struct bnx_softc *sc)
4102 {
4103         uint32_t status;
4104
4105         status = CSR_READ_4(sc, BGE_MAC_STS);
4106         if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) {
4107                 sc->bnx_link_evt = 0;
4108                 sc->bnx_link_upd(sc, status);
4109         }
4110 }
4111
4112 static void
4113 bnx_enable_msi(struct bnx_softc *sc)
4114 {
4115         uint32_t msi_mode;
4116
4117         msi_mode = CSR_READ_4(sc, BGE_MSI_MODE);
4118         msi_mode |= BGE_MSIMODE_ENABLE;
4119         if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
4120                 /*
4121                  * NOTE:
4122                  * 5718-PG105-R says that "one shot" mode
4123                  * does not work if MSI is used, however,
4124                  * it obviously works.
4125                  */
4126                 msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE;
4127         }
4128         CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode);
4129 }
4130
4131 static uint32_t
4132 bnx_dma_swap_options(struct bnx_softc *sc)
4133 {
4134         uint32_t dma_options;
4135
4136         dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
4137             BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
4138 #if BYTE_ORDER == BIG_ENDIAN
4139         dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
4140 #endif
4141         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
4142             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
4143                 dma_options |= BGE_MODECTL_BYTESWAP_B2HRX_DATA |
4144                     BGE_MODECTL_WORDSWAP_B2HRX_DATA | BGE_MODECTL_B2HRX_ENABLE |
4145                     BGE_MODECTL_HTX2B_ENABLE;
4146         }
4147         return dma_options;
4148 }
4149
4150 static int
4151 bnx_setup_tso(struct bnx_tx_ring *txr, struct mbuf **mp,
4152     uint16_t *mss0, uint16_t *flags0)
4153 {
4154         struct mbuf *m;
4155         struct ip *ip;
4156         struct tcphdr *th;
4157         int thoff, iphlen, hoff, hlen;
4158         uint16_t flags, mss;
4159
4160         m = *mp;
4161         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
4162
4163         hoff = m->m_pkthdr.csum_lhlen;
4164         iphlen = m->m_pkthdr.csum_iphlen;
4165         thoff = m->m_pkthdr.csum_thlen;
4166
4167         KASSERT(hoff > 0, ("invalid ether header len"));
4168         KASSERT(iphlen > 0, ("invalid ip header len"));
4169         KASSERT(thoff > 0, ("invalid tcp header len"));
4170
4171         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
4172                 m = m_pullup(m, hoff + iphlen + thoff);
4173                 if (m == NULL) {
4174                         *mp = NULL;
4175                         return ENOBUFS;
4176                 }
4177                 *mp = m;
4178         }
4179         ip = mtodoff(m, struct ip *, hoff);
4180         th = mtodoff(m, struct tcphdr *, hoff + iphlen);
4181
4182         mss = m->m_pkthdr.tso_segsz;
4183         flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA;
4184
4185         ip->ip_len = htons(mss + iphlen + thoff);
4186         th->th_sum = 0;
4187
4188         hlen = (iphlen + thoff) >> 2;
4189         mss |= ((hlen & 0x3) << 14);
4190         flags |= ((hlen & 0xf8) << 7) | ((hlen & 0x4) << 2);
4191
4192         *mss0 = mss;
4193         *flags0 = flags;
4194
4195         return 0;
4196 }
4197
4198 static int
4199 bnx_create_tx_ring(struct bnx_tx_ring *txr)
4200 {
4201         bus_size_t txmaxsz, txmaxsegsz;
4202         int i, error;
4203
4204         /*
4205          * Create DMA tag and maps for TX mbufs.
4206          */
4207         if (txr->bnx_sc->bnx_flags & BNX_FLAG_TSO)
4208                 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header);
4209         else
4210                 txmaxsz = BNX_JUMBO_FRAMELEN;
4211         if (txr->bnx_sc->bnx_asicrev == BGE_ASICREV_BCM57766)
4212                 txmaxsegsz = MCLBYTES;
4213         else
4214                 txmaxsegsz = PAGE_SIZE;
4215         error = bus_dma_tag_create(txr->bnx_sc->bnx_cdata.bnx_parent_tag,
4216             1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
4217             txmaxsz, BNX_NSEG_NEW, txmaxsegsz,
4218             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
4219             &txr->bnx_tx_mtag);
4220         if (error) {
4221                 device_printf(txr->bnx_sc->bnx_dev,
4222                     "could not allocate TX mbuf dma tag\n");
4223                 return error;
4224         }
4225
4226         for (i = 0; i < BGE_TX_RING_CNT; i++) {
4227                 error = bus_dmamap_create(txr->bnx_tx_mtag,
4228                     BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
4229                     &txr->bnx_tx_dmamap[i]);
4230                 if (error) {
4231                         int j;
4232
4233                         for (j = 0; j < i; ++j) {
4234                                 bus_dmamap_destroy(txr->bnx_tx_mtag,
4235                                         txr->bnx_tx_dmamap[j]);
4236                         }
4237                         bus_dma_tag_destroy(txr->bnx_tx_mtag);
4238                         txr->bnx_tx_mtag = NULL;
4239
4240                         device_printf(txr->bnx_sc->bnx_dev,
4241                             "could not create DMA map for TX\n");
4242                         return error;
4243                 }
4244         }
4245
4246         /*
4247          * Create DMA stuffs for TX ring.
4248          */
4249         error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ,
4250             &txr->bnx_tx_ring_tag, &txr->bnx_tx_ring_map,
4251             (void *)&txr->bnx_tx_ring, &txr->bnx_tx_ring_paddr);
4252         if (error) {
4253                 device_printf(txr->bnx_sc->bnx_dev,
4254                     "could not create TX ring\n");
4255                 return error;
4256         }
4257
4258         txr->bnx_tx_flags |= BNX_TX_FLAG_SHORTDMA;
4259         txr->bnx_tx_wreg = BNX_TX_WREG_NSEGS;
4260
4261         return 0;
4262 }
4263
4264 static void
4265 bnx_destroy_tx_ring(struct bnx_tx_ring *txr)
4266 {
4267         /* Destroy TX mbuf DMA stuffs. */
4268         if (txr->bnx_tx_mtag != NULL) {
4269                 int i;
4270
4271                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
4272                         bus_dmamap_destroy(txr->bnx_tx_mtag,
4273                             txr->bnx_tx_dmamap[i]);
4274                 }
4275                 bus_dma_tag_destroy(txr->bnx_tx_mtag);
4276         }
4277
4278         /* Destroy TX ring */
4279         bnx_dma_block_free(txr->bnx_tx_ring_tag,
4280             txr->bnx_tx_ring_map, txr->bnx_tx_ring);
4281 }
4282
4283 static int
4284 bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
4285 {
4286         struct bnx_softc *sc = (void *)arg1;
4287         struct ifnet *ifp = &sc->arpcom.ac_if;
4288         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
4289         int error, defrag, i;
4290
4291         if (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG)
4292                 defrag = 1;
4293         else
4294                 defrag = 0;
4295
4296         error = sysctl_handle_int(oidp, &defrag, 0, req);
4297         if (error || req->newptr == NULL)
4298                 return error;
4299
4300         lwkt_serialize_enter(ifp->if_serializer);
4301         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4302                 txr = &sc->bnx_tx_ring[i];
4303                 if (defrag)
4304                         txr->bnx_tx_flags |= BNX_TX_FLAG_FORCE_DEFRAG;
4305                 else
4306                         txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG;
4307         }
4308         lwkt_serialize_exit(ifp->if_serializer);
4309
4310         return 0;
4311 }
4312
4313 static int
4314 bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS)
4315 {
4316         struct bnx_softc *sc = (void *)arg1;
4317         struct ifnet *ifp = &sc->arpcom.ac_if;
4318         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
4319         int error, tx_wreg, i;
4320
4321         tx_wreg = txr->bnx_tx_wreg;
4322         error = sysctl_handle_int(oidp, &tx_wreg, 0, req);
4323         if (error || req->newptr == NULL)
4324                 return error;
4325
4326         lwkt_serialize_enter(ifp->if_serializer);
4327         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
4328                 sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg;
4329         lwkt_serialize_exit(ifp->if_serializer);
4330
4331         return 0;
4332 }