sys/dev/netif/bnx/if_bnx.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  */
  35
  36 #include "opt_bnx.h"
  37 #include "opt_ifpoll.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/kernel.h>
  43 #include <sys/interrupt.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/queue.h>
  47 #include <sys/rman.h>
  48 #include <sys/serialize.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <netinet/ip.h>
  54 #include <netinet/tcp.h>
  55
  56 #include <net/bpf.h>
  57 #include <net/ethernet.h>
  58 #include <net/if.h>
  59 #include <net/if_arp.h>
  60 #include <net/if_dl.h>
  61 #include <net/if_media.h>
  62 #include <net/if_poll.h>
  63 #include <net/if_types.h>
  64 #include <net/ifq_var.h>
  65 #include <net/toeplitz.h>
  66 #include <net/toeplitz2.h>
  67 #include <net/vlan/if_vlan_var.h>
  68 #include <net/vlan/if_vlan_ether.h>
  69
  70 #include <dev/netif/mii_layer/mii.h>
  71 #include <dev/netif/mii_layer/miivar.h>
  72 #include <dev/netif/mii_layer/brgphyreg.h>
  73
  74 #include <bus/pci/pcidevs.h>
  75 #include <bus/pci/pcireg.h>
  76 #include <bus/pci/pcivar.h>
  77
  78 #include <dev/netif/bge/if_bgereg.h>
  79 #include <dev/netif/bnx/if_bnxvar.h>
  80
  81 /* "device miibus" required.  See GENERIC if you get errors here. */
  82 #include "miibus_if.h"
  83
  84 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
  85
  86 #define BNX_INTR_CKINTVL        ((10 * hz) / 1000)      /* 10ms */
  87
  88 #ifdef BNX_RSS_DEBUG
  89 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) \
  90 do { \
  91         if (sc->bnx_rss_debug >= lvl) \
  92                 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
  93 } while (0)
  94 #else   /* !BNX_RSS_DEBUG */
  95 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...)      ((void)0)
  96 #endif  /* BNX_RSS_DEBUG */
  97
  98 static const struct bnx_type {
  99         uint16_t                bnx_vid;
 100         uint16_t                bnx_did;
 101         char                    *bnx_name;
 102 } bnx_devs[] = {
 103         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
 104                 "Broadcom BCM5717 Gigabit Ethernet" },
 105         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C,
 106                 "Broadcom BCM5717C Gigabit Ethernet" },
 107         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
 108                 "Broadcom BCM5718 Gigabit Ethernet" },
 109         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
 110                 "Broadcom BCM5719 Gigabit Ethernet" },
 111         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
 112                 "Broadcom BCM5720 Gigabit Ethernet" },
 113
 114         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725,
 115                 "Broadcom BCM5725 Gigabit Ethernet" },
 116         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727,
 117                 "Broadcom BCM5727 Gigabit Ethernet" },
 118         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762,
 119                 "Broadcom BCM5762 Gigabit Ethernet" },
 120
 121         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
 122                 "Broadcom BCM57761 Gigabit Ethernet" },
 123         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762,
 124                 "Broadcom BCM57762 Gigabit Ethernet" },
 125         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
 126                 "Broadcom BCM57765 Gigabit Ethernet" },
 127         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766,
 128                 "Broadcom BCM57766 Gigabit Ethernet" },
 129         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
 130                 "Broadcom BCM57781 Gigabit Ethernet" },
 131         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782,
 132                 "Broadcom BCM57782 Gigabit Ethernet" },
 133         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
 134                 "Broadcom BCM57785 Gigabit Ethernet" },
 135         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786,
 136                 "Broadcom BCM57786 Gigabit Ethernet" },
 137         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
 138                 "Broadcom BCM57791 Fast Ethernet" },
 139         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
 140                 "Broadcom BCM57795 Fast Ethernet" },
 141
 142         { 0, 0, NULL }
 143 };
 144
 145 static const int bnx_tx_mailbox[BNX_TX_RING_MAX] = {
 146         BGE_MBX_TX_HOST_PROD0_LO,
 147         BGE_MBX_TX_HOST_PROD0_HI,
 148         BGE_MBX_TX_HOST_PROD1_LO,
 149         BGE_MBX_TX_HOST_PROD1_HI
 150 };
 151
 152 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
 153 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
 154 #define BNX_IS_57765_PLUS(sc)           ((sc)->bnx_flags & BNX_FLAG_57765_PLUS)
 155 #define BNX_IS_57765_FAMILY(sc)  \
 156         ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY)
 157
 158 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
 159
 160 static int      bnx_probe(device_t);
 161 static int      bnx_attach(device_t);
 162 static int      bnx_detach(device_t);
 163 static void     bnx_shutdown(device_t);
 164 static int      bnx_suspend(device_t);
 165 static int      bnx_resume(device_t);
 166 static int      bnx_miibus_readreg(device_t, int, int);
 167 static int      bnx_miibus_writereg(device_t, int, int, int);
 168 static void     bnx_miibus_statchg(device_t);
 169
 170 static void     bnx_handle_status(struct bnx_softc *);
 171 #ifdef IFPOLL_ENABLE
 172 static void     bnx_npoll(struct ifnet *, struct ifpoll_info *);
 173 static void     bnx_npoll_rx(struct ifnet *, void *, int);
 174 static void     bnx_npoll_tx(struct ifnet *, void *, int);
 175 static void     bnx_npoll_tx_notag(struct ifnet *, void *, int);
 176 static void     bnx_npoll_status(struct ifnet *);
 177 static void     bnx_npoll_status_notag(struct ifnet *);
 178 #endif
 179 static void     bnx_intr_legacy(void *);
 180 static void     bnx_msi(void *);
 181 static void     bnx_intr(struct bnx_softc *);
 182 static void     bnx_msix_status(void *);
 183 static void     bnx_msix_tx_status(void *);
 184 static void     bnx_msix_rx(void *);
 185 static void     bnx_msix_rxtx(void *);
 186 static void     bnx_enable_intr(struct bnx_softc *);
 187 static void     bnx_disable_intr(struct bnx_softc *);
 188 static void     bnx_txeof(struct bnx_tx_ring *, uint16_t);
 189 static void     bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int);
 190 static int      bnx_alloc_intr(struct bnx_softc *);
 191 static int      bnx_setup_intr(struct bnx_softc *);
 192 static void     bnx_free_intr(struct bnx_softc *);
 193 static void     bnx_teardown_intr(struct bnx_softc *, int);
 194 static int      bnx_alloc_msix(struct bnx_softc *);
 195 static void     bnx_free_msix(struct bnx_softc *, boolean_t);
 196 static void     bnx_check_intr_rxtx(void *);
 197 static void     bnx_check_intr_rx(void *);
 198 static void     bnx_check_intr_tx(void *);
 199 static void     bnx_rx_std_refill_ithread(void *);
 200 static void     bnx_rx_std_refill(void *, void *);
 201 static void     bnx_rx_std_refill_sched_ipi(void *);
 202 static void     bnx_rx_std_refill_stop(void *);
 203 static void     bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *,
 204                     struct bnx_rx_std_ring *);
 205
 206 static void     bnx_start(struct ifnet *, struct ifaltq_subque *);
 207 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 208 static void     bnx_init(void *);
 209 static void     bnx_stop(struct bnx_softc *);
 210 static void     bnx_watchdog(struct ifaltq_subque *);
 211 static int      bnx_ifmedia_upd(struct ifnet *);
 212 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 213 static void     bnx_tick(void *);
 214 static void     bnx_serialize(struct ifnet *, enum ifnet_serialize);
 215 static void     bnx_deserialize(struct ifnet *, enum ifnet_serialize);
 216 static int      bnx_tryserialize(struct ifnet *, enum ifnet_serialize);
 217 #ifdef INVARIANTS
 218 static void     bnx_serialize_assert(struct ifnet *, enum ifnet_serialize,
 219                     boolean_t);
 220 #endif
 221 static void     bnx_serialize_skipmain(struct bnx_softc *);
 222 static void     bnx_deserialize_skipmain(struct bnx_softc *sc);
 223
 224 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
 225 static void     bnx_free_jumbo_mem(struct bnx_softc *);
 226 static struct bnx_jslot
 227                 *bnx_jalloc(struct bnx_softc *);
 228 static void     bnx_jfree(void *);
 229 static void     bnx_jref(void *);
 230 static int      bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int);
 231 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
 232 static void     bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int);
 233 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
 234 static int      bnx_init_rx_ring_std(struct bnx_rx_std_ring *);
 235 static void     bnx_free_rx_ring_std(struct bnx_rx_std_ring *);
 236 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
 237 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
 238 static void     bnx_free_tx_ring(struct bnx_tx_ring *);
 239 static int      bnx_init_tx_ring(struct bnx_tx_ring *);
 240 static int      bnx_create_tx_ring(struct bnx_tx_ring *);
 241 static void     bnx_destroy_tx_ring(struct bnx_tx_ring *);
 242 static int      bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *);
 243 static void     bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *);
 244 static int      bnx_dma_alloc(device_t);
 245 static void     bnx_dma_free(struct bnx_softc *);
 246 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
 247                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
 248 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 249 static struct mbuf *
 250                 bnx_defrag_shortdma(struct mbuf *);
 251 static int      bnx_encap(struct bnx_tx_ring *, struct mbuf **,
 252                     uint32_t *, int *);
 253 static int      bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
 254                     uint16_t *, uint16_t *);
 255 static void     bnx_setup_serialize(struct bnx_softc *);
 256 static void     bnx_set_tick_cpuid(struct bnx_softc *, boolean_t);
 257 static void     bnx_setup_ring_cnt(struct bnx_softc *);
 258
 259 static struct pktinfo *bnx_rss_info(struct pktinfo *,
 260                     const struct bge_rx_bd *);
 261 static void     bnx_init_rss(struct bnx_softc *);
 262 static void     bnx_reset(struct bnx_softc *);
 263 static int      bnx_chipinit(struct bnx_softc *);
 264 static int      bnx_blockinit(struct bnx_softc *);
 265 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
 266 static void     bnx_enable_msi(struct bnx_softc *, boolean_t);
 267 static void     bnx_setmulti(struct bnx_softc *);
 268 static void     bnx_setpromisc(struct bnx_softc *);
 269 static void     bnx_stats_update_regs(struct bnx_softc *);
 270 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
 271
 272 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
 273 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
 274 #ifdef notdef
 275 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
 276 #endif
 277 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
 278 static void     bnx_writembx(struct bnx_softc *, int, int);
 279 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
 280 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
 281 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
 282
 283 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
 284 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
 285 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
 286 static void     bnx_link_poll(struct bnx_softc *);
 287
 288 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
 289 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
 290 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
 291 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
 292
 293 static void     bnx_coal_change(struct bnx_softc *);
 294 static int      bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS);
 295 static int      bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS);
 296 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 297 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 298 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
 299 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
 300 static int      bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS);
 301 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 302 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 303 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
 304                     int, int, uint32_t);
 305 #ifdef IFPOLL_ENABLE
 306 static int      bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS);
 307 static int      bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS);
 308 static int      bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS);
 309 #endif
 310 static int      bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS);
 311
 312 static int      bnx_msi_enable = 1;
 313 static int      bnx_msix_enable = 1;
 314
 315 static int      bnx_rx_rings = 0; /* auto */
 316 static int      bnx_tx_rings = 0; /* auto */
 317
 318 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
 319 TUNABLE_INT("hw.bnx.msix.enable", &bnx_msix_enable);
 320 TUNABLE_INT("hw.bnx.rx_rings", &bnx_rx_rings);
 321 TUNABLE_INT("hw.bnx.tx_rings", &bnx_tx_rings);
 322
 323 static device_method_t bnx_methods[] = {
 324         /* Device interface */
 325         DEVMETHOD(device_probe,         bnx_probe),
 326         DEVMETHOD(device_attach,        bnx_attach),
 327         DEVMETHOD(device_detach,        bnx_detach),
 328         DEVMETHOD(device_shutdown,      bnx_shutdown),
 329         DEVMETHOD(device_suspend,       bnx_suspend),
 330         DEVMETHOD(device_resume,        bnx_resume),
 331
 332         /* bus interface */
 333         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 334         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 335
 336         /* MII interface */
 337         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
 338         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
 339         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
 340
 341         DEVMETHOD_END
 342 };
 343
 344 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
 345 static devclass_t bnx_devclass;
 346
 347 DECLARE_DUMMY_MODULE(if_bnx);
 348 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
 349 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
 350
 351 static uint32_t
 352 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
 353 {
 354         device_t dev = sc->bnx_dev;
 355         uint32_t val;
 356
 357         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 358         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 359         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 360         return (val);
 361 }
 362
 363 static void
 364 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
 365 {
 366         device_t dev = sc->bnx_dev;
 367
 368         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 369         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 370         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 371 }
 372
 373 static void
 374 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
 375 {
 376         CSR_WRITE_4(sc, off, val);
 377 }
 378
 379 static void
 380 bnx_writembx(struct bnx_softc *sc, int off, int val)
 381 {
 382         CSR_WRITE_4(sc, off, val);
 383 }
 384
 385 /*
 386  * Read a sequence of bytes from NVRAM.
 387  */
 388 static int
 389 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
 390 {
 391         return (1);
 392 }
 393
 394 /*
 395  * Read a byte of data stored in the EEPROM at address 'addr.' The
 396  * BCM570x supports both the traditional bitbang interface and an
 397  * auto access interface for reading the EEPROM. We use the auto
 398  * access method.
 399  */
 400 static uint8_t
 401 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
 402 {
 403         int i;
 404         uint32_t byte = 0;
 405
 406         /*
 407          * Enable use of auto EEPROM access so we can avoid
 408          * having to use the bitbang method.
 409          */
 410         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 411
 412         /* Reset the EEPROM, load the clock period. */
 413         CSR_WRITE_4(sc, BGE_EE_ADDR,
 414             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 415         DELAY(20);
 416
 417         /* Issue the read EEPROM command. */
 418         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 419
 420         /* Wait for completion */
 421         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
 422                 DELAY(10);
 423                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 424                         break;
 425         }
 426
 427         if (i == BNX_TIMEOUT) {
 428                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 429                 return(1);
 430         }
 431
 432         /* Get result. */
 433         byte = CSR_READ_4(sc, BGE_EE_DATA);
 434
 435         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 436
 437         return(0);
 438 }
 439
 440 /*
 441  * Read a sequence of bytes from the EEPROM.
 442  */
 443 static int
 444 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
 445 {
 446         size_t i;
 447         int err;
 448         uint8_t byte;
 449
 450         for (byte = 0, err = 0, i = 0; i < len; i++) {
 451                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
 452                 if (err)
 453                         break;
 454                 *(dest + i) = byte;
 455         }
 456
 457         return(err ? 1 : 0);
 458 }
 459
 460 static int
 461 bnx_miibus_readreg(device_t dev, int phy, int reg)
 462 {
 463         struct bnx_softc *sc = device_get_softc(dev);
 464         uint32_t val;
 465         int i;
 466
 467         KASSERT(phy == sc->bnx_phyno,
 468             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 469
 470         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 471         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 472                 CSR_WRITE_4(sc, BGE_MI_MODE,
 473                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 474                 DELAY(80);
 475         }
 476
 477         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
 478             BGE_MIPHY(phy) | BGE_MIREG(reg));
 479
 480         /* Poll for the PHY register access to complete. */
 481         for (i = 0; i < BNX_TIMEOUT; i++) {
 482                 DELAY(10);
 483                 val = CSR_READ_4(sc, BGE_MI_COMM);
 484                 if ((val & BGE_MICOMM_BUSY) == 0) {
 485                         DELAY(5);
 486                         val = CSR_READ_4(sc, BGE_MI_COMM);
 487                         break;
 488                 }
 489         }
 490         if (i == BNX_TIMEOUT) {
 491                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
 492                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 493                 val = 0;
 494         }
 495
 496         /* Restore the autopoll bit if necessary. */
 497         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 498                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 499                 DELAY(80);
 500         }
 501
 502         if (val & BGE_MICOMM_READFAIL)
 503                 return 0;
 504
 505         return (val & 0xFFFF);
 506 }
 507
 508 static int
 509 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
 510 {
 511         struct bnx_softc *sc = device_get_softc(dev);
 512         int i;
 513
 514         KASSERT(phy == sc->bnx_phyno,
 515             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 516
 517         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 518         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 519                 CSR_WRITE_4(sc, BGE_MI_MODE,
 520                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 521                 DELAY(80);
 522         }
 523
 524         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
 525             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
 526
 527         for (i = 0; i < BNX_TIMEOUT; i++) {
 528                 DELAY(10);
 529                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 530                         DELAY(5);
 531                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 532                         break;
 533                 }
 534         }
 535         if (i == BNX_TIMEOUT) {
 536                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 537                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
 538         }
 539
 540         /* Restore the autopoll bit if necessary. */
 541         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 542                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 543                 DELAY(80);
 544         }
 545
 546         return 0;
 547 }
 548
 549 static void
 550 bnx_miibus_statchg(device_t dev)
 551 {
 552         struct bnx_softc *sc;
 553         struct mii_data *mii;
 554
 555         sc = device_get_softc(dev);
 556         mii = device_get_softc(sc->bnx_miibus);
 557
 558         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 559             (IFM_ACTIVE | IFM_AVALID)) {
 560                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 561                 case IFM_10_T:
 562                 case IFM_100_TX:
 563                         sc->bnx_link = 1;
 564                         break;
 565                 case IFM_1000_T:
 566                 case IFM_1000_SX:
 567                 case IFM_2500_SX:
 568                         sc->bnx_link = 1;
 569                         break;
 570                 default:
 571                         sc->bnx_link = 0;
 572                         break;
 573                 }
 574         } else {
 575                 sc->bnx_link = 0;
 576         }
 577         if (sc->bnx_link == 0)
 578                 return;
 579
 580         BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
 581         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
 582             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
 583                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
 584         } else {
 585                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
 586         }
 587
 588         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
 589                 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 590         } else {
 591                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
 592         }
 593 }
 594
 595 /*
 596  * Memory management for jumbo frames.
 597  */
 598 static int
 599 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
 600 {
 601         struct ifnet *ifp = &sc->arpcom.ac_if;
 602         struct bnx_jslot *entry;
 603         uint8_t *ptr;
 604         bus_addr_t paddr;
 605         int i, error;
 606
 607         /*
 608          * Create tag for jumbo mbufs.
 609          * This is really a bit of a kludge. We allocate a special
 610          * jumbo buffer pool which (thanks to the way our DMA
 611          * memory allocation works) will consist of contiguous
 612          * pages. This means that even though a jumbo buffer might
 613          * be larger than a page size, we don't really need to
 614          * map it into more than one DMA segment. However, the
 615          * default mbuf tag will result in multi-segment mappings,
 616          * so we have to create a special jumbo mbuf tag that
 617          * lets us get away with mapping the jumbo buffers as
 618          * a single segment. I think eventually the driver should
 619          * be changed so that it uses ordinary mbufs and cluster
 620          * buffers, i.e. jumbo frames can span multiple DMA
 621          * descriptors. But that's a project for another day.
 622          */
 623
 624         /*
 625          * Create DMA stuffs for jumbo RX ring.
 626          */
 627         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 628                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 629                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 630                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
 631                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
 632         if (error) {
 633                 if_printf(ifp, "could not create jumbo RX ring\n");
 634                 return error;
 635         }
 636
 637         /*
 638          * Create DMA stuffs for jumbo buffer block.
 639          */
 640         error = bnx_dma_block_alloc(sc, BNX_JMEM,
 641                                     &sc->bnx_cdata.bnx_jumbo_tag,
 642                                     &sc->bnx_cdata.bnx_jumbo_map,
 643                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
 644                                     &paddr);
 645         if (error) {
 646                 if_printf(ifp, "could not create jumbo buffer\n");
 647                 return error;
 648         }
 649
 650         SLIST_INIT(&sc->bnx_jfree_listhead);
 651
 652         /*
 653          * Now divide it up into 9K pieces and save the addresses
 654          * in an array. Note that we play an evil trick here by using
 655          * the first few bytes in the buffer to hold the the address
 656          * of the softc structure for this interface. This is because
 657          * bnx_jfree() needs it, but it is called by the mbuf management
 658          * code which will not pass it to us explicitly.
 659          */
 660         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
 661                 entry = &sc->bnx_cdata.bnx_jslots[i];
 662                 entry->bnx_sc = sc;
 663                 entry->bnx_buf = ptr;
 664                 entry->bnx_paddr = paddr;
 665                 entry->bnx_inuse = 0;
 666                 entry->bnx_slot = i;
 667                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
 668
 669                 ptr += BNX_JLEN;
 670                 paddr += BNX_JLEN;
 671         }
 672         return 0;
 673 }
 674
 675 static void
 676 bnx_free_jumbo_mem(struct bnx_softc *sc)
 677 {
 678         /* Destroy jumbo RX ring. */
 679         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 680                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 681                            sc->bnx_ldata.bnx_rx_jumbo_ring);
 682
 683         /* Destroy jumbo buffer block. */
 684         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
 685                            sc->bnx_cdata.bnx_jumbo_map,
 686                            sc->bnx_ldata.bnx_jumbo_buf);
 687 }
 688
 689 /*
 690  * Allocate a jumbo buffer.
 691  */
 692 static struct bnx_jslot *
 693 bnx_jalloc(struct bnx_softc *sc)
 694 {
 695         struct bnx_jslot *entry;
 696
 697         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 698         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
 699         if (entry) {
 700                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
 701                 entry->bnx_inuse = 1;
 702         } else {
 703                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 704         }
 705         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 706         return(entry);
 707 }
 708
 709 /*
 710  * Adjust usage count on a jumbo buffer.
 711  */
 712 static void
 713 bnx_jref(void *arg)
 714 {
 715         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 716         struct bnx_softc *sc = entry->bnx_sc;
 717
 718         if (sc == NULL)
 719                 panic("bnx_jref: can't find softc pointer!");
 720
 721         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 722                 panic("bnx_jref: asked to reference buffer "
 723                     "that we don't manage!");
 724         } else if (entry->bnx_inuse == 0) {
 725                 panic("bnx_jref: buffer already free!");
 726         } else {
 727                 atomic_add_int(&entry->bnx_inuse, 1);
 728         }
 729 }
 730
 731 /*
 732  * Release a jumbo buffer.
 733  */
 734 static void
 735 bnx_jfree(void *arg)
 736 {
 737         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 738         struct bnx_softc *sc = entry->bnx_sc;
 739
 740         if (sc == NULL)
 741                 panic("bnx_jfree: can't find softc pointer!");
 742
 743         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 744                 panic("bnx_jfree: asked to free buffer that we don't manage!");
 745         } else if (entry->bnx_inuse == 0) {
 746                 panic("bnx_jfree: buffer already free!");
 747         } else {
 748                 /*
 749                  * Possible MP race to 0, use the serializer.  The atomic insn
 750                  * is still needed for races against bnx_jref().
 751                  */
 752                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 753                 atomic_subtract_int(&entry->bnx_inuse, 1);
 754                 if (entry->bnx_inuse == 0) {
 755                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead,
 756                                           entry, jslot_link);
 757                 }
 758                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 759         }
 760 }
 761
 762
 763 /*
 764  * Intialize a standard receive ring descriptor.
 765  */
 766 static int
 767 bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init)
 768 {
 769         struct mbuf *m_new = NULL;
 770         bus_dma_segment_t seg;
 771         bus_dmamap_t map;
 772         int error, nsegs;
 773         struct bnx_rx_buf *rb;
 774
 775         rb = &ret->bnx_std->bnx_rx_std_buf[i];
 776         KASSERT(!rb->bnx_rx_refilled, ("RX buf %dth has been refilled", i));
 777
 778         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
 779         if (m_new == NULL) {
 780                 error = ENOBUFS;
 781                 goto back;
 782         }
 783         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
 784         m_adj(m_new, ETHER_ALIGN);
 785
 786         error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag,
 787             ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT);
 788         if (error) {
 789                 m_freem(m_new);
 790                 goto back;
 791         }
 792
 793         if (!init) {
 794                 bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap,
 795                     BUS_DMASYNC_POSTREAD);
 796                 bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap);
 797         }
 798
 799         map = ret->bnx_rx_tmpmap;
 800         ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap;
 801
 802         rb->bnx_rx_dmamap = map;
 803         rb->bnx_rx_mbuf = m_new;
 804         rb->bnx_rx_paddr = seg.ds_addr;
 805         rb->bnx_rx_len = m_new->m_len;
 806 back:
 807         cpu_sfence();
 808         rb->bnx_rx_refilled = 1;
 809         return error;
 810 }
 811
 812 static void
 813 bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i)
 814 {
 815         struct bnx_rx_buf *rb;
 816         struct bge_rx_bd *r;
 817         bus_addr_t paddr;
 818         int len;
 819
 820         rb = &std->bnx_rx_std_buf[i];
 821         KASSERT(rb->bnx_rx_refilled, ("RX buf %dth is not refilled", i));
 822
 823         paddr = rb->bnx_rx_paddr;
 824         len = rb->bnx_rx_len;
 825
 826         cpu_mfence();
 827
 828         rb->bnx_rx_refilled = 0;
 829
 830         r = &std->bnx_rx_std_ring[i];
 831         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(paddr);
 832         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(paddr);
 833         r->bge_len = len;
 834         r->bge_idx = i;
 835         r->bge_flags = BGE_RXBDFLAG_END;
 836 }
 837
 838 /*
 839  * Initialize a jumbo receive ring descriptor. This allocates
 840  * a jumbo buffer from the pool managed internally by the driver.
 841  */
 842 static int
 843 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
 844 {
 845         struct mbuf *m_new = NULL;
 846         struct bnx_jslot *buf;
 847         bus_addr_t paddr;
 848
 849         /* Allocate the mbuf. */
 850         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
 851         if (m_new == NULL)
 852                 return ENOBUFS;
 853
 854         /* Allocate the jumbo buffer */
 855         buf = bnx_jalloc(sc);
 856         if (buf == NULL) {
 857                 m_freem(m_new);
 858                 return ENOBUFS;
 859         }
 860
 861         /* Attach the buffer to the mbuf. */
 862         m_new->m_ext.ext_arg = buf;
 863         m_new->m_ext.ext_buf = buf->bnx_buf;
 864         m_new->m_ext.ext_free = bnx_jfree;
 865         m_new->m_ext.ext_ref = bnx_jref;
 866         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
 867
 868         m_new->m_flags |= M_EXT;
 869
 870         m_new->m_data = m_new->m_ext.ext_buf;
 871         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
 872
 873         paddr = buf->bnx_paddr;
 874         m_adj(m_new, ETHER_ALIGN);
 875         paddr += ETHER_ALIGN;
 876
 877         /* Save necessary information */
 878         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new;
 879         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr;
 880
 881         /* Set up the descriptor. */
 882         bnx_setup_rxdesc_jumbo(sc, i);
 883         return 0;
 884 }
 885
 886 static void
 887 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
 888 {
 889         struct bge_rx_bd *r;
 890         struct bnx_rx_buf *rc;
 891
 892         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
 893         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 894
 895         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr);
 896         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr);
 897         r->bge_len = rc->bnx_rx_mbuf->m_len;
 898         r->bge_idx = i;
 899         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 900 }
 901
 902 static int
 903 bnx_init_rx_ring_std(struct bnx_rx_std_ring *std)
 904 {
 905         int i, error;
 906
 907         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 908                 /* Use the first RX return ring's tmp RX mbuf DMA map */
 909                 error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1);
 910                 if (error)
 911                         return error;
 912                 bnx_setup_rxdesc_std(std, i);
 913         }
 914
 915         std->bnx_rx_std_refill = 0;
 916         std->bnx_rx_std_running = 0;
 917         cpu_sfence();
 918         lwkt_serialize_handler_enable(&std->bnx_rx_std_serialize);
 919
 920         std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1;
 921         bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std);
 922
 923         return(0);
 924 }
 925
 926 static void
 927 bnx_free_rx_ring_std(struct bnx_rx_std_ring *std)
 928 {
 929         int i;
 930
 931         lwkt_serialize_handler_disable(&std->bnx_rx_std_serialize);
 932
 933         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 934                 struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i];
 935
 936                 rb->bnx_rx_refilled = 0;
 937                 if (rb->bnx_rx_mbuf != NULL) {
 938                         bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap);
 939                         m_freem(rb->bnx_rx_mbuf);
 940                         rb->bnx_rx_mbuf = NULL;
 941                 }
 942                 bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd));
 943         }
 944 }
 945
 946 static int
 947 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
 948 {
 949         struct bge_rcb *rcb;
 950         int i, error;
 951
 952         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 953                 error = bnx_newbuf_jumbo(sc, i, 1);
 954                 if (error)
 955                         return error;
 956         }
 957
 958         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
 959
 960         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
 961         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
 962         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
 963
 964         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
 965
 966         return(0);
 967 }
 968
 969 static void
 970 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
 971 {
 972         int i;
 973
 974         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 975                 struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 976
 977                 if (rc->bnx_rx_mbuf != NULL) {
 978                         m_freem(rc->bnx_rx_mbuf);
 979                         rc->bnx_rx_mbuf = NULL;
 980                 }
 981                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
 982                     sizeof(struct bge_rx_bd));
 983         }
 984 }
 985
 986 static void
 987 bnx_free_tx_ring(struct bnx_tx_ring *txr)
 988 {
 989         int i;
 990
 991         for (i = 0; i < BGE_TX_RING_CNT; i++) {
 992                 struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i];
 993
 994                 if (buf->bnx_tx_mbuf != NULL) {
 995                         bus_dmamap_unload(txr->bnx_tx_mtag,
 996                             buf->bnx_tx_dmamap);
 997                         m_freem(buf->bnx_tx_mbuf);
 998                         buf->bnx_tx_mbuf = NULL;
 999                 }
1000                 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd));
1001         }
1002         txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
1003 }
1004
1005 static int
1006 bnx_init_tx_ring(struct bnx_tx_ring *txr)
1007 {
1008         txr->bnx_tx_cnt = 0;
1009         txr->bnx_tx_saved_considx = 0;
1010         txr->bnx_tx_prodidx = 0;
1011
1012         /* Initialize transmit producer index for host-memory send ring. */
1013         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx);
1014
1015         return(0);
1016 }
1017
1018 static void
1019 bnx_setmulti(struct bnx_softc *sc)
1020 {
1021         struct ifnet *ifp;
1022         struct ifmultiaddr *ifma;
1023         uint32_t hashes[4] = { 0, 0, 0, 0 };
1024         int h, i;
1025
1026         ifp = &sc->arpcom.ac_if;
1027
1028         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1029                 for (i = 0; i < 4; i++)
1030                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1031                 return;
1032         }
1033
1034         /* First, zot all the existing filters. */
1035         for (i = 0; i < 4; i++)
1036                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1037
1038         /* Now program new ones. */
1039         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1040                 if (ifma->ifma_addr->sa_family != AF_LINK)
1041                         continue;
1042                 h = ether_crc32_le(
1043                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1044                     ETHER_ADDR_LEN) & 0x7f;
1045                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1046         }
1047
1048         for (i = 0; i < 4; i++)
1049                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1050 }
1051
1052 /*
1053  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1054  * self-test results.
1055  */
1056 static int
1057 bnx_chipinit(struct bnx_softc *sc)
1058 {
1059         uint32_t dma_rw_ctl, mode_ctl;
1060         int i;
1061
1062         /* Set endian type before we access any non-PCI registers. */
1063         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
1064             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
1065
1066         /* Clear the MAC control register */
1067         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
1068
1069         /*
1070          * Clear the MAC statistics block in the NIC's
1071          * internal memory.
1072          */
1073         for (i = BGE_STATS_BLOCK;
1074             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1075                 BNX_MEMWIN_WRITE(sc, i, 0);
1076
1077         for (i = BGE_STATUS_BLOCK;
1078             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1079                 BNX_MEMWIN_WRITE(sc, i, 0);
1080
1081         if (BNX_IS_57765_FAMILY(sc)) {
1082                 uint32_t val;
1083
1084                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) {
1085                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1086                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1087
1088                         /* Access the lower 1K of PL PCI-E block registers. */
1089                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1090                             val | BGE_MODECTL_PCIE_PL_SEL);
1091
1092                         val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5);
1093                         val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ;
1094                         CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val);
1095
1096                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1097                 }
1098                 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) {
1099                         /* Fix transmit hangs */
1100                         val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL);
1101                         val |= BGE_CPMU_PADRNG_CTL_RDIV2;
1102                         CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val);
1103
1104                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1105                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1106
1107                         /* Access the lower 1K of DL PCI-E block registers. */
1108                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1109                             val | BGE_MODECTL_PCIE_DL_SEL);
1110
1111                         val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX);
1112                         val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK;
1113                         val |= BGE_PCIE_DL_LO_FTSMAX_VAL;
1114                         CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val);
1115
1116                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1117                 }
1118
1119                 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK);
1120                 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK;
1121                 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25;
1122                 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val);
1123         }
1124
1125         /*
1126          * Set up the PCI DMA control register.
1127          */
1128         dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4);
1129         /*
1130          * Disable 32bytes cache alignment for DMA write to host memory
1131          *
1132          * NOTE:
1133          * 64bytes cache alignment for DMA write to host memory is still
1134          * enabled.
1135          */
1136         dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1137         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1138                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1139         /*
1140          * Enable HW workaround for controllers that misinterpret
1141          * a status tag update and leave interrupts permanently
1142          * disabled.
1143          */
1144         if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1145             sc->bnx_asicrev != BGE_ASICREV_BCM5762 &&
1146             !BNX_IS_57765_FAMILY(sc))
1147                 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1148         if (bootverbose) {
1149                 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n",
1150                     dma_rw_ctl);
1151         }
1152         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1153
1154         /*
1155          * Set up general mode register.
1156          */
1157         mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
1158             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1159         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1160
1161         /*
1162          * Disable memory write invalidate.  Apparently it is not supported
1163          * properly by these devices.  Also ensure that INTx isn't disabled,
1164          * as these chips need it even when using MSI.
1165          */
1166         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1167             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1168
1169         /* Set the timer prescaler (always 66Mhz) */
1170         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1171
1172         return(0);
1173 }
1174
1175 static int
1176 bnx_blockinit(struct bnx_softc *sc)
1177 {
1178         struct bnx_intr_data *intr;
1179         struct bge_rcb *rcb;
1180         bus_size_t vrcb;
1181         bge_hostaddr taddr;
1182         uint32_t val;
1183         int i, limit;
1184
1185         /*
1186          * Initialize the memory window pointer register so that
1187          * we can access the first 32K of internal NIC RAM. This will
1188          * allow us to set up the TX send ring RCBs and the RX return
1189          * ring RCBs, plus other things which live in NIC memory.
1190          */
1191         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1192
1193         /* Configure mbuf pool watermarks */
1194         if (BNX_IS_57765_PLUS(sc)) {
1195                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1196                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1197                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1198                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1199                 } else {
1200                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1201                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1202                 }
1203         } else {
1204                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1205                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1206                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1207         }
1208
1209         /* Configure DMA resource watermarks */
1210         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1211         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1212
1213         /* Enable buffer manager */
1214         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1215         /*
1216          * Change the arbitration algorithm of TXMBUF read request to
1217          * round-robin instead of priority based for BCM5719.  When
1218          * TXFIFO is almost empty, RDMA will hold its request until
1219          * TXFIFO is not almost empty.
1220          */
1221         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1222                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1223         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1224             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 ||
1225             sc->bnx_chipid == BGE_CHIPID_BCM5720_A0)
1226                 val |= BGE_BMANMODE_LOMBUF_ATTN;
1227         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1228
1229         /* Poll for buffer manager start indication */
1230         for (i = 0; i < BNX_TIMEOUT; i++) {
1231                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1232                         break;
1233                 DELAY(10);
1234         }
1235
1236         if (i == BNX_TIMEOUT) {
1237                 if_printf(&sc->arpcom.ac_if,
1238                           "buffer manager failed to start\n");
1239                 return(ENXIO);
1240         }
1241
1242         /* Enable flow-through queues */
1243         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1244         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1245
1246         /* Wait until queue initialization is complete */
1247         for (i = 0; i < BNX_TIMEOUT; i++) {
1248                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1249                         break;
1250                 DELAY(10);
1251         }
1252
1253         if (i == BNX_TIMEOUT) {
1254                 if_printf(&sc->arpcom.ac_if,
1255                           "flow-through queue init failed\n");
1256                 return(ENXIO);
1257         }
1258
1259         /*
1260          * Summary of rings supported by the controller:
1261          *
1262          * Standard Receive Producer Ring
1263          * - This ring is used to feed receive buffers for "standard"
1264          *   sized frames (typically 1536 bytes) to the controller.
1265          *
1266          * Jumbo Receive Producer Ring
1267          * - This ring is used to feed receive buffers for jumbo sized
1268          *   frames (i.e. anything bigger than the "standard" frames)
1269          *   to the controller.
1270          *
1271          * Mini Receive Producer Ring
1272          * - This ring is used to feed receive buffers for "mini"
1273          *   sized frames to the controller.
1274          * - This feature required external memory for the controller
1275          *   but was never used in a production system.  Should always
1276          *   be disabled.
1277          *
1278          * Receive Return Ring
1279          * - After the controller has placed an incoming frame into a
1280          *   receive buffer that buffer is moved into a receive return
1281          *   ring.  The driver is then responsible to passing the
1282          *   buffer up to the stack.  BCM5718/BCM57785 families support
1283          *   multiple receive return rings.
1284          *
1285          * Send Ring
1286          * - This ring is used for outgoing frames.  BCM5719/BCM5720
1287          *   support multiple send rings.
1288          */
1289
1290         /* Initialize the standard receive producer ring control block. */
1291         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1292         rcb->bge_hostaddr.bge_addr_lo =
1293             BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1294         rcb->bge_hostaddr.bge_addr_hi =
1295             BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1296         if (BNX_IS_57765_PLUS(sc)) {
1297                 /*
1298                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1299                  * Bits 15-2 : Maximum RX frame size
1300                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1301                  * Bit 0     : Reserved
1302                  */
1303                 rcb->bge_maxlen_flags =
1304                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1305         } else {
1306                 /*
1307                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1308                  * Bits 15-2 : Reserved (should be 0)
1309                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1310                  * Bit 0     : Reserved
1311                  */
1312                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1313         }
1314         if (BNX_IS_5717_PLUS(sc))
1315                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1316         else
1317                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1318         /* Write the standard receive producer ring control block. */
1319         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1320         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1321         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1322         if (!BNX_IS_5717_PLUS(sc))
1323                 CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1324         /* Reset the standard receive producer ring producer index. */
1325         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1326
1327         /*
1328          * Initialize the jumbo RX producer ring control
1329          * block.  We set the 'ring disabled' bit in the
1330          * flags field until we're actually ready to start
1331          * using this ring (i.e. once we set the MTU
1332          * high enough to require it).
1333          */
1334         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1335                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1336                 /* Get the jumbo receive producer ring RCB parameters. */
1337                 rcb->bge_hostaddr.bge_addr_lo =
1338                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1339                 rcb->bge_hostaddr.bge_addr_hi =
1340                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1341                 rcb->bge_maxlen_flags =
1342                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1343                     BGE_RCB_FLAG_RING_DISABLED);
1344                 if (BNX_IS_5717_PLUS(sc))
1345                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1346                 else
1347                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1348                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1349                     rcb->bge_hostaddr.bge_addr_hi);
1350                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1351                     rcb->bge_hostaddr.bge_addr_lo);
1352                 /* Program the jumbo receive producer ring RCB parameters. */
1353                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1354                     rcb->bge_maxlen_flags);
1355                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1356                 /* Reset the jumbo receive producer ring producer index. */
1357                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1358         }
1359
1360         /*
1361          * The BD ring replenish thresholds control how often the
1362          * hardware fetches new BD's from the producer rings in host
1363          * memory.  Setting the value too low on a busy system can
1364          * starve the hardware and recue the throughpout.
1365          *
1366          * Set the BD ring replentish thresholds. The recommended
1367          * values are 1/8th the number of descriptors allocated to
1368          * each ring.
1369          */
1370         val = 8;
1371         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1372         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1373                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1374                     BGE_JUMBO_RX_RING_CNT/8);
1375         }
1376         if (BNX_IS_57765_PLUS(sc)) {
1377                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1378                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1379         }
1380
1381         /*
1382          * Disable all send rings by setting the 'ring disabled' bit
1383          * in the flags field of all the TX send ring control blocks,
1384          * located in NIC memory.
1385          */
1386         if (BNX_IS_5717_PLUS(sc))
1387                 limit = 4;
1388         else if (BNX_IS_57765_FAMILY(sc) ||
1389             sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1390                 limit = 2;
1391         else
1392                 limit = 1;
1393         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1394         for (i = 0; i < limit; i++) {
1395                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1396                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1397                 vrcb += sizeof(struct bge_rcb);
1398         }
1399
1400         /*
1401          * Configure send ring RCBs
1402          */
1403         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1404         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
1405                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
1406
1407                 BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr);
1408                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi,
1409                     taddr.bge_addr_hi);
1410                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo,
1411                     taddr.bge_addr_lo);
1412                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1413                     BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1414                 vrcb += sizeof(struct bge_rcb);
1415         }
1416
1417         /*
1418          * Disable all receive return rings by setting the
1419          * 'ring disabled' bit in the flags field of all the receive
1420          * return ring control blocks, located in NIC memory.
1421          */
1422         if (BNX_IS_5717_PLUS(sc)) {
1423                 /* Should be 17, use 16 until we get an SRAM map. */
1424                 limit = 16;
1425         } else if (BNX_IS_57765_FAMILY(sc) ||
1426             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1427                 limit = 4;
1428         } else {
1429                 limit = 1;
1430         }
1431         /* Disable all receive return rings. */
1432         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1433         for (i = 0; i < limit; i++) {
1434                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1435                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1436                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1437                     BGE_RCB_FLAG_RING_DISABLED);
1438                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1439                     (i * (sizeof(uint64_t))), 0);
1440                 vrcb += sizeof(struct bge_rcb);
1441         }
1442
1443         /*
1444          * Set up receive return rings.
1445          */
1446         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1447         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
1448                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
1449
1450                 BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr);
1451                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi,
1452                     taddr.bge_addr_hi);
1453                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo,
1454                     taddr.bge_addr_lo);
1455                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1456                     BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0));
1457                 vrcb += sizeof(struct bge_rcb);
1458         }
1459
1460         /* Set random backoff seed for TX */
1461         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1462             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1463             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1464             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1465             BGE_TX_BACKOFF_SEED_MASK);
1466
1467         /* Set inter-packet gap */
1468         val = 0x2620;
1469         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1470             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1471                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1472                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1473         }
1474         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1475
1476         /*
1477          * Specify which ring to use for packets that don't match
1478          * any RX rules.
1479          */
1480         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1481
1482         /*
1483          * Configure number of RX lists. One interrupt distribution
1484          * list, sixteen active lists, one bad frames class.
1485          */
1486         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1487
1488         /* Inialize RX list placement stats mask. */
1489         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1490         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1491
1492         /* Disable host coalescing until we get it set up */
1493         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1494
1495         /* Poll to make sure it's shut down. */
1496         for (i = 0; i < BNX_TIMEOUT; i++) {
1497                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1498                         break;
1499                 DELAY(10);
1500         }
1501
1502         if (i == BNX_TIMEOUT) {
1503                 if_printf(&sc->arpcom.ac_if,
1504                           "host coalescing engine failed to idle\n");
1505                 return(ENXIO);
1506         }
1507
1508         /* Set up host coalescing defaults */
1509         sc->bnx_coal_chg = BNX_RX_COAL_TICKS_CHG |
1510             BNX_TX_COAL_TICKS_CHG |
1511             BNX_RX_COAL_BDS_CHG |
1512             BNX_TX_COAL_BDS_CHG |
1513             BNX_RX_COAL_BDS_INT_CHG |
1514             BNX_TX_COAL_BDS_INT_CHG;
1515         bnx_coal_change(sc);
1516
1517         /*
1518          * Set up addresses of status blocks
1519          */
1520         intr = &sc->bnx_intr_data[0];
1521         bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ);
1522         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1523             BGE_ADDR_HI(intr->bnx_status_block_paddr));
1524         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1525             BGE_ADDR_LO(intr->bnx_status_block_paddr));
1526         for (i = 1; i < sc->bnx_intr_cnt; ++i) {
1527                 intr = &sc->bnx_intr_data[i];
1528                 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ);
1529                 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_HI + ((i - 1) * 8),
1530                     BGE_ADDR_HI(intr->bnx_status_block_paddr));
1531                 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_LO + ((i - 1) * 8),
1532                     BGE_ADDR_LO(intr->bnx_status_block_paddr));
1533         }
1534
1535         /* Set up status block partail update size. */
1536         val = BGE_STATBLKSZ_32BYTE;
1537 #if 0
1538         /*
1539          * Does not seem to have visible effect in both
1540          * bulk data (1472B UDP datagram) and tiny data
1541          * (18B UDP datagram) TX tests.
1542          */
1543         val |= BGE_HCCMODE_CLRTICK_TX;
1544 #endif
1545         /* Turn on host coalescing state machine */
1546         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1547
1548         /* Turn on RX BD completion state machine and enable attentions */
1549         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1550             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1551
1552         /* Turn on RX list placement state machine */
1553         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1554
1555         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1556             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1557             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1558             BGE_MACMODE_FRMHDR_DMA_ENB;
1559
1560         if (sc->bnx_flags & BNX_FLAG_TBI)
1561                 val |= BGE_PORTMODE_TBI;
1562         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1563                 val |= BGE_PORTMODE_GMII;
1564         else
1565                 val |= BGE_PORTMODE_MII;
1566
1567         /* Turn on DMA, clear stats */
1568         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1569
1570         /* Set misc. local control, enable interrupts on attentions */
1571         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1572
1573 #ifdef notdef
1574         /* Assert GPIO pins for PHY reset */
1575         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1576             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1577         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1578             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1579 #endif
1580
1581         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSIX)
1582                 bnx_enable_msi(sc, TRUE);
1583
1584         /* Turn on write DMA state machine */
1585         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1586         /* Enable host coalescing bug fix. */
1587         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1588         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1589                 /* Request larger DMA burst size to get better performance. */
1590                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1591         }
1592         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1593         DELAY(40);
1594
1595         if (BNX_IS_57765_PLUS(sc)) {
1596                 uint32_t dmactl, dmactl_reg;
1597
1598                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1599                         dmactl_reg = BGE_RDMA_RSRVCTRL2;
1600                 else
1601                         dmactl_reg = BGE_RDMA_RSRVCTRL;
1602
1603                 dmactl = CSR_READ_4(sc, dmactl_reg);
1604                 /*
1605                  * Adjust tx margin to prevent TX data corruption and
1606                  * fix internal FIFO overflow.
1607                  */
1608                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1609                     sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1610                     sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1611                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1612                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1613                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1614                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1615                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1616                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1617                 }
1618                 /*
1619                  * Enable fix for read DMA FIFO overruns.
1620                  * The fix is to limit the number of RX BDs
1621                  * the hardware would fetch at a fime.
1622                  */
1623                 CSR_WRITE_4(sc, dmactl_reg,
1624                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1625         }
1626
1627         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1628                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1629                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1630                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1631                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1632         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1633             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1634                 uint32_t ctrl_reg;
1635
1636                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1637                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2;
1638                 else
1639                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL;
1640
1641                 /*
1642                  * Allow 4KB burst length reads for non-LSO frames.
1643                  * Enable 512B burst length reads for buffer descriptors.
1644                  */
1645                 CSR_WRITE_4(sc, ctrl_reg,
1646                     CSR_READ_4(sc, ctrl_reg) |
1647                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1648                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1649         }
1650
1651         /* Turn on read DMA state machine */
1652         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1653         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1654                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1655         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1656             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1657             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1658                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1659                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1660                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1661         }
1662         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1663             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1664                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1665                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1666                 /*
1667                  * Allow multiple outstanding read requests from
1668                  * non-LSO read DMA engine.
1669                  */
1670                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1671         }
1672         if (sc->bnx_asicrev == BGE_ASICREV_BCM57766)
1673                 val |= BGE_RDMAMODE_JMB_2K_MMRR;
1674         if (sc->bnx_flags & BNX_FLAG_TSO)
1675                 val |= BGE_RDMAMODE_TSO4_ENABLE;
1676         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1677         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1678         DELAY(40);
1679
1680         /* Turn on RX data completion state machine */
1681         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1682
1683         /* Turn on RX BD initiator state machine */
1684         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1685
1686         /* Turn on RX data and RX BD initiator state machine */
1687         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1688
1689         /* Turn on send BD completion state machine */
1690         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1691
1692         /* Turn on send data completion state machine */
1693         val = BGE_SDCMODE_ENABLE;
1694         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1695                 val |= BGE_SDCMODE_CDELAY;
1696         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1697
1698         /* Turn on send data initiator state machine */
1699         if (sc->bnx_flags & BNX_FLAG_TSO) {
1700                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
1701                     BGE_SDIMODE_HW_LSO_PRE_DMA);
1702         } else {
1703                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1704         }
1705
1706         /* Turn on send BD initiator state machine */
1707         val = BGE_SBDIMODE_ENABLE;
1708         if (sc->bnx_tx_ringcnt > 1)
1709                 val |= BGE_SBDIMODE_MULTI_TXR;
1710         CSR_WRITE_4(sc, BGE_SBDI_MODE, val);
1711
1712         /* Turn on send BD selector state machine */
1713         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1714
1715         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1716         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1717             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1718
1719         /* ack/clear link change events */
1720         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1721             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1722             BGE_MACSTAT_LINK_CHANGED);
1723         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1724
1725         /*
1726          * Enable attention when the link has changed state for
1727          * devices that use auto polling.
1728          */
1729         if (sc->bnx_flags & BNX_FLAG_TBI) {
1730                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1731         } else {
1732                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1733                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1734                         DELAY(80);
1735                 }
1736         }
1737
1738         /*
1739          * Clear any pending link state attention.
1740          * Otherwise some link state change events may be lost until attention
1741          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1742          * It's not necessary on newer BCM chips - perhaps enabling link
1743          * state change attentions implies clearing pending attention.
1744          */
1745         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1746             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1747             BGE_MACSTAT_LINK_CHANGED);
1748
1749         /* Enable link state change attentions. */
1750         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1751
1752         return(0);
1753 }
1754
1755 /*
1756  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1757  * against our list and return its name if we find a match. Note
1758  * that since the Broadcom controller contains VPD support, we
1759  * can get the device name string from the controller itself instead
1760  * of the compiled-in string. This is a little slow, but it guarantees
1761  * we'll always announce the right product name.
1762  */
1763 static int
1764 bnx_probe(device_t dev)
1765 {
1766         const struct bnx_type *t;
1767         uint16_t product, vendor;
1768
1769         if (!pci_is_pcie(dev))
1770                 return ENXIO;
1771
1772         product = pci_get_device(dev);
1773         vendor = pci_get_vendor(dev);
1774
1775         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1776                 if (vendor == t->bnx_vid && product == t->bnx_did)
1777                         break;
1778         }
1779         if (t->bnx_name == NULL)
1780                 return ENXIO;
1781
1782         device_set_desc(dev, t->bnx_name);
1783         return 0;
1784 }
1785
1786 static int
1787 bnx_attach(device_t dev)
1788 {
1789         struct ifnet *ifp;
1790         struct bnx_softc *sc;
1791         struct bnx_rx_std_ring *std;
1792         uint32_t hwcfg = 0;
1793         int error = 0, rid, capmask, i, std_cpuid, std_cpuid_def;
1794         uint8_t ether_addr[ETHER_ADDR_LEN];
1795         uint16_t product;
1796         uintptr_t mii_priv = 0;
1797 #if defined(BNX_TSO_DEBUG) || defined(BNX_RSS_DEBUG) || defined(BNX_TSS_DEBUG)
1798         char desc[32];
1799 #endif
1800 #ifdef IFPOLL_ENABLE
1801         int offset, offset_def;
1802 #endif
1803
1804         sc = device_get_softc(dev);
1805         sc->bnx_dev = dev;
1806         callout_init_mp(&sc->bnx_tick_timer);
1807         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1808         lwkt_serialize_init(&sc->bnx_main_serialize);
1809
1810         /* Always setup interrupt mailboxes */
1811         for (i = 0; i < BNX_INTR_MAX; ++i) {
1812                 callout_init_mp(&sc->bnx_intr_data[i].bnx_intr_timer);
1813                 sc->bnx_intr_data[i].bnx_sc = sc;
1814                 sc->bnx_intr_data[i].bnx_intr_mbx = BGE_MBX_IRQ0_LO + (i * 8);
1815                 sc->bnx_intr_data[i].bnx_intr_rid = -1;
1816                 sc->bnx_intr_data[i].bnx_intr_cpuid = -1;
1817         }
1818
1819         product = pci_get_device(dev);
1820
1821 #ifndef BURN_BRIDGES
1822         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1823                 uint32_t irq, mem;
1824
1825                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1826                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1827
1828                 device_printf(dev, "chip is in D%d power mode "
1829                     "-- setting to D0\n", pci_get_powerstate(dev));
1830
1831                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1832
1833                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1834                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1835         }
1836 #endif  /* !BURN_BRIDGE */
1837
1838         /*
1839          * Map control/status registers.
1840          */
1841         pci_enable_busmaster(dev);
1842
1843         rid = BGE_PCI_BAR0;
1844         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1845             RF_ACTIVE);
1846
1847         if (sc->bnx_res == NULL) {
1848                 device_printf(dev, "couldn't map memory\n");
1849                 return ENXIO;
1850         }
1851
1852         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1853         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1854
1855         /* Save various chip information */
1856         sc->bnx_chipid =
1857             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1858             BGE_PCIMISCCTL_ASICREV_SHIFT;
1859         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1860                 /* All chips having dedicated ASICREV register have CPMU */
1861                 sc->bnx_flags |= BNX_FLAG_CPMU;
1862
1863                 switch (product) {
1864                 case PCI_PRODUCT_BROADCOM_BCM5717:
1865                 case PCI_PRODUCT_BROADCOM_BCM5717C:
1866                 case PCI_PRODUCT_BROADCOM_BCM5718:
1867                 case PCI_PRODUCT_BROADCOM_BCM5719:
1868                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1869                 case PCI_PRODUCT_BROADCOM_BCM5725:
1870                 case PCI_PRODUCT_BROADCOM_BCM5727:
1871                 case PCI_PRODUCT_BROADCOM_BCM5762:
1872                         sc->bnx_chipid = pci_read_config(dev,
1873                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1874                         break;
1875
1876                 case PCI_PRODUCT_BROADCOM_BCM57761:
1877                 case PCI_PRODUCT_BROADCOM_BCM57762:
1878                 case PCI_PRODUCT_BROADCOM_BCM57765:
1879                 case PCI_PRODUCT_BROADCOM_BCM57766:
1880                 case PCI_PRODUCT_BROADCOM_BCM57781:
1881                 case PCI_PRODUCT_BROADCOM_BCM57782:
1882                 case PCI_PRODUCT_BROADCOM_BCM57785:
1883                 case PCI_PRODUCT_BROADCOM_BCM57786:
1884                 case PCI_PRODUCT_BROADCOM_BCM57791:
1885                 case PCI_PRODUCT_BROADCOM_BCM57795:
1886                         sc->bnx_chipid = pci_read_config(dev,
1887                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1888                         break;
1889
1890                 default:
1891                         sc->bnx_chipid = pci_read_config(dev,
1892                             BGE_PCI_PRODID_ASICREV, 4);
1893                         break;
1894                 }
1895         }
1896         if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0)
1897                 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0;
1898
1899         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1900         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1901
1902         switch (sc->bnx_asicrev) {
1903         case BGE_ASICREV_BCM5717:
1904         case BGE_ASICREV_BCM5719:
1905         case BGE_ASICREV_BCM5720:
1906                 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS;
1907                 break;
1908
1909         case BGE_ASICREV_BCM5762:
1910                 sc->bnx_flags |= BNX_FLAG_57765_PLUS;
1911                 break;
1912
1913         case BGE_ASICREV_BCM57765:
1914         case BGE_ASICREV_BCM57766:
1915                 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS;
1916                 break;
1917         }
1918
1919         sc->bnx_flags |= BNX_FLAG_TSO;
1920         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 &&
1921             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0)
1922                 sc->bnx_flags &= ~BNX_FLAG_TSO;
1923
1924         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1925             BNX_IS_57765_FAMILY(sc)) {
1926                 /*
1927                  * All BCM57785 and BCM5718 families chips have a bug that
1928                  * under certain situation interrupt will not be enabled
1929                  * even if status tag is written to interrupt mailbox.
1930                  *
1931                  * While BCM5719 and BCM5720 have a hardware workaround
1932                  * which could fix the above bug.
1933                  * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in
1934                  * bnx_chipinit().
1935                  *
1936                  * For the rest of the chips in these two families, we will
1937                  * have to poll the status block at high rate (10ms currently)
1938                  * to check whether the interrupt is hosed or not.
1939                  * See bnx_check_intr_*() for details.
1940                  */
1941                 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
1942         }
1943
1944         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
1945         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1946             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1947                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
1948         else
1949                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
1950         device_printf(dev, "CHIP ID 0x%08x; "
1951                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
1952                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
1953
1954         /*
1955          * Set various PHY quirk flags.
1956          */
1957
1958         capmask = MII_CAPMASK_DEFAULT;
1959         if (product == PCI_PRODUCT_BROADCOM_BCM57791 ||
1960             product == PCI_PRODUCT_BROADCOM_BCM57795) {
1961                 /* 10/100 only */
1962                 capmask &= ~BMSR_EXTSTAT;
1963         }
1964
1965         mii_priv |= BRGPHY_FLAG_WIRESPEED;
1966         if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0)
1967                 mii_priv |= BRGPHY_FLAG_5762_A0;
1968
1969         /* Initialize if_name earlier, so if_printf could be used */
1970         ifp = &sc->arpcom.ac_if;
1971         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1972
1973         /* Try to reset the chip. */
1974         bnx_reset(sc);
1975
1976         if (bnx_chipinit(sc)) {
1977                 device_printf(dev, "chip initialization failed\n");
1978                 error = ENXIO;
1979                 goto fail;
1980         }
1981
1982         /*
1983          * Get station address
1984          */
1985         error = bnx_get_eaddr(sc, ether_addr);
1986         if (error) {
1987                 device_printf(dev, "failed to read station address\n");
1988                 goto fail;
1989         }
1990
1991         /* Setup RX/TX and interrupt count */
1992         bnx_setup_ring_cnt(sc);
1993
1994         if ((sc->bnx_rx_retcnt == 1 && sc->bnx_tx_ringcnt == 1) ||
1995             (sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt > 1)) {
1996                 /*
1997                  * The RX ring and the corresponding TX ring processing
1998                  * should be on the same CPU, since they share the same
1999                  * status block.
2000                  */
2001                 sc->bnx_flags |= BNX_FLAG_RXTX_BUNDLE;
2002                 if (bootverbose)
2003                         device_printf(dev, "RX/TX bundle\n");
2004                 if (sc->bnx_tx_ringcnt > 1) {
2005                         /*
2006                          * Multiple TX rings do not share status block
2007                          * with link status, so link status will have
2008                          * to save its own status_tag.
2009                          */
2010                         sc->bnx_flags |= BNX_FLAG_STATUS_HASTAG;
2011                         if (bootverbose)
2012                                 device_printf(dev, "status needs tag\n");
2013                 }
2014         } else {
2015                 KKASSERT(sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt == 1);
2016                 if (bootverbose)
2017                         device_printf(dev, "RX/TX not bundled\n");
2018         }
2019
2020         error = bnx_dma_alloc(dev);
2021         if (error)
2022                 goto fail;
2023
2024 #ifdef IFPOLL_ENABLE
2025         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2026                 /*
2027                  * NPOLLING RX/TX CPU offset
2028                  */
2029                 if (sc->bnx_rx_retcnt == ncpus2) {
2030                         offset = 0;
2031                 } else {
2032                         offset_def =
2033                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
2034                         offset = device_getenv_int(dev, "npoll.offset",
2035                             offset_def);
2036                         if (offset >= ncpus2 ||
2037                             offset % sc->bnx_rx_retcnt != 0) {
2038                                 device_printf(dev, "invalid npoll.offset %d, "
2039                                     "use %d\n", offset, offset_def);
2040                                 offset = offset_def;
2041                         }
2042                 }
2043                 sc->bnx_npoll_rxoff = offset;
2044                 sc->bnx_npoll_txoff = offset;
2045         } else {
2046                 /*
2047                  * NPOLLING RX CPU offset
2048                  */
2049                 if (sc->bnx_rx_retcnt == ncpus2) {
2050                         offset = 0;
2051                 } else {
2052                         offset_def =
2053                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
2054                         offset = device_getenv_int(dev, "npoll.rxoff",
2055                             offset_def);
2056                         if (offset >= ncpus2 ||
2057                             offset % sc->bnx_rx_retcnt != 0) {
2058                                 device_printf(dev, "invalid npoll.rxoff %d, "
2059                                     "use %d\n", offset, offset_def);
2060                                 offset = offset_def;
2061                         }
2062                 }
2063                 sc->bnx_npoll_rxoff = offset;
2064
2065                 /*
2066                  * NPOLLING TX CPU offset
2067                  */
2068                 offset_def = device_get_unit(dev) % ncpus2;
2069                 offset = device_getenv_int(dev, "npoll.txoff", offset_def);
2070                 if (offset >= ncpus2) {
2071                         device_printf(dev, "invalid npoll.txoff %d, use %d\n",
2072                             offset, offset_def);
2073                         offset = offset_def;
2074                 }
2075                 sc->bnx_npoll_txoff = offset;
2076         }
2077 #endif  /* IFPOLL_ENABLE */
2078
2079         /*
2080          * Allocate interrupt
2081          */
2082         error = bnx_alloc_intr(sc);
2083         if (error)
2084                 goto fail;
2085
2086         /* Setup serializers */
2087         bnx_setup_serialize(sc);
2088
2089         /* Set default tuneable values. */
2090         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
2091         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
2092         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
2093         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
2094         sc->bnx_tx_coal_bds_poll = BNX_TX_COAL_BDS_POLL_DEF;
2095         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF;
2096         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF;
2097
2098         /* Set up ifnet structure */
2099         ifp->if_softc = sc;
2100         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2101         ifp->if_ioctl = bnx_ioctl;
2102         ifp->if_start = bnx_start;
2103 #ifdef IFPOLL_ENABLE
2104         ifp->if_npoll = bnx_npoll;
2105 #endif
2106         ifp->if_init = bnx_init;
2107         ifp->if_serialize = bnx_serialize;
2108         ifp->if_deserialize = bnx_deserialize;
2109         ifp->if_tryserialize = bnx_tryserialize;
2110 #ifdef INVARIANTS
2111         ifp->if_serialize_assert = bnx_serialize_assert;
2112 #endif
2113         ifp->if_mtu = ETHERMTU;
2114         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2115
2116         ifp->if_capabilities |= IFCAP_HWCSUM;
2117         ifp->if_hwassist = BNX_CSUM_FEATURES;
2118         if (sc->bnx_flags & BNX_FLAG_TSO) {
2119                 ifp->if_capabilities |= IFCAP_TSO;
2120                 ifp->if_hwassist |= CSUM_TSO;
2121         }
2122         if (BNX_RSS_ENABLED(sc))
2123                 ifp->if_capabilities |= IFCAP_RSS;
2124         ifp->if_capenable = ifp->if_capabilities;
2125
2126         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
2127         ifq_set_ready(&ifp->if_snd);
2128         ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt);
2129
2130         if (sc->bnx_tx_ringcnt > 1) {
2131                 ifp->if_mapsubq = ifq_mapsubq_mask;
2132                 ifq_set_subq_mask(&ifp->if_snd, sc->bnx_tx_ringcnt - 1);
2133         }
2134
2135         /*
2136          * Figure out what sort of media we have by checking the
2137          * hardware config word in the first 32k of NIC internal memory,
2138          * or fall back to examining the EEPROM if necessary.
2139          * Note: on some BCM5700 cards, this value appears to be unset.
2140          * If that's the case, we have to rely on identifying the NIC
2141          * by its PCI subsystem ID, as we do below for the SysKonnect
2142          * SK-9D41.
2143          */
2144         if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) {
2145                 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
2146         } else {
2147                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
2148                                     sizeof(hwcfg))) {
2149                         device_printf(dev, "failed to read EEPROM\n");
2150                         error = ENXIO;
2151                         goto fail;
2152                 }
2153                 hwcfg = ntohl(hwcfg);
2154         }
2155
2156         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
2157         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
2158             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
2159                 sc->bnx_flags |= BNX_FLAG_TBI;
2160
2161         /* Setup MI MODE */
2162         if (sc->bnx_flags & BNX_FLAG_CPMU)
2163                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
2164         else
2165                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
2166
2167         /* Setup link status update stuffs */
2168         if (sc->bnx_flags & BNX_FLAG_TBI) {
2169                 sc->bnx_link_upd = bnx_tbi_link_upd;
2170                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2171         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
2172                 sc->bnx_link_upd = bnx_autopoll_link_upd;
2173                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2174         } else {
2175                 sc->bnx_link_upd = bnx_copper_link_upd;
2176                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2177         }
2178
2179         /* Set default PHY address */
2180         sc->bnx_phyno = 1;
2181
2182         /*
2183          * PHY address mapping for various devices.
2184          *
2185          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
2186          * ---------+-------+-------+-------+-------+
2187          * BCM57XX  |   1   |   X   |   X   |   X   |
2188          * BCM5704  |   1   |   X   |   1   |   X   |
2189          * BCM5717  |   1   |   8   |   2   |   9   |
2190          * BCM5719  |   1   |   8   |   2   |   9   |
2191          * BCM5720  |   1   |   8   |   2   |   9   |
2192          *
2193          * Other addresses may respond but they are not
2194          * IEEE compliant PHYs and should be ignored.
2195          */
2196         if (BNX_IS_5717_PLUS(sc)) {
2197                 int f;
2198
2199                 f = pci_get_function(dev);
2200                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
2201                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
2202                             BGE_SGDIGSTS_IS_SERDES)
2203                                 sc->bnx_phyno = f + 8;
2204                         else
2205                                 sc->bnx_phyno = f + 1;
2206                 } else {
2207                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
2208                             BGE_CPMU_PHY_STRAP_IS_SERDES)
2209                                 sc->bnx_phyno = f + 8;
2210                         else
2211                                 sc->bnx_phyno = f + 1;
2212                 }
2213         }
2214
2215         if (sc->bnx_flags & BNX_FLAG_TBI) {
2216                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
2217                     bnx_ifmedia_upd, bnx_ifmedia_sts);
2218                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2219                 ifmedia_add(&sc->bnx_ifmedia,
2220                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2221                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2222                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2223                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2224         } else {
2225                 struct mii_probe_args mii_args;
2226
2227                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2228                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2229                 mii_args.mii_capmask = capmask;
2230                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2231                 mii_args.mii_priv = mii_priv;
2232
2233                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2234                 if (error) {
2235                         device_printf(dev, "MII without any PHY!\n");
2236                         goto fail;
2237                 }
2238         }
2239
2240         /*
2241          * Create sysctl nodes.
2242          */
2243         sysctl_ctx_init(&sc->bnx_sysctl_ctx);
2244         sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx,
2245                                               SYSCTL_STATIC_CHILDREN(_hw),
2246                                               OID_AUTO,
2247                                               device_get_nameunit(dev),
2248                                               CTLFLAG_RD, 0, "");
2249         if (sc->bnx_sysctl_tree == NULL) {
2250                 device_printf(dev, "can't add sysctl node\n");
2251                 error = ENXIO;
2252                 goto fail;
2253         }
2254
2255         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2256             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2257             "rx_rings", CTLFLAG_RD, &sc->bnx_rx_retcnt, 0, "# of RX rings");
2258         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2259             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2260             "tx_rings", CTLFLAG_RD, &sc->bnx_tx_ringcnt, 0, "# of TX rings");
2261
2262         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2263                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2264                         OID_AUTO, "rx_coal_ticks",
2265                         CTLTYPE_INT | CTLFLAG_RW,
2266                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2267                         "Receive coalescing ticks (usec).");
2268         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2269                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2270                         OID_AUTO, "tx_coal_ticks",
2271                         CTLTYPE_INT | CTLFLAG_RW,
2272                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2273                         "Transmit coalescing ticks (usec).");
2274         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2275                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2276                         OID_AUTO, "rx_coal_bds",
2277                         CTLTYPE_INT | CTLFLAG_RW,
2278                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2279                         "Receive max coalesced BD count.");
2280         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2281                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2282                         OID_AUTO, "tx_coal_bds",
2283                         CTLTYPE_INT | CTLFLAG_RW,
2284                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2285                         "Transmit max coalesced BD count.");
2286         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2287                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2288                         OID_AUTO, "tx_coal_bds_poll",
2289                         CTLTYPE_INT | CTLFLAG_RW,
2290                         sc, 0, bnx_sysctl_tx_coal_bds_poll, "I",
2291                         "Transmit max coalesced BD count in polling.");
2292         /*
2293          * A common design characteristic for many Broadcom
2294          * client controllers is that they only support a
2295          * single outstanding DMA read operation on the PCIe
2296          * bus. This means that it will take twice as long to
2297          * fetch a TX frame that is split into header and
2298          * payload buffers as it does to fetch a single,
2299          * contiguous TX frame (2 reads vs. 1 read). For these
2300          * controllers, coalescing buffers to reduce the number
2301          * of memory reads is effective way to get maximum
2302          * performance(about 940Mbps).  Without collapsing TX
2303          * buffers the maximum TCP bulk transfer performance
2304          * is about 850Mbps. However forcing coalescing mbufs
2305          * consumes a lot of CPU cycles, so leave it off by
2306          * default.
2307          */
2308         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2309             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2310             "force_defrag", CTLTYPE_INT | CTLFLAG_RW,
2311             sc, 0, bnx_sysctl_force_defrag, "I",
2312             "Force defragment on TX path");
2313
2314         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2315             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2316             "tx_wreg", CTLTYPE_INT | CTLFLAG_RW,
2317             sc, 0, bnx_sysctl_tx_wreg, "I",
2318             "# of segments before writing to hardware register");
2319
2320         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2321             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2322             "std_refill", CTLTYPE_INT | CTLFLAG_RW,
2323             sc, 0, bnx_sysctl_std_refill, "I",
2324             "# of packets received before scheduling standard refilling");
2325
2326         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2327             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2328             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2329             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2330             "Receive max coalesced BD count during interrupt.");
2331         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2332             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2333             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2334             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2335             "Transmit max coalesced BD count during interrupt.");
2336
2337 #ifdef IFPOLL_ENABLE
2338         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2339                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2340                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2341                     "npoll_offset", CTLTYPE_INT | CTLFLAG_RW,
2342                     sc, 0, bnx_sysctl_npoll_offset, "I",
2343                     "NPOLLING cpu offset");
2344         } else {
2345                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2346                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2347                     "npoll_rxoff", CTLTYPE_INT | CTLFLAG_RW,
2348                     sc, 0, bnx_sysctl_npoll_rxoff, "I",
2349                     "NPOLLING RX cpu offset");
2350                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2351                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2352                     "npoll_txoff", CTLTYPE_INT | CTLFLAG_RW,
2353                     sc, 0, bnx_sysctl_npoll_txoff, "I",
2354                     "NPOLLING TX cpu offset");
2355         }
2356 #endif
2357
2358 #ifdef BNX_RSS_DEBUG
2359         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2360             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2361             "std_refill_mask", CTLFLAG_RD,
2362             &sc->bnx_rx_std_ring.bnx_rx_std_refill, 0, "");
2363         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2364             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2365             "rss_debug", CTLFLAG_RW, &sc->bnx_rss_debug, 0, "");
2366         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
2367                 ksnprintf(desc, sizeof(desc), "rx_pkt%d", i);
2368                 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2369                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2370                     desc, CTLFLAG_RW, &sc->bnx_rx_ret_ring[i].bnx_rx_pkt, "");
2371         }
2372 #endif
2373 #ifdef BNX_TSS_DEBUG
2374         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2375                 ksnprintf(desc, sizeof(desc), "tx_pkt%d", i);
2376                 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2377                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2378                     desc, CTLFLAG_RW, &sc->bnx_tx_ring[i].bnx_tx_pkt, "");
2379         }
2380 #endif
2381
2382         SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2383             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2384             "norxbds", CTLFLAG_RW, &sc->bnx_norxbds, "");
2385
2386         SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2387             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2388             "errors", CTLFLAG_RW, &sc->bnx_errors, "");
2389
2390 #ifdef BNX_TSO_DEBUG
2391         for (i = 0; i < BNX_TSO_NSTATS; ++i) {
2392                 ksnprintf(desc, sizeof(desc), "tso%d", i + 1);
2393                 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2394                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2395                     desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], "");
2396         }
2397 #endif
2398
2399         /*
2400          * Call MI attach routine.
2401          */
2402         ether_ifattach(ifp, ether_addr, NULL);
2403
2404         /* Setup TX rings and subqueues */
2405         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2406                 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
2407                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
2408
2409                 ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid);
2410                 ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize);
2411                 ifsq_set_priv(ifsq, txr);
2412                 txr->bnx_ifsq = ifsq;
2413
2414                 ifsq_watchdog_init(&txr->bnx_tx_watchdog, ifsq, bnx_watchdog);
2415
2416                 if (bootverbose) {
2417                         device_printf(dev, "txr %d -> cpu%d\n", i,
2418                             txr->bnx_tx_cpuid);
2419                 }
2420         }
2421
2422         error = bnx_setup_intr(sc);
2423         if (error) {
2424                 ether_ifdetach(ifp);
2425                 goto fail;
2426         }
2427         bnx_set_tick_cpuid(sc, FALSE);
2428
2429         /*
2430          * Create RX standard ring refilling thread
2431          */
2432         std_cpuid_def = device_get_unit(dev) % ncpus;
2433         std_cpuid = device_getenv_int(dev, "std.cpuid", std_cpuid_def);
2434         if (std_cpuid < 0 || std_cpuid >= ncpus) {
2435                 device_printf(dev, "invalid std.cpuid %d, use %d\n",
2436                     std_cpuid, std_cpuid_def);
2437                 std_cpuid = std_cpuid_def;
2438         }
2439
2440         std = &sc->bnx_rx_std_ring;
2441         lwkt_create(bnx_rx_std_refill_ithread, std, NULL,
2442             &std->bnx_rx_std_ithread, TDF_NOSTART | TDF_INTTHREAD, std_cpuid,
2443             "%s std", device_get_nameunit(dev));
2444         lwkt_setpri(&std->bnx_rx_std_ithread, TDPRI_INT_MED);
2445         std->bnx_rx_std_ithread.td_preemptable = lwkt_preempt;
2446         sc->bnx_flags |= BNX_FLAG_STD_THREAD;
2447
2448         return(0);
2449 fail:
2450         bnx_detach(dev);
2451         return(error);
2452 }
2453
2454 static int
2455 bnx_detach(device_t dev)
2456 {
2457         struct bnx_softc *sc = device_get_softc(dev);
2458
2459         if (device_is_attached(dev)) {
2460                 struct ifnet *ifp = &sc->arpcom.ac_if;
2461
2462                 ifnet_serialize_all(ifp);
2463                 bnx_stop(sc);
2464                 bnx_reset(sc);
2465                 bnx_teardown_intr(sc, sc->bnx_intr_cnt);
2466                 ifnet_deserialize_all(ifp);
2467
2468                 ether_ifdetach(ifp);
2469         }
2470
2471         if (sc->bnx_flags & BNX_FLAG_STD_THREAD) {
2472                 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
2473
2474                 tsleep_interlock(std, 0);
2475
2476                 if (std->bnx_rx_std_ithread.td_gd == mycpu) {
2477                         bnx_rx_std_refill_stop(std);
2478                 } else {
2479                         lwkt_send_ipiq(std->bnx_rx_std_ithread.td_gd,
2480                             bnx_rx_std_refill_stop, std);
2481                 }
2482
2483                 tsleep(std, PINTERLOCKED, "bnx_detach", 0);
2484                 if (bootverbose)
2485                         device_printf(dev, "RX std ithread exited\n");
2486
2487                 lwkt_synchronize_ipiqs("bnx_detach_ipiq");
2488         }
2489
2490         if (sc->bnx_flags & BNX_FLAG_TBI)
2491                 ifmedia_removeall(&sc->bnx_ifmedia);
2492         if (sc->bnx_miibus)
2493                 device_delete_child(dev, sc->bnx_miibus);
2494         bus_generic_detach(dev);
2495
2496         bnx_free_intr(sc);
2497
2498         if (sc->bnx_msix_mem_res != NULL) {
2499                 bus_release_resource(dev, SYS_RES_MEMORY, sc->bnx_msix_mem_rid,
2500                     sc->bnx_msix_mem_res);
2501         }
2502         if (sc->bnx_res != NULL) {
2503                 bus_release_resource(dev, SYS_RES_MEMORY,
2504                     BGE_PCI_BAR0, sc->bnx_res);
2505         }
2506
2507         if (sc->bnx_sysctl_tree != NULL)
2508                 sysctl_ctx_free(&sc->bnx_sysctl_ctx);
2509
2510         bnx_dma_free(sc);
2511
2512         if (sc->bnx_serialize != NULL)
2513                 kfree(sc->bnx_serialize, M_DEVBUF);
2514
2515         return 0;
2516 }
2517
2518 static void
2519 bnx_reset(struct bnx_softc *sc)
2520 {
2521         device_t dev;
2522         uint32_t cachesize, command, pcistate, reset;
2523         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2524         int i, val = 0;
2525         uint16_t devctl;
2526
2527         dev = sc->bnx_dev;
2528
2529         write_op = bnx_writemem_direct;
2530
2531         /* Save some important PCI state. */
2532         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2533         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2534         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2535
2536         pci_write_config(dev, BGE_PCI_MISC_CTL,
2537             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2538             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2539             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2540
2541         /* Disable fastboot on controllers that support it. */
2542         if (bootverbose)
2543                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2544         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2545
2546         /*
2547          * Write the magic number to SRAM at offset 0xB50.
2548          * When firmware finishes its initialization it will
2549          * write ~BGE_MAGIC_NUMBER to the same location.
2550          */
2551         bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2552
2553         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2554
2555         /* XXX: Broadcom Linux driver. */
2556         /* Force PCI-E 1.0a mode */
2557         if (!BNX_IS_57765_PLUS(sc) &&
2558             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2559             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2560              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2561                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2562                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2563         }
2564         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2565                 /* Prevent PCIE link training during global reset */
2566                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2567                 reset |= (1<<29);
2568         }
2569
2570         /*
2571          * Set GPHY Power Down Override to leave GPHY
2572          * powered up in D0 uninitialized.
2573          */
2574         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2575                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2576
2577         /* Issue global reset */
2578         write_op(sc, BGE_MISC_CFG, reset);
2579
2580         DELAY(1000);
2581
2582         /* XXX: Broadcom Linux driver. */
2583         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2584                 uint32_t v;
2585
2586                 DELAY(500000); /* wait for link training to complete */
2587                 v = pci_read_config(dev, 0xc4, 4);
2588                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2589         }
2590
2591         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2592
2593         /* Disable no snoop and disable relaxed ordering. */
2594         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2595
2596         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2597         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2598                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2599                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2600         }
2601
2602         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2603             devctl, 2);
2604
2605         /* Clear error status. */
2606         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2607             PCIEM_DEVSTS_CORR_ERR |
2608             PCIEM_DEVSTS_NFATAL_ERR |
2609             PCIEM_DEVSTS_FATAL_ERR |
2610             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2611
2612         /* Reset some of the PCI state that got zapped by reset */
2613         pci_write_config(dev, BGE_PCI_MISC_CTL,
2614             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2615             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2616             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2617         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2618         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2619         write_op(sc, BGE_MISC_CFG, (65 << 1));
2620
2621         /* Enable memory arbiter */
2622         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2623
2624         /*
2625          * Poll until we see the 1's complement of the magic number.
2626          * This indicates that the firmware initialization is complete.
2627          */
2628         for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2629                 val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2630                 if (val == ~BGE_MAGIC_NUMBER)
2631                         break;
2632                 DELAY(10);
2633         }
2634         if (i == BNX_FIRMWARE_TIMEOUT) {
2635                 if_printf(&sc->arpcom.ac_if, "firmware handshake "
2636                           "timed out, found 0x%08x\n", val);
2637         }
2638
2639         /* BCM57765 A0 needs additional time before accessing. */
2640         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2641                 DELAY(10 * 1000);
2642
2643         /*
2644          * XXX Wait for the value of the PCISTATE register to
2645          * return to its original pre-reset state. This is a
2646          * fairly good indicator of reset completion. If we don't
2647          * wait for the reset to fully complete, trying to read
2648          * from the device's non-PCI registers may yield garbage
2649          * results.
2650          */
2651         for (i = 0; i < BNX_TIMEOUT; i++) {
2652                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2653                         break;
2654                 DELAY(10);
2655         }
2656
2657         /* Fix up byte swapping */
2658         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2659
2660         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2661
2662         /*
2663          * The 5704 in TBI mode apparently needs some special
2664          * adjustment to insure the SERDES drive level is set
2665          * to 1.2V.
2666          */
2667         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2668             (sc->bnx_flags & BNX_FLAG_TBI)) {
2669                 uint32_t serdescfg;
2670
2671                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2672                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2673                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2674         }
2675
2676         CSR_WRITE_4(sc, BGE_MI_MODE,
2677             sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
2678         DELAY(80);
2679
2680         /* XXX: Broadcom Linux driver. */
2681         if (!BNX_IS_57765_PLUS(sc)) {
2682                 uint32_t v;
2683
2684                 /* Enable Data FIFO protection. */
2685                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
2686                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
2687         }
2688
2689         DELAY(10000);
2690
2691         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2692                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2693                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2694         }
2695 }
2696
2697 /*
2698  * Frame reception handling. This is called if there's a frame
2699  * on the receive return list.
2700  *
2701  * Note: we have to be able to handle two possibilities here:
2702  * 1) the frame is from the jumbo recieve ring
2703  * 2) the frame is from the standard receive ring
2704  */
2705
2706 static void
2707 bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count)
2708 {
2709         struct bnx_softc *sc = ret->bnx_sc;
2710         struct bnx_rx_std_ring *std = ret->bnx_std;
2711         struct ifnet *ifp = &sc->arpcom.ac_if;
2712
2713         while (ret->bnx_rx_saved_considx != rx_prod && count != 0) {
2714                 struct pktinfo pi0, *pi = NULL;
2715                 struct bge_rx_bd *cur_rx;
2716                 struct bnx_rx_buf *rb;
2717                 uint32_t rxidx;
2718                 struct mbuf *m = NULL;
2719                 uint16_t vlan_tag = 0;
2720                 int have_tag = 0;
2721
2722                 --count;
2723
2724                 cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx];
2725
2726                 rxidx = cur_rx->bge_idx;
2727                 KKASSERT(rxidx < BGE_STD_RX_RING_CNT);
2728
2729                 BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
2730 #ifdef BNX_RSS_DEBUG
2731                 ret->bnx_rx_pkt++;
2732 #endif
2733
2734                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2735                         have_tag = 1;
2736                         vlan_tag = cur_rx->bge_vlan_tag;
2737                 }
2738
2739                 if (ret->bnx_rx_cnt >= ret->bnx_rx_cntmax)
2740                         bnx_rx_std_refill_sched(ret, std);
2741                 ret->bnx_rx_cnt++;
2742
2743                 rb = &std->bnx_rx_std_buf[rxidx];
2744                 m = rb->bnx_rx_mbuf;
2745                 if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2746                         IFNET_STAT_INC(ifp, ierrors, 1);
2747                         cpu_sfence();
2748                         rb->bnx_rx_refilled = 1;
2749                         continue;
2750                 }
2751                 if (bnx_newbuf_std(ret, rxidx, 0)) {
2752                         IFNET_STAT_INC(ifp, ierrors, 1);
2753                         continue;
2754                 }
2755
2756                 IFNET_STAT_INC(ifp, ipackets, 1);
2757                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2758                 m->m_pkthdr.rcvif = ifp;
2759
2760                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2761                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2762                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2763                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2764                                 if ((cur_rx->bge_error_flag &
2765                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2766                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2767                         }
2768                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2769                                 m->m_pkthdr.csum_data =
2770                                     cur_rx->bge_tcp_udp_csum;
2771                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2772                                     CSUM_PSEUDO_HDR;
2773                         }
2774                 }
2775                 if (ifp->if_capenable & IFCAP_RSS) {
2776                         pi = bnx_rss_info(&pi0, cur_rx);
2777                         if (pi != NULL &&
2778                             (cur_rx->bge_flags & BGE_RXBDFLAG_RSS_HASH)) {
2779                                 m->m_flags |= M_HASH;
2780                                 m->m_pkthdr.hash =
2781                                     toeplitz_hash(cur_rx->bge_hash);
2782                         }
2783                 }
2784
2785                 /*
2786                  * If we received a packet with a vlan tag, pass it
2787                  * to vlan_input() instead of ether_input().
2788                  */
2789                 if (have_tag) {
2790                         m->m_flags |= M_VLANTAG;
2791                         m->m_pkthdr.ether_vlantag = vlan_tag;
2792                 }
2793                 ether_input_pkt(ifp, m, pi);
2794         }
2795         bnx_writembx(sc, ret->bnx_rx_mbx, ret->bnx_rx_saved_considx);
2796
2797         if (ret->bnx_rx_cnt > 0)
2798                 bnx_rx_std_refill_sched(ret, std);
2799 }
2800
2801 static void
2802 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
2803 {
2804         struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if;
2805
2806         /*
2807          * Go through our tx ring and free mbufs for those
2808          * frames that have been sent.
2809          */
2810         while (txr->bnx_tx_saved_considx != tx_cons) {
2811                 struct bnx_tx_buf *buf;
2812                 uint32_t idx = 0;
2813
2814                 idx = txr->bnx_tx_saved_considx;
2815                 buf = &txr->bnx_tx_buf[idx];
2816                 if (buf->bnx_tx_mbuf != NULL) {
2817                         IFNET_STAT_INC(ifp, opackets, 1);
2818 #ifdef BNX_TSS_DEBUG
2819                         txr->bnx_tx_pkt++;
2820 #endif
2821                         bus_dmamap_unload(txr->bnx_tx_mtag,
2822                             buf->bnx_tx_dmamap);
2823                         m_freem(buf->bnx_tx_mbuf);
2824                         buf->bnx_tx_mbuf = NULL;
2825                 }
2826                 txr->bnx_tx_cnt--;
2827                 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2828         }
2829
2830         if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >=
2831             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2832                 ifsq_clr_oactive(txr->bnx_ifsq);
2833
2834         if (txr->bnx_tx_cnt == 0)
2835                 txr->bnx_tx_watchdog.wd_timer = 0;
2836
2837         if (!ifsq_is_empty(txr->bnx_ifsq))
2838                 ifsq_devstart(txr->bnx_ifsq);
2839 }
2840
2841 static void
2842 bnx_handle_status(struct bnx_softc *sc)
2843 {
2844         uint32_t status;
2845
2846         status = *sc->bnx_hw_status;
2847
2848         if (status & BGE_STATFLAG_ERROR) {
2849                 uint32_t val;
2850                 int reset = 0;
2851
2852                 sc->bnx_errors++;
2853
2854                 val = CSR_READ_4(sc, BGE_FLOW_ATTN);
2855                 if (val & ~BGE_FLOWATTN_MB_LOWAT) {
2856                         if_printf(&sc->arpcom.ac_if,
2857                             "flow attn 0x%08x\n", val);
2858                         reset = 1;
2859                 }
2860
2861                 val = CSR_READ_4(sc, BGE_MSI_STATUS);
2862                 if (val & ~BGE_MSISTAT_MSI_PCI_REQ) {
2863                         if_printf(&sc->arpcom.ac_if,
2864                             "msi status 0x%08x\n", val);
2865                         reset = 1;
2866                 }
2867
2868                 val = CSR_READ_4(sc, BGE_RDMA_STATUS);
2869                 if (val) {
2870                         if_printf(&sc->arpcom.ac_if,
2871                             "rmda status 0x%08x\n", val);
2872                         reset = 1;
2873                 }
2874
2875                 val = CSR_READ_4(sc, BGE_WDMA_STATUS);
2876                 if (val) {
2877                         if_printf(&sc->arpcom.ac_if,
2878                             "wdma status 0x%08x\n", val);
2879                         reset = 1;
2880                 }
2881
2882                 if (reset) {
2883                         bnx_serialize_skipmain(sc);
2884                         bnx_init(sc);
2885                         bnx_deserialize_skipmain(sc);
2886                 }
2887         }
2888
2889         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
2890                 bnx_link_poll(sc);
2891 }
2892
2893 #ifdef IFPOLL_ENABLE
2894
2895 static void
2896 bnx_npoll_rx(struct ifnet *ifp __unused, void *xret, int cycle)
2897 {
2898         struct bnx_rx_ret_ring *ret = xret;
2899         uint16_t rx_prod;
2900
2901         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
2902
2903         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
2904         cpu_lfence();
2905
2906         rx_prod = *ret->bnx_rx_considx;
2907         if (ret->bnx_rx_saved_considx != rx_prod)
2908                 bnx_rxeof(ret, rx_prod, cycle);
2909 }
2910
2911 static void
2912 bnx_npoll_tx_notag(struct ifnet *ifp __unused, void *xtxr, int cycle __unused)
2913 {
2914         struct bnx_tx_ring *txr = xtxr;
2915         uint16_t tx_cons;
2916
2917         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
2918
2919         tx_cons = *txr->bnx_tx_considx;
2920         if (txr->bnx_tx_saved_considx != tx_cons)
2921                 bnx_txeof(txr, tx_cons);
2922 }
2923
2924 static void
2925 bnx_npoll_tx(struct ifnet *ifp, void *xtxr, int cycle)
2926 {
2927         struct bnx_tx_ring *txr = xtxr;
2928
2929         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
2930
2931         txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag;
2932         cpu_lfence();
2933         bnx_npoll_tx_notag(ifp, txr, cycle);
2934 }
2935
2936 static void
2937 bnx_npoll_status_notag(struct ifnet *ifp)
2938 {
2939         struct bnx_softc *sc = ifp->if_softc;
2940
2941         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
2942
2943         bnx_handle_status(sc);
2944 }
2945
2946 static void
2947 bnx_npoll_status(struct ifnet *ifp)
2948 {
2949         struct bnx_softc *sc = ifp->if_softc;
2950
2951         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
2952
2953         sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag;
2954         cpu_lfence();
2955         bnx_npoll_status_notag(ifp);
2956 }
2957
2958 static void
2959 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
2960 {
2961         struct bnx_softc *sc = ifp->if_softc;
2962         int i;
2963
2964         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2965
2966         if (info != NULL) {
2967                 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG)
2968                         info->ifpi_status.status_func = bnx_npoll_status;
2969                 else
2970                         info->ifpi_status.status_func = bnx_npoll_status_notag;
2971                 info->ifpi_status.serializer = &sc->bnx_main_serialize;
2972
2973                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2974                         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
2975                         int idx = i + sc->bnx_npoll_txoff;
2976
2977                         KKASSERT(idx < ncpus2);
2978                         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2979                                 info->ifpi_tx[idx].poll_func =
2980                                     bnx_npoll_tx_notag;
2981                         } else {
2982                                 info->ifpi_tx[idx].poll_func = bnx_npoll_tx;
2983                         }
2984                         info->ifpi_tx[idx].arg = txr;
2985                         info->ifpi_tx[idx].serializer = &txr->bnx_tx_serialize;
2986                         ifsq_set_cpuid(txr->bnx_ifsq, idx);
2987                 }
2988
2989                 for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
2990                         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
2991                         int idx = i + sc->bnx_npoll_rxoff;
2992
2993                         KKASSERT(idx < ncpus2);
2994                         info->ifpi_rx[idx].poll_func = bnx_npoll_rx;
2995                         info->ifpi_rx[idx].arg = ret;
2996                         info->ifpi_rx[idx].serializer =
2997                             &ret->bnx_rx_ret_serialize;
2998                 }
2999
3000                 if (ifp->if_flags & IFF_RUNNING) {
3001                         bnx_disable_intr(sc);
3002                         bnx_set_tick_cpuid(sc, TRUE);
3003
3004                         sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG;
3005                         bnx_coal_change(sc);
3006                 }
3007         } else {
3008                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3009                         ifsq_set_cpuid(sc->bnx_tx_ring[i].bnx_ifsq,
3010                             sc->bnx_tx_ring[i].bnx_tx_cpuid);
3011                 }
3012                 if (ifp->if_flags & IFF_RUNNING) {
3013                         sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG;
3014                         bnx_coal_change(sc);
3015
3016                         bnx_enable_intr(sc);
3017                         bnx_set_tick_cpuid(sc, FALSE);
3018                 }
3019         }
3020 }
3021
3022 #endif  /* IFPOLL_ENABLE */
3023
3024 static void
3025 bnx_intr_legacy(void *xsc)
3026 {
3027         struct bnx_softc *sc = xsc;
3028         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
3029
3030         if (ret->bnx_saved_status_tag == *ret->bnx_hw_status_tag) {
3031                 uint32_t val;
3032
3033                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
3034                 if (val & BGE_PCISTAT_INTR_NOTACT)
3035                         return;
3036         }
3037
3038         /*
3039          * NOTE:
3040          * Interrupt will have to be disabled if tagged status
3041          * is used, else interrupt will always be asserted on
3042          * certain chips (at least on BCM5750 AX/BX).
3043          */
3044         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3045
3046         bnx_intr(sc);
3047 }
3048
3049 static void
3050 bnx_msi(void *xsc)
3051 {
3052         bnx_intr(xsc);
3053 }
3054
3055 static void
3056 bnx_intr(struct bnx_softc *sc)
3057 {
3058         struct ifnet *ifp = &sc->arpcom.ac_if;
3059         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
3060
3061         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3062
3063         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3064         /*
3065          * Use a load fence to ensure that status_tag is saved
3066          * before rx_prod, tx_cons and status.
3067          */
3068         cpu_lfence();
3069
3070         bnx_handle_status(sc);
3071
3072         if (ifp->if_flags & IFF_RUNNING) {
3073                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
3074                 uint16_t rx_prod, tx_cons;
3075
3076                 lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
3077                 rx_prod = *ret->bnx_rx_considx;
3078                 if (ret->bnx_rx_saved_considx != rx_prod)
3079                         bnx_rxeof(ret, rx_prod, -1);
3080                 lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
3081
3082                 lwkt_serialize_enter(&txr->bnx_tx_serialize);
3083                 tx_cons = *txr->bnx_tx_considx;
3084                 if (txr->bnx_tx_saved_considx != tx_cons)
3085                         bnx_txeof(txr, tx_cons);
3086                 lwkt_serialize_exit(&txr->bnx_tx_serialize);
3087         }
3088
3089         bnx_writembx(sc, BGE_MBX_IRQ0_LO, ret->bnx_saved_status_tag << 24);
3090 }
3091
3092 static void
3093 bnx_msix_tx_status(void *xtxr)
3094 {
3095         struct bnx_tx_ring *txr = xtxr;
3096         struct bnx_softc *sc = txr->bnx_sc;
3097         struct ifnet *ifp = &sc->arpcom.ac_if;
3098
3099         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3100
3101         txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag;
3102         /*
3103          * Use a load fence to ensure that status_tag is saved
3104          * before tx_cons and status.
3105          */
3106         cpu_lfence();
3107
3108         bnx_handle_status(sc);
3109
3110         if (ifp->if_flags & IFF_RUNNING) {
3111                 uint16_t tx_cons;
3112
3113                 lwkt_serialize_enter(&txr->bnx_tx_serialize);
3114                 tx_cons = *txr->bnx_tx_considx;
3115                 if (txr->bnx_tx_saved_considx != tx_cons)
3116                         bnx_txeof(txr, tx_cons);
3117                 lwkt_serialize_exit(&txr->bnx_tx_serialize);
3118         }
3119
3120         bnx_writembx(sc, BGE_MBX_IRQ0_LO, txr->bnx_saved_status_tag << 24);
3121 }
3122
3123 static void
3124 bnx_msix_rx(void *xret)
3125 {
3126         struct bnx_rx_ret_ring *ret = xret;
3127         uint16_t rx_prod;
3128
3129         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
3130
3131         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3132         /*
3133          * Use a load fence to ensure that status_tag is saved
3134          * before rx_prod.
3135          */
3136         cpu_lfence();
3137
3138         rx_prod = *ret->bnx_rx_considx;
3139         if (ret->bnx_rx_saved_considx != rx_prod)
3140                 bnx_rxeof(ret, rx_prod, -1);
3141
3142         bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx,
3143             ret->bnx_saved_status_tag << 24);
3144 }
3145
3146 static void
3147 bnx_msix_rxtx(void *xret)
3148 {
3149         struct bnx_rx_ret_ring *ret = xret;
3150         struct bnx_tx_ring *txr = ret->bnx_txr;
3151         uint16_t rx_prod, tx_cons;
3152
3153         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
3154
3155         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3156         /*
3157          * Use a load fence to ensure that status_tag is saved
3158          * before rx_prod and tx_cons.
3159          */
3160         cpu_lfence();
3161
3162         rx_prod = *ret->bnx_rx_considx;
3163         if (ret->bnx_rx_saved_considx != rx_prod)
3164                 bnx_rxeof(ret, rx_prod, -1);
3165
3166         lwkt_serialize_enter(&txr->bnx_tx_serialize);
3167         tx_cons = *txr->bnx_tx_considx;
3168         if (txr->bnx_tx_saved_considx != tx_cons)
3169                 bnx_txeof(txr, tx_cons);
3170         lwkt_serialize_exit(&txr->bnx_tx_serialize);
3171
3172         bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx,
3173             ret->bnx_saved_status_tag << 24);
3174 }
3175
3176 static void
3177 bnx_msix_status(void *xsc)
3178 {
3179         struct bnx_softc *sc = xsc;
3180
3181         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3182
3183         sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag;
3184         /*
3185          * Use a load fence to ensure that status_tag is saved
3186          * before status.
3187          */
3188         cpu_lfence();
3189
3190         bnx_handle_status(sc);
3191
3192         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_saved_status_tag << 24);
3193 }
3194
3195 static void
3196 bnx_tick(void *xsc)
3197 {
3198         struct bnx_softc *sc = xsc;
3199
3200         lwkt_serialize_enter(&sc->bnx_main_serialize);
3201
3202         bnx_stats_update_regs(sc);
3203
3204         if (sc->bnx_flags & BNX_FLAG_TBI) {
3205                 /*
3206                  * Since in TBI mode auto-polling can't be used we should poll
3207                  * link status manually. Here we register pending link event
3208                  * and trigger interrupt.
3209                  */
3210                 sc->bnx_link_evt++;
3211                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3212         } else if (!sc->bnx_link) {
3213                 mii_tick(device_get_softc(sc->bnx_miibus));
3214         }
3215
3216         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
3217             sc->bnx_tick_cpuid);
3218
3219         lwkt_serialize_exit(&sc->bnx_main_serialize);
3220 }
3221
3222 static void
3223 bnx_stats_update_regs(struct bnx_softc *sc)
3224 {
3225         struct ifnet *ifp = &sc->arpcom.ac_if;
3226         struct bge_mac_stats_regs stats;
3227         uint32_t *s, val;
3228         int i;
3229
3230         s = (uint32_t *)&stats;
3231         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
3232                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
3233                 s++;
3234         }
3235
3236         IFNET_STAT_SET(ifp, collisions,
3237            (stats.dot3StatsSingleCollisionFrames +
3238            stats.dot3StatsMultipleCollisionFrames +
3239            stats.dot3StatsExcessiveCollisions +
3240            stats.dot3StatsLateCollisions));
3241
3242         val = CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS);
3243         sc->bnx_norxbds += val;
3244 }
3245
3246 /*
3247  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
3248  * pointers to descriptors.
3249  */
3250 static int
3251 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
3252     int *segs_used)
3253 {
3254         struct bge_tx_bd *d = NULL;
3255         uint16_t csum_flags = 0, vlan_tag = 0, mss = 0;
3256         bus_dma_segment_t segs[BNX_NSEG_NEW];
3257         bus_dmamap_t map;
3258         int error, maxsegs, nsegs, idx, i;
3259         struct mbuf *m_head = *m_head0, *m_new;
3260
3261         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3262 #ifdef BNX_TSO_DEBUG
3263                 int tso_nsegs;
3264 #endif
3265
3266                 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags);
3267                 if (error)
3268                         return error;
3269                 m_head = *m_head0;
3270
3271 #ifdef BNX_TSO_DEBUG
3272                 tso_nsegs = (m_head->m_pkthdr.len /
3273                     m_head->m_pkthdr.tso_segsz) - 1;
3274                 if (tso_nsegs > (BNX_TSO_NSTATS - 1))
3275                         tso_nsegs = BNX_TSO_NSTATS - 1;
3276                 else if (tso_nsegs < 0)
3277                         tso_nsegs = 0;
3278                 txr->bnx_sc->bnx_tsosegs[tso_nsegs]++;
3279 #endif
3280         } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) {
3281                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
3282                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
3283                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
3284                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
3285                 if (m_head->m_flags & M_LASTFRAG)
3286                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
3287                 else if (m_head->m_flags & M_FRAG)
3288                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
3289         }
3290         if (m_head->m_flags & M_VLANTAG) {
3291                 csum_flags |= BGE_TXBDFLAG_VLAN_TAG;
3292                 vlan_tag = m_head->m_pkthdr.ether_vlantag;
3293         }
3294
3295         idx = *txidx;
3296         map = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
3297
3298         maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD;
3299         KASSERT(maxsegs >= BNX_NSEG_SPARE,
3300                 ("not enough segments %d", maxsegs));
3301
3302         if (maxsegs > BNX_NSEG_NEW)
3303                 maxsegs = BNX_NSEG_NEW;
3304
3305         /*
3306          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
3307          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
3308          * but when such padded frames employ the bge IP/TCP checksum
3309          * offload, the hardware checksum assist gives incorrect results
3310          * (possibly from incorporating its own padding into the UDP/TCP
3311          * checksum; who knows).  If we pad such runts with zeros, the
3312          * onboard checksum comes out correct.
3313          */
3314         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
3315             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
3316                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
3317                 if (error)
3318                         goto back;
3319         }
3320
3321         if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) &&
3322             m_head->m_next != NULL) {
3323                 m_new = bnx_defrag_shortdma(m_head);
3324                 if (m_new == NULL) {
3325                         error = ENOBUFS;
3326                         goto back;
3327                 }
3328                 *m_head0 = m_head = m_new;
3329         }
3330         if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 &&
3331             (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) &&
3332             m_head->m_next != NULL) {
3333                 /*
3334                  * Forcefully defragment mbuf chain to overcome hardware
3335                  * limitation which only support a single outstanding
3336                  * DMA read operation.  If it fails, keep moving on using
3337                  * the original mbuf chain.
3338                  */
3339                 m_new = m_defrag(m_head, MB_DONTWAIT);
3340                 if (m_new != NULL)
3341                         *m_head0 = m_head = m_new;
3342         }
3343
3344         error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map,
3345             m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
3346         if (error)
3347                 goto back;
3348         *segs_used += nsegs;
3349
3350         m_head = *m_head0;
3351         bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
3352
3353         for (i = 0; ; i++) {
3354                 d = &txr->bnx_tx_ring[idx];
3355
3356                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
3357                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
3358                 d->bge_len = segs[i].ds_len;
3359                 d->bge_flags = csum_flags;
3360                 d->bge_vlan_tag = vlan_tag;
3361                 d->bge_mss = mss;
3362
3363                 if (i == nsegs - 1)
3364                         break;
3365                 BNX_INC(idx, BGE_TX_RING_CNT);
3366         }
3367         /* Mark the last segment as end of packet... */
3368         d->bge_flags |= BGE_TXBDFLAG_END;
3369
3370         /*
3371          * Insure that the map for this transmission is placed at
3372          * the array index of the last descriptor in this chain.
3373          */
3374         txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
3375         txr->bnx_tx_buf[idx].bnx_tx_dmamap = map;
3376         txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head;
3377         txr->bnx_tx_cnt += nsegs;
3378
3379         BNX_INC(idx, BGE_TX_RING_CNT);
3380         *txidx = idx;
3381 back:
3382         if (error) {
3383                 m_freem(*m_head0);
3384                 *m_head0 = NULL;
3385         }
3386         return error;
3387 }
3388
3389 /*
3390  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
3391  * to the mbuf data regions directly in the transmit descriptors.
3392  */
3393 static void
3394 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3395 {
3396         struct bnx_tx_ring *txr = ifsq_get_priv(ifsq);
3397         struct mbuf *m_head = NULL;
3398         uint32_t prodidx;
3399         int nsegs = 0;
3400
3401         KKASSERT(txr->bnx_ifsq == ifsq);
3402         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
3403
3404         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
3405                 return;
3406
3407         prodidx = txr->bnx_tx_prodidx;
3408
3409         while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) {
3410                 /*
3411                  * Sanity check: avoid coming within BGE_NSEG_RSVD
3412                  * descriptors of the end of the ring.  Also make
3413                  * sure there are BGE_NSEG_SPARE descriptors for
3414                  * jumbo buffers' or TSO segments' defragmentation.
3415                  */
3416                 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) <
3417                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
3418                         ifsq_set_oactive(ifsq);
3419                         break;
3420                 }
3421
3422                 m_head = ifsq_dequeue(ifsq, NULL);
3423                 if (m_head == NULL)
3424                         break;
3425
3426                 /*
3427                  * Pack the data into the transmit ring. If we
3428                  * don't have room, set the OACTIVE flag and wait
3429                  * for the NIC to drain the ring.
3430                  */
3431                 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) {
3432                         ifsq_set_oactive(ifsq);
3433                         IFNET_STAT_INC(ifp, oerrors, 1);
3434                         break;
3435                 }
3436
3437                 if (nsegs >= txr->bnx_tx_wreg) {
3438                         /* Transmit */
3439                         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3440                         nsegs = 0;
3441                 }
3442
3443                 ETHER_BPF_MTAP(ifp, m_head);
3444
3445                 /*
3446                  * Set a timeout in case the chip goes out to lunch.
3447                  */
3448                 txr->bnx_tx_watchdog.wd_timer = 5;
3449         }
3450
3451         if (nsegs > 0) {
3452                 /* Transmit */
3453                 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3454         }
3455         txr->bnx_tx_prodidx = prodidx;
3456 }
3457
3458 static void
3459 bnx_init(void *xsc)
3460 {
3461         struct bnx_softc *sc = xsc;
3462         struct ifnet *ifp = &sc->arpcom.ac_if;
3463         uint16_t *m;
3464         uint32_t mode;
3465         int i;
3466         boolean_t polling;
3467
3468         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3469
3470         /* Cancel pending I/O and flush buffers. */
3471         bnx_stop(sc);
3472         bnx_reset(sc);
3473         bnx_chipinit(sc);
3474
3475         /*
3476          * Init the various state machines, ring
3477          * control blocks and firmware.
3478          */
3479         if (bnx_blockinit(sc)) {
3480                 if_printf(ifp, "initialization failure\n");
3481                 bnx_stop(sc);
3482                 return;
3483         }
3484
3485         /* Specify MTU. */
3486         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
3487             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
3488
3489         /* Load our MAC address. */
3490         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
3491         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
3492         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
3493
3494         /* Enable or disable promiscuous mode as needed. */
3495         bnx_setpromisc(sc);
3496
3497         /* Program multicast filter. */
3498         bnx_setmulti(sc);
3499
3500         /* Init RX ring. */
3501         if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) {
3502                 if_printf(ifp, "RX ring initialization failed\n");
3503                 bnx_stop(sc);
3504                 return;
3505         }
3506
3507         /* Init jumbo RX ring. */
3508         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
3509                 if (bnx_init_rx_ring_jumbo(sc)) {
3510                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
3511                         bnx_stop(sc);
3512                         return;
3513                 }
3514         }
3515
3516         /* Init our RX return ring index */
3517         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
3518                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
3519
3520                 ret->bnx_rx_saved_considx = 0;
3521                 ret->bnx_rx_cnt = 0;
3522         }
3523
3524         /* Init TX ring. */
3525         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3526                 bnx_init_tx_ring(&sc->bnx_tx_ring[i]);
3527
3528         /* Enable TX MAC state machine lockup fix. */
3529         mode = CSR_READ_4(sc, BGE_TX_MODE);
3530         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3531         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
3532             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
3533                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3534                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
3535                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3536         }
3537         /* Turn on transmitter */
3538         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3539
3540         /* Initialize RSS */
3541         mode = BGE_RXMODE_ENABLE;
3542         if (BNX_RSS_ENABLED(sc)) {
3543                 bnx_init_rss(sc);
3544                 mode |= BGE_RXMODE_RSS_ENABLE |
3545                     BGE_RXMODE_RSS_HASH_MASK_BITS |
3546                     BGE_RXMODE_RSS_IPV4_HASH |
3547                     BGE_RXMODE_RSS_TCP_IPV4_HASH;
3548         }
3549         /* Turn on receiver */
3550         BNX_SETBIT(sc, BGE_RX_MODE, mode);
3551
3552         /*
3553          * Set the number of good frames to receive after RX MBUF
3554          * Low Watermark has been reached.  After the RX MAC receives
3555          * this number of frames, it will drop subsequent incoming
3556          * frames until the MBUF High Watermark is reached.
3557          */
3558         if (BNX_IS_57765_FAMILY(sc))
3559                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3560         else
3561                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3562
3563         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI ||
3564             sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) {
3565                 if (bootverbose) {
3566                         if_printf(ifp, "MSI_MODE: %#x\n",
3567                             CSR_READ_4(sc, BGE_MSI_MODE));
3568                 }
3569         }
3570
3571         /* Tell firmware we're alive. */
3572         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3573
3574         /* Enable host interrupts if polling(4) is not enabled. */
3575         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3576
3577         polling = FALSE;
3578 #ifdef IFPOLL_ENABLE
3579         if (ifp->if_flags & IFF_NPOLLING)
3580                 polling = TRUE;
3581 #endif
3582         if (polling)
3583                 bnx_disable_intr(sc);
3584         else
3585                 bnx_enable_intr(sc);
3586         bnx_set_tick_cpuid(sc, polling);
3587
3588         bnx_ifmedia_upd(ifp);
3589
3590         ifp->if_flags |= IFF_RUNNING;
3591         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3592                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3593
3594                 ifsq_clr_oactive(txr->bnx_ifsq);
3595                 ifsq_watchdog_start(&txr->bnx_tx_watchdog);
3596         }
3597
3598         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
3599             sc->bnx_tick_cpuid);
3600 }
3601
3602 /*
3603  * Set media options.
3604  */
3605 static int
3606 bnx_ifmedia_upd(struct ifnet *ifp)
3607 {
3608         struct bnx_softc *sc = ifp->if_softc;
3609
3610         /* If this is a 1000baseX NIC, enable the TBI port. */
3611         if (sc->bnx_flags & BNX_FLAG_TBI) {
3612                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3613
3614                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3615                         return(EINVAL);
3616
3617                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3618                 case IFM_AUTO:
3619                         break;
3620
3621                 case IFM_1000_SX:
3622                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3623                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3624                                     BGE_MACMODE_HALF_DUPLEX);
3625                         } else {
3626                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3627                                     BGE_MACMODE_HALF_DUPLEX);
3628                         }
3629                         break;
3630                 default:
3631                         return(EINVAL);
3632                 }
3633         } else {
3634                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3635
3636                 sc->bnx_link_evt++;
3637                 sc->bnx_link = 0;
3638                 if (mii->mii_instance) {
3639                         struct mii_softc *miisc;
3640
3641                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3642                                 mii_phy_reset(miisc);
3643                 }
3644                 mii_mediachg(mii);
3645
3646                 /*
3647                  * Force an interrupt so that we will call bnx_link_upd
3648                  * if needed and clear any pending link state attention.
3649                  * Without this we are not getting any further interrupts
3650                  * for link state changes and thus will not UP the link and
3651                  * not be able to send in bnx_start.  The only way to get
3652                  * things working was to receive a packet and get an RX
3653                  * intr.
3654                  *
3655                  * bnx_tick should help for fiber cards and we might not
3656                  * need to do this here if BNX_FLAG_TBI is set but as
3657                  * we poll for fiber anyway it should not harm.
3658                  */
3659                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3660         }
3661         return(0);
3662 }
3663
3664 /*
3665  * Report current media status.
3666  */
3667 static void
3668 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3669 {
3670         struct bnx_softc *sc = ifp->if_softc;
3671
3672         if (sc->bnx_flags & BNX_FLAG_TBI) {
3673                 ifmr->ifm_status = IFM_AVALID;
3674                 ifmr->ifm_active = IFM_ETHER;
3675                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3676                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3677                         ifmr->ifm_status |= IFM_ACTIVE;
3678                 } else {
3679                         ifmr->ifm_active |= IFM_NONE;
3680                         return;
3681                 }
3682
3683                 ifmr->ifm_active |= IFM_1000_SX;
3684                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3685                         ifmr->ifm_active |= IFM_HDX;
3686                 else
3687                         ifmr->ifm_active |= IFM_FDX;
3688         } else {
3689                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3690
3691                 mii_pollstat(mii);
3692                 ifmr->ifm_active = mii->mii_media_active;
3693                 ifmr->ifm_status = mii->mii_media_status;
3694         }
3695 }
3696
3697 static int
3698 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3699 {
3700         struct bnx_softc *sc = ifp->if_softc;
3701         struct ifreq *ifr = (struct ifreq *)data;
3702         int mask, error = 0;
3703
3704         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3705
3706         switch (command) {
3707         case SIOCSIFMTU:
3708                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3709                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3710                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3711                         error = EINVAL;
3712                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3713                         ifp->if_mtu = ifr->ifr_mtu;
3714                         if (ifp->if_flags & IFF_RUNNING)
3715                                 bnx_init(sc);
3716                 }
3717                 break;
3718         case SIOCSIFFLAGS:
3719                 if (ifp->if_flags & IFF_UP) {
3720                         if (ifp->if_flags & IFF_RUNNING) {
3721                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3722
3723                                 /*
3724                                  * If only the state of the PROMISC flag
3725                                  * changed, then just use the 'set promisc
3726                                  * mode' command instead of reinitializing
3727                                  * the entire NIC. Doing a full re-init
3728                                  * means reloading the firmware and waiting
3729                                  * for it to start up, which may take a
3730                                  * second or two.  Similarly for ALLMULTI.
3731                                  */
3732                                 if (mask & IFF_PROMISC)
3733                                         bnx_setpromisc(sc);
3734                                 if (mask & IFF_ALLMULTI)
3735                                         bnx_setmulti(sc);
3736                         } else {
3737                                 bnx_init(sc);
3738                         }
3739                 } else if (ifp->if_flags & IFF_RUNNING) {
3740                         bnx_stop(sc);
3741                 }
3742                 sc->bnx_if_flags = ifp->if_flags;
3743                 break;
3744         case SIOCADDMULTI:
3745         case SIOCDELMULTI:
3746                 if (ifp->if_flags & IFF_RUNNING)
3747                         bnx_setmulti(sc);
3748                 break;
3749         case SIOCSIFMEDIA:
3750         case SIOCGIFMEDIA:
3751                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3752                         error = ifmedia_ioctl(ifp, ifr,
3753                             &sc->bnx_ifmedia, command);
3754                 } else {
3755                         struct mii_data *mii;
3756
3757                         mii = device_get_softc(sc->bnx_miibus);
3758                         error = ifmedia_ioctl(ifp, ifr,
3759                                               &mii->mii_media, command);
3760                 }
3761                 break;
3762         case SIOCSIFCAP:
3763                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3764                 if (mask & IFCAP_HWCSUM) {
3765                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3766                         if (ifp->if_capenable & IFCAP_TXCSUM)
3767                                 ifp->if_hwassist |= BNX_CSUM_FEATURES;
3768                         else
3769                                 ifp->if_hwassist &= ~BNX_CSUM_FEATURES;
3770                 }
3771                 if (mask & IFCAP_TSO) {
3772                         ifp->if_capenable ^= (mask & IFCAP_TSO);
3773                         if (ifp->if_capenable & IFCAP_TSO)
3774                                 ifp->if_hwassist |= CSUM_TSO;
3775                         else
3776                                 ifp->if_hwassist &= ~CSUM_TSO;
3777                 }
3778                 if (mask & IFCAP_RSS)
3779                         ifp->if_capenable ^= IFCAP_RSS;
3780                 break;
3781         default:
3782                 error = ether_ioctl(ifp, command, data);
3783                 break;
3784         }
3785         return error;
3786 }
3787
3788 static void
3789 bnx_watchdog(struct ifaltq_subque *ifsq)
3790 {
3791         struct ifnet *ifp = ifsq_get_ifp(ifsq);
3792         struct bnx_softc *sc = ifp->if_softc;
3793         int i;
3794
3795         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3796
3797         if_printf(ifp, "watchdog timeout -- resetting\n");
3798
3799         bnx_init(sc);
3800
3801         IFNET_STAT_INC(ifp, oerrors, 1);
3802
3803         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3804                 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
3805 }
3806
3807 /*
3808  * Stop the adapter and free any mbufs allocated to the
3809  * RX and TX lists.
3810  */
3811 static void
3812 bnx_stop(struct bnx_softc *sc)
3813 {
3814         struct ifnet *ifp = &sc->arpcom.ac_if;
3815         int i;
3816
3817         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3818
3819         callout_stop(&sc->bnx_tick_timer);
3820
3821         /*
3822          * Disable all of the receiver blocks
3823          */
3824         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3825         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3826         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3827         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3828         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3829         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3830
3831         /*
3832          * Disable all of the transmit blocks
3833          */
3834         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3835         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3836         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3837         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3838         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3839         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3840
3841         /*
3842          * Shut down all of the memory managers and related
3843          * state machines.
3844          */
3845         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3846         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3847         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3848         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3849
3850         /* Disable host interrupts. */
3851         bnx_disable_intr(sc);
3852
3853         /*
3854          * Tell firmware we're shutting down.
3855          */
3856         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3857
3858         /* Free the RX lists. */
3859         bnx_free_rx_ring_std(&sc->bnx_rx_std_ring);
3860
3861         /* Free jumbo RX list. */
3862         if (BNX_IS_JUMBO_CAPABLE(sc))
3863                 bnx_free_rx_ring_jumbo(sc);
3864
3865         /* Free TX buffers. */
3866         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3867                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3868
3869                 txr->bnx_saved_status_tag = 0;
3870                 bnx_free_tx_ring(txr);
3871         }
3872
3873         /* Clear saved status tag */
3874         for (i = 0; i < sc->bnx_rx_retcnt; ++i)
3875                 sc->bnx_rx_ret_ring[i].bnx_saved_status_tag = 0;
3876
3877         sc->bnx_link = 0;
3878         sc->bnx_coal_chg = 0;
3879
3880         ifp->if_flags &= ~IFF_RUNNING;
3881         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3882                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3883
3884                 ifsq_clr_oactive(txr->bnx_ifsq);
3885                 ifsq_watchdog_stop(&txr->bnx_tx_watchdog);
3886         }
3887 }
3888
3889 /*
3890  * Stop all chip I/O so that the kernel's probe routines don't
3891  * get confused by errant DMAs when rebooting.
3892  */
3893 static void
3894 bnx_shutdown(device_t dev)
3895 {
3896         struct bnx_softc *sc = device_get_softc(dev);
3897         struct ifnet *ifp = &sc->arpcom.ac_if;
3898
3899         ifnet_serialize_all(ifp);
3900         bnx_stop(sc);
3901         bnx_reset(sc);
3902         ifnet_deserialize_all(ifp);
3903 }
3904
3905 static int
3906 bnx_suspend(device_t dev)
3907 {
3908         struct bnx_softc *sc = device_get_softc(dev);
3909         struct ifnet *ifp = &sc->arpcom.ac_if;
3910
3911         ifnet_serialize_all(ifp);
3912         bnx_stop(sc);
3913         ifnet_deserialize_all(ifp);
3914
3915         return 0;
3916 }
3917
3918 static int
3919 bnx_resume(device_t dev)
3920 {
3921         struct bnx_softc *sc = device_get_softc(dev);
3922         struct ifnet *ifp = &sc->arpcom.ac_if;
3923
3924         ifnet_serialize_all(ifp);
3925
3926         if (ifp->if_flags & IFF_UP) {
3927                 int i;
3928
3929                 bnx_init(sc);
3930                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3931                         ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
3932         }
3933
3934         ifnet_deserialize_all(ifp);
3935
3936         return 0;
3937 }
3938
3939 static void
3940 bnx_setpromisc(struct bnx_softc *sc)
3941 {
3942         struct ifnet *ifp = &sc->arpcom.ac_if;
3943
3944         if (ifp->if_flags & IFF_PROMISC)
3945                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3946         else
3947                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3948 }
3949
3950 static void
3951 bnx_dma_free(struct bnx_softc *sc)
3952 {
3953         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
3954         int i;
3955
3956         /* Destroy RX return rings */
3957         if (sc->bnx_rx_ret_ring != NULL) {
3958                 for (i = 0; i < sc->bnx_rx_retcnt; ++i)
3959                         bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]);
3960                 kfree(sc->bnx_rx_ret_ring, M_DEVBUF);
3961         }
3962
3963         /* Destroy RX mbuf DMA stuffs. */
3964         if (std->bnx_rx_mtag != NULL) {
3965                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3966                         KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL);
3967                         bus_dmamap_destroy(std->bnx_rx_mtag,
3968                             std->bnx_rx_std_buf[i].bnx_rx_dmamap);
3969                 }
3970                 bus_dma_tag_destroy(std->bnx_rx_mtag);
3971         }
3972
3973         /* Destroy standard RX ring */
3974         bnx_dma_block_free(std->bnx_rx_std_ring_tag,
3975             std->bnx_rx_std_ring_map, std->bnx_rx_std_ring);
3976
3977         /* Destroy TX rings */
3978         if (sc->bnx_tx_ring != NULL) {
3979                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3980                         bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]);
3981                 kfree(sc->bnx_tx_ring, M_DEVBUF);
3982         }
3983
3984         if (BNX_IS_JUMBO_CAPABLE(sc))
3985                 bnx_free_jumbo_mem(sc);
3986
3987         /* Destroy status blocks */
3988         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
3989                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
3990
3991                 bnx_dma_block_free(intr->bnx_status_tag,
3992                     intr->bnx_status_map, intr->bnx_status_block);
3993         }
3994
3995         /* Destroy the parent tag */
3996         if (sc->bnx_cdata.bnx_parent_tag != NULL)
3997                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
3998 }
3999
4000 static int
4001 bnx_dma_alloc(device_t dev)
4002 {
4003         struct bnx_softc *sc = device_get_softc(dev);
4004         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
4005         int i, error, mbx;
4006
4007         /*
4008          * Allocate the parent bus DMA tag appropriate for PCI.
4009          *
4010          * All of the NetExtreme/NetLink controllers have 4GB boundary
4011          * DMA bug.
4012          * Whenever an address crosses a multiple of the 4GB boundary
4013          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
4014          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
4015          * state machine will lockup and cause the device to hang.
4016          */
4017         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
4018             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
4019             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
4020             0, &sc->bnx_cdata.bnx_parent_tag);
4021         if (error) {
4022                 device_printf(dev, "could not create parent DMA tag\n");
4023                 return error;
4024         }
4025
4026         /*
4027          * Create DMA stuffs for status blocks.
4028          */
4029         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4030                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4031
4032                 error = bnx_dma_block_alloc(sc,
4033                     __VM_CACHELINE_ALIGN(BGE_STATUS_BLK_SZ),
4034                     &intr->bnx_status_tag, &intr->bnx_status_map,
4035                     (void *)&intr->bnx_status_block,
4036                     &intr->bnx_status_block_paddr);
4037                 if (error) {
4038                         device_printf(dev,
4039                             "could not create %dth status block\n", i);
4040                         return error;
4041                 }
4042         }
4043         sc->bnx_hw_status = &sc->bnx_intr_data[0].bnx_status_block->bge_status;
4044         if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) {
4045                 sc->bnx_hw_status_tag =
4046                     &sc->bnx_intr_data[0].bnx_status_block->bge_status_tag;
4047         }
4048
4049         /*
4050          * Create DMA tag and maps for RX mbufs.
4051          */
4052         std->bnx_sc = sc;
4053         lwkt_serialize_init(&std->bnx_rx_std_serialize);
4054         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
4055             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4056             NULL, NULL, MCLBYTES, 1, MCLBYTES,
4057             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag);
4058         if (error) {
4059                 device_printf(dev, "could not create RX mbuf DMA tag\n");
4060                 return error;
4061         }
4062
4063         for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) {
4064                 error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK,
4065                     &std->bnx_rx_std_buf[i].bnx_rx_dmamap);
4066                 if (error) {
4067                         int j;
4068
4069                         for (j = 0; j < i; ++j) {
4070                                 bus_dmamap_destroy(std->bnx_rx_mtag,
4071                                     std->bnx_rx_std_buf[j].bnx_rx_dmamap);
4072                         }
4073                         bus_dma_tag_destroy(std->bnx_rx_mtag);
4074                         std->bnx_rx_mtag = NULL;
4075
4076                         device_printf(dev,
4077                             "could not create %dth RX mbuf DMA map\n", i);
4078                         return error;
4079                 }
4080         }
4081
4082         /*
4083          * Create DMA stuffs for standard RX ring.
4084          */
4085         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
4086             &std->bnx_rx_std_ring_tag,
4087             &std->bnx_rx_std_ring_map,
4088             (void *)&std->bnx_rx_std_ring,
4089             &std->bnx_rx_std_ring_paddr);
4090         if (error) {
4091                 device_printf(dev, "could not create std RX ring\n");
4092                 return error;
4093         }
4094
4095         /*
4096          * Create RX return rings
4097          */
4098         mbx = BGE_MBX_RX_CONS0_LO;
4099         sc->bnx_rx_ret_ring = kmalloc_cachealign(
4100             sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF,
4101             M_WAITOK | M_ZERO);
4102         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4103                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
4104                 struct bnx_intr_data *intr;
4105
4106                 ret->bnx_sc = sc;
4107                 ret->bnx_std = std;
4108                 ret->bnx_rx_mbx = mbx;
4109                 ret->bnx_rx_cntmax = (BGE_STD_RX_RING_CNT / 4) /
4110                     sc->bnx_rx_retcnt;
4111                 ret->bnx_rx_mask = 1 << i;
4112
4113                 if (!BNX_RSS_ENABLED(sc)) {
4114                         intr = &sc->bnx_intr_data[0];
4115                 } else {
4116                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
4117                         intr = &sc->bnx_intr_data[i + 1];
4118                 }
4119
4120                 if (i == 0) {
4121                         ret->bnx_rx_considx =
4122                             &intr->bnx_status_block->bge_idx[0].bge_rx_prod_idx;
4123                 } else if (i == 1) {
4124                         ret->bnx_rx_considx =
4125                             &intr->bnx_status_block->bge_rx_jumbo_cons_idx;
4126                 } else if (i == 2) {
4127                         ret->bnx_rx_considx =
4128                             &intr->bnx_status_block->bge_rsvd1;
4129                 } else if (i == 3) {
4130                         ret->bnx_rx_considx =
4131                             &intr->bnx_status_block->bge_rx_mini_cons_idx;
4132                 } else {
4133                         panic("unknown RX return ring %d\n", i);
4134                 }
4135                 ret->bnx_hw_status_tag =
4136                     &intr->bnx_status_block->bge_status_tag;
4137
4138                 error = bnx_create_rx_ret_ring(ret);
4139                 if (error) {
4140                         device_printf(dev,
4141                             "could not create %dth RX ret ring\n", i);
4142                         return error;
4143                 }
4144                 mbx += 8;
4145         }
4146
4147         /*
4148          * Create TX rings
4149          */
4150         sc->bnx_tx_ring = kmalloc_cachealign(
4151             sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF,
4152             M_WAITOK | M_ZERO);
4153         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4154                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
4155                 struct bnx_intr_data *intr;
4156
4157                 txr->bnx_sc = sc;
4158                 txr->bnx_tx_mbx = bnx_tx_mailbox[i];
4159
4160                 if (sc->bnx_tx_ringcnt == 1) {
4161                         intr = &sc->bnx_intr_data[0];
4162                 } else {
4163                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
4164                         intr = &sc->bnx_intr_data[i + 1];
4165                 }
4166
4167                 if ((sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) == 0) {
4168                         txr->bnx_hw_status_tag =
4169                             &intr->bnx_status_block->bge_status_tag;
4170                 }
4171                 txr->bnx_tx_considx =
4172                     &intr->bnx_status_block->bge_idx[0].bge_tx_cons_idx;
4173
4174                 error = bnx_create_tx_ring(txr);
4175                 if (error) {
4176                         device_printf(dev,
4177                             "could not create %dth TX ring\n", i);
4178                         return error;
4179                 }
4180         }
4181
4182         /*
4183          * Create jumbo buffer pool.
4184          */
4185         if (BNX_IS_JUMBO_CAPABLE(sc)) {
4186                 error = bnx_alloc_jumbo_mem(sc);
4187                 if (error) {
4188                         device_printf(dev,
4189                             "could not create jumbo buffer pool\n");
4190                         return error;
4191                 }
4192         }
4193
4194         return 0;
4195 }
4196
4197 static int
4198 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
4199                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
4200 {
4201         bus_dmamem_t dmem;
4202         int error;
4203
4204         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
4205                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4206                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
4207         if (error)
4208                 return error;
4209
4210         *tag = dmem.dmem_tag;
4211         *map = dmem.dmem_map;
4212         *addr = dmem.dmem_addr;
4213         *paddr = dmem.dmem_busaddr;
4214
4215         return 0;
4216 }
4217
4218 static void
4219 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
4220 {
4221         if (tag != NULL) {
4222                 bus_dmamap_unload(tag, map);
4223                 bus_dmamem_free(tag, addr, map);
4224                 bus_dma_tag_destroy(tag);
4225         }
4226 }
4227
4228 static void
4229 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
4230 {
4231         struct ifnet *ifp = &sc->arpcom.ac_if;
4232
4233 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
4234
4235         /*
4236          * Sometimes PCS encoding errors are detected in
4237          * TBI mode (on fiber NICs), and for some reason
4238          * the chip will signal them as link changes.
4239          * If we get a link change event, but the 'PCS
4240          * encoding error' bit in the MAC status register
4241          * is set, don't bother doing a link check.
4242          * This avoids spurious "gigabit link up" messages
4243          * that sometimes appear on fiber NICs during
4244          * periods of heavy traffic.
4245          */
4246         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
4247                 if (!sc->bnx_link) {
4248                         sc->bnx_link++;
4249                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
4250                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
4251                                     BGE_MACMODE_TBI_SEND_CFGS);
4252                         }
4253                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
4254
4255                         if (bootverbose)
4256                                 if_printf(ifp, "link UP\n");
4257
4258                         ifp->if_link_state = LINK_STATE_UP;
4259                         if_link_state_change(ifp);
4260                 }
4261         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
4262                 if (sc->bnx_link) {
4263                         sc->bnx_link = 0;
4264
4265                         if (bootverbose)
4266                                 if_printf(ifp, "link DOWN\n");
4267
4268                         ifp->if_link_state = LINK_STATE_DOWN;
4269                         if_link_state_change(ifp);
4270                 }
4271         }
4272
4273 #undef PCS_ENCODE_ERR
4274
4275         /* Clear the attention. */
4276         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4277             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4278             BGE_MACSTAT_LINK_CHANGED);
4279 }
4280
4281 static void
4282 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
4283 {
4284         struct ifnet *ifp = &sc->arpcom.ac_if;
4285         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
4286
4287         mii_pollstat(mii);
4288         bnx_miibus_statchg(sc->bnx_dev);
4289
4290         if (bootverbose) {
4291                 if (sc->bnx_link)
4292                         if_printf(ifp, "link UP\n");
4293                 else
4294                         if_printf(ifp, "link DOWN\n");
4295         }
4296
4297         /* Clear the attention. */
4298         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4299             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4300             BGE_MACSTAT_LINK_CHANGED);
4301 }
4302
4303 static void
4304 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
4305 {
4306         struct ifnet *ifp = &sc->arpcom.ac_if;
4307         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
4308
4309         mii_pollstat(mii);
4310
4311         if (!sc->bnx_link &&
4312             (mii->mii_media_status & IFM_ACTIVE) &&
4313             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
4314                 sc->bnx_link++;
4315                 if (bootverbose)
4316                         if_printf(ifp, "link UP\n");
4317         } else if (sc->bnx_link &&
4318             (!(mii->mii_media_status & IFM_ACTIVE) ||
4319             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
4320                 sc->bnx_link = 0;
4321                 if (bootverbose)
4322                         if_printf(ifp, "link DOWN\n");
4323         }
4324
4325         /* Clear the attention. */
4326         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4327             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4328             BGE_MACSTAT_LINK_CHANGED);
4329 }
4330
4331 static int
4332 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
4333 {
4334         struct bnx_softc *sc = arg1;
4335
4336         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4337             &sc->bnx_rx_coal_ticks,
4338             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
4339             BNX_RX_COAL_TICKS_CHG);
4340 }
4341
4342 static int
4343 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
4344 {
4345         struct bnx_softc *sc = arg1;
4346
4347         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4348             &sc->bnx_tx_coal_ticks,
4349             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
4350             BNX_TX_COAL_TICKS_CHG);
4351 }
4352
4353 static int
4354 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
4355 {
4356         struct bnx_softc *sc = arg1;
4357
4358         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4359             &sc->bnx_rx_coal_bds,
4360             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
4361             BNX_RX_COAL_BDS_CHG);
4362 }
4363
4364 static int
4365 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
4366 {
4367         struct bnx_softc *sc = arg1;
4368
4369         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4370             &sc->bnx_tx_coal_bds,
4371             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4372             BNX_TX_COAL_BDS_CHG);
4373 }
4374
4375 static int
4376 bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS)
4377 {
4378         struct bnx_softc *sc = arg1;
4379
4380         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4381             &sc->bnx_tx_coal_bds_poll,
4382             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4383             BNX_TX_COAL_BDS_CHG);
4384 }
4385
4386 static int
4387 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4388 {
4389         struct bnx_softc *sc = arg1;
4390
4391         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4392             &sc->bnx_rx_coal_bds_int,
4393             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
4394             BNX_RX_COAL_BDS_INT_CHG);
4395 }
4396
4397 static int
4398 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4399 {
4400         struct bnx_softc *sc = arg1;
4401
4402         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4403             &sc->bnx_tx_coal_bds_int,
4404             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4405             BNX_TX_COAL_BDS_INT_CHG);
4406 }
4407
4408 static int
4409 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
4410     int coal_min, int coal_max, uint32_t coal_chg_mask)
4411 {
4412         struct bnx_softc *sc = arg1;
4413         struct ifnet *ifp = &sc->arpcom.ac_if;
4414         int error = 0, v;
4415
4416         ifnet_serialize_all(ifp);
4417
4418         v = *coal;
4419         error = sysctl_handle_int(oidp, &v, 0, req);
4420         if (!error && req->newptr != NULL) {
4421                 if (v < coal_min || v > coal_max) {
4422                         error = EINVAL;
4423                 } else {
4424                         *coal = v;
4425                         sc->bnx_coal_chg |= coal_chg_mask;
4426
4427                         /* Commit changes */
4428                         bnx_coal_change(sc);
4429                 }
4430         }
4431
4432         ifnet_deserialize_all(ifp);
4433         return error;
4434 }
4435
4436 static void
4437 bnx_coal_change(struct bnx_softc *sc)
4438 {
4439         struct ifnet *ifp = &sc->arpcom.ac_if;
4440         int i;
4441
4442         ASSERT_IFNET_SERIALIZED_ALL(ifp);
4443
4444         if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
4445                 if (sc->bnx_rx_retcnt == 1) {
4446                         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
4447                             sc->bnx_rx_coal_ticks);
4448                         i = 0;
4449                 } else {
4450                         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 0);
4451                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4452                                 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS +
4453                                     (i * BGE_VEC_COALSET_SIZE),
4454                                     sc->bnx_rx_coal_ticks);
4455                         }
4456                 }
4457                 for (; i < BNX_INTR_MAX - 1; ++i) {
4458                         CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS +
4459                             (i * BGE_VEC_COALSET_SIZE), 0);
4460                 }
4461                 if (bootverbose) {
4462                         if_printf(ifp, "rx_coal_ticks -> %u\n",
4463                             sc->bnx_rx_coal_ticks);
4464                 }
4465         }
4466
4467         if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) {
4468                 if (sc->bnx_tx_ringcnt == 1) {
4469                         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
4470                             sc->bnx_tx_coal_ticks);
4471                         i = 0;
4472                 } else {
4473                         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 0);
4474                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4475                                 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS +
4476                                     (i * BGE_VEC_COALSET_SIZE),
4477                                     sc->bnx_tx_coal_ticks);
4478                         }
4479                 }
4480                 for (; i < BNX_INTR_MAX - 1; ++i) {
4481                         CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS +
4482                             (i * BGE_VEC_COALSET_SIZE), 0);
4483                 }
4484                 if (bootverbose) {
4485                         if_printf(ifp, "tx_coal_ticks -> %u\n",
4486                             sc->bnx_tx_coal_ticks);
4487                 }
4488         }
4489
4490         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) {
4491                 if (sc->bnx_rx_retcnt == 1) {
4492                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
4493                             sc->bnx_rx_coal_bds);
4494                         i = 0;
4495                 } else {
4496                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, 0);
4497                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4498                                 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS +
4499                                     (i * BGE_VEC_COALSET_SIZE),
4500                                     sc->bnx_rx_coal_bds);
4501                         }
4502                 }
4503                 for (; i < BNX_INTR_MAX - 1; ++i) {
4504                         CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS +
4505                             (i * BGE_VEC_COALSET_SIZE), 0);
4506                 }
4507                 if (bootverbose) {
4508                         if_printf(ifp, "rx_coal_bds -> %u\n",
4509                             sc->bnx_rx_coal_bds);
4510                 }
4511         }
4512
4513         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) {
4514                 uint32_t tx_coal_bds;
4515
4516                 if (ifp->if_flags & IFF_NPOLLING)
4517                         tx_coal_bds = sc->bnx_tx_coal_bds_poll;
4518                 else
4519                         tx_coal_bds = sc->bnx_tx_coal_bds;
4520
4521                 if (sc->bnx_tx_ringcnt == 1) {
4522                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, tx_coal_bds);
4523                         i = 0;
4524                 } else {
4525                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, 0);
4526                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4527                                 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS +
4528                                     (i * BGE_VEC_COALSET_SIZE), tx_coal_bds);
4529                         }
4530                 }
4531                 for (; i < BNX_INTR_MAX - 1; ++i) {
4532                         CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS +
4533                             (i * BGE_VEC_COALSET_SIZE), 0);
4534                 }
4535                 if (bootverbose) {
4536                         if_printf(ifp, "%stx_coal_bds -> %u\n",
4537                             (ifp->if_flags & IFF_NPOLLING) ? "polling " : "",
4538                             tx_coal_bds);
4539                 }
4540         }
4541
4542         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) {
4543                 if (sc->bnx_rx_retcnt == 1) {
4544                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
4545                             sc->bnx_rx_coal_bds_int);
4546                         i = 0;
4547                 } else {
4548                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 0);
4549                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4550                                 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT +
4551                                     (i * BGE_VEC_COALSET_SIZE),
4552                                     sc->bnx_rx_coal_bds_int);
4553                         }
4554                 }
4555                 for (; i < BNX_INTR_MAX - 1; ++i) {
4556                         CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT +
4557                             (i * BGE_VEC_COALSET_SIZE), 0);
4558                 }
4559                 if (bootverbose) {
4560                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
4561                             sc->bnx_rx_coal_bds_int);
4562                 }
4563         }
4564
4565         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) {
4566                 if (sc->bnx_tx_ringcnt == 1) {
4567                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
4568                             sc->bnx_tx_coal_bds_int);
4569                         i = 0;
4570                 } else {
4571                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 0);
4572                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4573                                 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT +
4574                                     (i * BGE_VEC_COALSET_SIZE),
4575                                     sc->bnx_tx_coal_bds_int);
4576                         }
4577                 }
4578                 for (; i < BNX_INTR_MAX - 1; ++i) {
4579                         CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT +
4580                             (i * BGE_VEC_COALSET_SIZE), 0);
4581                 }
4582                 if (bootverbose) {
4583                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
4584                             sc->bnx_tx_coal_bds_int);
4585                 }
4586         }
4587
4588         sc->bnx_coal_chg = 0;
4589 }
4590
4591 static void
4592 bnx_check_intr_rxtx(void *xintr)
4593 {
4594         struct bnx_intr_data *intr = xintr;
4595         struct bnx_rx_ret_ring *ret;
4596         struct bnx_tx_ring *txr;
4597         struct ifnet *ifp;
4598
4599         lwkt_serialize_enter(intr->bnx_intr_serialize);
4600
4601         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4602
4603         ifp = &intr->bnx_sc->arpcom.ac_if;
4604         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4605                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4606                 return;
4607         }
4608
4609         txr = intr->bnx_txr;
4610         ret = intr->bnx_ret;
4611
4612         if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx ||
4613             *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
4614                 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx &&
4615                     intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
4616                         if (!intr->bnx_intr_maylose) {
4617                                 intr->bnx_intr_maylose = TRUE;
4618                                 goto done;
4619                         }
4620                         if (bootverbose)
4621                                 if_printf(ifp, "lost interrupt\n");
4622                         intr->bnx_intr_func(intr->bnx_intr_arg);
4623                 }
4624         }
4625         intr->bnx_intr_maylose = FALSE;
4626         intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
4627         intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
4628
4629 done:
4630         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4631             intr->bnx_intr_check, intr);
4632         lwkt_serialize_exit(intr->bnx_intr_serialize);
4633 }
4634
4635 static void
4636 bnx_check_intr_tx(void *xintr)
4637 {
4638         struct bnx_intr_data *intr = xintr;
4639         struct bnx_tx_ring *txr;
4640         struct ifnet *ifp;
4641
4642         lwkt_serialize_enter(intr->bnx_intr_serialize);
4643
4644         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4645
4646         ifp = &intr->bnx_sc->arpcom.ac_if;
4647         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4648                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4649                 return;
4650         }
4651
4652         txr = intr->bnx_txr;
4653
4654         if (*txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
4655                 if (intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
4656                         if (!intr->bnx_intr_maylose) {
4657                                 intr->bnx_intr_maylose = TRUE;
4658                                 goto done;
4659                         }
4660                         if (bootverbose)
4661                                 if_printf(ifp, "lost interrupt\n");
4662                         intr->bnx_intr_func(intr->bnx_intr_arg);
4663                 }
4664         }
4665         intr->bnx_intr_maylose = FALSE;
4666         intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
4667
4668 done:
4669         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4670             intr->bnx_intr_check, intr);
4671         lwkt_serialize_exit(intr->bnx_intr_serialize);
4672 }
4673
4674 static void
4675 bnx_check_intr_rx(void *xintr)
4676 {
4677         struct bnx_intr_data *intr = xintr;
4678         struct bnx_rx_ret_ring *ret;
4679         struct ifnet *ifp;
4680
4681         lwkt_serialize_enter(intr->bnx_intr_serialize);
4682
4683         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4684
4685         ifp = &intr->bnx_sc->arpcom.ac_if;
4686         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4687                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4688                 return;
4689         }
4690
4691         ret = intr->bnx_ret;
4692
4693         if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx) {
4694                 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx) {
4695                         if (!intr->bnx_intr_maylose) {
4696                                 intr->bnx_intr_maylose = TRUE;
4697                                 goto done;
4698                         }
4699                         if (bootverbose)
4700                                 if_printf(ifp, "lost interrupt\n");
4701                         intr->bnx_intr_func(intr->bnx_intr_arg);
4702                 }
4703         }
4704         intr->bnx_intr_maylose = FALSE;
4705         intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
4706
4707 done:
4708         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4709             intr->bnx_intr_check, intr);
4710         lwkt_serialize_exit(intr->bnx_intr_serialize);
4711 }
4712
4713 static void
4714 bnx_enable_intr(struct bnx_softc *sc)
4715 {
4716         struct ifnet *ifp = &sc->arpcom.ac_if;
4717         int i;
4718
4719         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4720                 lwkt_serialize_handler_enable(
4721                     sc->bnx_intr_data[i].bnx_intr_serialize);
4722         }
4723
4724         /*
4725          * Enable interrupt.
4726          */
4727         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4728                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4729
4730                 bnx_writembx(sc, intr->bnx_intr_mbx,
4731                     (*intr->bnx_saved_status_tag) << 24);
4732                 /* XXX Linux driver */
4733                 bnx_writembx(sc, intr->bnx_intr_mbx,
4734                     (*intr->bnx_saved_status_tag) << 24);
4735         }
4736
4737         /*
4738          * Unmask the interrupt when we stop polling.
4739          */
4740         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
4741             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4742
4743         /*
4744          * Trigger another interrupt, since above writing
4745          * to interrupt mailbox0 may acknowledge pending
4746          * interrupt.
4747          */
4748         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
4749
4750         if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) {
4751                 if (bootverbose)
4752                         if_printf(ifp, "status tag bug workaround\n");
4753
4754                 for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4755                         struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4756
4757                         if (intr->bnx_intr_check == NULL)
4758                                 continue;
4759                         intr->bnx_intr_maylose = FALSE;
4760                         intr->bnx_rx_check_considx = 0;
4761                         intr->bnx_tx_check_considx = 0;
4762                         callout_reset_bycpu(&intr->bnx_intr_timer,
4763                             BNX_INTR_CKINTVL, intr->bnx_intr_check, intr,
4764                             intr->bnx_intr_cpuid);
4765                 }
4766         }
4767 }
4768
4769 static void
4770 bnx_disable_intr(struct bnx_softc *sc)
4771 {
4772         int i;
4773
4774         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4775                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4776
4777                 callout_stop(&intr->bnx_intr_timer);
4778                 intr->bnx_intr_maylose = FALSE;
4779                 intr->bnx_rx_check_considx = 0;
4780                 intr->bnx_tx_check_considx = 0;
4781         }
4782
4783         /*
4784          * Mask the interrupt when we start polling.
4785          */
4786         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
4787             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4788
4789         /*
4790          * Acknowledge possible asserted interrupt.
4791          */
4792         for (i = 0; i < BNX_INTR_MAX; ++i)
4793                 bnx_writembx(sc, sc->bnx_intr_data[i].bnx_intr_mbx, 1);
4794
4795         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4796                 lwkt_serialize_handler_disable(
4797                     sc->bnx_intr_data[i].bnx_intr_serialize);
4798         }
4799 }
4800
4801 static int
4802 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[])
4803 {
4804         uint32_t mac_addr;
4805         int ret = 1;
4806
4807         mac_addr = bnx_readmem_ind(sc, 0x0c14);
4808         if ((mac_addr >> 16) == 0x484b) {
4809                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
4810                 ether_addr[1] = (uint8_t)mac_addr;
4811                 mac_addr = bnx_readmem_ind(sc, 0x0c18);
4812                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
4813                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
4814                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
4815                 ether_addr[5] = (uint8_t)mac_addr;
4816                 ret = 0;
4817         }
4818         return ret;
4819 }
4820
4821 static int
4822 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[])
4823 {
4824         int mac_offset = BGE_EE_MAC_OFFSET;
4825
4826         if (BNX_IS_5717_PLUS(sc)) {
4827                 int f;
4828
4829                 f = pci_get_function(sc->bnx_dev);
4830                 if (f & 1)
4831                         mac_offset = BGE_EE_MAC_OFFSET_5717;
4832                 if (f > 1)
4833                         mac_offset += BGE_EE_MAC_OFFSET_5717_OFF;
4834         }
4835
4836         return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN);
4837 }
4838
4839 static int
4840 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[])
4841 {
4842         if (sc->bnx_flags & BNX_FLAG_NO_EEPROM)
4843                 return 1;
4844
4845         return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
4846                                ETHER_ADDR_LEN);
4847 }
4848
4849 static int
4850 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[])
4851 {
4852         static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = {
4853                 /* NOTE: Order is critical */
4854                 bnx_get_eaddr_mem,
4855                 bnx_get_eaddr_nvram,
4856                 bnx_get_eaddr_eeprom,
4857                 NULL
4858         };
4859         const bnx_eaddr_fcn_t *func;
4860
4861         for (func = bnx_eaddr_funcs; *func != NULL; ++func) {
4862                 if ((*func)(sc, eaddr) == 0)
4863                         break;
4864         }
4865         return (*func == NULL ? ENXIO : 0);
4866 }
4867
4868 /*
4869  * NOTE: 'm' is not freed upon failure
4870  */
4871 struct mbuf *
4872 bnx_defrag_shortdma(struct mbuf *m)
4873 {
4874         struct mbuf *n;
4875         int found;
4876
4877         /*
4878          * If device receive two back-to-back send BDs with less than
4879          * or equal to 8 total bytes then the device may hang.  The two
4880          * back-to-back send BDs must in the same frame for this failure
4881          * to occur.  Scan mbuf chains and see whether two back-to-back
4882          * send BDs are there.  If this is the case, allocate new mbuf
4883          * and copy the frame to workaround the silicon bug.
4884          */
4885         for (n = m, found = 0; n != NULL; n = n->m_next) {
4886                 if (n->m_len < 8) {
4887                         found++;
4888                         if (found > 1)
4889                                 break;
4890                         continue;
4891                 }
4892                 found = 0;
4893         }
4894
4895         if (found > 1)
4896                 n = m_defrag(m, MB_DONTWAIT);
4897         else
4898                 n = m;
4899         return n;
4900 }
4901
4902 static void
4903 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit)
4904 {
4905         int i;
4906
4907         BNX_CLRBIT(sc, reg, bit);
4908         for (i = 0; i < BNX_TIMEOUT; i++) {
4909                 if ((CSR_READ_4(sc, reg) & bit) == 0)
4910                         return;
4911                 DELAY(100);
4912         }
4913 }
4914
4915 static void
4916 bnx_link_poll(struct bnx_softc *sc)
4917 {
4918         uint32_t status;
4919
4920         status = CSR_READ_4(sc, BGE_MAC_STS);
4921         if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) {
4922                 sc->bnx_link_evt = 0;
4923                 sc->bnx_link_upd(sc, status);
4924         }
4925 }
4926
4927 static void
4928 bnx_enable_msi(struct bnx_softc *sc, boolean_t is_msix)
4929 {
4930         uint32_t msi_mode;
4931
4932         msi_mode = CSR_READ_4(sc, BGE_MSI_MODE);
4933         msi_mode |= BGE_MSIMODE_ENABLE;
4934         /*
4935          * NOTE:
4936          * 5718-PG105-R says that "one shot" mode does not work
4937          * if MSI is used, however, it obviously works.
4938          */
4939         msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE;
4940         if (is_msix)
4941                 msi_mode |= BGE_MSIMODE_MSIX_MULTIMODE;
4942         else
4943                 msi_mode &= ~BGE_MSIMODE_MSIX_MULTIMODE;
4944         CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode);
4945 }
4946
4947 static uint32_t
4948 bnx_dma_swap_options(struct bnx_softc *sc)
4949 {
4950         uint32_t dma_options;
4951
4952         dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
4953             BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
4954 #if BYTE_ORDER == BIG_ENDIAN
4955         dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
4956 #endif
4957         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
4958             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
4959                 dma_options |= BGE_MODECTL_BYTESWAP_B2HRX_DATA |
4960                     BGE_MODECTL_WORDSWAP_B2HRX_DATA | BGE_MODECTL_B2HRX_ENABLE |
4961                     BGE_MODECTL_HTX2B_ENABLE;
4962         }
4963         return dma_options;
4964 }
4965
4966 static int
4967 bnx_setup_tso(struct bnx_tx_ring *txr, struct mbuf **mp,
4968     uint16_t *mss0, uint16_t *flags0)
4969 {
4970         struct mbuf *m;
4971         struct ip *ip;
4972         struct tcphdr *th;
4973         int thoff, iphlen, hoff, hlen;
4974         uint16_t flags, mss;
4975
4976         m = *mp;
4977         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
4978
4979         hoff = m->m_pkthdr.csum_lhlen;
4980         iphlen = m->m_pkthdr.csum_iphlen;
4981         thoff = m->m_pkthdr.csum_thlen;
4982
4983         KASSERT(hoff > 0, ("invalid ether header len"));
4984         KASSERT(iphlen > 0, ("invalid ip header len"));
4985         KASSERT(thoff > 0, ("invalid tcp header len"));
4986
4987         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
4988                 m = m_pullup(m, hoff + iphlen + thoff);
4989                 if (m == NULL) {
4990                         *mp = NULL;
4991                         return ENOBUFS;
4992                 }
4993                 *mp = m;
4994         }
4995         ip = mtodoff(m, struct ip *, hoff);
4996         th = mtodoff(m, struct tcphdr *, hoff + iphlen);
4997
4998         mss = m->m_pkthdr.tso_segsz;
4999         flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA;
5000
5001         ip->ip_len = htons(mss + iphlen + thoff);
5002         th->th_sum = 0;
5003
5004         hlen = (iphlen + thoff) >> 2;
5005         mss |= ((hlen & 0x3) << 14);
5006         flags |= ((hlen & 0xf8) << 7) | ((hlen & 0x4) << 2);
5007
5008         *mss0 = mss;
5009         *flags0 = flags;
5010
5011         return 0;
5012 }
5013
5014 static int
5015 bnx_create_tx_ring(struct bnx_tx_ring *txr)
5016 {
5017         bus_size_t txmaxsz, txmaxsegsz;
5018         int i, error;
5019
5020         lwkt_serialize_init(&txr->bnx_tx_serialize);
5021
5022         /*
5023          * Create DMA tag and maps for TX mbufs.
5024          */
5025         if (txr->bnx_sc->bnx_flags & BNX_FLAG_TSO)
5026                 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header);
5027         else
5028                 txmaxsz = BNX_JUMBO_FRAMELEN;
5029         if (txr->bnx_sc->bnx_asicrev == BGE_ASICREV_BCM57766)
5030                 txmaxsegsz = MCLBYTES;
5031         else
5032                 txmaxsegsz = PAGE_SIZE;
5033         error = bus_dma_tag_create(txr->bnx_sc->bnx_cdata.bnx_parent_tag,
5034             1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
5035             txmaxsz, BNX_NSEG_NEW, txmaxsegsz,
5036             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
5037             &txr->bnx_tx_mtag);
5038         if (error) {
5039                 device_printf(txr->bnx_sc->bnx_dev,
5040                     "could not create TX mbuf DMA tag\n");
5041                 return error;
5042         }
5043
5044         for (i = 0; i < BGE_TX_RING_CNT; i++) {
5045                 error = bus_dmamap_create(txr->bnx_tx_mtag,
5046                     BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
5047                     &txr->bnx_tx_buf[i].bnx_tx_dmamap);
5048                 if (error) {
5049                         int j;
5050
5051                         for (j = 0; j < i; ++j) {
5052                                 bus_dmamap_destroy(txr->bnx_tx_mtag,
5053                                     txr->bnx_tx_buf[j].bnx_tx_dmamap);
5054                         }
5055                         bus_dma_tag_destroy(txr->bnx_tx_mtag);
5056                         txr->bnx_tx_mtag = NULL;
5057
5058                         device_printf(txr->bnx_sc->bnx_dev,
5059                             "could not create TX mbuf DMA map\n");
5060                         return error;
5061                 }
5062         }
5063
5064         /*
5065          * Create DMA stuffs for TX ring.
5066          */
5067         error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ,
5068             &txr->bnx_tx_ring_tag,
5069             &txr->bnx_tx_ring_map,
5070             (void *)&txr->bnx_tx_ring,
5071             &txr->bnx_tx_ring_paddr);
5072         if (error) {
5073                 device_printf(txr->bnx_sc->bnx_dev,
5074                     "could not create TX ring\n");
5075                 return error;
5076         }
5077
5078         txr->bnx_tx_flags |= BNX_TX_FLAG_SHORTDMA;
5079         txr->bnx_tx_wreg = BNX_TX_WREG_NSEGS;
5080
5081         return 0;
5082 }
5083
5084 static void
5085 bnx_destroy_tx_ring(struct bnx_tx_ring *txr)
5086 {
5087         /* Destroy TX mbuf DMA stuffs. */
5088         if (txr->bnx_tx_mtag != NULL) {
5089                 int i;
5090
5091                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
5092                         KKASSERT(txr->bnx_tx_buf[i].bnx_tx_mbuf == NULL);
5093                         bus_dmamap_destroy(txr->bnx_tx_mtag,
5094                             txr->bnx_tx_buf[i].bnx_tx_dmamap);
5095                 }
5096                 bus_dma_tag_destroy(txr->bnx_tx_mtag);
5097         }
5098
5099         /* Destroy TX ring */
5100         bnx_dma_block_free(txr->bnx_tx_ring_tag,
5101             txr->bnx_tx_ring_map, txr->bnx_tx_ring);
5102 }
5103
5104 static int
5105 bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
5106 {
5107         struct bnx_softc *sc = (void *)arg1;
5108         struct ifnet *ifp = &sc->arpcom.ac_if;
5109         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
5110         int error, defrag, i;
5111
5112         if (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG)
5113                 defrag = 1;
5114         else
5115                 defrag = 0;
5116
5117         error = sysctl_handle_int(oidp, &defrag, 0, req);
5118         if (error || req->newptr == NULL)
5119                 return error;
5120
5121         ifnet_serialize_all(ifp);
5122         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
5123                 txr = &sc->bnx_tx_ring[i];
5124                 if (defrag)
5125                         txr->bnx_tx_flags |= BNX_TX_FLAG_FORCE_DEFRAG;
5126                 else
5127                         txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG;
5128         }
5129         ifnet_deserialize_all(ifp);
5130
5131         return 0;
5132 }
5133
5134 static int
5135 bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS)
5136 {
5137         struct bnx_softc *sc = (void *)arg1;
5138         struct ifnet *ifp = &sc->arpcom.ac_if;
5139         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
5140         int error, tx_wreg, i;
5141
5142         tx_wreg = txr->bnx_tx_wreg;
5143         error = sysctl_handle_int(oidp, &tx_wreg, 0, req);
5144         if (error || req->newptr == NULL)
5145                 return error;
5146
5147         ifnet_serialize_all(ifp);
5148         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
5149                 sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg;
5150         ifnet_deserialize_all(ifp);
5151
5152         return 0;
5153 }
5154
5155 static int
5156 bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret)
5157 {
5158         int error;
5159
5160         lwkt_serialize_init(&ret->bnx_rx_ret_serialize);
5161
5162         /*
5163          * Create DMA stuffs for RX return ring.
5164          */
5165         error = bnx_dma_block_alloc(ret->bnx_sc,
5166             BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT),
5167             &ret->bnx_rx_ret_ring_tag,
5168             &ret->bnx_rx_ret_ring_map,
5169             (void *)&ret->bnx_rx_ret_ring,
5170             &ret->bnx_rx_ret_ring_paddr);
5171         if (error) {
5172                 device_printf(ret->bnx_sc->bnx_dev,
5173                     "could not create RX ret ring\n");
5174                 return error;
5175         }
5176
5177         /* Shadow standard ring's RX mbuf DMA tag */
5178         ret->bnx_rx_mtag = ret->bnx_std->bnx_rx_mtag;
5179
5180         /*
5181          * Create tmp DMA map for RX mbufs.
5182          */
5183         error = bus_dmamap_create(ret->bnx_rx_mtag, BUS_DMA_WAITOK,
5184             &ret->bnx_rx_tmpmap);
5185         if (error) {
5186                 device_printf(ret->bnx_sc->bnx_dev,
5187                     "could not create tmp RX mbuf DMA map\n");
5188                 ret->bnx_rx_mtag = NULL;
5189                 return error;
5190         }
5191         return 0;
5192 }
5193
5194 static void
5195 bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret)
5196 {
5197         /* Destroy tmp RX mbuf DMA map */
5198         if (ret->bnx_rx_mtag != NULL)
5199                 bus_dmamap_destroy(ret->bnx_rx_mtag, ret->bnx_rx_tmpmap);
5200
5201         /* Destroy RX return ring */
5202         bnx_dma_block_free(ret->bnx_rx_ret_ring_tag,
5203             ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring);
5204 }
5205
5206 static int
5207 bnx_alloc_intr(struct bnx_softc *sc)
5208 {
5209         struct bnx_intr_data *intr;
5210         u_int intr_flags;
5211         int error;
5212
5213         if (sc->bnx_intr_cnt > 1) {
5214                 error = bnx_alloc_msix(sc);
5215                 if (error)
5216                         return error;
5217                 KKASSERT(sc->bnx_intr_type == PCI_INTR_TYPE_MSIX);
5218                 return 0;
5219         }
5220
5221         KKASSERT(sc->bnx_intr_cnt == 1);
5222
5223         intr = &sc->bnx_intr_data[0];
5224         intr->bnx_ret = &sc->bnx_rx_ret_ring[0];
5225         intr->bnx_txr = &sc->bnx_tx_ring[0];
5226         intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5227         intr->bnx_intr_check = bnx_check_intr_rxtx;
5228         intr->bnx_saved_status_tag = &intr->bnx_ret->bnx_saved_status_tag;
5229
5230         sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable,
5231             &intr->bnx_intr_rid, &intr_flags);
5232
5233         intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ,
5234             &intr->bnx_intr_rid, intr_flags);
5235         if (intr->bnx_intr_res == NULL) {
5236                 device_printf(sc->bnx_dev, "could not alloc interrupt\n");
5237                 return ENXIO;
5238         }
5239
5240         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
5241                 bnx_enable_msi(sc, FALSE);
5242                 intr->bnx_intr_func = bnx_msi;
5243                 if (bootverbose)
5244                         device_printf(sc->bnx_dev, "oneshot MSI\n");
5245         } else {
5246                 intr->bnx_intr_func = bnx_intr_legacy;
5247         }
5248         intr->bnx_intr_arg = sc;
5249         intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res);
5250
5251         intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
5252
5253         return 0;
5254 }
5255
5256 static int
5257 bnx_setup_intr(struct bnx_softc *sc)
5258 {
5259         int error, i;
5260
5261         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
5262                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
5263
5264                 error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res,
5265                     INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg,
5266                     &intr->bnx_intr_hand, intr->bnx_intr_serialize,
5267                     intr->bnx_intr_desc);
5268                 if (error) {
5269                         device_printf(sc->bnx_dev,
5270                             "could not set up %dth intr\n", i);
5271                         bnx_teardown_intr(sc, i);
5272                         return error;
5273                 }
5274         }
5275         return 0;
5276 }
5277
5278 static void
5279 bnx_teardown_intr(struct bnx_softc *sc, int cnt)
5280 {
5281         int i;
5282
5283         for (i = 0; i < cnt; ++i) {
5284                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
5285
5286                 bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res,
5287                     intr->bnx_intr_hand);
5288         }
5289 }
5290
5291 static void
5292 bnx_free_intr(struct bnx_softc *sc)
5293 {
5294         if (sc->bnx_intr_type != PCI_INTR_TYPE_MSIX) {
5295                 struct bnx_intr_data *intr;
5296
5297                 KKASSERT(sc->bnx_intr_cnt <= 1);
5298                 intr = &sc->bnx_intr_data[0];
5299
5300                 if (intr->bnx_intr_res != NULL) {
5301                         bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
5302                             intr->bnx_intr_rid, intr->bnx_intr_res);
5303                 }
5304                 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI)
5305                         pci_release_msi(sc->bnx_dev);
5306         } else {
5307                 bnx_free_msix(sc, TRUE);
5308         }
5309 }
5310
5311 static void
5312 bnx_setup_serialize(struct bnx_softc *sc)
5313 {
5314         int i, j;
5315
5316         /*
5317          * Allocate serializer array
5318          */
5319
5320         /* Main + RX STD + TX + RX RET */
5321         sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt;
5322
5323         sc->bnx_serialize =
5324             kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *),
5325                 M_DEVBUF, M_WAITOK | M_ZERO);
5326
5327         /*
5328          * Setup serializers
5329          *
5330          * NOTE: Order is critical
5331          */
5332
5333         i = 0;
5334
5335         KKASSERT(i < sc->bnx_serialize_cnt);
5336         sc->bnx_serialize[i++] = &sc->bnx_main_serialize;
5337
5338         KKASSERT(i < sc->bnx_serialize_cnt);
5339         sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize;
5340
5341         for (j = 0; j < sc->bnx_rx_retcnt; ++j) {
5342                 KKASSERT(i < sc->bnx_serialize_cnt);
5343                 sc->bnx_serialize[i++] =
5344                     &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize;
5345         }
5346
5347         for (j = 0; j < sc->bnx_tx_ringcnt; ++j) {
5348                 KKASSERT(i < sc->bnx_serialize_cnt);
5349                 sc->bnx_serialize[i++] =
5350                     &sc->bnx_tx_ring[j].bnx_tx_serialize;
5351         }
5352
5353         KKASSERT(i == sc->bnx_serialize_cnt);
5354 }
5355
5356 static void
5357 bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
5358 {
5359         struct bnx_softc *sc = ifp->if_softc;
5360
5361         ifnet_serialize_array_enter(sc->bnx_serialize,
5362             sc->bnx_serialize_cnt, slz);
5363 }
5364
5365 static void
5366 bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
5367 {
5368         struct bnx_softc *sc = ifp->if_softc;
5369
5370         ifnet_serialize_array_exit(sc->bnx_serialize,
5371             sc->bnx_serialize_cnt, slz);
5372 }
5373
5374 static int
5375 bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
5376 {
5377         struct bnx_softc *sc = ifp->if_softc;
5378
5379         return ifnet_serialize_array_try(sc->bnx_serialize,
5380             sc->bnx_serialize_cnt, slz);
5381 }
5382
5383 #ifdef INVARIANTS
5384
5385 static void
5386 bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
5387     boolean_t serialized)
5388 {
5389         struct bnx_softc *sc = ifp->if_softc;
5390
5391         ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt,
5392             slz, serialized);
5393 }
5394
5395 #endif  /* INVARIANTS */
5396
5397 #ifdef IFPOLL_ENABLE
5398
5399 static int
5400 bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS)
5401 {
5402         struct bnx_softc *sc = (void *)arg1;
5403         struct ifnet *ifp = &sc->arpcom.ac_if;
5404         int error, off;
5405
5406         off = sc->bnx_npoll_rxoff;
5407         error = sysctl_handle_int(oidp, &off, 0, req);
5408         if (error || req->newptr == NULL)
5409                 return error;
5410         if (off < 0)
5411                 return EINVAL;
5412
5413         ifnet_serialize_all(ifp);
5414         if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) {
5415                 error = EINVAL;
5416         } else {
5417                 error = 0;
5418                 sc->bnx_npoll_txoff = off;
5419                 sc->bnx_npoll_rxoff = off;
5420         }
5421         ifnet_deserialize_all(ifp);
5422
5423         return error;
5424 }
5425
5426 static int
5427 bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS)
5428 {
5429         struct bnx_softc *sc = (void *)arg1;
5430         struct ifnet *ifp = &sc->arpcom.ac_if;
5431         int error, off;
5432
5433         off = sc->bnx_npoll_rxoff;
5434         error = sysctl_handle_int(oidp, &off, 0, req);
5435         if (error || req->newptr == NULL)
5436                 return error;
5437         if (off < 0)
5438                 return EINVAL;
5439
5440         ifnet_serialize_all(ifp);
5441         if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) {
5442                 error = EINVAL;
5443         } else {
5444                 error = 0;
5445                 sc->bnx_npoll_rxoff = off;
5446         }
5447         ifnet_deserialize_all(ifp);
5448
5449         return error;
5450 }
5451
5452 static int
5453 bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS)
5454 {
5455         struct bnx_softc *sc = (void *)arg1;
5456         struct ifnet *ifp = &sc->arpcom.ac_if;
5457         int error, off;
5458
5459         off = sc->bnx_npoll_txoff;
5460         error = sysctl_handle_int(oidp, &off, 0, req);
5461         if (error || req->newptr == NULL)
5462                 return error;
5463         if (off < 0)
5464                 return EINVAL;
5465
5466         ifnet_serialize_all(ifp);
5467         if (off >= ncpus2) {
5468                 error = EINVAL;
5469         } else {
5470                 error = 0;
5471                 sc->bnx_npoll_txoff = off;
5472         }
5473         ifnet_deserialize_all(ifp);
5474
5475         return error;
5476 }
5477
5478 #endif  /* IFPOLL_ENABLE */
5479
5480 static void
5481 bnx_set_tick_cpuid(struct bnx_softc *sc, boolean_t polling)
5482 {
5483         if (polling)
5484                 sc->bnx_tick_cpuid = 0; /* XXX */
5485         else
5486                 sc->bnx_tick_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid;
5487 }
5488
5489 static void
5490 bnx_rx_std_refill_ithread(void *xstd)
5491 {
5492         struct bnx_rx_std_ring *std = xstd;
5493         struct globaldata *gd = mycpu;
5494
5495         crit_enter_gd(gd);
5496
5497         while (!std->bnx_rx_std_stop) {
5498                 if (std->bnx_rx_std_refill) {
5499                         lwkt_serialize_handler_call(
5500                             &std->bnx_rx_std_serialize,
5501                             bnx_rx_std_refill, std, NULL);
5502                 }
5503
5504                 crit_exit_gd(gd);
5505                 crit_enter_gd(gd);
5506
5507                 atomic_poll_release_int(&std->bnx_rx_std_running);
5508                 cpu_mfence();
5509
5510                 if (!std->bnx_rx_std_refill && !std->bnx_rx_std_stop) {
5511                         lwkt_deschedule_self(gd->gd_curthread);
5512                         lwkt_switch();
5513                 }
5514         }
5515
5516         crit_exit_gd(gd);
5517
5518         wakeup(std);
5519
5520         lwkt_exit();
5521 }
5522
5523 static void
5524 bnx_rx_std_refill(void *xstd, void *frame __unused)
5525 {
5526         struct bnx_rx_std_ring *std = xstd;
5527         int cnt, refill_mask;
5528
5529 again:
5530         cnt = 0;
5531
5532         cpu_lfence();
5533         refill_mask = std->bnx_rx_std_refill;
5534         atomic_clear_int(&std->bnx_rx_std_refill, refill_mask);
5535
5536         while (refill_mask) {
5537                 uint16_t check_idx = std->bnx_rx_std;
5538                 int ret_idx;
5539
5540                 ret_idx = bsfl(refill_mask);
5541                 for (;;) {
5542                         struct bnx_rx_buf *rb;
5543                         int refilled;
5544
5545                         BNX_INC(check_idx, BGE_STD_RX_RING_CNT);
5546                         rb = &std->bnx_rx_std_buf[check_idx];
5547                         refilled = rb->bnx_rx_refilled;
5548                         cpu_lfence();
5549                         if (refilled) {
5550                                 bnx_setup_rxdesc_std(std, check_idx);
5551                                 std->bnx_rx_std = check_idx;
5552                                 ++cnt;
5553                                 if (cnt >= 8) {
5554                                         bnx_writembx(std->bnx_sc,
5555                                             BGE_MBX_RX_STD_PROD_LO,
5556                                             std->bnx_rx_std);
5557                                         cnt = 0;
5558                                 }
5559                         } else {
5560                                 break;
5561                         }
5562                 }
5563                 refill_mask &= ~(1 << ret_idx);
5564         }
5565
5566         if (cnt) {
5567                 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO,
5568                     std->bnx_rx_std);
5569         }
5570
5571         if (std->bnx_rx_std_refill)
5572                 goto again;
5573
5574         atomic_poll_release_int(&std->bnx_rx_std_running);
5575         cpu_mfence();
5576
5577         if (std->bnx_rx_std_refill)
5578                 goto again;
5579 }
5580
5581 static int
5582 bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS)
5583 {
5584         struct bnx_softc *sc = (void *)arg1;
5585         struct ifnet *ifp = &sc->arpcom.ac_if;
5586         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
5587         int error, cntmax, i;
5588
5589         cntmax = ret->bnx_rx_cntmax;
5590         error = sysctl_handle_int(oidp, &cntmax, 0, req);
5591         if (error || req->newptr == NULL)
5592                 return error;
5593
5594         ifnet_serialize_all(ifp);
5595
5596         if ((cntmax * sc->bnx_rx_retcnt) > BGE_STD_RX_RING_CNT / 2) {
5597                 error = EINVAL;
5598                 goto back;
5599         }
5600
5601         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
5602                 sc->bnx_rx_ret_ring[i].bnx_rx_cntmax = cntmax;
5603         error = 0;
5604
5605 back:
5606         ifnet_deserialize_all(ifp);
5607
5608         return error;
5609 }
5610
5611 static void
5612 bnx_init_rss(struct bnx_softc *sc)
5613 {
5614         uint8_t key[BGE_RSS_KEYREG_CNT * BGE_RSS_KEYREG_SIZE];
5615         int i, j, r;
5616
5617         KKASSERT(BNX_RSS_ENABLED(sc));
5618
5619         r = 0;
5620         for (j = 0; j < BGE_RSS_INDIR_TBL_CNT; ++j) {
5621                 uint32_t tbl = 0;
5622
5623                 for (i = 0; i < BGE_RSS_INDIR_TBLENT_CNT; ++i) {
5624                         uint32_t q;
5625
5626                         q = r % sc->bnx_rx_retcnt;
5627                         tbl |= q << (BGE_RSS_INDIR_TBLENT_SHIFT *
5628                             (BGE_RSS_INDIR_TBLENT_CNT - i - 1));
5629                         ++r;
5630                 }
5631
5632                 BNX_RSS_DPRINTF(sc, 1, "tbl%d %08x\n", j, tbl);
5633                 CSR_WRITE_4(sc, BGE_RSS_INDIR_TBL(j), tbl);
5634         }
5635
5636         toeplitz_get_key(key, sizeof(key));
5637         for (i = 0; i < BGE_RSS_KEYREG_CNT; ++i) {
5638                 uint32_t keyreg;
5639
5640                 keyreg = BGE_RSS_KEYREG_VAL(key, i);
5641
5642                 BNX_RSS_DPRINTF(sc, 1, "key%d %08x\n", i, keyreg);
5643                 CSR_WRITE_4(sc, BGE_RSS_KEYREG(i), keyreg);
5644         }
5645 }
5646
5647 static void
5648 bnx_setup_ring_cnt(struct bnx_softc *sc)
5649 {
5650         int msix_enable, i, msix_cnt, msix_cnt2, ring_max;
5651
5652         sc->bnx_tx_ringcnt = 1;
5653         sc->bnx_rx_retcnt = 1;
5654         sc->bnx_intr_cnt = 1;
5655
5656         msix_enable = device_getenv_int(sc->bnx_dev, "msix.enable",
5657             bnx_msix_enable);
5658         if (!msix_enable)
5659                 return;
5660
5661         if (ncpus2 == 1)
5662                 return;
5663
5664         msix_cnt = pci_msix_count(sc->bnx_dev);
5665         if (msix_cnt <= 1)
5666                 return;
5667
5668         i = 0;
5669         while ((1 << (i + 1)) <= msix_cnt)
5670                 ++i;
5671         msix_cnt2 = 1 << i;
5672
5673         /*
5674          * One MSI-X vector is dedicated to status or single TX queue,
5675          * so make sure that there are enough MSI-X vectors.
5676          */
5677         if (msix_cnt == msix_cnt2) {
5678                 /*
5679                  * XXX
5680                  * This probably will not happen; 57785/5718 families
5681                  * come with at least 5 MSI-X vectors.
5682                  */
5683                 msix_cnt2 >>= 1;
5684                 if (msix_cnt2 <= 1) {
5685                         device_printf(sc->bnx_dev,
5686                             "MSI-X count %d could not be used\n", msix_cnt);
5687                         return;
5688                 }
5689                 device_printf(sc->bnx_dev, "MSI-X count %d is power of 2\n",
5690                     msix_cnt);
5691         }
5692
5693         /*
5694          * Setup RX ring count
5695          */
5696         ring_max = BNX_RX_RING_MAX;
5697         if (ring_max > msix_cnt2)
5698                 ring_max = msix_cnt2;
5699         sc->bnx_rx_retcnt = device_getenv_int(sc->bnx_dev, "rx_rings",
5700             bnx_rx_rings);
5701         sc->bnx_rx_retcnt = if_ring_count2(sc->bnx_rx_retcnt, ring_max);
5702
5703         if (sc->bnx_rx_retcnt == 1)
5704                 return;
5705
5706         /*
5707          * We need one extra MSI-X vector for link status or
5708          * TX ring (if only one TX ring is enabled).
5709          */
5710         sc->bnx_intr_cnt = sc->bnx_rx_retcnt + 1;
5711
5712         /*
5713          * Setup TX ring count
5714          *
5715          * Currently only BCM5719 and BCM5720 support multiple TX rings
5716          * and the TX ring count must be less than the RX ring count.
5717          */
5718         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
5719             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
5720                 ring_max = BNX_TX_RING_MAX;
5721                 if (ring_max > msix_cnt2)
5722                         ring_max = msix_cnt2;
5723                 if (ring_max > sc->bnx_rx_retcnt)
5724                         ring_max = sc->bnx_rx_retcnt;
5725                 sc->bnx_tx_ringcnt = device_getenv_int(sc->bnx_dev, "tx_rings",
5726                     bnx_tx_rings);
5727                 sc->bnx_tx_ringcnt = if_ring_count2(sc->bnx_tx_ringcnt,
5728                     ring_max);
5729         }
5730 }
5731
5732 static int
5733 bnx_alloc_msix(struct bnx_softc *sc)
5734 {
5735         struct bnx_intr_data *intr;
5736         boolean_t setup = FALSE;
5737         int error, i, offset, offset_def;
5738
5739         KKASSERT(sc->bnx_intr_cnt > 1);
5740         KKASSERT(sc->bnx_intr_cnt == sc->bnx_rx_retcnt + 1);
5741
5742         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
5743                 /*
5744                  * Link status
5745                  */
5746                 intr = &sc->bnx_intr_data[0];
5747
5748                 intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5749                 intr->bnx_saved_status_tag = &sc->bnx_saved_status_tag;
5750
5751                 intr->bnx_intr_func = bnx_msix_status;
5752                 intr->bnx_intr_arg = sc;
5753                 intr->bnx_intr_cpuid = 0; /* XXX */
5754
5755                 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0),
5756                     "%s sts", device_get_nameunit(sc->bnx_dev));
5757                 intr->bnx_intr_desc = intr->bnx_intr_desc0;
5758
5759                 /*
5760                  * RX/TX rings
5761                  */
5762                 if (sc->bnx_rx_retcnt == ncpus2) {
5763                         offset = 0;
5764                 } else {
5765                         offset_def = (sc->bnx_rx_retcnt *
5766                             device_get_unit(sc->bnx_dev)) % ncpus2;
5767
5768                         offset = device_getenv_int(sc->bnx_dev,
5769                             "msix.offset", offset_def);
5770                         if (offset >= ncpus2 ||
5771                             offset % sc->bnx_rx_retcnt != 0) {
5772                                 device_printf(sc->bnx_dev,
5773                                     "invalid msix.offset %d, use %d\n",
5774                                     offset, offset_def);
5775                                 offset = offset_def;
5776                         }
5777                 }
5778
5779                 for (i = 1; i < sc->bnx_intr_cnt; ++i) {
5780                         int idx = i - 1;
5781
5782                         intr = &sc->bnx_intr_data[i];
5783
5784                         KKASSERT(idx < sc->bnx_rx_retcnt);
5785                         intr->bnx_ret = &sc->bnx_rx_ret_ring[idx];
5786                         if (idx < sc->bnx_tx_ringcnt) {
5787                                 intr->bnx_txr = &sc->bnx_tx_ring[idx];
5788                                 intr->bnx_ret->bnx_txr = intr->bnx_txr;
5789                         }
5790
5791                         intr->bnx_intr_serialize =
5792                             &intr->bnx_ret->bnx_rx_ret_serialize;
5793                         intr->bnx_saved_status_tag =
5794                             &intr->bnx_ret->bnx_saved_status_tag;
5795
5796                         intr->bnx_intr_arg = intr->bnx_ret;
5797                         KKASSERT(idx + offset < ncpus2);
5798                         intr->bnx_intr_cpuid = idx + offset;
5799
5800                         if (intr->bnx_txr == NULL) {
5801                                 intr->bnx_intr_check = bnx_check_intr_rx;
5802                                 intr->bnx_intr_func = bnx_msix_rx;
5803                                 ksnprintf(intr->bnx_intr_desc0,
5804                                     sizeof(intr->bnx_intr_desc0), "%s rx%d",
5805                                     device_get_nameunit(sc->bnx_dev), idx);
5806                         } else {
5807                                 intr->bnx_intr_check = bnx_check_intr_rxtx;
5808                                 intr->bnx_intr_func = bnx_msix_rxtx;
5809                                 ksnprintf(intr->bnx_intr_desc0,
5810                                     sizeof(intr->bnx_intr_desc0), "%s rxtx%d",
5811                                     device_get_nameunit(sc->bnx_dev), idx);
5812
5813                                 intr->bnx_txr->bnx_tx_cpuid =
5814                                     intr->bnx_intr_cpuid;
5815                         }
5816                         intr->bnx_intr_desc = intr->bnx_intr_desc0;
5817
5818                         intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx;
5819                 }
5820         } else {
5821                 /*
5822                  * TX ring and link status
5823                  */
5824                 offset_def = device_get_unit(sc->bnx_dev) % ncpus2;
5825                 offset = device_getenv_int(sc->bnx_dev, "msix.txoff",
5826                     offset_def);
5827                 if (offset >= ncpus2) {
5828                         device_printf(sc->bnx_dev,
5829                             "invalid msix.txoff %d, use %d\n",
5830                             offset, offset_def);
5831                         offset = offset_def;
5832                 }
5833
5834                 intr = &sc->bnx_intr_data[0];
5835
5836                 intr->bnx_txr = &sc->bnx_tx_ring[0];
5837                 intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5838                 intr->bnx_intr_check = bnx_check_intr_tx;
5839                 intr->bnx_saved_status_tag =
5840                     &intr->bnx_txr->bnx_saved_status_tag;
5841
5842                 intr->bnx_intr_func = bnx_msix_tx_status;
5843                 intr->bnx_intr_arg = intr->bnx_txr;
5844                 intr->bnx_intr_cpuid = offset;
5845
5846                 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0),
5847                     "%s ststx", device_get_nameunit(sc->bnx_dev));
5848                 intr->bnx_intr_desc = intr->bnx_intr_desc0;
5849
5850                 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
5851
5852                 /*
5853                  * RX rings
5854                  */
5855                 if (sc->bnx_rx_retcnt == ncpus2) {
5856                         offset = 0;
5857                 } else {
5858                         offset_def = (sc->bnx_rx_retcnt *
5859                             device_get_unit(sc->bnx_dev)) % ncpus2;
5860
5861                         offset = device_getenv_int(sc->bnx_dev,
5862                             "msix.rxoff", offset_def);
5863                         if (offset >= ncpus2 ||
5864                             offset % sc->bnx_rx_retcnt != 0) {
5865                                 device_printf(sc->bnx_dev,
5866                                     "invalid msix.rxoff %d, use %d\n",
5867                                     offset, offset_def);
5868                                 offset = offset_def;
5869                         }
5870                 }
5871
5872                 for (i = 1; i < sc->bnx_intr_cnt; ++i) {
5873                         int idx = i - 1;
5874
5875                         intr = &sc->bnx_intr_data[i];
5876
5877                         KKASSERT(idx < sc->bnx_rx_retcnt);
5878                         intr->bnx_ret = &sc->bnx_rx_ret_ring[idx];
5879                         intr->bnx_intr_serialize =
5880                             &intr->bnx_ret->bnx_rx_ret_serialize;
5881                         intr->bnx_intr_check = bnx_check_intr_rx;
5882                         intr->bnx_saved_status_tag =
5883                             &intr->bnx_ret->bnx_saved_status_tag;
5884
5885                         intr->bnx_intr_func = bnx_msix_rx;
5886                         intr->bnx_intr_arg = intr->bnx_ret;
5887                         KKASSERT(idx + offset < ncpus2);
5888                         intr->bnx_intr_cpuid = idx + offset;
5889
5890                         ksnprintf(intr->bnx_intr_desc0,
5891                             sizeof(intr->bnx_intr_desc0), "%s rx%d",
5892                             device_get_nameunit(sc->bnx_dev), idx);
5893                         intr->bnx_intr_desc = intr->bnx_intr_desc0;
5894
5895                         intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx;
5896                 }
5897         }
5898
5899         sc->bnx_msix_mem_rid = PCIR_BAR(4);
5900         sc->bnx_msix_mem_res = bus_alloc_resource_any(sc->bnx_dev,
5901             SYS_RES_MEMORY, &sc->bnx_msix_mem_rid, RF_ACTIVE);
5902         if (sc->bnx_msix_mem_res == NULL) {
5903                 device_printf(sc->bnx_dev, "could not alloc MSI-X table\n");
5904                 return ENXIO;
5905         }
5906
5907         bnx_enable_msi(sc, TRUE);
5908
5909         error = pci_setup_msix(sc->bnx_dev);
5910         if (error) {
5911                 device_printf(sc->bnx_dev, "could not setup MSI-X\n");
5912                 goto back;
5913         }
5914         setup = TRUE;
5915
5916         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
5917                 intr = &sc->bnx_intr_data[i];
5918
5919                 error = pci_alloc_msix_vector(sc->bnx_dev, i,
5920                     &intr->bnx_intr_rid, intr->bnx_intr_cpuid);
5921                 if (error) {
5922                         device_printf(sc->bnx_dev,
5923                             "could not alloc MSI-X %d on cpu%d\n",
5924                             i, intr->bnx_intr_cpuid);
5925                         goto back;
5926                 }
5927
5928                 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev,
5929                     SYS_RES_IRQ, &intr->bnx_intr_rid, RF_ACTIVE);
5930                 if (intr->bnx_intr_res == NULL) {
5931                         device_printf(sc->bnx_dev,
5932                             "could not alloc MSI-X %d resource\n", i);
5933                         error = ENXIO;
5934                         goto back;
5935                 }
5936         }
5937
5938         pci_enable_msix(sc->bnx_dev);
5939         sc->bnx_intr_type = PCI_INTR_TYPE_MSIX;
5940 back:
5941         if (error)
5942                 bnx_free_msix(sc, setup);
5943         return error;
5944 }
5945
5946 static void
5947 bnx_free_msix(struct bnx_softc *sc, boolean_t setup)
5948 {
5949         int i;
5950
5951         KKASSERT(sc->bnx_intr_cnt > 1);
5952
5953         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
5954                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
5955
5956                 if (intr->bnx_intr_res != NULL) {
5957                         bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
5958                             intr->bnx_intr_rid, intr->bnx_intr_res);
5959                 }
5960                 if (intr->bnx_intr_rid >= 0) {
5961                         pci_release_msix_vector(sc->bnx_dev,
5962                             intr->bnx_intr_rid);
5963                 }
5964         }
5965         if (setup)
5966                 pci_teardown_msix(sc->bnx_dev);
5967 }
5968
5969 static void
5970 bnx_rx_std_refill_sched_ipi(void *xret)
5971 {
5972         struct bnx_rx_ret_ring *ret = xret;
5973         struct bnx_rx_std_ring *std = ret->bnx_std;
5974         struct globaldata *gd = mycpu;
5975
5976         crit_enter_gd(gd);
5977
5978         atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask);
5979         cpu_sfence();
5980
5981         KKASSERT(std->bnx_rx_std_ithread.td_gd == gd);
5982         lwkt_schedule(&std->bnx_rx_std_ithread);
5983
5984         crit_exit_gd(gd);
5985 }
5986
5987 static void
5988 bnx_rx_std_refill_stop(void *xstd)
5989 {
5990         struct bnx_rx_std_ring *std = xstd;
5991         struct globaldata *gd = mycpu;
5992
5993         crit_enter_gd(gd);
5994
5995         std->bnx_rx_std_stop = 1;
5996         cpu_sfence();
5997
5998         KKASSERT(std->bnx_rx_std_ithread.td_gd == gd);
5999         lwkt_schedule(&std->bnx_rx_std_ithread);
6000
6001         crit_exit_gd(gd);
6002 }
6003
6004 static void
6005 bnx_serialize_skipmain(struct bnx_softc *sc)
6006 {
6007         lwkt_serialize_array_enter(sc->bnx_serialize,
6008             sc->bnx_serialize_cnt, 1);
6009 }
6010
6011 static void
6012 bnx_deserialize_skipmain(struct bnx_softc *sc)
6013 {
6014         lwkt_serialize_array_exit(sc->bnx_serialize,
6015             sc->bnx_serialize_cnt, 1);
6016 }
6017
6018 static void
6019 bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *ret,
6020     struct bnx_rx_std_ring *std)
6021 {
6022         struct globaldata *gd = mycpu;
6023
6024         ret->bnx_rx_cnt = 0;
6025         cpu_sfence();
6026
6027         crit_enter_gd(gd);
6028
6029         atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask);
6030         cpu_sfence();
6031         if (atomic_poll_acquire_int(&std->bnx_rx_std_running)) {
6032                 if (std->bnx_rx_std_ithread.td_gd == gd) {
6033                         lwkt_schedule(&std->bnx_rx_std_ithread);
6034                 } else {
6035                         lwkt_send_ipiq(
6036                             std->bnx_rx_std_ithread.td_gd,
6037                             bnx_rx_std_refill_sched_ipi, ret);
6038                 }
6039         }
6040
6041         crit_exit_gd(gd);
6042 }
6043
6044 static struct pktinfo *
6045 bnx_rss_info(struct pktinfo *pi, const struct bge_rx_bd *cur_rx)
6046 {
6047         /* Don't pick up IPv6 packet */
6048         if (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6)
6049                 return NULL;
6050
6051         /* Don't pick up IP packet w/o IP checksum */
6052         if ((cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) == 0 ||
6053             (cur_rx->bge_error_flag & BGE_RXERRFLAG_IP_CSUM_NOK))
6054                 return NULL;
6055
6056         /* Don't pick up IP packet w/o TCP/UDP checksum */
6057         if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) == 0)
6058                 return NULL;
6059
6060         /* May be IP fragment */
6061         if (cur_rx->bge_tcp_udp_csum != 0xffff)
6062                 return NULL;
6063
6064         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_IS_TCP)
6065                 pi->pi_l3proto = IPPROTO_TCP;
6066         else
6067                 pi->pi_l3proto = IPPROTO_UDP;
6068         pi->pi_netisr = NETISR_IP;
6069         pi->pi_flags = 0;
6070
6071         return pi;
6072 }