sys/dev/netif/bnx/if_bnx.c

   1 /*
   2  * Copyright (c) 2001 Wind River Systems
   3  * Copyright (c) 1997, 1998, 1999, 2001
   4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  * 3. All advertising materials mentioning features or use of this software
  15  *    must display the following acknowledgement:
  16  *      This product includes software developed by Bill Paul.
  17  * 4. Neither the name of the author nor the names of any co-contributors
  18  *    may be used to endorse or promote products derived from this software
  19  *    without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  31  * THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
  34  */
  35
  36 #include "opt_bnx.h"
  37 #include "opt_ifpoll.h"
  38
  39 #include <sys/param.h>
  40 #include <sys/bus.h>
  41 #include <sys/endian.h>
  42 #include <sys/kernel.h>
  43 #include <sys/interrupt.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/malloc.h>
  46 #include <sys/queue.h>
  47 #include <sys/rman.h>
  48 #include <sys/serialize.h>
  49 #include <sys/socket.h>
  50 #include <sys/sockio.h>
  51 #include <sys/sysctl.h>
  52
  53 #include <netinet/ip.h>
  54 #include <netinet/tcp.h>
  55
  56 #include <net/bpf.h>
  57 #include <net/ethernet.h>
  58 #include <net/if.h>
  59 #include <net/if_arp.h>
  60 #include <net/if_dl.h>
  61 #include <net/if_media.h>
  62 #include <net/if_poll.h>
  63 #include <net/if_types.h>
  64 #include <net/ifq_var.h>
  65 #include <net/toeplitz.h>
  66 #include <net/toeplitz2.h>
  67 #include <net/vlan/if_vlan_var.h>
  68 #include <net/vlan/if_vlan_ether.h>
  69
  70 #include <dev/netif/mii_layer/mii.h>
  71 #include <dev/netif/mii_layer/miivar.h>
  72 #include <dev/netif/mii_layer/brgphyreg.h>
  73
  74 #include "pcidevs.h"
  75 #include <bus/pci/pcireg.h>
  76 #include <bus/pci/pcivar.h>
  77
  78 #include <dev/netif/bge/if_bgereg.h>
  79 #include <dev/netif/bnx/if_bnxvar.h>
  80
  81 /* "device miibus" required.  See GENERIC if you get errors here. */
  82 #include "miibus_if.h"
  83
  84 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
  85
  86 #define BNX_RESET_SHUTDOWN      0
  87 #define BNX_RESET_START         1
  88 #define BNX_RESET_SUSPEND       2
  89
  90 #define BNX_INTR_CKINTVL        ((10 * hz) / 1000)      /* 10ms */
  91
  92 #ifdef BNX_RSS_DEBUG
  93 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...) \
  94 do { \
  95         if (sc->bnx_rss_debug >= lvl) \
  96                 if_printf(&sc->arpcom.ac_if, fmt, __VA_ARGS__); \
  97 } while (0)
  98 #else   /* !BNX_RSS_DEBUG */
  99 #define BNX_RSS_DPRINTF(sc, lvl, fmt, ...)      ((void)0)
 100 #endif  /* BNX_RSS_DEBUG */
 101
 102 static const struct bnx_type {
 103         uint16_t                bnx_vid;
 104         uint16_t                bnx_did;
 105         char                    *bnx_name;
 106 } bnx_devs[] = {
 107         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
 108                 "Broadcom BCM5717 Gigabit Ethernet" },
 109         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C,
 110                 "Broadcom BCM5717C Gigabit Ethernet" },
 111         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
 112                 "Broadcom BCM5718 Gigabit Ethernet" },
 113         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
 114                 "Broadcom BCM5719 Gigabit Ethernet" },
 115         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
 116                 "Broadcom BCM5720 Gigabit Ethernet" },
 117
 118         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725,
 119                 "Broadcom BCM5725 Gigabit Ethernet" },
 120         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727,
 121                 "Broadcom BCM5727 Gigabit Ethernet" },
 122         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762,
 123                 "Broadcom BCM5762 Gigabit Ethernet" },
 124
 125         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
 126                 "Broadcom BCM57761 Gigabit Ethernet" },
 127         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762,
 128                 "Broadcom BCM57762 Gigabit Ethernet" },
 129         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
 130                 "Broadcom BCM57765 Gigabit Ethernet" },
 131         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766,
 132                 "Broadcom BCM57766 Gigabit Ethernet" },
 133         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
 134                 "Broadcom BCM57781 Gigabit Ethernet" },
 135         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782,
 136                 "Broadcom BCM57782 Gigabit Ethernet" },
 137         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
 138                 "Broadcom BCM57785 Gigabit Ethernet" },
 139         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786,
 140                 "Broadcom BCM57786 Gigabit Ethernet" },
 141         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
 142                 "Broadcom BCM57791 Fast Ethernet" },
 143         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
 144                 "Broadcom BCM57795 Fast Ethernet" },
 145
 146         { 0, 0, NULL }
 147 };
 148
 149 static const int bnx_tx_mailbox[BNX_TX_RING_MAX] = {
 150         BGE_MBX_TX_HOST_PROD0_LO,
 151         BGE_MBX_TX_HOST_PROD0_HI,
 152         BGE_MBX_TX_HOST_PROD1_LO,
 153         BGE_MBX_TX_HOST_PROD1_HI
 154 };
 155
 156 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
 157 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
 158 #define BNX_IS_57765_PLUS(sc)           ((sc)->bnx_flags & BNX_FLAG_57765_PLUS)
 159 #define BNX_IS_57765_FAMILY(sc)  \
 160         ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY)
 161
 162 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
 163
 164 static int      bnx_probe(device_t);
 165 static int      bnx_attach(device_t);
 166 static int      bnx_detach(device_t);
 167 static void     bnx_shutdown(device_t);
 168 static int      bnx_suspend(device_t);
 169 static int      bnx_resume(device_t);
 170 static int      bnx_miibus_readreg(device_t, int, int);
 171 static int      bnx_miibus_writereg(device_t, int, int, int);
 172 static void     bnx_miibus_statchg(device_t);
 173
 174 static int      bnx_handle_status(struct bnx_softc *);
 175 #ifdef IFPOLL_ENABLE
 176 static void     bnx_npoll(struct ifnet *, struct ifpoll_info *);
 177 static void     bnx_npoll_rx(struct ifnet *, void *, int);
 178 static void     bnx_npoll_tx(struct ifnet *, void *, int);
 179 static void     bnx_npoll_tx_notag(struct ifnet *, void *, int);
 180 static void     bnx_npoll_status(struct ifnet *);
 181 static void     bnx_npoll_status_notag(struct ifnet *);
 182 #endif
 183 static void     bnx_intr_legacy(void *);
 184 static void     bnx_msi(void *);
 185 static void     bnx_intr(struct bnx_softc *);
 186 static void     bnx_msix_status(void *);
 187 static void     bnx_msix_tx_status(void *);
 188 static void     bnx_msix_rx(void *);
 189 static void     bnx_msix_rxtx(void *);
 190 static void     bnx_enable_intr(struct bnx_softc *);
 191 static void     bnx_disable_intr(struct bnx_softc *);
 192 static void     bnx_txeof(struct bnx_tx_ring *, uint16_t);
 193 static void     bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int);
 194 static int      bnx_alloc_intr(struct bnx_softc *);
 195 static int      bnx_setup_intr(struct bnx_softc *);
 196 static void     bnx_free_intr(struct bnx_softc *);
 197 static void     bnx_teardown_intr(struct bnx_softc *, int);
 198 static int      bnx_alloc_msix(struct bnx_softc *);
 199 static void     bnx_free_msix(struct bnx_softc *, boolean_t);
 200 static void     bnx_check_intr_rxtx(void *);
 201 static void     bnx_check_intr_rx(void *);
 202 static void     bnx_check_intr_tx(void *);
 203 static void     bnx_rx_std_refill_ithread(void *);
 204 static void     bnx_rx_std_refill(void *, void *);
 205 static void     bnx_rx_std_refill_sched_ipi(void *);
 206 static void     bnx_rx_std_refill_stop(void *);
 207 static void     bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *,
 208                     struct bnx_rx_std_ring *);
 209
 210 static void     bnx_start(struct ifnet *, struct ifaltq_subque *);
 211 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 212 static void     bnx_init(void *);
 213 static void     bnx_stop(struct bnx_softc *);
 214 static void     bnx_watchdog(struct ifaltq_subque *);
 215 static int      bnx_ifmedia_upd(struct ifnet *);
 216 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 217 static void     bnx_tick(void *);
 218 static void     bnx_serialize(struct ifnet *, enum ifnet_serialize);
 219 static void     bnx_deserialize(struct ifnet *, enum ifnet_serialize);
 220 static int      bnx_tryserialize(struct ifnet *, enum ifnet_serialize);
 221 #ifdef INVARIANTS
 222 static void     bnx_serialize_assert(struct ifnet *, enum ifnet_serialize,
 223                     boolean_t);
 224 #endif
 225 static void     bnx_serialize_skipmain(struct bnx_softc *);
 226 static void     bnx_deserialize_skipmain(struct bnx_softc *sc);
 227
 228 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
 229 static void     bnx_free_jumbo_mem(struct bnx_softc *);
 230 static struct bnx_jslot
 231                 *bnx_jalloc(struct bnx_softc *);
 232 static void     bnx_jfree(void *);
 233 static void     bnx_jref(void *);
 234 static int      bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int);
 235 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
 236 static void     bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int);
 237 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
 238 static int      bnx_init_rx_ring_std(struct bnx_rx_std_ring *);
 239 static void     bnx_free_rx_ring_std(struct bnx_rx_std_ring *);
 240 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
 241 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
 242 static void     bnx_free_tx_ring(struct bnx_tx_ring *);
 243 static int      bnx_init_tx_ring(struct bnx_tx_ring *);
 244 static int      bnx_create_tx_ring(struct bnx_tx_ring *);
 245 static void     bnx_destroy_tx_ring(struct bnx_tx_ring *);
 246 static int      bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *);
 247 static void     bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *);
 248 static int      bnx_dma_alloc(device_t);
 249 static void     bnx_dma_free(struct bnx_softc *);
 250 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
 251                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
 252 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
 253 static struct mbuf *
 254                 bnx_defrag_shortdma(struct mbuf *);
 255 static int      bnx_encap(struct bnx_tx_ring *, struct mbuf **,
 256                     uint32_t *, int *);
 257 static int      bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
 258                     uint16_t *, uint16_t *);
 259 static void     bnx_setup_serialize(struct bnx_softc *);
 260 static void     bnx_set_tick_cpuid(struct bnx_softc *, boolean_t);
 261 static void     bnx_setup_ring_cnt(struct bnx_softc *);
 262
 263 static struct pktinfo *bnx_rss_info(struct pktinfo *,
 264                     const struct bge_rx_bd *);
 265 static void     bnx_init_rss(struct bnx_softc *);
 266 static void     bnx_reset(struct bnx_softc *);
 267 static int      bnx_chipinit(struct bnx_softc *);
 268 static int      bnx_blockinit(struct bnx_softc *);
 269 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
 270 static void     bnx_enable_msi(struct bnx_softc *, boolean_t);
 271 static void     bnx_setmulti(struct bnx_softc *);
 272 static void     bnx_setpromisc(struct bnx_softc *);
 273 static void     bnx_stats_update_regs(struct bnx_softc *);
 274 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
 275
 276 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
 277 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
 278 #ifdef notdef
 279 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
 280 #endif
 281 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
 282 static void     bnx_writembx(struct bnx_softc *, int, int);
 283 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
 284 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
 285 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
 286
 287 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
 288 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
 289 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
 290 static void     bnx_link_poll(struct bnx_softc *);
 291
 292 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
 293 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
 294 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
 295 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
 296
 297 static void     bnx_coal_change(struct bnx_softc *);
 298 static int      bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS);
 299 static int      bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS);
 300 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
 301 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
 302 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
 303 static int      bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS);
 304 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
 305 static int      bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS);
 306 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 307 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
 308 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
 309                     int, int, uint32_t);
 310 #ifdef IFPOLL_ENABLE
 311 static int      bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS);
 312 static int      bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS);
 313 static int      bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS);
 314 #endif
 315 static int      bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS);
 316
 317 static void     bnx_sig_post_reset(struct bnx_softc *, int);
 318 static void     bnx_sig_pre_reset(struct bnx_softc *, int);
 319 static void     bnx_ape_lock_init(struct bnx_softc *);
 320 static void     bnx_ape_read_fw_ver(struct bnx_softc *);
 321 static int      bnx_ape_lock(struct bnx_softc *, int);
 322 static void     bnx_ape_unlock(struct bnx_softc *, int);
 323 static void     bnx_ape_send_event(struct bnx_softc *, uint32_t);
 324 static void     bnx_ape_driver_state_change(struct bnx_softc *, int);
 325
 326 static int      bnx_msi_enable = 1;
 327 static int      bnx_msix_enable = 1;
 328
 329 static int      bnx_rx_rings = 0; /* auto */
 330 static int      bnx_tx_rings = 0; /* auto */
 331
 332 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
 333 TUNABLE_INT("hw.bnx.msix.enable", &bnx_msix_enable);
 334 TUNABLE_INT("hw.bnx.rx_rings", &bnx_rx_rings);
 335 TUNABLE_INT("hw.bnx.tx_rings", &bnx_tx_rings);
 336
 337 static device_method_t bnx_methods[] = {
 338         /* Device interface */
 339         DEVMETHOD(device_probe,         bnx_probe),
 340         DEVMETHOD(device_attach,        bnx_attach),
 341         DEVMETHOD(device_detach,        bnx_detach),
 342         DEVMETHOD(device_shutdown,      bnx_shutdown),
 343         DEVMETHOD(device_suspend,       bnx_suspend),
 344         DEVMETHOD(device_resume,        bnx_resume),
 345
 346         /* bus interface */
 347         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 348         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 349
 350         /* MII interface */
 351         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
 352         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
 353         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
 354
 355         DEVMETHOD_END
 356 };
 357
 358 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
 359 static devclass_t bnx_devclass;
 360
 361 DECLARE_DUMMY_MODULE(if_bnx);
 362 MODULE_DEPEND(if_bnx, miibus, 1, 1, 1);
 363 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
 364 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
 365
 366 static uint32_t
 367 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
 368 {
 369         device_t dev = sc->bnx_dev;
 370         uint32_t val;
 371
 372         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 373         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
 374         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 375         return (val);
 376 }
 377
 378 static void
 379 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
 380 {
 381         device_t dev = sc->bnx_dev;
 382
 383         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
 384         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
 385         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
 386 }
 387
 388 static void
 389 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
 390 {
 391         CSR_WRITE_4(sc, off, val);
 392 }
 393
 394 static void
 395 bnx_writembx(struct bnx_softc *sc, int off, int val)
 396 {
 397         CSR_WRITE_4(sc, off, val);
 398 }
 399
 400 /*
 401  * Read a sequence of bytes from NVRAM.
 402  */
 403 static int
 404 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
 405 {
 406         return (1);
 407 }
 408
 409 /*
 410  * Read a byte of data stored in the EEPROM at address 'addr.' The
 411  * BCM570x supports both the traditional bitbang interface and an
 412  * auto access interface for reading the EEPROM. We use the auto
 413  * access method.
 414  */
 415 static uint8_t
 416 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
 417 {
 418         int i;
 419         uint32_t byte = 0;
 420
 421         /*
 422          * Enable use of auto EEPROM access so we can avoid
 423          * having to use the bitbang method.
 424          */
 425         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
 426
 427         /* Reset the EEPROM, load the clock period. */
 428         CSR_WRITE_4(sc, BGE_EE_ADDR,
 429             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
 430         DELAY(20);
 431
 432         /* Issue the read EEPROM command. */
 433         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
 434
 435         /* Wait for completion */
 436         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
 437                 DELAY(10);
 438                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
 439                         break;
 440         }
 441
 442         if (i == BNX_TIMEOUT) {
 443                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
 444                 return(1);
 445         }
 446
 447         /* Get result. */
 448         byte = CSR_READ_4(sc, BGE_EE_DATA);
 449
 450         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
 451
 452         return(0);
 453 }
 454
 455 /*
 456  * Read a sequence of bytes from the EEPROM.
 457  */
 458 static int
 459 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
 460 {
 461         size_t i;
 462         int err;
 463         uint8_t byte;
 464
 465         for (byte = 0, err = 0, i = 0; i < len; i++) {
 466                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
 467                 if (err)
 468                         break;
 469                 *(dest + i) = byte;
 470         }
 471
 472         return(err ? 1 : 0);
 473 }
 474
 475 static int
 476 bnx_miibus_readreg(device_t dev, int phy, int reg)
 477 {
 478         struct bnx_softc *sc = device_get_softc(dev);
 479         uint32_t val;
 480         int i;
 481
 482         KASSERT(phy == sc->bnx_phyno,
 483             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 484
 485         if (bnx_ape_lock(sc, sc->bnx_phy_ape_lock) != 0)
 486                 return 0;
 487
 488         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 489         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 490                 CSR_WRITE_4(sc, BGE_MI_MODE,
 491                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 492                 DELAY(80);
 493         }
 494
 495         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
 496             BGE_MIPHY(phy) | BGE_MIREG(reg));
 497
 498         /* Poll for the PHY register access to complete. */
 499         for (i = 0; i < BNX_TIMEOUT; i++) {
 500                 DELAY(10);
 501                 val = CSR_READ_4(sc, BGE_MI_COMM);
 502                 if ((val & BGE_MICOMM_BUSY) == 0) {
 503                         DELAY(5);
 504                         val = CSR_READ_4(sc, BGE_MI_COMM);
 505                         break;
 506                 }
 507         }
 508         if (i == BNX_TIMEOUT) {
 509                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
 510                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
 511                 val = 0;
 512         }
 513
 514         /* Restore the autopoll bit if necessary. */
 515         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 516                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 517                 DELAY(80);
 518         }
 519
 520         bnx_ape_unlock(sc, sc->bnx_phy_ape_lock);
 521
 522         if (val & BGE_MICOMM_READFAIL)
 523                 return 0;
 524
 525         return (val & 0xFFFF);
 526 }
 527
 528 static int
 529 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
 530 {
 531         struct bnx_softc *sc = device_get_softc(dev);
 532         int i;
 533
 534         KASSERT(phy == sc->bnx_phyno,
 535             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
 536
 537         if (bnx_ape_lock(sc, sc->bnx_phy_ape_lock) != 0)
 538                 return 0;
 539
 540         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
 541         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 542                 CSR_WRITE_4(sc, BGE_MI_MODE,
 543                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
 544                 DELAY(80);
 545         }
 546
 547         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
 548             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
 549
 550         for (i = 0; i < BNX_TIMEOUT; i++) {
 551                 DELAY(10);
 552                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
 553                         DELAY(5);
 554                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
 555                         break;
 556                 }
 557         }
 558         if (i == BNX_TIMEOUT) {
 559                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
 560                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
 561         }
 562
 563         /* Restore the autopoll bit if necessary. */
 564         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
 565                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
 566                 DELAY(80);
 567         }
 568
 569         bnx_ape_unlock(sc, sc->bnx_phy_ape_lock);
 570
 571         return 0;
 572 }
 573
 574 static void
 575 bnx_miibus_statchg(device_t dev)
 576 {
 577         struct bnx_softc *sc;
 578         struct mii_data *mii;
 579         uint32_t mac_mode;
 580
 581         sc = device_get_softc(dev);
 582         if ((sc->arpcom.ac_if.if_flags & IFF_RUNNING) == 0)
 583                 return;
 584
 585         mii = device_get_softc(sc->bnx_miibus);
 586
 587         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
 588             (IFM_ACTIVE | IFM_AVALID)) {
 589                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 590                 case IFM_10_T:
 591                 case IFM_100_TX:
 592                         sc->bnx_link = 1;
 593                         break;
 594                 case IFM_1000_T:
 595                 case IFM_1000_SX:
 596                 case IFM_2500_SX:
 597                         sc->bnx_link = 1;
 598                         break;
 599                 default:
 600                         sc->bnx_link = 0;
 601                         break;
 602                 }
 603         } else {
 604                 sc->bnx_link = 0;
 605         }
 606         if (sc->bnx_link == 0)
 607                 return;
 608
 609         /*
 610          * APE firmware touches these registers to keep the MAC
 611          * connected to the outside world.  Try to keep the
 612          * accesses atomic.
 613          */
 614
 615         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) &
 616             ~(BGE_MACMODE_PORTMODE | BGE_MACMODE_HALF_DUPLEX);
 617
 618         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
 619             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX)
 620                 mac_mode |= BGE_PORTMODE_GMII;
 621         else
 622                 mac_mode |= BGE_PORTMODE_MII;
 623
 624         if ((mii->mii_media_active & IFM_GMASK) != IFM_FDX)
 625                 mac_mode |= BGE_MACMODE_HALF_DUPLEX;
 626
 627         CSR_WRITE_4(sc, BGE_MAC_MODE, mac_mode);
 628         DELAY(40);
 629 }
 630
 631 /*
 632  * Memory management for jumbo frames.
 633  */
 634 static int
 635 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
 636 {
 637         struct ifnet *ifp = &sc->arpcom.ac_if;
 638         struct bnx_jslot *entry;
 639         uint8_t *ptr;
 640         bus_addr_t paddr;
 641         int i, error;
 642
 643         /*
 644          * Create tag for jumbo mbufs.
 645          * This is really a bit of a kludge. We allocate a special
 646          * jumbo buffer pool which (thanks to the way our DMA
 647          * memory allocation works) will consist of contiguous
 648          * pages. This means that even though a jumbo buffer might
 649          * be larger than a page size, we don't really need to
 650          * map it into more than one DMA segment. However, the
 651          * default mbuf tag will result in multi-segment mappings,
 652          * so we have to create a special jumbo mbuf tag that
 653          * lets us get away with mapping the jumbo buffers as
 654          * a single segment. I think eventually the driver should
 655          * be changed so that it uses ordinary mbufs and cluster
 656          * buffers, i.e. jumbo frames can span multiple DMA
 657          * descriptors. But that's a project for another day.
 658          */
 659
 660         /*
 661          * Create DMA stuffs for jumbo RX ring.
 662          */
 663         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
 664                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 665                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 666                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
 667                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
 668         if (error) {
 669                 if_printf(ifp, "could not create jumbo RX ring\n");
 670                 return error;
 671         }
 672
 673         /*
 674          * Create DMA stuffs for jumbo buffer block.
 675          */
 676         error = bnx_dma_block_alloc(sc, BNX_JMEM,
 677                                     &sc->bnx_cdata.bnx_jumbo_tag,
 678                                     &sc->bnx_cdata.bnx_jumbo_map,
 679                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
 680                                     &paddr);
 681         if (error) {
 682                 if_printf(ifp, "could not create jumbo buffer\n");
 683                 return error;
 684         }
 685
 686         SLIST_INIT(&sc->bnx_jfree_listhead);
 687
 688         /*
 689          * Now divide it up into 9K pieces and save the addresses
 690          * in an array. Note that we play an evil trick here by using
 691          * the first few bytes in the buffer to hold the the address
 692          * of the softc structure for this interface. This is because
 693          * bnx_jfree() needs it, but it is called by the mbuf management
 694          * code which will not pass it to us explicitly.
 695          */
 696         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
 697                 entry = &sc->bnx_cdata.bnx_jslots[i];
 698                 entry->bnx_sc = sc;
 699                 entry->bnx_buf = ptr;
 700                 entry->bnx_paddr = paddr;
 701                 entry->bnx_inuse = 0;
 702                 entry->bnx_slot = i;
 703                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
 704
 705                 ptr += BNX_JLEN;
 706                 paddr += BNX_JLEN;
 707         }
 708         return 0;
 709 }
 710
 711 static void
 712 bnx_free_jumbo_mem(struct bnx_softc *sc)
 713 {
 714         /* Destroy jumbo RX ring. */
 715         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
 716                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
 717                            sc->bnx_ldata.bnx_rx_jumbo_ring);
 718
 719         /* Destroy jumbo buffer block. */
 720         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
 721                            sc->bnx_cdata.bnx_jumbo_map,
 722                            sc->bnx_ldata.bnx_jumbo_buf);
 723 }
 724
 725 /*
 726  * Allocate a jumbo buffer.
 727  */
 728 static struct bnx_jslot *
 729 bnx_jalloc(struct bnx_softc *sc)
 730 {
 731         struct bnx_jslot *entry;
 732
 733         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 734         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
 735         if (entry) {
 736                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
 737                 entry->bnx_inuse = 1;
 738         } else {
 739                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
 740         }
 741         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 742         return(entry);
 743 }
 744
 745 /*
 746  * Adjust usage count on a jumbo buffer.
 747  */
 748 static void
 749 bnx_jref(void *arg)
 750 {
 751         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 752         struct bnx_softc *sc = entry->bnx_sc;
 753
 754         if (sc == NULL)
 755                 panic("bnx_jref: can't find softc pointer!");
 756
 757         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 758                 panic("bnx_jref: asked to reference buffer "
 759                     "that we don't manage!");
 760         } else if (entry->bnx_inuse == 0) {
 761                 panic("bnx_jref: buffer already free!");
 762         } else {
 763                 atomic_add_int(&entry->bnx_inuse, 1);
 764         }
 765 }
 766
 767 /*
 768  * Release a jumbo buffer.
 769  */
 770 static void
 771 bnx_jfree(void *arg)
 772 {
 773         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
 774         struct bnx_softc *sc = entry->bnx_sc;
 775
 776         if (sc == NULL)
 777                 panic("bnx_jfree: can't find softc pointer!");
 778
 779         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
 780                 panic("bnx_jfree: asked to free buffer that we don't manage!");
 781         } else if (entry->bnx_inuse == 0) {
 782                 panic("bnx_jfree: buffer already free!");
 783         } else {
 784                 /*
 785                  * Possible MP race to 0, use the serializer.  The atomic insn
 786                  * is still needed for races against bnx_jref().
 787                  */
 788                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
 789                 atomic_subtract_int(&entry->bnx_inuse, 1);
 790                 if (entry->bnx_inuse == 0) {
 791                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead,
 792                                           entry, jslot_link);
 793                 }
 794                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
 795         }
 796 }
 797
 798
 799 /*
 800  * Intialize a standard receive ring descriptor.
 801  */
 802 static int
 803 bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init)
 804 {
 805         struct mbuf *m_new = NULL;
 806         bus_dma_segment_t seg;
 807         bus_dmamap_t map;
 808         int error, nsegs;
 809         struct bnx_rx_buf *rb;
 810
 811         rb = &ret->bnx_std->bnx_rx_std_buf[i];
 812         KASSERT(!rb->bnx_rx_refilled, ("RX buf %dth has been refilled", i));
 813
 814         m_new = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR);
 815         if (m_new == NULL) {
 816                 error = ENOBUFS;
 817                 goto back;
 818         }
 819         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
 820         m_adj(m_new, ETHER_ALIGN);
 821
 822         error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag,
 823             ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT);
 824         if (error) {
 825                 m_freem(m_new);
 826                 goto back;
 827         }
 828
 829         if (!init) {
 830                 bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap,
 831                     BUS_DMASYNC_POSTREAD);
 832                 bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap);
 833         }
 834
 835         map = ret->bnx_rx_tmpmap;
 836         ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap;
 837
 838         rb->bnx_rx_dmamap = map;
 839         rb->bnx_rx_mbuf = m_new;
 840         rb->bnx_rx_paddr = seg.ds_addr;
 841         rb->bnx_rx_len = m_new->m_len;
 842 back:
 843         cpu_sfence();
 844         rb->bnx_rx_refilled = 1;
 845         return error;
 846 }
 847
 848 static void
 849 bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i)
 850 {
 851         struct bnx_rx_buf *rb;
 852         struct bge_rx_bd *r;
 853         bus_addr_t paddr;
 854         int len;
 855
 856         rb = &std->bnx_rx_std_buf[i];
 857         KASSERT(rb->bnx_rx_refilled, ("RX buf %dth is not refilled", i));
 858
 859         paddr = rb->bnx_rx_paddr;
 860         len = rb->bnx_rx_len;
 861
 862         cpu_mfence();
 863
 864         rb->bnx_rx_refilled = 0;
 865
 866         r = &std->bnx_rx_std_ring[i];
 867         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(paddr);
 868         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(paddr);
 869         r->bge_len = len;
 870         r->bge_idx = i;
 871         r->bge_flags = BGE_RXBDFLAG_END;
 872 }
 873
 874 /*
 875  * Initialize a jumbo receive ring descriptor. This allocates
 876  * a jumbo buffer from the pool managed internally by the driver.
 877  */
 878 static int
 879 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
 880 {
 881         struct mbuf *m_new = NULL;
 882         struct bnx_jslot *buf;
 883         bus_addr_t paddr;
 884
 885         /* Allocate the mbuf. */
 886         MGETHDR(m_new, init ? M_WAITOK : M_NOWAIT, MT_DATA);
 887         if (m_new == NULL)
 888                 return ENOBUFS;
 889
 890         /* Allocate the jumbo buffer */
 891         buf = bnx_jalloc(sc);
 892         if (buf == NULL) {
 893                 m_freem(m_new);
 894                 return ENOBUFS;
 895         }
 896
 897         /* Attach the buffer to the mbuf. */
 898         m_new->m_ext.ext_arg = buf;
 899         m_new->m_ext.ext_buf = buf->bnx_buf;
 900         m_new->m_ext.ext_free = bnx_jfree;
 901         m_new->m_ext.ext_ref = bnx_jref;
 902         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
 903
 904         m_new->m_flags |= M_EXT;
 905
 906         m_new->m_data = m_new->m_ext.ext_buf;
 907         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
 908
 909         paddr = buf->bnx_paddr;
 910         m_adj(m_new, ETHER_ALIGN);
 911         paddr += ETHER_ALIGN;
 912
 913         /* Save necessary information */
 914         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new;
 915         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr;
 916
 917         /* Set up the descriptor. */
 918         bnx_setup_rxdesc_jumbo(sc, i);
 919         return 0;
 920 }
 921
 922 static void
 923 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
 924 {
 925         struct bge_rx_bd *r;
 926         struct bnx_rx_buf *rc;
 927
 928         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
 929         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
 930
 931         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr);
 932         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr);
 933         r->bge_len = rc->bnx_rx_mbuf->m_len;
 934         r->bge_idx = i;
 935         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
 936 }
 937
 938 static int
 939 bnx_init_rx_ring_std(struct bnx_rx_std_ring *std)
 940 {
 941         int i, error;
 942
 943         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 944                 /* Use the first RX return ring's tmp RX mbuf DMA map */
 945                 error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1);
 946                 if (error)
 947                         return error;
 948                 bnx_setup_rxdesc_std(std, i);
 949         }
 950
 951         std->bnx_rx_std_used = 0;
 952         std->bnx_rx_std_refill = 0;
 953         std->bnx_rx_std_running = 0;
 954         cpu_sfence();
 955         lwkt_serialize_handler_enable(&std->bnx_rx_std_serialize);
 956
 957         std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1;
 958         bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std);
 959
 960         return(0);
 961 }
 962
 963 static void
 964 bnx_free_rx_ring_std(struct bnx_rx_std_ring *std)
 965 {
 966         int i;
 967
 968         lwkt_serialize_handler_disable(&std->bnx_rx_std_serialize);
 969
 970         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
 971                 struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i];
 972
 973                 rb->bnx_rx_refilled = 0;
 974                 if (rb->bnx_rx_mbuf != NULL) {
 975                         bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap);
 976                         m_freem(rb->bnx_rx_mbuf);
 977                         rb->bnx_rx_mbuf = NULL;
 978                 }
 979                 bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd));
 980         }
 981 }
 982
 983 static int
 984 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
 985 {
 986         struct bge_rcb *rcb;
 987         int i, error;
 988
 989         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
 990                 error = bnx_newbuf_jumbo(sc, i, 1);
 991                 if (error)
 992                         return error;
 993         }
 994
 995         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
 996
 997         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
 998         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
 999         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1000
1001         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
1002
1003         return(0);
1004 }
1005
1006 static void
1007 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
1008 {
1009         int i;
1010
1011         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
1012                 struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
1013
1014                 if (rc->bnx_rx_mbuf != NULL) {
1015                         m_freem(rc->bnx_rx_mbuf);
1016                         rc->bnx_rx_mbuf = NULL;
1017                 }
1018                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
1019                     sizeof(struct bge_rx_bd));
1020         }
1021 }
1022
1023 static void
1024 bnx_free_tx_ring(struct bnx_tx_ring *txr)
1025 {
1026         int i;
1027
1028         for (i = 0; i < BGE_TX_RING_CNT; i++) {
1029                 struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i];
1030
1031                 if (buf->bnx_tx_mbuf != NULL) {
1032                         bus_dmamap_unload(txr->bnx_tx_mtag,
1033                             buf->bnx_tx_dmamap);
1034                         m_freem(buf->bnx_tx_mbuf);
1035                         buf->bnx_tx_mbuf = NULL;
1036                 }
1037                 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd));
1038         }
1039         txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
1040 }
1041
1042 static int
1043 bnx_init_tx_ring(struct bnx_tx_ring *txr)
1044 {
1045         txr->bnx_tx_cnt = 0;
1046         txr->bnx_tx_saved_considx = 0;
1047         txr->bnx_tx_prodidx = 0;
1048
1049         /* Initialize transmit producer index for host-memory send ring. */
1050         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx);
1051
1052         return(0);
1053 }
1054
1055 static void
1056 bnx_setmulti(struct bnx_softc *sc)
1057 {
1058         struct ifnet *ifp;
1059         struct ifmultiaddr *ifma;
1060         uint32_t hashes[4] = { 0, 0, 0, 0 };
1061         int h, i;
1062
1063         ifp = &sc->arpcom.ac_if;
1064
1065         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1066                 for (i = 0; i < 4; i++)
1067                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1068                 return;
1069         }
1070
1071         /* First, zot all the existing filters. */
1072         for (i = 0; i < 4; i++)
1073                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1074
1075         /* Now program new ones. */
1076         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1077                 if (ifma->ifma_addr->sa_family != AF_LINK)
1078                         continue;
1079                 h = ether_crc32_le(
1080                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1081                     ETHER_ADDR_LEN) & 0x7f;
1082                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1083         }
1084
1085         for (i = 0; i < 4; i++)
1086                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1087 }
1088
1089 /*
1090  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1091  * self-test results.
1092  */
1093 static int
1094 bnx_chipinit(struct bnx_softc *sc)
1095 {
1096         uint32_t dma_rw_ctl, mode_ctl;
1097         int i;
1098
1099         /* Set endian type before we access any non-PCI registers. */
1100         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
1101             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
1102
1103         /*
1104          * Clear the MAC statistics block in the NIC's
1105          * internal memory.
1106          */
1107         for (i = BGE_STATS_BLOCK;
1108             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1109                 BNX_MEMWIN_WRITE(sc, i, 0);
1110
1111         for (i = BGE_STATUS_BLOCK;
1112             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1113                 BNX_MEMWIN_WRITE(sc, i, 0);
1114
1115         if (BNX_IS_57765_FAMILY(sc)) {
1116                 uint32_t val;
1117
1118                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) {
1119                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1120                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1121
1122                         /* Access the lower 1K of PL PCI-E block registers. */
1123                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1124                             val | BGE_MODECTL_PCIE_PL_SEL);
1125
1126                         val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5);
1127                         val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ;
1128                         CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val);
1129
1130                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1131                 }
1132                 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) {
1133                         /* Fix transmit hangs */
1134                         val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL);
1135                         val |= BGE_CPMU_PADRNG_CTL_RDIV2;
1136                         CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val);
1137
1138                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1139                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1140
1141                         /* Access the lower 1K of DL PCI-E block registers. */
1142                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1143                             val | BGE_MODECTL_PCIE_DL_SEL);
1144
1145                         val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX);
1146                         val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK;
1147                         val |= BGE_PCIE_DL_LO_FTSMAX_VAL;
1148                         CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val);
1149
1150                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1151                 }
1152
1153                 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK);
1154                 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK;
1155                 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25;
1156                 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val);
1157         }
1158
1159         /*
1160          * Set up the PCI DMA control register.
1161          */
1162         dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4);
1163         /*
1164          * Disable 32bytes cache alignment for DMA write to host memory
1165          *
1166          * NOTE:
1167          * 64bytes cache alignment for DMA write to host memory is still
1168          * enabled.
1169          */
1170         dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1171         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1172                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1173         /*
1174          * Enable HW workaround for controllers that misinterpret
1175          * a status tag update and leave interrupts permanently
1176          * disabled.
1177          */
1178         if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1179             sc->bnx_asicrev != BGE_ASICREV_BCM5762 &&
1180             !BNX_IS_57765_FAMILY(sc))
1181                 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1182         if (bootverbose) {
1183                 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n",
1184                     dma_rw_ctl);
1185         }
1186         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1187
1188         /*
1189          * Set up general mode register.
1190          */
1191         mode_ctl = bnx_dma_swap_options(sc);
1192         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1193             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1194                 /* Retain Host-2-BMC settings written by APE firmware. */
1195                 mode_ctl |= CSR_READ_4(sc, BGE_MODE_CTL) &
1196                     (BGE_MODECTL_BYTESWAP_B2HRX_DATA |
1197                     BGE_MODECTL_WORDSWAP_B2HRX_DATA |
1198                     BGE_MODECTL_B2HRX_ENABLE | BGE_MODECTL_HTX2B_ENABLE);
1199         }
1200         mode_ctl |= BGE_MODECTL_MAC_ATTN_INTR |
1201             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1202         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1203
1204         /*
1205          * Disable memory write invalidate.  Apparently it is not supported
1206          * properly by these devices.  Also ensure that INTx isn't disabled,
1207          * as these chips need it even when using MSI.
1208          */
1209         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1210             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1211
1212         /* Set the timer prescaler (always 66Mhz) */
1213         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1214
1215         return(0);
1216 }
1217
1218 static int
1219 bnx_blockinit(struct bnx_softc *sc)
1220 {
1221         struct bnx_intr_data *intr;
1222         struct bge_rcb *rcb;
1223         bus_size_t vrcb;
1224         bge_hostaddr taddr;
1225         uint32_t val;
1226         int i, limit;
1227
1228         /*
1229          * Initialize the memory window pointer register so that
1230          * we can access the first 32K of internal NIC RAM. This will
1231          * allow us to set up the TX send ring RCBs and the RX return
1232          * ring RCBs, plus other things which live in NIC memory.
1233          */
1234         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1235
1236         /* Configure mbuf pool watermarks */
1237         if (BNX_IS_57765_PLUS(sc)) {
1238                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1239                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1240                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1241                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1242                 } else {
1243                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1244                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1245                 }
1246         } else {
1247                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1248                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1249                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1250         }
1251
1252         /* Configure DMA resource watermarks */
1253         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1254         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1255
1256         /* Enable buffer manager */
1257         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1258         /*
1259          * Change the arbitration algorithm of TXMBUF read request to
1260          * round-robin instead of priority based for BCM5719.  When
1261          * TXFIFO is almost empty, RDMA will hold its request until
1262          * TXFIFO is not almost empty.
1263          */
1264         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1265                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1266         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1267             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 ||
1268             sc->bnx_chipid == BGE_CHIPID_BCM5720_A0)
1269                 val |= BGE_BMANMODE_LOMBUF_ATTN;
1270         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1271
1272         /* Poll for buffer manager start indication */
1273         for (i = 0; i < BNX_TIMEOUT; i++) {
1274                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1275                         break;
1276                 DELAY(10);
1277         }
1278
1279         if (i == BNX_TIMEOUT) {
1280                 if_printf(&sc->arpcom.ac_if,
1281                           "buffer manager failed to start\n");
1282                 return(ENXIO);
1283         }
1284
1285         /* Enable flow-through queues */
1286         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1287         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1288
1289         /* Wait until queue initialization is complete */
1290         for (i = 0; i < BNX_TIMEOUT; i++) {
1291                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1292                         break;
1293                 DELAY(10);
1294         }
1295
1296         if (i == BNX_TIMEOUT) {
1297                 if_printf(&sc->arpcom.ac_if,
1298                           "flow-through queue init failed\n");
1299                 return(ENXIO);
1300         }
1301
1302         /*
1303          * Summary of rings supported by the controller:
1304          *
1305          * Standard Receive Producer Ring
1306          * - This ring is used to feed receive buffers for "standard"
1307          *   sized frames (typically 1536 bytes) to the controller.
1308          *
1309          * Jumbo Receive Producer Ring
1310          * - This ring is used to feed receive buffers for jumbo sized
1311          *   frames (i.e. anything bigger than the "standard" frames)
1312          *   to the controller.
1313          *
1314          * Mini Receive Producer Ring
1315          * - This ring is used to feed receive buffers for "mini"
1316          *   sized frames to the controller.
1317          * - This feature required external memory for the controller
1318          *   but was never used in a production system.  Should always
1319          *   be disabled.
1320          *
1321          * Receive Return Ring
1322          * - After the controller has placed an incoming frame into a
1323          *   receive buffer that buffer is moved into a receive return
1324          *   ring.  The driver is then responsible to passing the
1325          *   buffer up to the stack.  BCM5718/BCM57785 families support
1326          *   multiple receive return rings.
1327          *
1328          * Send Ring
1329          * - This ring is used for outgoing frames.  BCM5719/BCM5720
1330          *   support multiple send rings.
1331          */
1332
1333         /* Initialize the standard receive producer ring control block. */
1334         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1335         rcb->bge_hostaddr.bge_addr_lo =
1336             BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1337         rcb->bge_hostaddr.bge_addr_hi =
1338             BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1339         if (BNX_IS_57765_PLUS(sc)) {
1340                 /*
1341                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1342                  * Bits 15-2 : Maximum RX frame size
1343                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1344                  * Bit 0     : Reserved
1345                  */
1346                 rcb->bge_maxlen_flags =
1347                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1348         } else {
1349                 /*
1350                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1351                  * Bits 15-2 : Reserved (should be 0)
1352                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1353                  * Bit 0     : Reserved
1354                  */
1355                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1356         }
1357         if (BNX_IS_5717_PLUS(sc))
1358                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1359         else
1360                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1361         /* Write the standard receive producer ring control block. */
1362         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1363         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1364         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1365         if (!BNX_IS_5717_PLUS(sc))
1366                 CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1367         /* Reset the standard receive producer ring producer index. */
1368         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1369
1370         /*
1371          * Initialize the jumbo RX producer ring control
1372          * block.  We set the 'ring disabled' bit in the
1373          * flags field until we're actually ready to start
1374          * using this ring (i.e. once we set the MTU
1375          * high enough to require it).
1376          */
1377         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1378                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1379                 /* Get the jumbo receive producer ring RCB parameters. */
1380                 rcb->bge_hostaddr.bge_addr_lo =
1381                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1382                 rcb->bge_hostaddr.bge_addr_hi =
1383                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1384                 rcb->bge_maxlen_flags =
1385                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1386                     BGE_RCB_FLAG_RING_DISABLED);
1387                 if (BNX_IS_5717_PLUS(sc))
1388                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1389                 else
1390                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1391                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1392                     rcb->bge_hostaddr.bge_addr_hi);
1393                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1394                     rcb->bge_hostaddr.bge_addr_lo);
1395                 /* Program the jumbo receive producer ring RCB parameters. */
1396                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1397                     rcb->bge_maxlen_flags);
1398                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1399                 /* Reset the jumbo receive producer ring producer index. */
1400                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1401         }
1402
1403         /*
1404          * The BD ring replenish thresholds control how often the
1405          * hardware fetches new BD's from the producer rings in host
1406          * memory.  Setting the value too low on a busy system can
1407          * starve the hardware and recue the throughpout.
1408          *
1409          * Set the BD ring replentish thresholds. The recommended
1410          * values are 1/8th the number of descriptors allocated to
1411          * each ring.
1412          */
1413         val = 8;
1414         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1415         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1416                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1417                     BGE_JUMBO_RX_RING_CNT/8);
1418         }
1419         if (BNX_IS_57765_PLUS(sc)) {
1420                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1421                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1422         }
1423
1424         /*
1425          * Disable all send rings by setting the 'ring disabled' bit
1426          * in the flags field of all the TX send ring control blocks,
1427          * located in NIC memory.
1428          */
1429         if (BNX_IS_5717_PLUS(sc))
1430                 limit = 4;
1431         else if (BNX_IS_57765_FAMILY(sc) ||
1432             sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1433                 limit = 2;
1434         else
1435                 limit = 1;
1436         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1437         for (i = 0; i < limit; i++) {
1438                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1439                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1440                 vrcb += sizeof(struct bge_rcb);
1441         }
1442
1443         /*
1444          * Configure send ring RCBs
1445          */
1446         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1447         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
1448                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
1449
1450                 BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr);
1451                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi,
1452                     taddr.bge_addr_hi);
1453                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo,
1454                     taddr.bge_addr_lo);
1455                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1456                     BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1457                 vrcb += sizeof(struct bge_rcb);
1458         }
1459
1460         /*
1461          * Disable all receive return rings by setting the
1462          * 'ring disabled' bit in the flags field of all the receive
1463          * return ring control blocks, located in NIC memory.
1464          */
1465         if (BNX_IS_5717_PLUS(sc)) {
1466                 /* Should be 17, use 16 until we get an SRAM map. */
1467                 limit = 16;
1468         } else if (BNX_IS_57765_FAMILY(sc) ||
1469             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1470                 limit = 4;
1471         } else {
1472                 limit = 1;
1473         }
1474         /* Disable all receive return rings. */
1475         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1476         for (i = 0; i < limit; i++) {
1477                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1478                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1479                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1480                     BGE_RCB_FLAG_RING_DISABLED);
1481                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1482                     (i * (sizeof(uint64_t))), 0);
1483                 vrcb += sizeof(struct bge_rcb);
1484         }
1485
1486         /*
1487          * Set up receive return rings.
1488          */
1489         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1490         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
1491                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
1492
1493                 BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr);
1494                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi,
1495                     taddr.bge_addr_hi);
1496                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo,
1497                     taddr.bge_addr_lo);
1498                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1499                     BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0));
1500                 vrcb += sizeof(struct bge_rcb);
1501         }
1502
1503         /* Set random backoff seed for TX */
1504         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1505             (sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1506              sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1507              sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5]) &
1508             BGE_TX_BACKOFF_SEED_MASK);
1509
1510         /* Set inter-packet gap */
1511         val = 0x2620;
1512         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1513             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1514                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1515                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1516         }
1517         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1518
1519         /*
1520          * Specify which ring to use for packets that don't match
1521          * any RX rules.
1522          */
1523         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1524
1525         /*
1526          * Configure number of RX lists. One interrupt distribution
1527          * list, sixteen active lists, one bad frames class.
1528          */
1529         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1530
1531         /* Inialize RX list placement stats mask. */
1532         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1533         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1534
1535         /* Disable host coalescing until we get it set up */
1536         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1537
1538         /* Poll to make sure it's shut down. */
1539         for (i = 0; i < BNX_TIMEOUT; i++) {
1540                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1541                         break;
1542                 DELAY(10);
1543         }
1544
1545         if (i == BNX_TIMEOUT) {
1546                 if_printf(&sc->arpcom.ac_if,
1547                           "host coalescing engine failed to idle\n");
1548                 return(ENXIO);
1549         }
1550
1551         /* Set up host coalescing defaults */
1552         sc->bnx_coal_chg = BNX_RX_COAL_TICKS_CHG |
1553             BNX_TX_COAL_TICKS_CHG |
1554             BNX_RX_COAL_BDS_CHG |
1555             BNX_TX_COAL_BDS_CHG |
1556             BNX_RX_COAL_BDS_INT_CHG |
1557             BNX_TX_COAL_BDS_INT_CHG;
1558         bnx_coal_change(sc);
1559
1560         /*
1561          * Set up addresses of status blocks
1562          */
1563         intr = &sc->bnx_intr_data[0];
1564         bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ);
1565         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1566             BGE_ADDR_HI(intr->bnx_status_block_paddr));
1567         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1568             BGE_ADDR_LO(intr->bnx_status_block_paddr));
1569         for (i = 1; i < sc->bnx_intr_cnt; ++i) {
1570                 intr = &sc->bnx_intr_data[i];
1571                 bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ);
1572                 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_HI + ((i - 1) * 8),
1573                     BGE_ADDR_HI(intr->bnx_status_block_paddr));
1574                 CSR_WRITE_4(sc, BGE_VEC1_STATUSBLK_ADDR_LO + ((i - 1) * 8),
1575                     BGE_ADDR_LO(intr->bnx_status_block_paddr));
1576         }
1577
1578         /* Set up status block partail update size. */
1579         val = BGE_STATBLKSZ_32BYTE;
1580 #if 0
1581         /*
1582          * Does not seem to have visible effect in both
1583          * bulk data (1472B UDP datagram) and tiny data
1584          * (18B UDP datagram) TX tests.
1585          */
1586         val |= BGE_HCCMODE_CLRTICK_TX;
1587 #endif
1588         /* Turn on host coalescing state machine */
1589         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1590
1591         /* Turn on RX BD completion state machine and enable attentions */
1592         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1593             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1594
1595         /* Turn on RX list placement state machine */
1596         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1597
1598         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1599             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1600             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1601             BGE_MACMODE_FRMHDR_DMA_ENB;
1602
1603         if (sc->bnx_flags & BNX_FLAG_TBI)
1604                 val |= BGE_PORTMODE_TBI;
1605         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1606                 val |= BGE_PORTMODE_GMII;
1607         else
1608                 val |= BGE_PORTMODE_MII;
1609
1610         /* Allow APE to send/receive frames. */
1611         if (sc->bnx_mfw_flags & BNX_MFW_ON_APE)
1612                 val |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
1613
1614         /* Turn on DMA, clear stats */
1615         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1616         DELAY(40);
1617
1618         /* Set misc. local control, enable interrupts on attentions */
1619         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1620
1621 #ifdef notdef
1622         /* Assert GPIO pins for PHY reset */
1623         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1624             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1625         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1626             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1627 #endif
1628
1629         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSIX)
1630                 bnx_enable_msi(sc, TRUE);
1631
1632         /* Turn on write DMA state machine */
1633         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1634         /* Enable host coalescing bug fix. */
1635         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1636         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1637                 /* Request larger DMA burst size to get better performance. */
1638                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1639         }
1640         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1641         DELAY(40);
1642
1643         if (BNX_IS_57765_PLUS(sc)) {
1644                 uint32_t dmactl, dmactl_reg;
1645
1646                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1647                         dmactl_reg = BGE_RDMA_RSRVCTRL2;
1648                 else
1649                         dmactl_reg = BGE_RDMA_RSRVCTRL;
1650
1651                 dmactl = CSR_READ_4(sc, dmactl_reg);
1652                 /*
1653                  * Adjust tx margin to prevent TX data corruption and
1654                  * fix internal FIFO overflow.
1655                  */
1656                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1657                     sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1658                     sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1659                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1660                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1661                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1662                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1663                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1664                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1665                 }
1666                 /*
1667                  * Enable fix for read DMA FIFO overruns.
1668                  * The fix is to limit the number of RX BDs
1669                  * the hardware would fetch at a fime.
1670                  */
1671                 CSR_WRITE_4(sc, dmactl_reg,
1672                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1673         }
1674
1675         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1676                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1677                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1678                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1679                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1680         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1681             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1682                 uint32_t ctrl_reg;
1683
1684                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1685                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2;
1686                 else
1687                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL;
1688
1689                 /*
1690                  * Allow 4KB burst length reads for non-LSO frames.
1691                  * Enable 512B burst length reads for buffer descriptors.
1692                  */
1693                 CSR_WRITE_4(sc, ctrl_reg,
1694                     CSR_READ_4(sc, ctrl_reg) |
1695                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1696                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1697         }
1698
1699         /* Turn on read DMA state machine */
1700         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1701         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1702                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1703         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1704             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1705             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1706                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1707                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1708                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1709         }
1710         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1711             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1712                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1713                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1714                 /*
1715                  * Allow multiple outstanding read requests from
1716                  * non-LSO read DMA engine.
1717                  */
1718                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1719         }
1720         if (sc->bnx_asicrev == BGE_ASICREV_BCM57766)
1721                 val |= BGE_RDMAMODE_JMB_2K_MMRR;
1722         if (sc->bnx_flags & BNX_FLAG_TSO)
1723                 val |= BGE_RDMAMODE_TSO4_ENABLE;
1724         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1725         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1726         DELAY(40);
1727
1728         /* Turn on RX data completion state machine */
1729         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1730
1731         /* Turn on RX BD initiator state machine */
1732         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1733
1734         /* Turn on RX data and RX BD initiator state machine */
1735         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1736
1737         /* Turn on send BD completion state machine */
1738         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1739
1740         /* Turn on send data completion state machine */
1741         val = BGE_SDCMODE_ENABLE;
1742         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1743                 val |= BGE_SDCMODE_CDELAY;
1744         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1745
1746         /* Turn on send data initiator state machine */
1747         if (sc->bnx_flags & BNX_FLAG_TSO) {
1748                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
1749                     BGE_SDIMODE_HW_LSO_PRE_DMA);
1750         } else {
1751                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1752         }
1753
1754         /* Turn on send BD initiator state machine */
1755         val = BGE_SBDIMODE_ENABLE;
1756         if (sc->bnx_tx_ringcnt > 1)
1757                 val |= BGE_SBDIMODE_MULTI_TXR;
1758         CSR_WRITE_4(sc, BGE_SBDI_MODE, val);
1759
1760         /* Turn on send BD selector state machine */
1761         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1762
1763         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1764         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1765             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1766
1767         /* ack/clear link change events */
1768         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1769             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1770             BGE_MACSTAT_LINK_CHANGED);
1771         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1772
1773         /*
1774          * Enable attention when the link has changed state for
1775          * devices that use auto polling.
1776          */
1777         if (sc->bnx_flags & BNX_FLAG_TBI) {
1778                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1779         } else {
1780                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1781                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1782                         DELAY(80);
1783                 }
1784         }
1785
1786         /*
1787          * Clear any pending link state attention.
1788          * Otherwise some link state change events may be lost until attention
1789          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1790          * It's not necessary on newer BCM chips - perhaps enabling link
1791          * state change attentions implies clearing pending attention.
1792          */
1793         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1794             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1795             BGE_MACSTAT_LINK_CHANGED);
1796
1797         /* Enable link state change attentions. */
1798         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1799
1800         return(0);
1801 }
1802
1803 /*
1804  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1805  * against our list and return its name if we find a match. Note
1806  * that since the Broadcom controller contains VPD support, we
1807  * can get the device name string from the controller itself instead
1808  * of the compiled-in string. This is a little slow, but it guarantees
1809  * we'll always announce the right product name.
1810  */
1811 static int
1812 bnx_probe(device_t dev)
1813 {
1814         const struct bnx_type *t;
1815         uint16_t product, vendor;
1816
1817         if (!pci_is_pcie(dev))
1818                 return ENXIO;
1819
1820         product = pci_get_device(dev);
1821         vendor = pci_get_vendor(dev);
1822
1823         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1824                 if (vendor == t->bnx_vid && product == t->bnx_did)
1825                         break;
1826         }
1827         if (t->bnx_name == NULL)
1828                 return ENXIO;
1829
1830         device_set_desc(dev, t->bnx_name);
1831         return 0;
1832 }
1833
1834 static int
1835 bnx_attach(device_t dev)
1836 {
1837         struct ifnet *ifp;
1838         struct bnx_softc *sc;
1839         struct bnx_rx_std_ring *std;
1840         struct sysctl_ctx_list *ctx;
1841         struct sysctl_oid_list *tree;
1842         uint32_t hwcfg = 0;
1843         int error = 0, rid, capmask, i, std_cpuid, std_cpuid_def;
1844         uint8_t ether_addr[ETHER_ADDR_LEN];
1845         uint16_t product;
1846         uintptr_t mii_priv = 0;
1847 #if defined(BNX_TSO_DEBUG) || defined(BNX_RSS_DEBUG) || defined(BNX_TSS_DEBUG)
1848         char desc[32];
1849 #endif
1850 #ifdef IFPOLL_ENABLE
1851         int offset, offset_def;
1852 #endif
1853
1854         sc = device_get_softc(dev);
1855         sc->bnx_dev = dev;
1856         callout_init_mp(&sc->bnx_tick_timer);
1857         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1858         lwkt_serialize_init(&sc->bnx_main_serialize);
1859
1860         /* Always setup interrupt mailboxes */
1861         for (i = 0; i < BNX_INTR_MAX; ++i) {
1862                 callout_init_mp(&sc->bnx_intr_data[i].bnx_intr_timer);
1863                 sc->bnx_intr_data[i].bnx_sc = sc;
1864                 sc->bnx_intr_data[i].bnx_intr_mbx = BGE_MBX_IRQ0_LO + (i * 8);
1865                 sc->bnx_intr_data[i].bnx_intr_rid = -1;
1866                 sc->bnx_intr_data[i].bnx_intr_cpuid = -1;
1867         }
1868
1869         sc->bnx_func_addr = pci_get_function(dev);
1870         product = pci_get_device(dev);
1871
1872 #ifndef BURN_BRIDGES
1873         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1874                 uint32_t irq, mem;
1875
1876                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1877                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1878
1879                 device_printf(dev, "chip is in D%d power mode "
1880                     "-- setting to D0\n", pci_get_powerstate(dev));
1881
1882                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1883
1884                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1885                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1886         }
1887 #endif  /* !BURN_BRIDGE */
1888
1889         /*
1890          * Map control/status registers.
1891          */
1892         pci_enable_busmaster(dev);
1893
1894         rid = BGE_PCI_BAR0;
1895         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1896             RF_ACTIVE);
1897
1898         if (sc->bnx_res == NULL) {
1899                 device_printf(dev, "couldn't map memory\n");
1900                 return ENXIO;
1901         }
1902
1903         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1904         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1905
1906         /* Save various chip information */
1907         sc->bnx_chipid =
1908             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1909             BGE_PCIMISCCTL_ASICREV_SHIFT;
1910         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1911                 /* All chips having dedicated ASICREV register have CPMU */
1912                 sc->bnx_flags |= BNX_FLAG_CPMU;
1913
1914                 switch (product) {
1915                 case PCI_PRODUCT_BROADCOM_BCM5717:
1916                 case PCI_PRODUCT_BROADCOM_BCM5717C:
1917                 case PCI_PRODUCT_BROADCOM_BCM5718:
1918                 case PCI_PRODUCT_BROADCOM_BCM5719:
1919                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1920                 case PCI_PRODUCT_BROADCOM_BCM5725:
1921                 case PCI_PRODUCT_BROADCOM_BCM5727:
1922                 case PCI_PRODUCT_BROADCOM_BCM5762:
1923                         sc->bnx_chipid = pci_read_config(dev,
1924                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1925                         break;
1926
1927                 case PCI_PRODUCT_BROADCOM_BCM57761:
1928                 case PCI_PRODUCT_BROADCOM_BCM57762:
1929                 case PCI_PRODUCT_BROADCOM_BCM57765:
1930                 case PCI_PRODUCT_BROADCOM_BCM57766:
1931                 case PCI_PRODUCT_BROADCOM_BCM57781:
1932                 case PCI_PRODUCT_BROADCOM_BCM57782:
1933                 case PCI_PRODUCT_BROADCOM_BCM57785:
1934                 case PCI_PRODUCT_BROADCOM_BCM57786:
1935                 case PCI_PRODUCT_BROADCOM_BCM57791:
1936                 case PCI_PRODUCT_BROADCOM_BCM57795:
1937                         sc->bnx_chipid = pci_read_config(dev,
1938                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1939                         break;
1940
1941                 default:
1942                         sc->bnx_chipid = pci_read_config(dev,
1943                             BGE_PCI_PRODID_ASICREV, 4);
1944                         break;
1945                 }
1946         }
1947         if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0)
1948                 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0;
1949
1950         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1951         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1952
1953         switch (sc->bnx_asicrev) {
1954         case BGE_ASICREV_BCM5717:
1955         case BGE_ASICREV_BCM5719:
1956         case BGE_ASICREV_BCM5720:
1957                 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS;
1958                 break;
1959
1960         case BGE_ASICREV_BCM5762:
1961                 sc->bnx_flags |= BNX_FLAG_57765_PLUS;
1962                 break;
1963
1964         case BGE_ASICREV_BCM57765:
1965         case BGE_ASICREV_BCM57766:
1966                 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS;
1967                 break;
1968         }
1969
1970         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1971             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1972             sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1973             sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1974                 sc->bnx_flags |= BNX_FLAG_APE;
1975
1976         sc->bnx_flags |= BNX_FLAG_TSO;
1977         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 &&
1978             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0)
1979                 sc->bnx_flags &= ~BNX_FLAG_TSO;
1980
1981         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1982             BNX_IS_57765_FAMILY(sc)) {
1983                 /*
1984                  * All BCM57785 and BCM5718 families chips have a bug that
1985                  * under certain situation interrupt will not be enabled
1986                  * even if status tag is written to interrupt mailbox.
1987                  *
1988                  * While BCM5719 and BCM5720 have a hardware workaround
1989                  * which could fix the above bug.
1990                  * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in
1991                  * bnx_chipinit().
1992                  *
1993                  * For the rest of the chips in these two families, we will
1994                  * have to poll the status block at high rate (10ms currently)
1995                  * to check whether the interrupt is hosed or not.
1996                  * See bnx_check_intr_*() for details.
1997                  */
1998                 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
1999         }
2000
2001         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
2002         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
2003             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
2004                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
2005         else
2006                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
2007         device_printf(dev, "CHIP ID 0x%08x; "
2008                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
2009                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
2010
2011         /*
2012          * Set various PHY quirk flags.
2013          */
2014
2015         capmask = MII_CAPMASK_DEFAULT;
2016         if (product == PCI_PRODUCT_BROADCOM_BCM57791 ||
2017             product == PCI_PRODUCT_BROADCOM_BCM57795) {
2018                 /* 10/100 only */
2019                 capmask &= ~BMSR_EXTSTAT;
2020         }
2021
2022         mii_priv |= BRGPHY_FLAG_WIRESPEED;
2023         if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0)
2024                 mii_priv |= BRGPHY_FLAG_5762_A0;
2025
2026         /*
2027          * Chips with APE need BAR2 access for APE registers/memory.
2028          */
2029         if (sc->bnx_flags & BNX_FLAG_APE) {
2030                 uint32_t pcistate;
2031
2032                 rid = PCIR_BAR(2);
2033                 sc->bnx_res2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
2034                     RF_ACTIVE);
2035                 if (sc->bnx_res2 == NULL) {
2036                         device_printf(dev, "couldn't map BAR2 memory\n");
2037                         error = ENXIO;
2038                         goto fail;
2039                 }
2040
2041                 /* Enable APE register/memory access by host driver. */
2042                 pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2043                 pcistate |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
2044                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
2045                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
2046                 pci_write_config(dev, BGE_PCI_PCISTATE, pcistate, 4);
2047
2048                 bnx_ape_lock_init(sc);
2049                 bnx_ape_read_fw_ver(sc);
2050         }
2051
2052         /* Initialize if_name earlier, so if_printf could be used */
2053         ifp = &sc->arpcom.ac_if;
2054         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2055
2056         /*
2057          * Try to reset the chip.
2058          */
2059         bnx_sig_pre_reset(sc, BNX_RESET_SHUTDOWN);
2060         bnx_reset(sc);
2061         bnx_sig_post_reset(sc, BNX_RESET_SHUTDOWN);
2062
2063         if (bnx_chipinit(sc)) {
2064                 device_printf(dev, "chip initialization failed\n");
2065                 error = ENXIO;
2066                 goto fail;
2067         }
2068
2069         /*
2070          * Get station address
2071          */
2072         error = bnx_get_eaddr(sc, ether_addr);
2073         if (error) {
2074                 device_printf(dev, "failed to read station address\n");
2075                 goto fail;
2076         }
2077
2078         /* Setup RX/TX and interrupt count */
2079         bnx_setup_ring_cnt(sc);
2080
2081         if ((sc->bnx_rx_retcnt == 1 && sc->bnx_tx_ringcnt == 1) ||
2082             (sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt > 1)) {
2083                 /*
2084                  * The RX ring and the corresponding TX ring processing
2085                  * should be on the same CPU, since they share the same
2086                  * status block.
2087                  */
2088                 sc->bnx_flags |= BNX_FLAG_RXTX_BUNDLE;
2089                 if (bootverbose)
2090                         device_printf(dev, "RX/TX bundle\n");
2091                 if (sc->bnx_tx_ringcnt > 1) {
2092                         /*
2093                          * Multiple TX rings do not share status block
2094                          * with link status, so link status will have
2095                          * to save its own status_tag.
2096                          */
2097                         sc->bnx_flags |= BNX_FLAG_STATUS_HASTAG;
2098                         if (bootverbose)
2099                                 device_printf(dev, "status needs tag\n");
2100                 }
2101         } else {
2102                 KKASSERT(sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt == 1);
2103                 if (bootverbose)
2104                         device_printf(dev, "RX/TX not bundled\n");
2105         }
2106
2107         error = bnx_dma_alloc(dev);
2108         if (error)
2109                 goto fail;
2110
2111 #ifdef IFPOLL_ENABLE
2112         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2113                 /*
2114                  * NPOLLING RX/TX CPU offset
2115                  */
2116                 if (sc->bnx_rx_retcnt == ncpus2) {
2117                         offset = 0;
2118                 } else {
2119                         offset_def =
2120                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
2121                         offset = device_getenv_int(dev, "npoll.offset",
2122                             offset_def);
2123                         if (offset >= ncpus2 ||
2124                             offset % sc->bnx_rx_retcnt != 0) {
2125                                 device_printf(dev, "invalid npoll.offset %d, "
2126                                     "use %d\n", offset, offset_def);
2127                                 offset = offset_def;
2128                         }
2129                 }
2130                 sc->bnx_npoll_rxoff = offset;
2131                 sc->bnx_npoll_txoff = offset;
2132         } else {
2133                 /*
2134                  * NPOLLING RX CPU offset
2135                  */
2136                 if (sc->bnx_rx_retcnt == ncpus2) {
2137                         offset = 0;
2138                 } else {
2139                         offset_def =
2140                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
2141                         offset = device_getenv_int(dev, "npoll.rxoff",
2142                             offset_def);
2143                         if (offset >= ncpus2 ||
2144                             offset % sc->bnx_rx_retcnt != 0) {
2145                                 device_printf(dev, "invalid npoll.rxoff %d, "
2146                                     "use %d\n", offset, offset_def);
2147                                 offset = offset_def;
2148                         }
2149                 }
2150                 sc->bnx_npoll_rxoff = offset;
2151
2152                 /*
2153                  * NPOLLING TX CPU offset
2154                  */
2155                 offset_def = device_get_unit(dev) % ncpus2;
2156                 offset = device_getenv_int(dev, "npoll.txoff", offset_def);
2157                 if (offset >= ncpus2) {
2158                         device_printf(dev, "invalid npoll.txoff %d, use %d\n",
2159                             offset, offset_def);
2160                         offset = offset_def;
2161                 }
2162                 sc->bnx_npoll_txoff = offset;
2163         }
2164 #endif  /* IFPOLL_ENABLE */
2165
2166         /*
2167          * Allocate interrupt
2168          */
2169         error = bnx_alloc_intr(sc);
2170         if (error)
2171                 goto fail;
2172
2173         /* Setup serializers */
2174         bnx_setup_serialize(sc);
2175
2176         /* Set default tuneable values. */
2177         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
2178         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
2179         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
2180         sc->bnx_rx_coal_bds_poll = sc->bnx_rx_ret_ring[0].bnx_rx_cntmax;
2181         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
2182         sc->bnx_tx_coal_bds_poll = BNX_TX_COAL_BDS_POLL_DEF;
2183         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF;
2184         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF;
2185
2186         /* Set up ifnet structure */
2187         ifp->if_softc = sc;
2188         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2189         ifp->if_ioctl = bnx_ioctl;
2190         ifp->if_start = bnx_start;
2191 #ifdef IFPOLL_ENABLE
2192         ifp->if_npoll = bnx_npoll;
2193 #endif
2194         ifp->if_init = bnx_init;
2195         ifp->if_serialize = bnx_serialize;
2196         ifp->if_deserialize = bnx_deserialize;
2197         ifp->if_tryserialize = bnx_tryserialize;
2198 #ifdef INVARIANTS
2199         ifp->if_serialize_assert = bnx_serialize_assert;
2200 #endif
2201         ifp->if_mtu = ETHERMTU;
2202         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2203
2204         ifp->if_capabilities |= IFCAP_HWCSUM;
2205         ifp->if_hwassist = BNX_CSUM_FEATURES;
2206         if (sc->bnx_flags & BNX_FLAG_TSO) {
2207                 ifp->if_capabilities |= IFCAP_TSO;
2208                 ifp->if_hwassist |= CSUM_TSO;
2209         }
2210         if (BNX_RSS_ENABLED(sc))
2211                 ifp->if_capabilities |= IFCAP_RSS;
2212         ifp->if_capenable = ifp->if_capabilities;
2213
2214         ifp->if_nmbclusters = BGE_STD_RX_RING_CNT;
2215
2216         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
2217         ifq_set_ready(&ifp->if_snd);
2218         ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt);
2219
2220         if (sc->bnx_tx_ringcnt > 1) {
2221                 ifp->if_mapsubq = ifq_mapsubq_mask;
2222                 ifq_set_subq_mask(&ifp->if_snd, sc->bnx_tx_ringcnt - 1);
2223         }
2224
2225         /*
2226          * Figure out what sort of media we have by checking the
2227          * hardware config word in the first 32k of NIC internal memory,
2228          * or fall back to examining the EEPROM if necessary.
2229          * Note: on some BCM5700 cards, this value appears to be unset.
2230          * If that's the case, we have to rely on identifying the NIC
2231          * by its PCI subsystem ID, as we do below for the SysKonnect
2232          * SK-9D41.
2233          */
2234         if (bnx_readmem_ind(sc, BGE_SRAM_DATA_SIG) == BGE_SRAM_DATA_SIG_MAGIC) {
2235                 hwcfg = bnx_readmem_ind(sc, BGE_SRAM_DATA_CFG);
2236         } else {
2237                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
2238                                     sizeof(hwcfg))) {
2239                         device_printf(dev, "failed to read EEPROM\n");
2240                         error = ENXIO;
2241                         goto fail;
2242                 }
2243                 hwcfg = ntohl(hwcfg);
2244         }
2245
2246         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
2247         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
2248             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
2249                 sc->bnx_flags |= BNX_FLAG_TBI;
2250
2251         /* Setup MI MODE */
2252         if (sc->bnx_flags & BNX_FLAG_CPMU)
2253                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
2254         else
2255                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
2256
2257         /* Setup link status update stuffs */
2258         if (sc->bnx_flags & BNX_FLAG_TBI) {
2259                 sc->bnx_link_upd = bnx_tbi_link_upd;
2260                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2261         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
2262                 sc->bnx_link_upd = bnx_autopoll_link_upd;
2263                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2264         } else {
2265                 sc->bnx_link_upd = bnx_copper_link_upd;
2266                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2267         }
2268
2269         /* Set default PHY address */
2270         sc->bnx_phyno = 1;
2271
2272         /*
2273          * PHY address mapping for various devices.
2274          *
2275          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
2276          * ---------+-------+-------+-------+-------+
2277          * BCM57XX  |   1   |   X   |   X   |   X   |
2278          * BCM5717  |   1   |   8   |   2   |   9   |
2279          * BCM5719  |   1   |   8   |   2   |   9   |
2280          * BCM5720  |   1   |   8   |   2   |   9   |
2281          *
2282          *          | F2 Cu | F2 Sr | F3 Cu | F3 Sr |
2283          * ---------+-------+-------+-------+-------+
2284          * BCM57XX  |   X   |   X   |   X   |   X   |
2285          * BCM5717  |   X   |   X   |   X   |   X   |
2286          * BCM5719  |   3   |   10  |   4   |   11  |
2287          * BCM5720  |   X   |   X   |   X   |   X   |
2288          *
2289          * Other addresses may respond but they are not
2290          * IEEE compliant PHYs and should be ignored.
2291          */
2292         if (BNX_IS_5717_PLUS(sc)) {
2293                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
2294                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
2295                             BGE_SGDIGSTS_IS_SERDES)
2296                                 sc->bnx_phyno = sc->bnx_func_addr + 8;
2297                         else
2298                                 sc->bnx_phyno = sc->bnx_func_addr + 1;
2299                 } else {
2300                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
2301                             BGE_CPMU_PHY_STRAP_IS_SERDES)
2302                                 sc->bnx_phyno = sc->bnx_func_addr + 8;
2303                         else
2304                                 sc->bnx_phyno = sc->bnx_func_addr + 1;
2305                 }
2306         }
2307
2308         if (sc->bnx_flags & BNX_FLAG_TBI) {
2309                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
2310                     bnx_ifmedia_upd, bnx_ifmedia_sts);
2311                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2312                 ifmedia_add(&sc->bnx_ifmedia,
2313                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2314                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2315                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2316                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2317         } else {
2318                 struct mii_probe_args mii_args;
2319
2320                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2321                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2322                 mii_args.mii_capmask = capmask;
2323                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2324                 mii_args.mii_priv = mii_priv;
2325
2326                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2327                 if (error) {
2328                         device_printf(dev, "MII without any PHY!\n");
2329                         goto fail;
2330                 }
2331         }
2332
2333         ctx = device_get_sysctl_ctx(sc->bnx_dev);
2334         tree = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->bnx_dev));
2335
2336         SYSCTL_ADD_INT(ctx, tree, OID_AUTO,
2337             "rx_rings", CTLFLAG_RD, &sc->bnx_rx_retcnt, 0, "# of RX rings");
2338         SYSCTL_ADD_INT(ctx, tree, OID_AUTO,
2339             "tx_rings", CTLFLAG_RD, &sc->bnx_tx_ringcnt, 0, "# of TX rings");
2340
2341         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_ticks",
2342                         CTLTYPE_INT | CTLFLAG_RW,
2343                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2344                         "Receive coalescing ticks (usec).");
2345         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_ticks",
2346                         CTLTYPE_INT | CTLFLAG_RW,
2347                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2348                         "Transmit coalescing ticks (usec).");
2349         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_bds",
2350                         CTLTYPE_INT | CTLFLAG_RW,
2351                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2352                         "Receive max coalesced BD count.");
2353         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "rx_coal_bds_poll",
2354                         CTLTYPE_INT | CTLFLAG_RW,
2355                         sc, 0, bnx_sysctl_rx_coal_bds_poll, "I",
2356                         "Receive max coalesced BD count in polling.");
2357         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_bds",
2358                         CTLTYPE_INT | CTLFLAG_RW,
2359                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2360                         "Transmit max coalesced BD count.");
2361         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "tx_coal_bds_poll",
2362                         CTLTYPE_INT | CTLFLAG_RW,
2363                         sc, 0, bnx_sysctl_tx_coal_bds_poll, "I",
2364                         "Transmit max coalesced BD count in polling.");
2365         /*
2366          * A common design characteristic for many Broadcom
2367          * client controllers is that they only support a
2368          * single outstanding DMA read operation on the PCIe
2369          * bus. This means that it will take twice as long to
2370          * fetch a TX frame that is split into header and
2371          * payload buffers as it does to fetch a single,
2372          * contiguous TX frame (2 reads vs. 1 read). For these
2373          * controllers, coalescing buffers to reduce the number
2374          * of memory reads is effective way to get maximum
2375          * performance(about 940Mbps).  Without collapsing TX
2376          * buffers the maximum TCP bulk transfer performance
2377          * is about 850Mbps. However forcing coalescing mbufs
2378          * consumes a lot of CPU cycles, so leave it off by
2379          * default.
2380          */
2381         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2382             "force_defrag", CTLTYPE_INT | CTLFLAG_RW,
2383             sc, 0, bnx_sysctl_force_defrag, "I",
2384             "Force defragment on TX path");
2385
2386         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2387             "tx_wreg", CTLTYPE_INT | CTLFLAG_RW,
2388             sc, 0, bnx_sysctl_tx_wreg, "I",
2389             "# of segments before writing to hardware register");
2390
2391         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2392             "std_refill", CTLTYPE_INT | CTLFLAG_RW,
2393             sc, 0, bnx_sysctl_std_refill, "I",
2394             "# of packets received before scheduling standard refilling");
2395
2396         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2397             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2398             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2399             "Receive max coalesced BD count during interrupt.");
2400         SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2401             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2402             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2403             "Transmit max coalesced BD count during interrupt.");
2404
2405 #ifdef IFPOLL_ENABLE
2406         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2407                 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2408                     "npoll_offset", CTLTYPE_INT | CTLFLAG_RW,
2409                     sc, 0, bnx_sysctl_npoll_offset, "I",
2410                     "NPOLLING cpu offset");
2411         } else {
2412                 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2413                     "npoll_rxoff", CTLTYPE_INT | CTLFLAG_RW,
2414                     sc, 0, bnx_sysctl_npoll_rxoff, "I",
2415                     "NPOLLING RX cpu offset");
2416                 SYSCTL_ADD_PROC(ctx, tree, OID_AUTO,
2417                     "npoll_txoff", CTLTYPE_INT | CTLFLAG_RW,
2418                     sc, 0, bnx_sysctl_npoll_txoff, "I",
2419                     "NPOLLING TX cpu offset");
2420         }
2421 #endif
2422
2423 #ifdef BNX_RSS_DEBUG
2424         SYSCTL_ADD_INT(ctx, tree, OID_AUTO,
2425             "std_refill_mask", CTLFLAG_RD,
2426             &sc->bnx_rx_std_ring.bnx_rx_std_refill, 0, "");
2427         SYSCTL_ADD_INT(ctx, tree, OID_AUTO,
2428             "std_used", CTLFLAG_RD,
2429             &sc->bnx_rx_std_ring.bnx_rx_std_used, 0, "");
2430         SYSCTL_ADD_INT(ctx, tree, OID_AUTO,
2431             "rss_debug", CTLFLAG_RW, &sc->bnx_rss_debug, 0, "");
2432         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
2433                 ksnprintf(desc, sizeof(desc), "rx_pkt%d", i);
2434                 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2435                     desc, CTLFLAG_RW, &sc->bnx_rx_ret_ring[i].bnx_rx_pkt, "");
2436
2437                 ksnprintf(desc, sizeof(desc), "rx_force_sched%d", i);
2438                 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2439                     desc, CTLFLAG_RW,
2440                     &sc->bnx_rx_ret_ring[i].bnx_rx_force_sched, "");
2441         }
2442 #endif
2443 #ifdef BNX_TSS_DEBUG
2444         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2445                 ksnprintf(desc, sizeof(desc), "tx_pkt%d", i);
2446                 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2447                     desc, CTLFLAG_RW, &sc->bnx_tx_ring[i].bnx_tx_pkt, "");
2448         }
2449 #endif
2450
2451         SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2452             "norxbds", CTLFLAG_RW, &sc->bnx_norxbds, "");
2453
2454         SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2455             "errors", CTLFLAG_RW, &sc->bnx_errors, "");
2456
2457 #ifdef BNX_TSO_DEBUG
2458         for (i = 0; i < BNX_TSO_NSTATS; ++i) {
2459                 ksnprintf(desc, sizeof(desc), "tso%d", i + 1);
2460                 SYSCTL_ADD_ULONG(ctx, tree, OID_AUTO,
2461                     desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], "");
2462         }
2463 #endif
2464
2465         /*
2466          * Call MI attach routine.
2467          */
2468         ether_ifattach(ifp, ether_addr, NULL);
2469
2470         /* Setup TX rings and subqueues */
2471         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2472                 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
2473                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
2474
2475                 ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid);
2476                 ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize);
2477                 ifsq_set_priv(ifsq, txr);
2478                 txr->bnx_ifsq = ifsq;
2479
2480                 ifsq_watchdog_init(&txr->bnx_tx_watchdog, ifsq, bnx_watchdog);
2481
2482                 if (bootverbose) {
2483                         device_printf(dev, "txr %d -> cpu%d\n", i,
2484                             txr->bnx_tx_cpuid);
2485                 }
2486         }
2487
2488         error = bnx_setup_intr(sc);
2489         if (error) {
2490                 ether_ifdetach(ifp);
2491                 goto fail;
2492         }
2493         bnx_set_tick_cpuid(sc, FALSE);
2494
2495         /*
2496          * Create RX standard ring refilling thread
2497          */
2498         std_cpuid_def = device_get_unit(dev) % ncpus;
2499         std_cpuid = device_getenv_int(dev, "std.cpuid", std_cpuid_def);
2500         if (std_cpuid < 0 || std_cpuid >= ncpus) {
2501                 device_printf(dev, "invalid std.cpuid %d, use %d\n",
2502                     std_cpuid, std_cpuid_def);
2503                 std_cpuid = std_cpuid_def;
2504         }
2505
2506         std = &sc->bnx_rx_std_ring;
2507         lwkt_create(bnx_rx_std_refill_ithread, std, NULL,
2508             &std->bnx_rx_std_ithread, TDF_NOSTART | TDF_INTTHREAD, std_cpuid,
2509             "%s std", device_get_nameunit(dev));
2510         lwkt_setpri(&std->bnx_rx_std_ithread, TDPRI_INT_MED);
2511         std->bnx_rx_std_ithread.td_preemptable = lwkt_preempt;
2512         sc->bnx_flags |= BNX_FLAG_STD_THREAD;
2513
2514         return(0);
2515 fail:
2516         bnx_detach(dev);
2517         return(error);
2518 }
2519
2520 static int
2521 bnx_detach(device_t dev)
2522 {
2523         struct bnx_softc *sc = device_get_softc(dev);
2524
2525         if (device_is_attached(dev)) {
2526                 struct ifnet *ifp = &sc->arpcom.ac_if;
2527
2528                 ifnet_serialize_all(ifp);
2529                 bnx_stop(sc);
2530                 bnx_teardown_intr(sc, sc->bnx_intr_cnt);
2531                 ifnet_deserialize_all(ifp);
2532
2533                 ether_ifdetach(ifp);
2534         }
2535
2536         if (sc->bnx_flags & BNX_FLAG_STD_THREAD) {
2537                 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
2538
2539                 tsleep_interlock(std, 0);
2540
2541                 if (std->bnx_rx_std_ithread.td_gd == mycpu) {
2542                         bnx_rx_std_refill_stop(std);
2543                 } else {
2544                         lwkt_send_ipiq(std->bnx_rx_std_ithread.td_gd,
2545                             bnx_rx_std_refill_stop, std);
2546                 }
2547
2548                 tsleep(std, PINTERLOCKED, "bnx_detach", 0);
2549                 if (bootverbose)
2550                         device_printf(dev, "RX std ithread exited\n");
2551
2552                 lwkt_synchronize_ipiqs("bnx_detach_ipiq");
2553         }
2554
2555         if (sc->bnx_flags & BNX_FLAG_TBI)
2556                 ifmedia_removeall(&sc->bnx_ifmedia);
2557         if (sc->bnx_miibus)
2558                 device_delete_child(dev, sc->bnx_miibus);
2559         bus_generic_detach(dev);
2560
2561         bnx_free_intr(sc);
2562
2563         if (sc->bnx_msix_mem_res != NULL) {
2564                 bus_release_resource(dev, SYS_RES_MEMORY, sc->bnx_msix_mem_rid,
2565                     sc->bnx_msix_mem_res);
2566         }
2567         if (sc->bnx_res != NULL) {
2568                 bus_release_resource(dev, SYS_RES_MEMORY,
2569                     BGE_PCI_BAR0, sc->bnx_res);
2570         }
2571         if (sc->bnx_res2 != NULL) {
2572                 bus_release_resource(dev, SYS_RES_MEMORY,
2573                     PCIR_BAR(2), sc->bnx_res2);
2574         }
2575
2576         bnx_dma_free(sc);
2577
2578         if (sc->bnx_serialize != NULL)
2579                 kfree(sc->bnx_serialize, M_DEVBUF);
2580
2581         return 0;
2582 }
2583
2584 static void
2585 bnx_reset(struct bnx_softc *sc)
2586 {
2587         device_t dev = sc->bnx_dev;
2588         uint32_t cachesize, command, reset, mac_mode, mac_mode_mask;
2589         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2590         int i, val = 0;
2591         uint16_t devctl;
2592
2593         mac_mode_mask = BGE_MACMODE_HALF_DUPLEX | BGE_MACMODE_PORTMODE;
2594         if (sc->bnx_mfw_flags & BNX_MFW_ON_APE)
2595                 mac_mode_mask |= BGE_MACMODE_APE_RX_EN | BGE_MACMODE_APE_TX_EN;
2596         mac_mode = CSR_READ_4(sc, BGE_MAC_MODE) & mac_mode_mask;
2597
2598         write_op = bnx_writemem_direct;
2599
2600         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
2601         for (i = 0; i < 8000; i++) {
2602                 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1)
2603                         break;
2604                 DELAY(20);
2605         }
2606         if (i == 8000)
2607                 if_printf(&sc->arpcom.ac_if, "NVRAM lock timedout!\n");
2608
2609         /* Take APE lock when performing reset. */
2610         bnx_ape_lock(sc, BGE_APE_LOCK_GRC);
2611
2612         /* Save some important PCI state. */
2613         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2614         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2615
2616         pci_write_config(dev, BGE_PCI_MISC_CTL,
2617             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2618             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2619             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2620
2621         /* Disable fastboot on controllers that support it. */
2622         if (bootverbose)
2623                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2624         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2625
2626         /*
2627          * Write the magic number to SRAM at offset 0xB50.
2628          * When firmware finishes its initialization it will
2629          * write ~BGE_SRAM_FW_MB_MAGIC to the same location.
2630          */
2631         bnx_writemem_ind(sc, BGE_SRAM_FW_MB, BGE_SRAM_FW_MB_MAGIC);
2632
2633         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2634
2635         /* XXX: Broadcom Linux driver. */
2636         /* Force PCI-E 1.0a mode */
2637         if (!BNX_IS_57765_PLUS(sc) &&
2638             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2639             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2640              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2641                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2642                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2643         }
2644         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2645                 /* Prevent PCIE link training during global reset */
2646                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2647                 reset |= (1<<29);
2648         }
2649
2650         /*
2651          * Set GPHY Power Down Override to leave GPHY
2652          * powered up in D0 uninitialized.
2653          */
2654         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2655                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2656
2657         /* Issue global reset */
2658         write_op(sc, BGE_MISC_CFG, reset);
2659
2660         DELAY(100 * 1000);
2661
2662         /* XXX: Broadcom Linux driver. */
2663         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2664                 uint32_t v;
2665
2666                 DELAY(500000); /* wait for link training to complete */
2667                 v = pci_read_config(dev, 0xc4, 4);
2668                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2669         }
2670
2671         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2672
2673         /* Disable no snoop and disable relaxed ordering. */
2674         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2675
2676         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2677         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2678                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2679                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2680         }
2681
2682         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2683             devctl, 2);
2684
2685         /* Clear error status. */
2686         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2687             PCIEM_DEVSTS_CORR_ERR |
2688             PCIEM_DEVSTS_NFATAL_ERR |
2689             PCIEM_DEVSTS_FATAL_ERR |
2690             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2691
2692         /* Reset some of the PCI state that got zapped by reset */
2693         pci_write_config(dev, BGE_PCI_MISC_CTL,
2694             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2695             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2696             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2697         val = BGE_PCISTATE_ROM_ENABLE | BGE_PCISTATE_ROM_RETRY_ENABLE;
2698         if (sc->bnx_mfw_flags & BNX_MFW_ON_APE) {
2699                 val |= BGE_PCISTATE_ALLOW_APE_CTLSPC_WR |
2700                     BGE_PCISTATE_ALLOW_APE_SHMEM_WR |
2701                     BGE_PCISTATE_ALLOW_APE_PSPACE_WR;
2702         }
2703         pci_write_config(dev, BGE_PCI_PCISTATE, val, 4);
2704         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2705         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2706
2707         /* Enable memory arbiter */
2708         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2709
2710         /* Fix up byte swapping */
2711         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2712
2713         val = CSR_READ_4(sc, BGE_MAC_MODE);
2714         val = (val & ~mac_mode_mask) | mac_mode;
2715         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
2716         DELAY(40);
2717
2718         bnx_ape_unlock(sc, BGE_APE_LOCK_GRC);
2719
2720         /*
2721          * Poll until we see the 1's complement of the magic number.
2722          * This indicates that the firmware initialization is complete.
2723          */
2724         for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2725                 val = bnx_readmem_ind(sc, BGE_SRAM_FW_MB);
2726                 if (val == ~BGE_SRAM_FW_MB_MAGIC)
2727                         break;
2728                 DELAY(10);
2729         }
2730         if (i == BNX_FIRMWARE_TIMEOUT) {
2731                 if_printf(&sc->arpcom.ac_if, "firmware handshake "
2732                           "timed out, found 0x%08x\n", val);
2733         }
2734
2735         /* BCM57765 A0 needs additional time before accessing. */
2736         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2737                 DELAY(10 * 1000);
2738
2739         /*
2740          * The 5704 in TBI mode apparently needs some special
2741          * adjustment to insure the SERDES drive level is set
2742          * to 1.2V.
2743          */
2744         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2745             (sc->bnx_flags & BNX_FLAG_TBI)) {
2746                 uint32_t serdescfg;
2747
2748                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2749                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2750                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2751         }
2752
2753         CSR_WRITE_4(sc, BGE_MI_MODE,
2754             sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
2755         DELAY(80);
2756
2757         /* XXX: Broadcom Linux driver. */
2758         if (!BNX_IS_57765_PLUS(sc)) {
2759                 uint32_t v;
2760
2761                 /* Enable Data FIFO protection. */
2762                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
2763                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
2764         }
2765
2766         DELAY(10000);
2767
2768         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2769                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2770                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2771         }
2772 }
2773
2774 /*
2775  * Frame reception handling. This is called if there's a frame
2776  * on the receive return list.
2777  *
2778  * Note: we have to be able to handle two possibilities here:
2779  * 1) the frame is from the jumbo recieve ring
2780  * 2) the frame is from the standard receive ring
2781  */
2782
2783 static void
2784 bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count)
2785 {
2786         struct bnx_softc *sc = ret->bnx_sc;
2787         struct bnx_rx_std_ring *std = ret->bnx_std;
2788         struct ifnet *ifp = &sc->arpcom.ac_if;
2789         int std_used = 0, cpuid = mycpuid;
2790
2791         while (ret->bnx_rx_saved_considx != rx_prod && count != 0) {
2792                 struct pktinfo pi0, *pi = NULL;
2793                 struct bge_rx_bd *cur_rx;
2794                 struct bnx_rx_buf *rb;
2795                 uint32_t rxidx;
2796                 struct mbuf *m = NULL;
2797                 uint16_t vlan_tag = 0;
2798                 int have_tag = 0;
2799
2800                 --count;
2801
2802                 cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx];
2803
2804                 rxidx = cur_rx->bge_idx;
2805                 KKASSERT(rxidx < BGE_STD_RX_RING_CNT);
2806
2807                 BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
2808 #ifdef BNX_RSS_DEBUG
2809                 ret->bnx_rx_pkt++;
2810 #endif
2811
2812                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2813                         have_tag = 1;
2814                         vlan_tag = cur_rx->bge_vlan_tag;
2815                 }
2816
2817                 if (ret->bnx_rx_cnt >= ret->bnx_rx_cntmax) {
2818                         atomic_add_int(&std->bnx_rx_std_used, std_used);
2819                         std_used = 0;
2820
2821                         bnx_rx_std_refill_sched(ret, std);
2822                 }
2823                 ret->bnx_rx_cnt++;
2824                 ++std_used;
2825
2826                 rb = &std->bnx_rx_std_buf[rxidx];
2827                 m = rb->bnx_rx_mbuf;
2828                 if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2829                         IFNET_STAT_INC(ifp, ierrors, 1);
2830                         cpu_sfence();
2831                         rb->bnx_rx_refilled = 1;
2832                         continue;
2833                 }
2834                 if (bnx_newbuf_std(ret, rxidx, 0)) {
2835                         IFNET_STAT_INC(ifp, ierrors, 1);
2836                         continue;
2837                 }
2838
2839                 IFNET_STAT_INC(ifp, ipackets, 1);
2840                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2841                 m->m_pkthdr.rcvif = ifp;
2842
2843                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2844                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2845                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2846                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2847                                 if ((cur_rx->bge_error_flag &
2848                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2849                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2850                         }
2851                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2852                                 m->m_pkthdr.csum_data =
2853                                     cur_rx->bge_tcp_udp_csum;
2854                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2855                                     CSUM_PSEUDO_HDR;
2856                         }
2857                 }
2858                 if (ifp->if_capenable & IFCAP_RSS) {
2859                         pi = bnx_rss_info(&pi0, cur_rx);
2860                         if (pi != NULL &&
2861                             (cur_rx->bge_flags & BGE_RXBDFLAG_RSS_HASH)) {
2862                                 m->m_flags |= M_HASH;
2863                                 m->m_pkthdr.hash =
2864                                     toeplitz_hash(cur_rx->bge_hash);
2865                         }
2866                 }
2867
2868                 /*
2869                  * If we received a packet with a vlan tag, pass it
2870                  * to vlan_input() instead of ether_input().
2871                  */
2872                 if (have_tag) {
2873                         m->m_flags |= M_VLANTAG;
2874                         m->m_pkthdr.ether_vlantag = vlan_tag;
2875                 }
2876                 ifp->if_input(ifp, m, pi, cpuid);
2877         }
2878         bnx_writembx(sc, ret->bnx_rx_mbx, ret->bnx_rx_saved_considx);
2879
2880         if (std_used > 0) {
2881                 int cur_std_used;
2882
2883                 cur_std_used = atomic_fetchadd_int(&std->bnx_rx_std_used,
2884                     std_used);
2885                 if (cur_std_used + std_used >= (BGE_STD_RX_RING_CNT / 2)) {
2886 #ifdef BNX_RSS_DEBUG
2887                         ret->bnx_rx_force_sched++;
2888 #endif
2889                         bnx_rx_std_refill_sched(ret, std);
2890                 }
2891         }
2892 }
2893
2894 static void
2895 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
2896 {
2897         struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if;
2898
2899         /*
2900          * Go through our tx ring and free mbufs for those
2901          * frames that have been sent.
2902          */
2903         while (txr->bnx_tx_saved_considx != tx_cons) {
2904                 struct bnx_tx_buf *buf;
2905                 uint32_t idx = 0;
2906
2907                 idx = txr->bnx_tx_saved_considx;
2908                 buf = &txr->bnx_tx_buf[idx];
2909                 if (buf->bnx_tx_mbuf != NULL) {
2910                         IFNET_STAT_INC(ifp, opackets, 1);
2911 #ifdef BNX_TSS_DEBUG
2912                         txr->bnx_tx_pkt++;
2913 #endif
2914                         bus_dmamap_unload(txr->bnx_tx_mtag,
2915                             buf->bnx_tx_dmamap);
2916                         m_freem(buf->bnx_tx_mbuf);
2917                         buf->bnx_tx_mbuf = NULL;
2918                 }
2919                 txr->bnx_tx_cnt--;
2920                 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2921         }
2922
2923         if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >=
2924             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2925                 ifsq_clr_oactive(txr->bnx_ifsq);
2926
2927         if (txr->bnx_tx_cnt == 0)
2928                 txr->bnx_tx_watchdog.wd_timer = 0;
2929
2930         if (!ifsq_is_empty(txr->bnx_ifsq))
2931                 ifsq_devstart(txr->bnx_ifsq);
2932 }
2933
2934 static int
2935 bnx_handle_status(struct bnx_softc *sc)
2936 {
2937         uint32_t status;
2938         int handle = 0;
2939
2940         status = *sc->bnx_hw_status;
2941
2942         if (status & BGE_STATFLAG_ERROR) {
2943                 uint32_t val;
2944                 int reset = 0;
2945
2946                 sc->bnx_errors++;
2947
2948                 val = CSR_READ_4(sc, BGE_FLOW_ATTN);
2949                 if (val & ~BGE_FLOWATTN_MB_LOWAT) {
2950                         if_printf(&sc->arpcom.ac_if,
2951                             "flow attn 0x%08x\n", val);
2952                         reset = 1;
2953                 }
2954
2955                 val = CSR_READ_4(sc, BGE_MSI_STATUS);
2956                 if (val & ~BGE_MSISTAT_MSI_PCI_REQ) {
2957                         if_printf(&sc->arpcom.ac_if,
2958                             "msi status 0x%08x\n", val);
2959                         reset = 1;
2960                 }
2961
2962                 val = CSR_READ_4(sc, BGE_RDMA_STATUS);
2963                 if (val) {
2964                         if_printf(&sc->arpcom.ac_if,
2965                             "rmda status 0x%08x\n", val);
2966                         reset = 1;
2967                 }
2968
2969                 val = CSR_READ_4(sc, BGE_WDMA_STATUS);
2970                 if (val) {
2971                         if_printf(&sc->arpcom.ac_if,
2972                             "wdma status 0x%08x\n", val);
2973                         reset = 1;
2974                 }
2975
2976                 if (reset) {
2977                         bnx_serialize_skipmain(sc);
2978                         bnx_init(sc);
2979                         bnx_deserialize_skipmain(sc);
2980                 }
2981                 handle = 1;
2982         }
2983
2984         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt) {
2985                 if (bootverbose) {
2986                         if_printf(&sc->arpcom.ac_if, "link change, "
2987                             "link_evt %d\n", sc->bnx_link_evt);
2988                 }
2989                 bnx_link_poll(sc);
2990                 handle = 1;
2991         }
2992
2993         return handle;
2994 }
2995
2996 #ifdef IFPOLL_ENABLE
2997
2998 static void
2999 bnx_npoll_rx(struct ifnet *ifp __unused, void *xret, int cycle)
3000 {
3001         struct bnx_rx_ret_ring *ret = xret;
3002         uint16_t rx_prod;
3003
3004         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
3005
3006         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3007         cpu_lfence();
3008
3009         rx_prod = *ret->bnx_rx_considx;
3010         if (ret->bnx_rx_saved_considx != rx_prod)
3011                 bnx_rxeof(ret, rx_prod, cycle);
3012 }
3013
3014 static void
3015 bnx_npoll_tx_notag(struct ifnet *ifp __unused, void *xtxr, int cycle __unused)
3016 {
3017         struct bnx_tx_ring *txr = xtxr;
3018         uint16_t tx_cons;
3019
3020         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
3021
3022         tx_cons = *txr->bnx_tx_considx;
3023         if (txr->bnx_tx_saved_considx != tx_cons)
3024                 bnx_txeof(txr, tx_cons);
3025 }
3026
3027 static void
3028 bnx_npoll_tx(struct ifnet *ifp, void *xtxr, int cycle)
3029 {
3030         struct bnx_tx_ring *txr = xtxr;
3031
3032         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
3033
3034         txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag;
3035         cpu_lfence();
3036         bnx_npoll_tx_notag(ifp, txr, cycle);
3037 }
3038
3039 static void
3040 bnx_npoll_status_notag(struct ifnet *ifp)
3041 {
3042         struct bnx_softc *sc = ifp->if_softc;
3043
3044         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3045
3046         if (bnx_handle_status(sc)) {
3047                 /*
3048                  * Status changes are handled; force the chip to
3049                  * update the status block to reflect whether there
3050                  * are more status changes or not, else staled status
3051                  * changes are always seen.
3052                  */
3053                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3054         }
3055 }
3056
3057 static void
3058 bnx_npoll_status(struct ifnet *ifp)
3059 {
3060         struct bnx_softc *sc = ifp->if_softc;
3061
3062         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3063
3064         sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag;
3065         cpu_lfence();
3066         bnx_npoll_status_notag(ifp);
3067 }
3068
3069 static void
3070 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
3071 {
3072         struct bnx_softc *sc = ifp->if_softc;
3073         int i;
3074
3075         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3076
3077         if (info != NULL) {
3078                 if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG)
3079                         info->ifpi_status.status_func = bnx_npoll_status;
3080                 else
3081                         info->ifpi_status.status_func = bnx_npoll_status_notag;
3082                 info->ifpi_status.serializer = &sc->bnx_main_serialize;
3083
3084                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3085                         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3086                         int idx = i + sc->bnx_npoll_txoff;
3087
3088                         KKASSERT(idx < ncpus2);
3089                         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
3090                                 info->ifpi_tx[idx].poll_func =
3091                                     bnx_npoll_tx_notag;
3092                         } else {
3093                                 info->ifpi_tx[idx].poll_func = bnx_npoll_tx;
3094                         }
3095                         info->ifpi_tx[idx].arg = txr;
3096                         info->ifpi_tx[idx].serializer = &txr->bnx_tx_serialize;
3097                         ifsq_set_cpuid(txr->bnx_ifsq, idx);
3098                 }
3099
3100                 for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
3101                         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
3102                         int idx = i + sc->bnx_npoll_rxoff;
3103
3104                         KKASSERT(idx < ncpus2);
3105                         info->ifpi_rx[idx].poll_func = bnx_npoll_rx;
3106                         info->ifpi_rx[idx].arg = ret;
3107                         info->ifpi_rx[idx].serializer =
3108                             &ret->bnx_rx_ret_serialize;
3109                 }
3110
3111                 if (ifp->if_flags & IFF_RUNNING) {
3112                         bnx_disable_intr(sc);
3113                         bnx_set_tick_cpuid(sc, TRUE);
3114
3115                         sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG |
3116                             BNX_RX_COAL_BDS_CHG;
3117                         bnx_coal_change(sc);
3118                 }
3119         } else {
3120                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3121                         ifsq_set_cpuid(sc->bnx_tx_ring[i].bnx_ifsq,
3122                             sc->bnx_tx_ring[i].bnx_tx_cpuid);
3123                 }
3124                 if (ifp->if_flags & IFF_RUNNING) {
3125                         sc->bnx_coal_chg = BNX_TX_COAL_BDS_CHG |
3126                             BNX_RX_COAL_BDS_CHG;
3127                         bnx_coal_change(sc);
3128
3129                         bnx_enable_intr(sc);
3130                         bnx_set_tick_cpuid(sc, FALSE);
3131                 }
3132         }
3133 }
3134
3135 #endif  /* IFPOLL_ENABLE */
3136
3137 static void
3138 bnx_intr_legacy(void *xsc)
3139 {
3140         struct bnx_softc *sc = xsc;
3141         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
3142
3143         if (ret->bnx_saved_status_tag == *ret->bnx_hw_status_tag) {
3144                 uint32_t val;
3145
3146                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
3147                 if (val & BGE_PCISTAT_INTR_NOTACT)
3148                         return;
3149         }
3150
3151         /*
3152          * NOTE:
3153          * Interrupt will have to be disabled if tagged status
3154          * is used, else interrupt will always be asserted on
3155          * certain chips (at least on BCM5750 AX/BX).
3156          */
3157         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
3158
3159         bnx_intr(sc);
3160 }
3161
3162 static void
3163 bnx_msi(void *xsc)
3164 {
3165         bnx_intr(xsc);
3166 }
3167
3168 static void
3169 bnx_intr(struct bnx_softc *sc)
3170 {
3171         struct ifnet *ifp = &sc->arpcom.ac_if;
3172         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
3173
3174         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3175
3176         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3177         /*
3178          * Use a load fence to ensure that status_tag is saved
3179          * before rx_prod, tx_cons and status.
3180          */
3181         cpu_lfence();
3182
3183         bnx_handle_status(sc);
3184
3185         if (ifp->if_flags & IFF_RUNNING) {
3186                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
3187                 uint16_t rx_prod, tx_cons;
3188
3189                 lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
3190                 rx_prod = *ret->bnx_rx_considx;
3191                 if (ret->bnx_rx_saved_considx != rx_prod)
3192                         bnx_rxeof(ret, rx_prod, -1);
3193                 lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
3194
3195                 lwkt_serialize_enter(&txr->bnx_tx_serialize);
3196                 tx_cons = *txr->bnx_tx_considx;
3197                 if (txr->bnx_tx_saved_considx != tx_cons)
3198                         bnx_txeof(txr, tx_cons);
3199                 lwkt_serialize_exit(&txr->bnx_tx_serialize);
3200         }
3201
3202         bnx_writembx(sc, BGE_MBX_IRQ0_LO, ret->bnx_saved_status_tag << 24);
3203 }
3204
3205 static void
3206 bnx_msix_tx_status(void *xtxr)
3207 {
3208         struct bnx_tx_ring *txr = xtxr;
3209         struct bnx_softc *sc = txr->bnx_sc;
3210         struct ifnet *ifp = &sc->arpcom.ac_if;
3211
3212         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3213
3214         txr->bnx_saved_status_tag = *txr->bnx_hw_status_tag;
3215         /*
3216          * Use a load fence to ensure that status_tag is saved
3217          * before tx_cons and status.
3218          */
3219         cpu_lfence();
3220
3221         bnx_handle_status(sc);
3222
3223         if (ifp->if_flags & IFF_RUNNING) {
3224                 uint16_t tx_cons;
3225
3226                 lwkt_serialize_enter(&txr->bnx_tx_serialize);
3227                 tx_cons = *txr->bnx_tx_considx;
3228                 if (txr->bnx_tx_saved_considx != tx_cons)
3229                         bnx_txeof(txr, tx_cons);
3230                 lwkt_serialize_exit(&txr->bnx_tx_serialize);
3231         }
3232
3233         bnx_writembx(sc, BGE_MBX_IRQ0_LO, txr->bnx_saved_status_tag << 24);
3234 }
3235
3236 static void
3237 bnx_msix_rx(void *xret)
3238 {
3239         struct bnx_rx_ret_ring *ret = xret;
3240         uint16_t rx_prod;
3241
3242         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
3243
3244         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3245         /*
3246          * Use a load fence to ensure that status_tag is saved
3247          * before rx_prod.
3248          */
3249         cpu_lfence();
3250
3251         rx_prod = *ret->bnx_rx_considx;
3252         if (ret->bnx_rx_saved_considx != rx_prod)
3253                 bnx_rxeof(ret, rx_prod, -1);
3254
3255         bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx,
3256             ret->bnx_saved_status_tag << 24);
3257 }
3258
3259 static void
3260 bnx_msix_rxtx(void *xret)
3261 {
3262         struct bnx_rx_ret_ring *ret = xret;
3263         struct bnx_tx_ring *txr = ret->bnx_txr;
3264         uint16_t rx_prod, tx_cons;
3265
3266         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
3267
3268         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
3269         /*
3270          * Use a load fence to ensure that status_tag is saved
3271          * before rx_prod and tx_cons.
3272          */
3273         cpu_lfence();
3274
3275         rx_prod = *ret->bnx_rx_considx;
3276         if (ret->bnx_rx_saved_considx != rx_prod)
3277                 bnx_rxeof(ret, rx_prod, -1);
3278
3279         lwkt_serialize_enter(&txr->bnx_tx_serialize);
3280         tx_cons = *txr->bnx_tx_considx;
3281         if (txr->bnx_tx_saved_considx != tx_cons)
3282                 bnx_txeof(txr, tx_cons);
3283         lwkt_serialize_exit(&txr->bnx_tx_serialize);
3284
3285         bnx_writembx(ret->bnx_sc, ret->bnx_msix_mbx,
3286             ret->bnx_saved_status_tag << 24);
3287 }
3288
3289 static void
3290 bnx_msix_status(void *xsc)
3291 {
3292         struct bnx_softc *sc = xsc;
3293
3294         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
3295
3296         sc->bnx_saved_status_tag = *sc->bnx_hw_status_tag;
3297         /*
3298          * Use a load fence to ensure that status_tag is saved
3299          * before status.
3300          */
3301         cpu_lfence();
3302
3303         bnx_handle_status(sc);
3304
3305         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_saved_status_tag << 24);
3306 }
3307
3308 static void
3309 bnx_tick(void *xsc)
3310 {
3311         struct bnx_softc *sc = xsc;
3312
3313         lwkt_serialize_enter(&sc->bnx_main_serialize);
3314
3315         bnx_stats_update_regs(sc);
3316
3317         if (sc->bnx_flags & BNX_FLAG_TBI) {
3318                 /*
3319                  * Since in TBI mode auto-polling can't be used we should poll
3320                  * link status manually. Here we register pending link event
3321                  * and trigger interrupt.
3322                  */
3323                 sc->bnx_link_evt++;
3324                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3325         } else if (!sc->bnx_link) {
3326                 mii_tick(device_get_softc(sc->bnx_miibus));
3327         }
3328
3329         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
3330             sc->bnx_tick_cpuid);
3331
3332         lwkt_serialize_exit(&sc->bnx_main_serialize);
3333 }
3334
3335 static void
3336 bnx_stats_update_regs(struct bnx_softc *sc)
3337 {
3338         struct ifnet *ifp = &sc->arpcom.ac_if;
3339         struct bge_mac_stats_regs stats;
3340         uint32_t *s, val;
3341         int i;
3342
3343         s = (uint32_t *)&stats;
3344         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
3345                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
3346                 s++;
3347         }
3348
3349         IFNET_STAT_SET(ifp, collisions,
3350            (stats.dot3StatsSingleCollisionFrames +
3351            stats.dot3StatsMultipleCollisionFrames +
3352            stats.dot3StatsExcessiveCollisions +
3353            stats.dot3StatsLateCollisions));
3354
3355         val = CSR_READ_4(sc, BGE_RXLP_LOCSTAT_OUT_OF_BDS);
3356         sc->bnx_norxbds += val;
3357 }
3358
3359 /*
3360  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
3361  * pointers to descriptors.
3362  */
3363 static int
3364 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
3365     int *segs_used)
3366 {
3367         struct bge_tx_bd *d = NULL;
3368         uint16_t csum_flags = 0, vlan_tag = 0, mss = 0;
3369         bus_dma_segment_t segs[BNX_NSEG_NEW];
3370         bus_dmamap_t map;
3371         int error, maxsegs, nsegs, idx, i;
3372         struct mbuf *m_head = *m_head0, *m_new;
3373
3374         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
3375 #ifdef BNX_TSO_DEBUG
3376                 int tso_nsegs;
3377 #endif
3378
3379                 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags);
3380                 if (error)
3381                         return error;
3382                 m_head = *m_head0;
3383
3384 #ifdef BNX_TSO_DEBUG
3385                 tso_nsegs = (m_head->m_pkthdr.len /
3386                     m_head->m_pkthdr.tso_segsz) - 1;
3387                 if (tso_nsegs > (BNX_TSO_NSTATS - 1))
3388                         tso_nsegs = BNX_TSO_NSTATS - 1;
3389                 else if (tso_nsegs < 0)
3390                         tso_nsegs = 0;
3391                 txr->bnx_sc->bnx_tsosegs[tso_nsegs]++;
3392 #endif
3393         } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) {
3394                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
3395                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
3396                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
3397                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
3398                 if (m_head->m_flags & M_LASTFRAG)
3399                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
3400                 else if (m_head->m_flags & M_FRAG)
3401                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
3402         }
3403         if (m_head->m_flags & M_VLANTAG) {
3404                 csum_flags |= BGE_TXBDFLAG_VLAN_TAG;
3405                 vlan_tag = m_head->m_pkthdr.ether_vlantag;
3406         }
3407
3408         idx = *txidx;
3409         map = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
3410
3411         maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD;
3412         KASSERT(maxsegs >= BNX_NSEG_SPARE,
3413                 ("not enough segments %d", maxsegs));
3414
3415         if (maxsegs > BNX_NSEG_NEW)
3416                 maxsegs = BNX_NSEG_NEW;
3417
3418         /*
3419          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
3420          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
3421          * but when such padded frames employ the bge IP/TCP checksum
3422          * offload, the hardware checksum assist gives incorrect results
3423          * (possibly from incorporating its own padding into the UDP/TCP
3424          * checksum; who knows).  If we pad such runts with zeros, the
3425          * onboard checksum comes out correct.
3426          */
3427         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
3428             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
3429                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
3430                 if (error)
3431                         goto back;
3432         }
3433
3434         if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) &&
3435             m_head->m_next != NULL) {
3436                 m_new = bnx_defrag_shortdma(m_head);
3437                 if (m_new == NULL) {
3438                         error = ENOBUFS;
3439                         goto back;
3440                 }
3441                 *m_head0 = m_head = m_new;
3442         }
3443         if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 &&
3444             (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) &&
3445             m_head->m_next != NULL) {
3446                 /*
3447                  * Forcefully defragment mbuf chain to overcome hardware
3448                  * limitation which only support a single outstanding
3449                  * DMA read operation.  If it fails, keep moving on using
3450                  * the original mbuf chain.
3451                  */
3452                 m_new = m_defrag(m_head, M_NOWAIT);
3453                 if (m_new != NULL)
3454                         *m_head0 = m_head = m_new;
3455         }
3456
3457         error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map,
3458             m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
3459         if (error)
3460                 goto back;
3461         *segs_used += nsegs;
3462
3463         m_head = *m_head0;
3464         bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
3465
3466         for (i = 0; ; i++) {
3467                 d = &txr->bnx_tx_ring[idx];
3468
3469                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
3470                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
3471                 d->bge_len = segs[i].ds_len;
3472                 d->bge_flags = csum_flags;
3473                 d->bge_vlan_tag = vlan_tag;
3474                 d->bge_mss = mss;
3475
3476                 if (i == nsegs - 1)
3477                         break;
3478                 BNX_INC(idx, BGE_TX_RING_CNT);
3479         }
3480         /* Mark the last segment as end of packet... */
3481         d->bge_flags |= BGE_TXBDFLAG_END;
3482
3483         /*
3484          * Insure that the map for this transmission is placed at
3485          * the array index of the last descriptor in this chain.
3486          */
3487         txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
3488         txr->bnx_tx_buf[idx].bnx_tx_dmamap = map;
3489         txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head;
3490         txr->bnx_tx_cnt += nsegs;
3491
3492         BNX_INC(idx, BGE_TX_RING_CNT);
3493         *txidx = idx;
3494 back:
3495         if (error) {
3496                 m_freem(*m_head0);
3497                 *m_head0 = NULL;
3498         }
3499         return error;
3500 }
3501
3502 /*
3503  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
3504  * to the mbuf data regions directly in the transmit descriptors.
3505  */
3506 static void
3507 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3508 {
3509         struct bnx_tx_ring *txr = ifsq_get_priv(ifsq);
3510         struct mbuf *m_head = NULL;
3511         uint32_t prodidx;
3512         int nsegs = 0;
3513
3514         KKASSERT(txr->bnx_ifsq == ifsq);
3515         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
3516
3517         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
3518                 return;
3519
3520         prodidx = txr->bnx_tx_prodidx;
3521
3522         while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) {
3523                 /*
3524                  * Sanity check: avoid coming within BGE_NSEG_RSVD
3525                  * descriptors of the end of the ring.  Also make
3526                  * sure there are BGE_NSEG_SPARE descriptors for
3527                  * jumbo buffers' or TSO segments' defragmentation.
3528                  */
3529                 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) <
3530                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
3531                         ifsq_set_oactive(ifsq);
3532                         break;
3533                 }
3534
3535                 m_head = ifsq_dequeue(ifsq);
3536                 if (m_head == NULL)
3537                         break;
3538
3539                 /*
3540                  * Pack the data into the transmit ring. If we
3541                  * don't have room, set the OACTIVE flag and wait
3542                  * for the NIC to drain the ring.
3543                  */
3544                 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) {
3545                         ifsq_set_oactive(ifsq);
3546                         IFNET_STAT_INC(ifp, oerrors, 1);
3547                         break;
3548                 }
3549
3550                 if (nsegs >= txr->bnx_tx_wreg) {
3551                         /* Transmit */
3552                         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3553                         nsegs = 0;
3554                 }
3555
3556                 ETHER_BPF_MTAP(ifp, m_head);
3557
3558                 /*
3559                  * Set a timeout in case the chip goes out to lunch.
3560                  */
3561                 txr->bnx_tx_watchdog.wd_timer = 5;
3562         }
3563
3564         if (nsegs > 0) {
3565                 /* Transmit */
3566                 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3567         }
3568         txr->bnx_tx_prodidx = prodidx;
3569 }
3570
3571 static void
3572 bnx_init(void *xsc)
3573 {
3574         struct bnx_softc *sc = xsc;
3575         struct ifnet *ifp = &sc->arpcom.ac_if;
3576         uint16_t *m;
3577         uint32_t mode;
3578         int i;
3579         boolean_t polling;
3580
3581         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3582
3583         /* Cancel pending I/O and flush buffers. */
3584         bnx_stop(sc);
3585
3586         bnx_sig_pre_reset(sc, BNX_RESET_START);
3587         bnx_reset(sc);
3588         bnx_sig_post_reset(sc, BNX_RESET_START);
3589
3590         bnx_chipinit(sc);
3591
3592         /*
3593          * Init the various state machines, ring
3594          * control blocks and firmware.
3595          */
3596         if (bnx_blockinit(sc)) {
3597                 if_printf(ifp, "initialization failure\n");
3598                 bnx_stop(sc);
3599                 return;
3600         }
3601
3602         /* Specify MTU. */
3603         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
3604             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
3605
3606         /* Load our MAC address. */
3607         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
3608         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
3609         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
3610
3611         /* Enable or disable promiscuous mode as needed. */
3612         bnx_setpromisc(sc);
3613
3614         /* Program multicast filter. */
3615         bnx_setmulti(sc);
3616
3617         /* Init RX ring. */
3618         if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) {
3619                 if_printf(ifp, "RX ring initialization failed\n");
3620                 bnx_stop(sc);
3621                 return;
3622         }
3623
3624         /* Init jumbo RX ring. */
3625         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
3626                 if (bnx_init_rx_ring_jumbo(sc)) {
3627                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
3628                         bnx_stop(sc);
3629                         return;
3630                 }
3631         }
3632
3633         /* Init our RX return ring index */
3634         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
3635                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
3636
3637                 ret->bnx_rx_saved_considx = 0;
3638                 ret->bnx_rx_cnt = 0;
3639         }
3640
3641         /* Init TX ring. */
3642         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3643                 bnx_init_tx_ring(&sc->bnx_tx_ring[i]);
3644
3645         /* Enable TX MAC state machine lockup fix. */
3646         mode = CSR_READ_4(sc, BGE_TX_MODE);
3647         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3648         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
3649             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
3650                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3651                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
3652                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3653         }
3654         /* Turn on transmitter */
3655         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3656         DELAY(100);
3657
3658         /* Initialize RSS */
3659         mode = BGE_RXMODE_ENABLE | BGE_RXMODE_IPV6_ENABLE;
3660         if (BNX_RSS_ENABLED(sc)) {
3661                 bnx_init_rss(sc);
3662                 mode |= BGE_RXMODE_RSS_ENABLE |
3663                     BGE_RXMODE_RSS_HASH_MASK_BITS |
3664                     BGE_RXMODE_RSS_IPV4_HASH |
3665                     BGE_RXMODE_RSS_TCP_IPV4_HASH;
3666         }
3667         /* Turn on receiver */
3668         BNX_SETBIT(sc, BGE_RX_MODE, mode);
3669         DELAY(10);
3670
3671         /*
3672          * Set the number of good frames to receive after RX MBUF
3673          * Low Watermark has been reached.  After the RX MAC receives
3674          * this number of frames, it will drop subsequent incoming
3675          * frames until the MBUF High Watermark is reached.
3676          */
3677         if (BNX_IS_57765_FAMILY(sc))
3678                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3679         else
3680                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3681
3682         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI ||
3683             sc->bnx_intr_type == PCI_INTR_TYPE_MSIX) {
3684                 if (bootverbose) {
3685                         if_printf(ifp, "MSI_MODE: %#x\n",
3686                             CSR_READ_4(sc, BGE_MSI_MODE));
3687                 }
3688         }
3689
3690         /* Tell firmware we're alive. */
3691         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3692
3693         /* Enable host interrupts if polling(4) is not enabled. */
3694         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3695
3696         polling = FALSE;
3697 #ifdef IFPOLL_ENABLE
3698         if (ifp->if_flags & IFF_NPOLLING)
3699                 polling = TRUE;
3700 #endif
3701         if (polling)
3702                 bnx_disable_intr(sc);
3703         else
3704                 bnx_enable_intr(sc);
3705         bnx_set_tick_cpuid(sc, polling);
3706
3707         ifp->if_flags |= IFF_RUNNING;
3708         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3709                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3710
3711                 ifsq_clr_oactive(txr->bnx_ifsq);
3712                 ifsq_watchdog_start(&txr->bnx_tx_watchdog);
3713         }
3714
3715         bnx_ifmedia_upd(ifp);
3716
3717         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
3718             sc->bnx_tick_cpuid);
3719 }
3720
3721 /*
3722  * Set media options.
3723  */
3724 static int
3725 bnx_ifmedia_upd(struct ifnet *ifp)
3726 {
3727         struct bnx_softc *sc = ifp->if_softc;
3728
3729         /* If this is a 1000baseX NIC, enable the TBI port. */
3730         if (sc->bnx_flags & BNX_FLAG_TBI) {
3731                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3732
3733                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3734                         return(EINVAL);
3735
3736                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3737                 case IFM_AUTO:
3738                         break;
3739
3740                 case IFM_1000_SX:
3741                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3742                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3743                                     BGE_MACMODE_HALF_DUPLEX);
3744                         } else {
3745                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3746                                     BGE_MACMODE_HALF_DUPLEX);
3747                         }
3748                         DELAY(40);
3749                         break;
3750                 default:
3751                         return(EINVAL);
3752                 }
3753         } else {
3754                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3755
3756                 sc->bnx_link_evt++;
3757                 sc->bnx_link = 0;
3758                 if (mii->mii_instance) {
3759                         struct mii_softc *miisc;
3760
3761                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3762                                 mii_phy_reset(miisc);
3763                 }
3764                 mii_mediachg(mii);
3765
3766                 /*
3767                  * Force an interrupt so that we will call bnx_link_upd
3768                  * if needed and clear any pending link state attention.
3769                  * Without this we are not getting any further interrupts
3770                  * for link state changes and thus will not UP the link and
3771                  * not be able to send in bnx_start.  The only way to get
3772                  * things working was to receive a packet and get an RX
3773                  * intr.
3774                  *
3775                  * bnx_tick should help for fiber cards and we might not
3776                  * need to do this here if BNX_FLAG_TBI is set but as
3777                  * we poll for fiber anyway it should not harm.
3778                  */
3779                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3780         }
3781         return(0);
3782 }
3783
3784 /*
3785  * Report current media status.
3786  */
3787 static void
3788 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3789 {
3790         struct bnx_softc *sc = ifp->if_softc;
3791
3792         if ((ifp->if_flags & IFF_RUNNING) == 0)
3793                 return;
3794
3795         if (sc->bnx_flags & BNX_FLAG_TBI) {
3796                 ifmr->ifm_status = IFM_AVALID;
3797                 ifmr->ifm_active = IFM_ETHER;
3798                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3799                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3800                         ifmr->ifm_status |= IFM_ACTIVE;
3801                 } else {
3802                         ifmr->ifm_active |= IFM_NONE;
3803                         return;
3804                 }
3805
3806                 ifmr->ifm_active |= IFM_1000_SX;
3807                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3808                         ifmr->ifm_active |= IFM_HDX;
3809                 else
3810                         ifmr->ifm_active |= IFM_FDX;
3811         } else {
3812                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3813
3814                 mii_pollstat(mii);
3815                 ifmr->ifm_active = mii->mii_media_active;
3816                 ifmr->ifm_status = mii->mii_media_status;
3817         }
3818 }
3819
3820 static int
3821 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3822 {
3823         struct bnx_softc *sc = ifp->if_softc;
3824         struct ifreq *ifr = (struct ifreq *)data;
3825         int mask, error = 0;
3826
3827         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3828
3829         switch (command) {
3830         case SIOCSIFMTU:
3831                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3832                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3833                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3834                         error = EINVAL;
3835                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3836                         ifp->if_mtu = ifr->ifr_mtu;
3837                         if (ifp->if_flags & IFF_RUNNING)
3838                                 bnx_init(sc);
3839                 }
3840                 break;
3841         case SIOCSIFFLAGS:
3842                 if (ifp->if_flags & IFF_UP) {
3843                         if (ifp->if_flags & IFF_RUNNING) {
3844                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3845
3846                                 /*
3847                                  * If only the state of the PROMISC flag
3848                                  * changed, then just use the 'set promisc
3849                                  * mode' command instead of reinitializing
3850                                  * the entire NIC. Doing a full re-init
3851                                  * means reloading the firmware and waiting
3852                                  * for it to start up, which may take a
3853                                  * second or two.  Similarly for ALLMULTI.
3854                                  */
3855                                 if (mask & IFF_PROMISC)
3856                                         bnx_setpromisc(sc);
3857                                 if (mask & IFF_ALLMULTI)
3858                                         bnx_setmulti(sc);
3859                         } else {
3860                                 bnx_init(sc);
3861                         }
3862                 } else if (ifp->if_flags & IFF_RUNNING) {
3863                         bnx_stop(sc);
3864                 }
3865                 sc->bnx_if_flags = ifp->if_flags;
3866                 break;
3867         case SIOCADDMULTI:
3868         case SIOCDELMULTI:
3869                 if (ifp->if_flags & IFF_RUNNING)
3870                         bnx_setmulti(sc);
3871                 break;
3872         case SIOCSIFMEDIA:
3873         case SIOCGIFMEDIA:
3874                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3875                         error = ifmedia_ioctl(ifp, ifr,
3876                             &sc->bnx_ifmedia, command);
3877                 } else {
3878                         struct mii_data *mii;
3879
3880                         mii = device_get_softc(sc->bnx_miibus);
3881                         error = ifmedia_ioctl(ifp, ifr,
3882                                               &mii->mii_media, command);
3883                 }
3884                 break;
3885         case SIOCSIFCAP:
3886                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3887                 if (mask & IFCAP_HWCSUM) {
3888                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3889                         if (ifp->if_capenable & IFCAP_TXCSUM)
3890                                 ifp->if_hwassist |= BNX_CSUM_FEATURES;
3891                         else
3892                                 ifp->if_hwassist &= ~BNX_CSUM_FEATURES;
3893                 }
3894                 if (mask & IFCAP_TSO) {
3895                         ifp->if_capenable ^= (mask & IFCAP_TSO);
3896                         if (ifp->if_capenable & IFCAP_TSO)
3897                                 ifp->if_hwassist |= CSUM_TSO;
3898                         else
3899                                 ifp->if_hwassist &= ~CSUM_TSO;
3900                 }
3901                 if (mask & IFCAP_RSS)
3902                         ifp->if_capenable ^= IFCAP_RSS;
3903                 break;
3904         default:
3905                 error = ether_ioctl(ifp, command, data);
3906                 break;
3907         }
3908         return error;
3909 }
3910
3911 static void
3912 bnx_watchdog(struct ifaltq_subque *ifsq)
3913 {
3914         struct ifnet *ifp = ifsq_get_ifp(ifsq);
3915         struct bnx_softc *sc = ifp->if_softc;
3916         int i;
3917
3918         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3919
3920         if_printf(ifp, "watchdog timeout -- resetting\n");
3921
3922         bnx_init(sc);
3923
3924         IFNET_STAT_INC(ifp, oerrors, 1);
3925
3926         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3927                 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
3928 }
3929
3930 /*
3931  * Stop the adapter and free any mbufs allocated to the
3932  * RX and TX lists.
3933  */
3934 static void
3935 bnx_stop(struct bnx_softc *sc)
3936 {
3937         struct ifnet *ifp = &sc->arpcom.ac_if;
3938         int i;
3939
3940         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3941
3942         callout_stop(&sc->bnx_tick_timer);
3943
3944         /* Disable host interrupts. */
3945         bnx_disable_intr(sc);
3946
3947         /*
3948          * Tell firmware we're shutting down.
3949          */
3950         bnx_sig_pre_reset(sc, BNX_RESET_SHUTDOWN);
3951
3952         /*
3953          * Disable all of the receiver blocks
3954          */
3955         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3956         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3957         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3958         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3959         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3960         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3961
3962         /*
3963          * Disable all of the transmit blocks
3964          */
3965         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3966         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3967         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3968         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3969         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3970         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3971
3972         /*
3973          * Shut down all of the memory managers and related
3974          * state machines.
3975          */
3976         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3977         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3978         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3979         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3980
3981         bnx_reset(sc);
3982         bnx_sig_post_reset(sc, BNX_RESET_SHUTDOWN);
3983
3984         /*
3985          * Tell firmware we're shutting down.
3986          */
3987         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3988
3989         /* Free the RX lists. */
3990         bnx_free_rx_ring_std(&sc->bnx_rx_std_ring);
3991
3992         /* Free jumbo RX list. */
3993         if (BNX_IS_JUMBO_CAPABLE(sc))
3994                 bnx_free_rx_ring_jumbo(sc);
3995
3996         /* Free TX buffers. */
3997         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3998                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3999
4000                 txr->bnx_saved_status_tag = 0;
4001                 bnx_free_tx_ring(txr);
4002         }
4003
4004         /* Clear saved status tag */
4005         for (i = 0; i < sc->bnx_rx_retcnt; ++i)
4006                 sc->bnx_rx_ret_ring[i].bnx_saved_status_tag = 0;
4007
4008         sc->bnx_link = 0;
4009         sc->bnx_coal_chg = 0;
4010
4011         ifp->if_flags &= ~IFF_RUNNING;
4012         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4013                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
4014
4015                 ifsq_clr_oactive(txr->bnx_ifsq);
4016                 ifsq_watchdog_stop(&txr->bnx_tx_watchdog);
4017         }
4018 }
4019
4020 /*
4021  * Stop all chip I/O so that the kernel's probe routines don't
4022  * get confused by errant DMAs when rebooting.
4023  */
4024 static void
4025 bnx_shutdown(device_t dev)
4026 {
4027         struct bnx_softc *sc = device_get_softc(dev);
4028         struct ifnet *ifp = &sc->arpcom.ac_if;
4029
4030         ifnet_serialize_all(ifp);
4031         bnx_stop(sc);
4032         ifnet_deserialize_all(ifp);
4033 }
4034
4035 static int
4036 bnx_suspend(device_t dev)
4037 {
4038         struct bnx_softc *sc = device_get_softc(dev);
4039         struct ifnet *ifp = &sc->arpcom.ac_if;
4040
4041         ifnet_serialize_all(ifp);
4042         bnx_stop(sc);
4043         ifnet_deserialize_all(ifp);
4044
4045         return 0;
4046 }
4047
4048 static int
4049 bnx_resume(device_t dev)
4050 {
4051         struct bnx_softc *sc = device_get_softc(dev);
4052         struct ifnet *ifp = &sc->arpcom.ac_if;
4053
4054         ifnet_serialize_all(ifp);
4055
4056         if (ifp->if_flags & IFF_UP) {
4057                 int i;
4058
4059                 bnx_init(sc);
4060                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
4061                         ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
4062         }
4063
4064         ifnet_deserialize_all(ifp);
4065
4066         return 0;
4067 }
4068
4069 static void
4070 bnx_setpromisc(struct bnx_softc *sc)
4071 {
4072         struct ifnet *ifp = &sc->arpcom.ac_if;
4073
4074         if (ifp->if_flags & IFF_PROMISC)
4075                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
4076         else
4077                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
4078 }
4079
4080 static void
4081 bnx_dma_free(struct bnx_softc *sc)
4082 {
4083         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
4084         int i;
4085
4086         /* Destroy RX return rings */
4087         if (sc->bnx_rx_ret_ring != NULL) {
4088                 for (i = 0; i < sc->bnx_rx_retcnt; ++i)
4089                         bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]);
4090                 kfree(sc->bnx_rx_ret_ring, M_DEVBUF);
4091         }
4092
4093         /* Destroy RX mbuf DMA stuffs. */
4094         if (std->bnx_rx_mtag != NULL) {
4095                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
4096                         KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL);
4097                         bus_dmamap_destroy(std->bnx_rx_mtag,
4098                             std->bnx_rx_std_buf[i].bnx_rx_dmamap);
4099                 }
4100                 bus_dma_tag_destroy(std->bnx_rx_mtag);
4101         }
4102
4103         /* Destroy standard RX ring */
4104         bnx_dma_block_free(std->bnx_rx_std_ring_tag,
4105             std->bnx_rx_std_ring_map, std->bnx_rx_std_ring);
4106
4107         /* Destroy TX rings */
4108         if (sc->bnx_tx_ring != NULL) {
4109                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
4110                         bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]);
4111                 kfree(sc->bnx_tx_ring, M_DEVBUF);
4112         }
4113
4114         if (BNX_IS_JUMBO_CAPABLE(sc))
4115                 bnx_free_jumbo_mem(sc);
4116
4117         /* Destroy status blocks */
4118         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4119                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4120
4121                 bnx_dma_block_free(intr->bnx_status_tag,
4122                     intr->bnx_status_map, intr->bnx_status_block);
4123         }
4124
4125         /* Destroy the parent tag */
4126         if (sc->bnx_cdata.bnx_parent_tag != NULL)
4127                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
4128 }
4129
4130 static int
4131 bnx_dma_alloc(device_t dev)
4132 {
4133         struct bnx_softc *sc = device_get_softc(dev);
4134         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
4135         int i, error, mbx;
4136
4137         /*
4138          * Allocate the parent bus DMA tag appropriate for PCI.
4139          *
4140          * All of the NetExtreme/NetLink controllers have 4GB boundary
4141          * DMA bug.
4142          * Whenever an address crosses a multiple of the 4GB boundary
4143          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
4144          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
4145          * state machine will lockup and cause the device to hang.
4146          */
4147         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
4148             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
4149             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
4150             0, &sc->bnx_cdata.bnx_parent_tag);
4151         if (error) {
4152                 device_printf(dev, "could not create parent DMA tag\n");
4153                 return error;
4154         }
4155
4156         /*
4157          * Create DMA stuffs for status blocks.
4158          */
4159         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4160                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4161
4162                 error = bnx_dma_block_alloc(sc,
4163                     __VM_CACHELINE_ALIGN(BGE_STATUS_BLK_SZ),
4164                     &intr->bnx_status_tag, &intr->bnx_status_map,
4165                     (void *)&intr->bnx_status_block,
4166                     &intr->bnx_status_block_paddr);
4167                 if (error) {
4168                         device_printf(dev,
4169                             "could not create %dth status block\n", i);
4170                         return error;
4171                 }
4172         }
4173         sc->bnx_hw_status = &sc->bnx_intr_data[0].bnx_status_block->bge_status;
4174         if (sc->bnx_flags & BNX_FLAG_STATUS_HASTAG) {
4175                 sc->bnx_hw_status_tag =
4176                     &sc->bnx_intr_data[0].bnx_status_block->bge_status_tag;
4177         }
4178
4179         /*
4180          * Create DMA tag and maps for RX mbufs.
4181          */
4182         std->bnx_sc = sc;
4183         lwkt_serialize_init(&std->bnx_rx_std_serialize);
4184         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
4185             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4186             NULL, NULL, MCLBYTES, 1, MCLBYTES,
4187             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag);
4188         if (error) {
4189                 device_printf(dev, "could not create RX mbuf DMA tag\n");
4190                 return error;
4191         }
4192
4193         for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) {
4194                 error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK,
4195                     &std->bnx_rx_std_buf[i].bnx_rx_dmamap);
4196                 if (error) {
4197                         int j;
4198
4199                         for (j = 0; j < i; ++j) {
4200                                 bus_dmamap_destroy(std->bnx_rx_mtag,
4201                                     std->bnx_rx_std_buf[j].bnx_rx_dmamap);
4202                         }
4203                         bus_dma_tag_destroy(std->bnx_rx_mtag);
4204                         std->bnx_rx_mtag = NULL;
4205
4206                         device_printf(dev,
4207                             "could not create %dth RX mbuf DMA map\n", i);
4208                         return error;
4209                 }
4210         }
4211
4212         /*
4213          * Create DMA stuffs for standard RX ring.
4214          */
4215         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
4216             &std->bnx_rx_std_ring_tag,
4217             &std->bnx_rx_std_ring_map,
4218             (void *)&std->bnx_rx_std_ring,
4219             &std->bnx_rx_std_ring_paddr);
4220         if (error) {
4221                 device_printf(dev, "could not create std RX ring\n");
4222                 return error;
4223         }
4224
4225         /*
4226          * Create RX return rings
4227          */
4228         mbx = BGE_MBX_RX_CONS0_LO;
4229         sc->bnx_rx_ret_ring = kmalloc_cachealign(
4230             sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF,
4231             M_WAITOK | M_ZERO);
4232         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4233                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
4234                 struct bnx_intr_data *intr;
4235
4236                 ret->bnx_sc = sc;
4237                 ret->bnx_std = std;
4238                 ret->bnx_rx_mbx = mbx;
4239                 ret->bnx_rx_cntmax = (BGE_STD_RX_RING_CNT / 4) /
4240                     sc->bnx_rx_retcnt;
4241                 ret->bnx_rx_mask = 1 << i;
4242
4243                 if (!BNX_RSS_ENABLED(sc)) {
4244                         intr = &sc->bnx_intr_data[0];
4245                 } else {
4246                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
4247                         intr = &sc->bnx_intr_data[i + 1];
4248                 }
4249
4250                 if (i == 0) {
4251                         ret->bnx_rx_considx =
4252                             &intr->bnx_status_block->bge_idx[0].bge_rx_prod_idx;
4253                 } else if (i == 1) {
4254                         ret->bnx_rx_considx =
4255                             &intr->bnx_status_block->bge_rx_jumbo_cons_idx;
4256                 } else if (i == 2) {
4257                         ret->bnx_rx_considx =
4258                             &intr->bnx_status_block->bge_rsvd1;
4259                 } else if (i == 3) {
4260                         ret->bnx_rx_considx =
4261                             &intr->bnx_status_block->bge_rx_mini_cons_idx;
4262                 } else {
4263                         panic("unknown RX return ring %d\n", i);
4264                 }
4265                 ret->bnx_hw_status_tag =
4266                     &intr->bnx_status_block->bge_status_tag;
4267
4268                 error = bnx_create_rx_ret_ring(ret);
4269                 if (error) {
4270                         device_printf(dev,
4271                             "could not create %dth RX ret ring\n", i);
4272                         return error;
4273                 }
4274                 mbx += 8;
4275         }
4276
4277         /*
4278          * Create TX rings
4279          */
4280         sc->bnx_tx_ring = kmalloc_cachealign(
4281             sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF,
4282             M_WAITOK | M_ZERO);
4283         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4284                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
4285                 struct bnx_intr_data *intr;
4286
4287                 txr->bnx_sc = sc;
4288                 txr->bnx_tx_mbx = bnx_tx_mailbox[i];
4289
4290                 if (sc->bnx_tx_ringcnt == 1) {
4291                         intr = &sc->bnx_intr_data[0];
4292                 } else {
4293                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
4294                         intr = &sc->bnx_intr_data[i + 1];
4295                 }
4296
4297                 if ((sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) == 0) {
4298                         txr->bnx_hw_status_tag =
4299                             &intr->bnx_status_block->bge_status_tag;
4300                 }
4301                 txr->bnx_tx_considx =
4302                     &intr->bnx_status_block->bge_idx[0].bge_tx_cons_idx;
4303
4304                 error = bnx_create_tx_ring(txr);
4305                 if (error) {
4306                         device_printf(dev,
4307                             "could not create %dth TX ring\n", i);
4308                         return error;
4309                 }
4310         }
4311
4312         /*
4313          * Create jumbo buffer pool.
4314          */
4315         if (BNX_IS_JUMBO_CAPABLE(sc)) {
4316                 error = bnx_alloc_jumbo_mem(sc);
4317                 if (error) {
4318                         device_printf(dev,
4319                             "could not create jumbo buffer pool\n");
4320                         return error;
4321                 }
4322         }
4323
4324         return 0;
4325 }
4326
4327 static int
4328 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
4329                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
4330 {
4331         bus_dmamem_t dmem;
4332         int error;
4333
4334         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
4335                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
4336                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
4337         if (error)
4338                 return error;
4339
4340         *tag = dmem.dmem_tag;
4341         *map = dmem.dmem_map;
4342         *addr = dmem.dmem_addr;
4343         *paddr = dmem.dmem_busaddr;
4344
4345         return 0;
4346 }
4347
4348 static void
4349 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
4350 {
4351         if (tag != NULL) {
4352                 bus_dmamap_unload(tag, map);
4353                 bus_dmamem_free(tag, addr, map);
4354                 bus_dma_tag_destroy(tag);
4355         }
4356 }
4357
4358 static void
4359 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
4360 {
4361         struct ifnet *ifp = &sc->arpcom.ac_if;
4362
4363 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
4364
4365         /*
4366          * Sometimes PCS encoding errors are detected in
4367          * TBI mode (on fiber NICs), and for some reason
4368          * the chip will signal them as link changes.
4369          * If we get a link change event, but the 'PCS
4370          * encoding error' bit in the MAC status register
4371          * is set, don't bother doing a link check.
4372          * This avoids spurious "gigabit link up" messages
4373          * that sometimes appear on fiber NICs during
4374          * periods of heavy traffic.
4375          */
4376         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
4377                 if (!sc->bnx_link) {
4378                         sc->bnx_link++;
4379                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
4380                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
4381                                     BGE_MACMODE_TBI_SEND_CFGS);
4382                                 DELAY(40);
4383                         }
4384                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
4385
4386                         if (bootverbose)
4387                                 if_printf(ifp, "link UP\n");
4388
4389                         ifp->if_link_state = LINK_STATE_UP;
4390                         if_link_state_change(ifp);
4391                 }
4392         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
4393                 if (sc->bnx_link) {
4394                         sc->bnx_link = 0;
4395
4396                         if (bootverbose)
4397                                 if_printf(ifp, "link DOWN\n");
4398
4399                         ifp->if_link_state = LINK_STATE_DOWN;
4400                         if_link_state_change(ifp);
4401                 }
4402         }
4403
4404 #undef PCS_ENCODE_ERR
4405
4406         /* Clear the attention. */
4407         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4408             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4409             BGE_MACSTAT_LINK_CHANGED);
4410 }
4411
4412 static void
4413 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
4414 {
4415         struct ifnet *ifp = &sc->arpcom.ac_if;
4416         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
4417
4418         mii_pollstat(mii);
4419         bnx_miibus_statchg(sc->bnx_dev);
4420
4421         if (bootverbose) {
4422                 if (sc->bnx_link)
4423                         if_printf(ifp, "link UP\n");
4424                 else
4425                         if_printf(ifp, "link DOWN\n");
4426         }
4427
4428         /* Clear the attention. */
4429         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4430             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4431             BGE_MACSTAT_LINK_CHANGED);
4432 }
4433
4434 static void
4435 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
4436 {
4437         struct ifnet *ifp = &sc->arpcom.ac_if;
4438         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
4439
4440         mii_pollstat(mii);
4441
4442         if (!sc->bnx_link &&
4443             (mii->mii_media_status & IFM_ACTIVE) &&
4444             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
4445                 sc->bnx_link++;
4446                 if (bootverbose)
4447                         if_printf(ifp, "link UP\n");
4448         } else if (sc->bnx_link &&
4449             (!(mii->mii_media_status & IFM_ACTIVE) ||
4450             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
4451                 sc->bnx_link = 0;
4452                 if (bootverbose)
4453                         if_printf(ifp, "link DOWN\n");
4454         }
4455
4456         /* Clear the attention. */
4457         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
4458             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
4459             BGE_MACSTAT_LINK_CHANGED);
4460 }
4461
4462 static int
4463 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
4464 {
4465         struct bnx_softc *sc = arg1;
4466
4467         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4468             &sc->bnx_rx_coal_ticks,
4469             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
4470             BNX_RX_COAL_TICKS_CHG);
4471 }
4472
4473 static int
4474 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
4475 {
4476         struct bnx_softc *sc = arg1;
4477
4478         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4479             &sc->bnx_tx_coal_ticks,
4480             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
4481             BNX_TX_COAL_TICKS_CHG);
4482 }
4483
4484 static int
4485 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
4486 {
4487         struct bnx_softc *sc = arg1;
4488
4489         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4490             &sc->bnx_rx_coal_bds,
4491             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
4492             BNX_RX_COAL_BDS_CHG);
4493 }
4494
4495 static int
4496 bnx_sysctl_rx_coal_bds_poll(SYSCTL_HANDLER_ARGS)
4497 {
4498         struct bnx_softc *sc = arg1;
4499
4500         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4501             &sc->bnx_rx_coal_bds_poll,
4502             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
4503             BNX_RX_COAL_BDS_CHG);
4504 }
4505
4506 static int
4507 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
4508 {
4509         struct bnx_softc *sc = arg1;
4510
4511         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4512             &sc->bnx_tx_coal_bds,
4513             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4514             BNX_TX_COAL_BDS_CHG);
4515 }
4516
4517 static int
4518 bnx_sysctl_tx_coal_bds_poll(SYSCTL_HANDLER_ARGS)
4519 {
4520         struct bnx_softc *sc = arg1;
4521
4522         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4523             &sc->bnx_tx_coal_bds_poll,
4524             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4525             BNX_TX_COAL_BDS_CHG);
4526 }
4527
4528 static int
4529 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4530 {
4531         struct bnx_softc *sc = arg1;
4532
4533         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4534             &sc->bnx_rx_coal_bds_int,
4535             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
4536             BNX_RX_COAL_BDS_INT_CHG);
4537 }
4538
4539 static int
4540 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
4541 {
4542         struct bnx_softc *sc = arg1;
4543
4544         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4545             &sc->bnx_tx_coal_bds_int,
4546             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
4547             BNX_TX_COAL_BDS_INT_CHG);
4548 }
4549
4550 static int
4551 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
4552     int coal_min, int coal_max, uint32_t coal_chg_mask)
4553 {
4554         struct bnx_softc *sc = arg1;
4555         struct ifnet *ifp = &sc->arpcom.ac_if;
4556         int error = 0, v;
4557
4558         ifnet_serialize_all(ifp);
4559
4560         v = *coal;
4561         error = sysctl_handle_int(oidp, &v, 0, req);
4562         if (!error && req->newptr != NULL) {
4563                 if (v < coal_min || v > coal_max) {
4564                         error = EINVAL;
4565                 } else {
4566                         *coal = v;
4567                         sc->bnx_coal_chg |= coal_chg_mask;
4568
4569                         /* Commit changes */
4570                         bnx_coal_change(sc);
4571                 }
4572         }
4573
4574         ifnet_deserialize_all(ifp);
4575         return error;
4576 }
4577
4578 static void
4579 bnx_coal_change(struct bnx_softc *sc)
4580 {
4581         struct ifnet *ifp = &sc->arpcom.ac_if;
4582         int i;
4583
4584         ASSERT_IFNET_SERIALIZED_ALL(ifp);
4585
4586         if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
4587                 if (sc->bnx_rx_retcnt == 1) {
4588                         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
4589                             sc->bnx_rx_coal_ticks);
4590                         i = 0;
4591                 } else {
4592                         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, 0);
4593                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4594                                 CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS +
4595                                     (i * BGE_VEC_COALSET_SIZE),
4596                                     sc->bnx_rx_coal_ticks);
4597                         }
4598                 }
4599                 for (; i < BNX_INTR_MAX - 1; ++i) {
4600                         CSR_WRITE_4(sc, BGE_VEC1_RX_COAL_TICKS +
4601                             (i * BGE_VEC_COALSET_SIZE), 0);
4602                 }
4603                 if (bootverbose) {
4604                         if_printf(ifp, "rx_coal_ticks -> %u\n",
4605                             sc->bnx_rx_coal_ticks);
4606                 }
4607         }
4608
4609         if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) {
4610                 if (sc->bnx_tx_ringcnt == 1) {
4611                         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
4612                             sc->bnx_tx_coal_ticks);
4613                         i = 0;
4614                 } else {
4615                         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, 0);
4616                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4617                                 CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS +
4618                                     (i * BGE_VEC_COALSET_SIZE),
4619                                     sc->bnx_tx_coal_ticks);
4620                         }
4621                 }
4622                 for (; i < BNX_INTR_MAX - 1; ++i) {
4623                         CSR_WRITE_4(sc, BGE_VEC1_TX_COAL_TICKS +
4624                             (i * BGE_VEC_COALSET_SIZE), 0);
4625                 }
4626                 if (bootverbose) {
4627                         if_printf(ifp, "tx_coal_ticks -> %u\n",
4628                             sc->bnx_tx_coal_ticks);
4629                 }
4630         }
4631
4632         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) {
4633                 uint32_t rx_coal_bds;
4634
4635                 if (ifp->if_flags & IFF_NPOLLING)
4636                         rx_coal_bds = sc->bnx_rx_coal_bds_poll;
4637                 else
4638                         rx_coal_bds = sc->bnx_rx_coal_bds;
4639
4640                 if (sc->bnx_rx_retcnt == 1) {
4641                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, rx_coal_bds);
4642                         i = 0;
4643                 } else {
4644                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, 0);
4645                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4646                                 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS +
4647                                     (i * BGE_VEC_COALSET_SIZE), rx_coal_bds);
4648                         }
4649                 }
4650                 for (; i < BNX_INTR_MAX - 1; ++i) {
4651                         CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS +
4652                             (i * BGE_VEC_COALSET_SIZE), 0);
4653                 }
4654                 if (bootverbose) {
4655                         if_printf(ifp, "%srx_coal_bds -> %u\n",
4656                             (ifp->if_flags & IFF_NPOLLING) ? "polling " : "",
4657                             rx_coal_bds);
4658                 }
4659         }
4660
4661         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) {
4662                 uint32_t tx_coal_bds;
4663
4664                 if (ifp->if_flags & IFF_NPOLLING)
4665                         tx_coal_bds = sc->bnx_tx_coal_bds_poll;
4666                 else
4667                         tx_coal_bds = sc->bnx_tx_coal_bds;
4668
4669                 if (sc->bnx_tx_ringcnt == 1) {
4670                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, tx_coal_bds);
4671                         i = 0;
4672                 } else {
4673                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, 0);
4674                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4675                                 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS +
4676                                     (i * BGE_VEC_COALSET_SIZE), tx_coal_bds);
4677                         }
4678                 }
4679                 for (; i < BNX_INTR_MAX - 1; ++i) {
4680                         CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS +
4681                             (i * BGE_VEC_COALSET_SIZE), 0);
4682                 }
4683                 if (bootverbose) {
4684                         if_printf(ifp, "%stx_coal_bds -> %u\n",
4685                             (ifp->if_flags & IFF_NPOLLING) ? "polling " : "",
4686                             tx_coal_bds);
4687                 }
4688         }
4689
4690         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) {
4691                 if (sc->bnx_rx_retcnt == 1) {
4692                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
4693                             sc->bnx_rx_coal_bds_int);
4694                         i = 0;
4695                 } else {
4696                         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, 0);
4697                         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
4698                                 CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT +
4699                                     (i * BGE_VEC_COALSET_SIZE),
4700                                     sc->bnx_rx_coal_bds_int);
4701                         }
4702                 }
4703                 for (; i < BNX_INTR_MAX - 1; ++i) {
4704                         CSR_WRITE_4(sc, BGE_VEC1_RX_MAX_COAL_BDS_INT +
4705                             (i * BGE_VEC_COALSET_SIZE), 0);
4706                 }
4707                 if (bootverbose) {
4708                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
4709                             sc->bnx_rx_coal_bds_int);
4710                 }
4711         }
4712
4713         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) {
4714                 if (sc->bnx_tx_ringcnt == 1) {
4715                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
4716                             sc->bnx_tx_coal_bds_int);
4717                         i = 0;
4718                 } else {
4719                         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, 0);
4720                         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
4721                                 CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT +
4722                                     (i * BGE_VEC_COALSET_SIZE),
4723                                     sc->bnx_tx_coal_bds_int);
4724                         }
4725                 }
4726                 for (; i < BNX_INTR_MAX - 1; ++i) {
4727                         CSR_WRITE_4(sc, BGE_VEC1_TX_MAX_COAL_BDS_INT +
4728                             (i * BGE_VEC_COALSET_SIZE), 0);
4729                 }
4730                 if (bootverbose) {
4731                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
4732                             sc->bnx_tx_coal_bds_int);
4733                 }
4734         }
4735
4736         sc->bnx_coal_chg = 0;
4737 }
4738
4739 static void
4740 bnx_check_intr_rxtx(void *xintr)
4741 {
4742         struct bnx_intr_data *intr = xintr;
4743         struct bnx_rx_ret_ring *ret;
4744         struct bnx_tx_ring *txr;
4745         struct ifnet *ifp;
4746
4747         lwkt_serialize_enter(intr->bnx_intr_serialize);
4748
4749         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4750
4751         ifp = &intr->bnx_sc->arpcom.ac_if;
4752         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4753                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4754                 return;
4755         }
4756
4757         txr = intr->bnx_txr;
4758         ret = intr->bnx_ret;
4759
4760         if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx ||
4761             *txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
4762                 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx &&
4763                     intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
4764                         if (!intr->bnx_intr_maylose) {
4765                                 intr->bnx_intr_maylose = TRUE;
4766                                 goto done;
4767                         }
4768                         if (bootverbose)
4769                                 if_printf(ifp, "lost interrupt\n");
4770                         intr->bnx_intr_func(intr->bnx_intr_arg);
4771                 }
4772         }
4773         intr->bnx_intr_maylose = FALSE;
4774         intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
4775         intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
4776
4777 done:
4778         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4779             intr->bnx_intr_check, intr);
4780         lwkt_serialize_exit(intr->bnx_intr_serialize);
4781 }
4782
4783 static void
4784 bnx_check_intr_tx(void *xintr)
4785 {
4786         struct bnx_intr_data *intr = xintr;
4787         struct bnx_tx_ring *txr;
4788         struct ifnet *ifp;
4789
4790         lwkt_serialize_enter(intr->bnx_intr_serialize);
4791
4792         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4793
4794         ifp = &intr->bnx_sc->arpcom.ac_if;
4795         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4796                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4797                 return;
4798         }
4799
4800         txr = intr->bnx_txr;
4801
4802         if (*txr->bnx_tx_considx != txr->bnx_tx_saved_considx) {
4803                 if (intr->bnx_tx_check_considx == txr->bnx_tx_saved_considx) {
4804                         if (!intr->bnx_intr_maylose) {
4805                                 intr->bnx_intr_maylose = TRUE;
4806                                 goto done;
4807                         }
4808                         if (bootverbose)
4809                                 if_printf(ifp, "lost interrupt\n");
4810                         intr->bnx_intr_func(intr->bnx_intr_arg);
4811                 }
4812         }
4813         intr->bnx_intr_maylose = FALSE;
4814         intr->bnx_tx_check_considx = txr->bnx_tx_saved_considx;
4815
4816 done:
4817         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4818             intr->bnx_intr_check, intr);
4819         lwkt_serialize_exit(intr->bnx_intr_serialize);
4820 }
4821
4822 static void
4823 bnx_check_intr_rx(void *xintr)
4824 {
4825         struct bnx_intr_data *intr = xintr;
4826         struct bnx_rx_ret_ring *ret;
4827         struct ifnet *ifp;
4828
4829         lwkt_serialize_enter(intr->bnx_intr_serialize);
4830
4831         KKASSERT(mycpuid == intr->bnx_intr_cpuid);
4832
4833         ifp = &intr->bnx_sc->arpcom.ac_if;
4834         if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) != IFF_RUNNING) {
4835                 lwkt_serialize_exit(intr->bnx_intr_serialize);
4836                 return;
4837         }
4838
4839         ret = intr->bnx_ret;
4840
4841         if (*ret->bnx_rx_considx != ret->bnx_rx_saved_considx) {
4842                 if (intr->bnx_rx_check_considx == ret->bnx_rx_saved_considx) {
4843                         if (!intr->bnx_intr_maylose) {
4844                                 intr->bnx_intr_maylose = TRUE;
4845                                 goto done;
4846                         }
4847                         if (bootverbose)
4848                                 if_printf(ifp, "lost interrupt\n");
4849                         intr->bnx_intr_func(intr->bnx_intr_arg);
4850                 }
4851         }
4852         intr->bnx_intr_maylose = FALSE;
4853         intr->bnx_rx_check_considx = ret->bnx_rx_saved_considx;
4854
4855 done:
4856         callout_reset(&intr->bnx_intr_timer, BNX_INTR_CKINTVL,
4857             intr->bnx_intr_check, intr);
4858         lwkt_serialize_exit(intr->bnx_intr_serialize);
4859 }
4860
4861 static void
4862 bnx_enable_intr(struct bnx_softc *sc)
4863 {
4864         struct ifnet *ifp = &sc->arpcom.ac_if;
4865         int i;
4866
4867         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4868                 lwkt_serialize_handler_enable(
4869                     sc->bnx_intr_data[i].bnx_intr_serialize);
4870         }
4871
4872         /*
4873          * Enable interrupt.
4874          */
4875         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4876                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4877
4878                 bnx_writembx(sc, intr->bnx_intr_mbx,
4879                     (*intr->bnx_saved_status_tag) << 24);
4880                 /* XXX Linux driver */
4881                 bnx_writembx(sc, intr->bnx_intr_mbx,
4882                     (*intr->bnx_saved_status_tag) << 24);
4883         }
4884
4885         /*
4886          * Unmask the interrupt when we stop polling.
4887          */
4888         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
4889             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4890
4891         /*
4892          * Trigger another interrupt, since above writing
4893          * to interrupt mailbox0 may acknowledge pending
4894          * interrupt.
4895          */
4896         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_SET);
4897
4898         if (sc->bnx_flags & BNX_FLAG_STATUSTAG_BUG) {
4899                 if (bootverbose)
4900                         if_printf(ifp, "status tag bug workaround\n");
4901
4902                 for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4903                         struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4904
4905                         if (intr->bnx_intr_check == NULL)
4906                                 continue;
4907                         intr->bnx_intr_maylose = FALSE;
4908                         intr->bnx_rx_check_considx = 0;
4909                         intr->bnx_tx_check_considx = 0;
4910                         callout_reset_bycpu(&intr->bnx_intr_timer,
4911                             BNX_INTR_CKINTVL, intr->bnx_intr_check, intr,
4912                             intr->bnx_intr_cpuid);
4913                 }
4914         }
4915 }
4916
4917 static void
4918 bnx_disable_intr(struct bnx_softc *sc)
4919 {
4920         int i;
4921
4922         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4923                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
4924
4925                 callout_stop(&intr->bnx_intr_timer);
4926                 intr->bnx_intr_maylose = FALSE;
4927                 intr->bnx_rx_check_considx = 0;
4928                 intr->bnx_tx_check_considx = 0;
4929         }
4930
4931         /*
4932          * Mask the interrupt when we start polling.
4933          */
4934         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL,
4935             BGE_PCIMISCCTL_MASK_PCI_INTR, 4);
4936
4937         /*
4938          * Acknowledge possible asserted interrupt.
4939          */
4940         for (i = 0; i < BNX_INTR_MAX; ++i)
4941                 bnx_writembx(sc, sc->bnx_intr_data[i].bnx_intr_mbx, 1);
4942
4943         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
4944                 lwkt_serialize_handler_disable(
4945                     sc->bnx_intr_data[i].bnx_intr_serialize);
4946         }
4947 }
4948
4949 static int
4950 bnx_get_eaddr_mem(struct bnx_softc *sc, uint8_t ether_addr[])
4951 {
4952         uint32_t mac_addr;
4953         int ret = 1;
4954
4955         mac_addr = bnx_readmem_ind(sc, 0x0c14);
4956         if ((mac_addr >> 16) == 0x484b) {
4957                 ether_addr[0] = (uint8_t)(mac_addr >> 8);
4958                 ether_addr[1] = (uint8_t)mac_addr;
4959                 mac_addr = bnx_readmem_ind(sc, 0x0c18);
4960                 ether_addr[2] = (uint8_t)(mac_addr >> 24);
4961                 ether_addr[3] = (uint8_t)(mac_addr >> 16);
4962                 ether_addr[4] = (uint8_t)(mac_addr >> 8);
4963                 ether_addr[5] = (uint8_t)mac_addr;
4964                 ret = 0;
4965         }
4966         return ret;
4967 }
4968
4969 static int
4970 bnx_get_eaddr_nvram(struct bnx_softc *sc, uint8_t ether_addr[])
4971 {
4972         int mac_offset = BGE_EE_MAC_OFFSET;
4973
4974         if (BNX_IS_5717_PLUS(sc)) {
4975                 int f;
4976
4977                 f = pci_get_function(sc->bnx_dev);
4978                 if (f & 1)
4979                         mac_offset = BGE_EE_MAC_OFFSET_5717;
4980                 if (f > 1)
4981                         mac_offset += BGE_EE_MAC_OFFSET_5717_OFF;
4982         }
4983
4984         return bnx_read_nvram(sc, ether_addr, mac_offset + 2, ETHER_ADDR_LEN);
4985 }
4986
4987 static int
4988 bnx_get_eaddr_eeprom(struct bnx_softc *sc, uint8_t ether_addr[])
4989 {
4990         if (sc->bnx_flags & BNX_FLAG_NO_EEPROM)
4991                 return 1;
4992
4993         return bnx_read_eeprom(sc, ether_addr, BGE_EE_MAC_OFFSET + 2,
4994                                ETHER_ADDR_LEN);
4995 }
4996
4997 static int
4998 bnx_get_eaddr(struct bnx_softc *sc, uint8_t eaddr[])
4999 {
5000         static const bnx_eaddr_fcn_t bnx_eaddr_funcs[] = {
5001                 /* NOTE: Order is critical */
5002                 bnx_get_eaddr_mem,
5003                 bnx_get_eaddr_nvram,
5004                 bnx_get_eaddr_eeprom,
5005                 NULL
5006         };
5007         const bnx_eaddr_fcn_t *func;
5008
5009         for (func = bnx_eaddr_funcs; *func != NULL; ++func) {
5010                 if ((*func)(sc, eaddr) == 0)
5011                         break;
5012         }
5013         return (*func == NULL ? ENXIO : 0);
5014 }
5015
5016 /*
5017  * NOTE: 'm' is not freed upon failure
5018  */
5019 struct mbuf *
5020 bnx_defrag_shortdma(struct mbuf *m)
5021 {
5022         struct mbuf *n;
5023         int found;
5024
5025         /*
5026          * If device receive two back-to-back send BDs with less than
5027          * or equal to 8 total bytes then the device may hang.  The two
5028          * back-to-back send BDs must in the same frame for this failure
5029          * to occur.  Scan mbuf chains and see whether two back-to-back
5030          * send BDs are there.  If this is the case, allocate new mbuf
5031          * and copy the frame to workaround the silicon bug.
5032          */
5033         for (n = m, found = 0; n != NULL; n = n->m_next) {
5034                 if (n->m_len < 8) {
5035                         found++;
5036                         if (found > 1)
5037                                 break;
5038                         continue;
5039                 }
5040                 found = 0;
5041         }
5042
5043         if (found > 1)
5044                 n = m_defrag(m, M_NOWAIT);
5045         else
5046                 n = m;
5047         return n;
5048 }
5049
5050 static void
5051 bnx_stop_block(struct bnx_softc *sc, bus_size_t reg, uint32_t bit)
5052 {
5053         int i;
5054
5055         BNX_CLRBIT(sc, reg, bit);
5056         for (i = 0; i < BNX_TIMEOUT; i++) {
5057                 if ((CSR_READ_4(sc, reg) & bit) == 0)
5058                         return;
5059                 DELAY(100);
5060         }
5061 }
5062
5063 static void
5064 bnx_link_poll(struct bnx_softc *sc)
5065 {
5066         uint32_t status;
5067
5068         status = CSR_READ_4(sc, BGE_MAC_STS);
5069         if ((status & sc->bnx_link_chg) || sc->bnx_link_evt) {
5070                 sc->bnx_link_evt = 0;
5071                 sc->bnx_link_upd(sc, status);
5072         }
5073 }
5074
5075 static void
5076 bnx_enable_msi(struct bnx_softc *sc, boolean_t is_msix)
5077 {
5078         uint32_t msi_mode;
5079
5080         msi_mode = CSR_READ_4(sc, BGE_MSI_MODE);
5081         msi_mode |= BGE_MSIMODE_ENABLE;
5082         /*
5083          * NOTE:
5084          * 5718-PG105-R says that "one shot" mode does not work
5085          * if MSI is used, however, it obviously works.
5086          */
5087         msi_mode &= ~BGE_MSIMODE_ONESHOT_DISABLE;
5088         if (is_msix)
5089                 msi_mode |= BGE_MSIMODE_MSIX_MULTIMODE;
5090         else
5091                 msi_mode &= ~BGE_MSIMODE_MSIX_MULTIMODE;
5092         CSR_WRITE_4(sc, BGE_MSI_MODE, msi_mode);
5093 }
5094
5095 static uint32_t
5096 bnx_dma_swap_options(struct bnx_softc *sc)
5097 {
5098         uint32_t dma_options;
5099
5100         dma_options = BGE_MODECTL_WORDSWAP_NONFRAME |
5101             BGE_MODECTL_BYTESWAP_DATA | BGE_MODECTL_WORDSWAP_DATA;
5102 #if BYTE_ORDER == BIG_ENDIAN
5103         dma_options |= BGE_MODECTL_BYTESWAP_NONFRAME;
5104 #endif
5105         return dma_options;
5106 }
5107
5108 static int
5109 bnx_setup_tso(struct bnx_tx_ring *txr, struct mbuf **mp,
5110     uint16_t *mss0, uint16_t *flags0)
5111 {
5112         struct mbuf *m;
5113         struct ip *ip;
5114         struct tcphdr *th;
5115         int thoff, iphlen, hoff, hlen;
5116         uint16_t flags, mss;
5117
5118         m = *mp;
5119         KASSERT(M_WRITABLE(m), ("TSO mbuf not writable"));
5120
5121         hoff = m->m_pkthdr.csum_lhlen;
5122         iphlen = m->m_pkthdr.csum_iphlen;
5123         thoff = m->m_pkthdr.csum_thlen;
5124
5125         KASSERT(hoff > 0, ("invalid ether header len"));
5126         KASSERT(iphlen > 0, ("invalid ip header len"));
5127         KASSERT(thoff > 0, ("invalid tcp header len"));
5128
5129         if (__predict_false(m->m_len < hoff + iphlen + thoff)) {
5130                 m = m_pullup(m, hoff + iphlen + thoff);
5131                 if (m == NULL) {
5132                         *mp = NULL;
5133                         return ENOBUFS;
5134                 }
5135                 *mp = m;
5136         }
5137         ip = mtodoff(m, struct ip *, hoff);
5138         th = mtodoff(m, struct tcphdr *, hoff + iphlen);
5139
5140         mss = m->m_pkthdr.tso_segsz;
5141         flags = BGE_TXBDFLAG_CPU_PRE_DMA | BGE_TXBDFLAG_CPU_POST_DMA;
5142
5143         ip->ip_len = htons(mss + iphlen + thoff);
5144         th->th_sum = 0;
5145
5146         hlen = (iphlen + thoff) >> 2;
5147         mss |= ((hlen & 0x3) << 14);
5148         flags |= ((hlen & 0xf8) << 7) | ((hlen & 0x4) << 2);
5149
5150         *mss0 = mss;
5151         *flags0 = flags;
5152
5153         return 0;
5154 }
5155
5156 static int
5157 bnx_create_tx_ring(struct bnx_tx_ring *txr)
5158 {
5159         bus_size_t txmaxsz, txmaxsegsz;
5160         int i, error;
5161
5162         lwkt_serialize_init(&txr->bnx_tx_serialize);
5163
5164         /*
5165          * Create DMA tag and maps for TX mbufs.
5166          */
5167         if (txr->bnx_sc->bnx_flags & BNX_FLAG_TSO)
5168                 txmaxsz = IP_MAXPACKET + sizeof(struct ether_vlan_header);
5169         else
5170                 txmaxsz = BNX_JUMBO_FRAMELEN;
5171         if (txr->bnx_sc->bnx_asicrev == BGE_ASICREV_BCM57766)
5172                 txmaxsegsz = MCLBYTES;
5173         else
5174                 txmaxsegsz = PAGE_SIZE;
5175         error = bus_dma_tag_create(txr->bnx_sc->bnx_cdata.bnx_parent_tag,
5176             1, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
5177             txmaxsz, BNX_NSEG_NEW, txmaxsegsz,
5178             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
5179             &txr->bnx_tx_mtag);
5180         if (error) {
5181                 device_printf(txr->bnx_sc->bnx_dev,
5182                     "could not create TX mbuf DMA tag\n");
5183                 return error;
5184         }
5185
5186         for (i = 0; i < BGE_TX_RING_CNT; i++) {
5187                 error = bus_dmamap_create(txr->bnx_tx_mtag,
5188                     BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
5189                     &txr->bnx_tx_buf[i].bnx_tx_dmamap);
5190                 if (error) {
5191                         int j;
5192
5193                         for (j = 0; j < i; ++j) {
5194                                 bus_dmamap_destroy(txr->bnx_tx_mtag,
5195                                     txr->bnx_tx_buf[j].bnx_tx_dmamap);
5196                         }
5197                         bus_dma_tag_destroy(txr->bnx_tx_mtag);
5198                         txr->bnx_tx_mtag = NULL;
5199
5200                         device_printf(txr->bnx_sc->bnx_dev,
5201                             "could not create TX mbuf DMA map\n");
5202                         return error;
5203                 }
5204         }
5205
5206         /*
5207          * Create DMA stuffs for TX ring.
5208          */
5209         error = bnx_dma_block_alloc(txr->bnx_sc, BGE_TX_RING_SZ,
5210             &txr->bnx_tx_ring_tag,
5211             &txr->bnx_tx_ring_map,
5212             (void *)&txr->bnx_tx_ring,
5213             &txr->bnx_tx_ring_paddr);
5214         if (error) {
5215                 device_printf(txr->bnx_sc->bnx_dev,
5216                     "could not create TX ring\n");
5217                 return error;
5218         }
5219
5220         txr->bnx_tx_flags |= BNX_TX_FLAG_SHORTDMA;
5221         txr->bnx_tx_wreg = BNX_TX_WREG_NSEGS;
5222
5223         return 0;
5224 }
5225
5226 static void
5227 bnx_destroy_tx_ring(struct bnx_tx_ring *txr)
5228 {
5229         /* Destroy TX mbuf DMA stuffs. */
5230         if (txr->bnx_tx_mtag != NULL) {
5231                 int i;
5232
5233                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
5234                         KKASSERT(txr->bnx_tx_buf[i].bnx_tx_mbuf == NULL);
5235                         bus_dmamap_destroy(txr->bnx_tx_mtag,
5236                             txr->bnx_tx_buf[i].bnx_tx_dmamap);
5237                 }
5238                 bus_dma_tag_destroy(txr->bnx_tx_mtag);
5239         }
5240
5241         /* Destroy TX ring */
5242         bnx_dma_block_free(txr->bnx_tx_ring_tag,
5243             txr->bnx_tx_ring_map, txr->bnx_tx_ring);
5244 }
5245
5246 static int
5247 bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS)
5248 {
5249         struct bnx_softc *sc = (void *)arg1;
5250         struct ifnet *ifp = &sc->arpcom.ac_if;
5251         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
5252         int error, defrag, i;
5253
5254         if (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG)
5255                 defrag = 1;
5256         else
5257                 defrag = 0;
5258
5259         error = sysctl_handle_int(oidp, &defrag, 0, req);
5260         if (error || req->newptr == NULL)
5261                 return error;
5262
5263         ifnet_serialize_all(ifp);
5264         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
5265                 txr = &sc->bnx_tx_ring[i];
5266                 if (defrag)
5267                         txr->bnx_tx_flags |= BNX_TX_FLAG_FORCE_DEFRAG;
5268                 else
5269                         txr->bnx_tx_flags &= ~BNX_TX_FLAG_FORCE_DEFRAG;
5270         }
5271         ifnet_deserialize_all(ifp);
5272
5273         return 0;
5274 }
5275
5276 static int
5277 bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS)
5278 {
5279         struct bnx_softc *sc = (void *)arg1;
5280         struct ifnet *ifp = &sc->arpcom.ac_if;
5281         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
5282         int error, tx_wreg, i;
5283
5284         tx_wreg = txr->bnx_tx_wreg;
5285         error = sysctl_handle_int(oidp, &tx_wreg, 0, req);
5286         if (error || req->newptr == NULL)
5287                 return error;
5288
5289         ifnet_serialize_all(ifp);
5290         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
5291                 sc->bnx_tx_ring[i].bnx_tx_wreg = tx_wreg;
5292         ifnet_deserialize_all(ifp);
5293
5294         return 0;
5295 }
5296
5297 static int
5298 bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *ret)
5299 {
5300         int error;
5301
5302         lwkt_serialize_init(&ret->bnx_rx_ret_serialize);
5303
5304         /*
5305          * Create DMA stuffs for RX return ring.
5306          */
5307         error = bnx_dma_block_alloc(ret->bnx_sc,
5308             BGE_RX_RTN_RING_SZ(BNX_RETURN_RING_CNT),
5309             &ret->bnx_rx_ret_ring_tag,
5310             &ret->bnx_rx_ret_ring_map,
5311             (void *)&ret->bnx_rx_ret_ring,
5312             &ret->bnx_rx_ret_ring_paddr);
5313         if (error) {
5314                 device_printf(ret->bnx_sc->bnx_dev,
5315                     "could not create RX ret ring\n");
5316                 return error;
5317         }
5318
5319         /* Shadow standard ring's RX mbuf DMA tag */
5320         ret->bnx_rx_mtag = ret->bnx_std->bnx_rx_mtag;
5321
5322         /*
5323          * Create tmp DMA map for RX mbufs.
5324          */
5325         error = bus_dmamap_create(ret->bnx_rx_mtag, BUS_DMA_WAITOK,
5326             &ret->bnx_rx_tmpmap);
5327         if (error) {
5328                 device_printf(ret->bnx_sc->bnx_dev,
5329                     "could not create tmp RX mbuf DMA map\n");
5330                 ret->bnx_rx_mtag = NULL;
5331                 return error;
5332         }
5333         return 0;
5334 }
5335
5336 static void
5337 bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *ret)
5338 {
5339         /* Destroy tmp RX mbuf DMA map */
5340         if (ret->bnx_rx_mtag != NULL)
5341                 bus_dmamap_destroy(ret->bnx_rx_mtag, ret->bnx_rx_tmpmap);
5342
5343         /* Destroy RX return ring */
5344         bnx_dma_block_free(ret->bnx_rx_ret_ring_tag,
5345             ret->bnx_rx_ret_ring_map, ret->bnx_rx_ret_ring);
5346 }
5347
5348 static int
5349 bnx_alloc_intr(struct bnx_softc *sc)
5350 {
5351         struct bnx_intr_data *intr;
5352         u_int intr_flags;
5353         int error;
5354
5355         if (sc->bnx_intr_cnt > 1) {
5356                 error = bnx_alloc_msix(sc);
5357                 if (error)
5358                         return error;
5359                 KKASSERT(sc->bnx_intr_type == PCI_INTR_TYPE_MSIX);
5360                 return 0;
5361         }
5362
5363         KKASSERT(sc->bnx_intr_cnt == 1);
5364
5365         intr = &sc->bnx_intr_data[0];
5366         intr->bnx_ret = &sc->bnx_rx_ret_ring[0];
5367         intr->bnx_txr = &sc->bnx_tx_ring[0];
5368         intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5369         intr->bnx_intr_check = bnx_check_intr_rxtx;
5370         intr->bnx_saved_status_tag = &intr->bnx_ret->bnx_saved_status_tag;
5371
5372         sc->bnx_intr_type = pci_alloc_1intr(sc->bnx_dev, bnx_msi_enable,
5373             &intr->bnx_intr_rid, &intr_flags);
5374
5375         intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev, SYS_RES_IRQ,
5376             &intr->bnx_intr_rid, intr_flags);
5377         if (intr->bnx_intr_res == NULL) {
5378                 device_printf(sc->bnx_dev, "could not alloc interrupt\n");
5379                 return ENXIO;
5380         }
5381
5382         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
5383                 bnx_enable_msi(sc, FALSE);
5384                 intr->bnx_intr_func = bnx_msi;
5385                 if (bootverbose)
5386                         device_printf(sc->bnx_dev, "oneshot MSI\n");
5387         } else {
5388                 intr->bnx_intr_func = bnx_intr_legacy;
5389         }
5390         intr->bnx_intr_arg = sc;
5391         intr->bnx_intr_cpuid = rman_get_cpuid(intr->bnx_intr_res);
5392
5393         intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
5394
5395         return 0;
5396 }
5397
5398 static int
5399 bnx_setup_intr(struct bnx_softc *sc)
5400 {
5401         int error, i;
5402
5403         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
5404                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
5405
5406                 error = bus_setup_intr_descr(sc->bnx_dev, intr->bnx_intr_res,
5407                     INTR_MPSAFE, intr->bnx_intr_func, intr->bnx_intr_arg,
5408                     &intr->bnx_intr_hand, intr->bnx_intr_serialize,
5409                     intr->bnx_intr_desc);
5410                 if (error) {
5411                         device_printf(sc->bnx_dev,
5412                             "could not set up %dth intr\n", i);
5413                         bnx_teardown_intr(sc, i);
5414                         return error;
5415                 }
5416         }
5417         return 0;
5418 }
5419
5420 static void
5421 bnx_teardown_intr(struct bnx_softc *sc, int cnt)
5422 {
5423         int i;
5424
5425         for (i = 0; i < cnt; ++i) {
5426                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
5427
5428                 bus_teardown_intr(sc->bnx_dev, intr->bnx_intr_res,
5429                     intr->bnx_intr_hand);
5430         }
5431 }
5432
5433 static void
5434 bnx_free_intr(struct bnx_softc *sc)
5435 {
5436         if (sc->bnx_intr_type != PCI_INTR_TYPE_MSIX) {
5437                 struct bnx_intr_data *intr;
5438
5439                 KKASSERT(sc->bnx_intr_cnt <= 1);
5440                 intr = &sc->bnx_intr_data[0];
5441
5442                 if (intr->bnx_intr_res != NULL) {
5443                         bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
5444                             intr->bnx_intr_rid, intr->bnx_intr_res);
5445                 }
5446                 if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI)
5447                         pci_release_msi(sc->bnx_dev);
5448         } else {
5449                 bnx_free_msix(sc, TRUE);
5450         }
5451 }
5452
5453 static void
5454 bnx_setup_serialize(struct bnx_softc *sc)
5455 {
5456         int i, j;
5457
5458         /*
5459          * Allocate serializer array
5460          */
5461
5462         /* Main + RX STD + TX + RX RET */
5463         sc->bnx_serialize_cnt = 1 + 1 + sc->bnx_tx_ringcnt + sc->bnx_rx_retcnt;
5464
5465         sc->bnx_serialize =
5466             kmalloc(sc->bnx_serialize_cnt * sizeof(struct lwkt_serialize *),
5467                 M_DEVBUF, M_WAITOK | M_ZERO);
5468
5469         /*
5470          * Setup serializers
5471          *
5472          * NOTE: Order is critical
5473          */
5474
5475         i = 0;
5476
5477         KKASSERT(i < sc->bnx_serialize_cnt);
5478         sc->bnx_serialize[i++] = &sc->bnx_main_serialize;
5479
5480         KKASSERT(i < sc->bnx_serialize_cnt);
5481         sc->bnx_serialize[i++] = &sc->bnx_rx_std_ring.bnx_rx_std_serialize;
5482
5483         for (j = 0; j < sc->bnx_rx_retcnt; ++j) {
5484                 KKASSERT(i < sc->bnx_serialize_cnt);
5485                 sc->bnx_serialize[i++] =
5486                     &sc->bnx_rx_ret_ring[j].bnx_rx_ret_serialize;
5487         }
5488
5489         for (j = 0; j < sc->bnx_tx_ringcnt; ++j) {
5490                 KKASSERT(i < sc->bnx_serialize_cnt);
5491                 sc->bnx_serialize[i++] =
5492                     &sc->bnx_tx_ring[j].bnx_tx_serialize;
5493         }
5494
5495         KKASSERT(i == sc->bnx_serialize_cnt);
5496 }
5497
5498 static void
5499 bnx_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
5500 {
5501         struct bnx_softc *sc = ifp->if_softc;
5502
5503         ifnet_serialize_array_enter(sc->bnx_serialize,
5504             sc->bnx_serialize_cnt, slz);
5505 }
5506
5507 static void
5508 bnx_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
5509 {
5510         struct bnx_softc *sc = ifp->if_softc;
5511
5512         ifnet_serialize_array_exit(sc->bnx_serialize,
5513             sc->bnx_serialize_cnt, slz);
5514 }
5515
5516 static int
5517 bnx_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
5518 {
5519         struct bnx_softc *sc = ifp->if_softc;
5520
5521         return ifnet_serialize_array_try(sc->bnx_serialize,
5522             sc->bnx_serialize_cnt, slz);
5523 }
5524
5525 #ifdef INVARIANTS
5526
5527 static void
5528 bnx_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
5529     boolean_t serialized)
5530 {
5531         struct bnx_softc *sc = ifp->if_softc;
5532
5533         ifnet_serialize_array_assert(sc->bnx_serialize, sc->bnx_serialize_cnt,
5534             slz, serialized);
5535 }
5536
5537 #endif  /* INVARIANTS */
5538
5539 #ifdef IFPOLL_ENABLE
5540
5541 static int
5542 bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS)
5543 {
5544         struct bnx_softc *sc = (void *)arg1;
5545         struct ifnet *ifp = &sc->arpcom.ac_if;
5546         int error, off;
5547
5548         off = sc->bnx_npoll_rxoff;
5549         error = sysctl_handle_int(oidp, &off, 0, req);
5550         if (error || req->newptr == NULL)
5551                 return error;
5552         if (off < 0)
5553                 return EINVAL;
5554
5555         ifnet_serialize_all(ifp);
5556         if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) {
5557                 error = EINVAL;
5558         } else {
5559                 error = 0;
5560                 sc->bnx_npoll_txoff = off;
5561                 sc->bnx_npoll_rxoff = off;
5562         }
5563         ifnet_deserialize_all(ifp);
5564
5565         return error;
5566 }
5567
5568 static int
5569 bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS)
5570 {
5571         struct bnx_softc *sc = (void *)arg1;
5572         struct ifnet *ifp = &sc->arpcom.ac_if;
5573         int error, off;
5574
5575         off = sc->bnx_npoll_rxoff;
5576         error = sysctl_handle_int(oidp, &off, 0, req);
5577         if (error || req->newptr == NULL)
5578                 return error;
5579         if (off < 0)
5580                 return EINVAL;
5581
5582         ifnet_serialize_all(ifp);
5583         if (off >= ncpus2 || off % sc->bnx_rx_retcnt != 0) {
5584                 error = EINVAL;
5585         } else {
5586                 error = 0;
5587                 sc->bnx_npoll_rxoff = off;
5588         }
5589         ifnet_deserialize_all(ifp);
5590
5591         return error;
5592 }
5593
5594 static int
5595 bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS)
5596 {
5597         struct bnx_softc *sc = (void *)arg1;
5598         struct ifnet *ifp = &sc->arpcom.ac_if;
5599         int error, off;
5600
5601         off = sc->bnx_npoll_txoff;
5602         error = sysctl_handle_int(oidp, &off, 0, req);
5603         if (error || req->newptr == NULL)
5604                 return error;
5605         if (off < 0)
5606                 return EINVAL;
5607
5608         ifnet_serialize_all(ifp);
5609         if (off >= ncpus2) {
5610                 error = EINVAL;
5611         } else {
5612                 error = 0;
5613                 sc->bnx_npoll_txoff = off;
5614         }
5615         ifnet_deserialize_all(ifp);
5616
5617         return error;
5618 }
5619
5620 #endif  /* IFPOLL_ENABLE */
5621
5622 static void
5623 bnx_set_tick_cpuid(struct bnx_softc *sc, boolean_t polling)
5624 {
5625         if (polling)
5626                 sc->bnx_tick_cpuid = 0; /* XXX */
5627         else
5628                 sc->bnx_tick_cpuid = sc->bnx_intr_data[0].bnx_intr_cpuid;
5629 }
5630
5631 static void
5632 bnx_rx_std_refill_ithread(void *xstd)
5633 {
5634         struct bnx_rx_std_ring *std = xstd;
5635         struct globaldata *gd = mycpu;
5636
5637         crit_enter_gd(gd);
5638
5639         while (!std->bnx_rx_std_stop) {
5640                 if (std->bnx_rx_std_refill) {
5641                         lwkt_serialize_handler_call(
5642                             &std->bnx_rx_std_serialize,
5643                             bnx_rx_std_refill, std, NULL);
5644                 }
5645
5646                 crit_exit_gd(gd);
5647                 crit_enter_gd(gd);
5648
5649                 atomic_poll_release_int(&std->bnx_rx_std_running);
5650                 cpu_mfence();
5651
5652                 if (!std->bnx_rx_std_refill && !std->bnx_rx_std_stop) {
5653                         lwkt_deschedule_self(gd->gd_curthread);
5654                         lwkt_switch();
5655                 }
5656         }
5657
5658         crit_exit_gd(gd);
5659
5660         wakeup(std);
5661
5662         lwkt_exit();
5663 }
5664
5665 static void
5666 bnx_rx_std_refill(void *xstd, void *frame __unused)
5667 {
5668         struct bnx_rx_std_ring *std = xstd;
5669         int cnt, refill_mask;
5670
5671 again:
5672         cnt = 0;
5673
5674         cpu_lfence();
5675         refill_mask = std->bnx_rx_std_refill;
5676         atomic_clear_int(&std->bnx_rx_std_refill, refill_mask);
5677
5678         while (refill_mask) {
5679                 uint16_t check_idx = std->bnx_rx_std;
5680                 int ret_idx;
5681
5682                 ret_idx = bsfl(refill_mask);
5683                 for (;;) {
5684                         struct bnx_rx_buf *rb;
5685                         int refilled;
5686
5687                         BNX_INC(check_idx, BGE_STD_RX_RING_CNT);
5688                         rb = &std->bnx_rx_std_buf[check_idx];
5689                         refilled = rb->bnx_rx_refilled;
5690                         cpu_lfence();
5691                         if (refilled) {
5692                                 bnx_setup_rxdesc_std(std, check_idx);
5693                                 std->bnx_rx_std = check_idx;
5694                                 ++cnt;
5695                                 if (cnt >= 8) {
5696                                         atomic_subtract_int(
5697                                             &std->bnx_rx_std_used, cnt);
5698                                         bnx_writembx(std->bnx_sc,
5699                                             BGE_MBX_RX_STD_PROD_LO,
5700                                             std->bnx_rx_std);
5701                                         cnt = 0;
5702                                 }
5703                         } else {
5704                                 break;
5705                         }
5706                 }
5707                 refill_mask &= ~(1 << ret_idx);
5708         }
5709
5710         if (cnt) {
5711                 atomic_subtract_int(&std->bnx_rx_std_used, cnt);
5712                 bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO,
5713                     std->bnx_rx_std);
5714         }
5715
5716         if (std->bnx_rx_std_refill)
5717                 goto again;
5718
5719         atomic_poll_release_int(&std->bnx_rx_std_running);
5720         cpu_mfence();
5721
5722         if (std->bnx_rx_std_refill)
5723                 goto again;
5724 }
5725
5726 static int
5727 bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS)
5728 {
5729         struct bnx_softc *sc = (void *)arg1;
5730         struct ifnet *ifp = &sc->arpcom.ac_if;
5731         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
5732         int error, cntmax, i;
5733
5734         cntmax = ret->bnx_rx_cntmax;
5735         error = sysctl_handle_int(oidp, &cntmax, 0, req);
5736         if (error || req->newptr == NULL)
5737                 return error;
5738
5739         ifnet_serialize_all(ifp);
5740
5741         if ((cntmax * sc->bnx_rx_retcnt) >= BGE_STD_RX_RING_CNT / 2) {
5742                 error = EINVAL;
5743                 goto back;
5744         }
5745
5746         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
5747                 sc->bnx_rx_ret_ring[i].bnx_rx_cntmax = cntmax;
5748         error = 0;
5749
5750 back:
5751         ifnet_deserialize_all(ifp);
5752
5753         return error;
5754 }
5755
5756 static void
5757 bnx_init_rss(struct bnx_softc *sc)
5758 {
5759         uint8_t key[BGE_RSS_KEYREG_CNT * BGE_RSS_KEYREG_SIZE];
5760         int i, j, r;
5761
5762         KKASSERT(BNX_RSS_ENABLED(sc));
5763
5764         /*
5765          * Configure RSS redirect table in following fashion:
5766          * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
5767          */
5768         r = 0;
5769         for (j = 0; j < BGE_RSS_INDIR_TBL_CNT; ++j) {
5770                 uint32_t tbl = 0;
5771
5772                 for (i = 0; i < BGE_RSS_INDIR_TBLENT_CNT; ++i) {
5773                         uint32_t q;
5774
5775                         q = r % sc->bnx_rx_retcnt;
5776                         tbl |= q << (BGE_RSS_INDIR_TBLENT_SHIFT *
5777                             (BGE_RSS_INDIR_TBLENT_CNT - i - 1));
5778                         ++r;
5779                 }
5780
5781                 BNX_RSS_DPRINTF(sc, 1, "tbl%d %08x\n", j, tbl);
5782                 CSR_WRITE_4(sc, BGE_RSS_INDIR_TBL(j), tbl);
5783         }
5784
5785         toeplitz_get_key(key, sizeof(key));
5786         for (i = 0; i < BGE_RSS_KEYREG_CNT; ++i) {
5787                 uint32_t keyreg;
5788
5789                 keyreg = BGE_RSS_KEYREG_VAL(key, i);
5790
5791                 BNX_RSS_DPRINTF(sc, 1, "key%d %08x\n", i, keyreg);
5792                 CSR_WRITE_4(sc, BGE_RSS_KEYREG(i), keyreg);
5793         }
5794 }
5795
5796 static void
5797 bnx_setup_ring_cnt(struct bnx_softc *sc)
5798 {
5799         int msix_enable, i, msix_cnt, msix_cnt2, ring_max;
5800
5801         sc->bnx_tx_ringcnt = 1;
5802         sc->bnx_rx_retcnt = 1;
5803         sc->bnx_intr_cnt = 1;
5804
5805         msix_enable = device_getenv_int(sc->bnx_dev, "msix.enable",
5806             bnx_msix_enable);
5807         if (!msix_enable)
5808                 return;
5809
5810         if (ncpus2 == 1)
5811                 return;
5812
5813         msix_cnt = pci_msix_count(sc->bnx_dev);
5814         if (msix_cnt <= 1)
5815                 return;
5816
5817         i = 0;
5818         while ((1 << (i + 1)) <= msix_cnt)
5819                 ++i;
5820         msix_cnt2 = 1 << i;
5821
5822         /*
5823          * One MSI-X vector is dedicated to status or single TX queue,
5824          * so make sure that there are enough MSI-X vectors.
5825          */
5826         if (msix_cnt == msix_cnt2) {
5827                 /*
5828                  * XXX
5829                  * This probably will not happen; 57785/5718 families
5830                  * come with at least 5 MSI-X vectors.
5831                  */
5832                 msix_cnt2 >>= 1;
5833                 if (msix_cnt2 <= 1) {
5834                         device_printf(sc->bnx_dev,
5835                             "MSI-X count %d could not be used\n", msix_cnt);
5836                         return;
5837                 }
5838                 device_printf(sc->bnx_dev, "MSI-X count %d is power of 2\n",
5839                     msix_cnt);
5840         }
5841
5842         /*
5843          * Setup RX ring count
5844          */
5845         ring_max = BNX_RX_RING_MAX;
5846         if (ring_max > msix_cnt2)
5847                 ring_max = msix_cnt2;
5848         sc->bnx_rx_retcnt = device_getenv_int(sc->bnx_dev, "rx_rings",
5849             bnx_rx_rings);
5850         sc->bnx_rx_retcnt = if_ring_count2(sc->bnx_rx_retcnt, ring_max);
5851
5852         if (sc->bnx_rx_retcnt == 1)
5853                 return;
5854
5855         /*
5856          * We need one extra MSI-X vector for link status or
5857          * TX ring (if only one TX ring is enabled).
5858          */
5859         sc->bnx_intr_cnt = sc->bnx_rx_retcnt + 1;
5860
5861         /*
5862          * Setup TX ring count
5863          *
5864          * Currently only BCM5719 and BCM5720 support multiple TX rings
5865          * and the TX ring count must be less than the RX ring count.
5866          */
5867         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
5868             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
5869                 ring_max = BNX_TX_RING_MAX;
5870                 if (ring_max > msix_cnt2)
5871                         ring_max = msix_cnt2;
5872                 if (ring_max > sc->bnx_rx_retcnt)
5873                         ring_max = sc->bnx_rx_retcnt;
5874                 sc->bnx_tx_ringcnt = device_getenv_int(sc->bnx_dev, "tx_rings",
5875                     bnx_tx_rings);
5876                 sc->bnx_tx_ringcnt = if_ring_count2(sc->bnx_tx_ringcnt,
5877                     ring_max);
5878         }
5879 }
5880
5881 static int
5882 bnx_alloc_msix(struct bnx_softc *sc)
5883 {
5884         struct bnx_intr_data *intr;
5885         boolean_t setup = FALSE;
5886         int error, i, offset, offset_def;
5887
5888         KKASSERT(sc->bnx_intr_cnt > 1);
5889         KKASSERT(sc->bnx_intr_cnt == sc->bnx_rx_retcnt + 1);
5890
5891         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
5892                 /*
5893                  * Link status
5894                  */
5895                 intr = &sc->bnx_intr_data[0];
5896
5897                 intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5898                 intr->bnx_saved_status_tag = &sc->bnx_saved_status_tag;
5899
5900                 intr->bnx_intr_func = bnx_msix_status;
5901                 intr->bnx_intr_arg = sc;
5902                 intr->bnx_intr_cpuid = 0; /* XXX */
5903
5904                 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0),
5905                     "%s sts", device_get_nameunit(sc->bnx_dev));
5906                 intr->bnx_intr_desc = intr->bnx_intr_desc0;
5907
5908                 /*
5909                  * RX/TX rings
5910                  */
5911                 if (sc->bnx_rx_retcnt == ncpus2) {
5912                         offset = 0;
5913                 } else {
5914                         offset_def = (sc->bnx_rx_retcnt *
5915                             device_get_unit(sc->bnx_dev)) % ncpus2;
5916
5917                         offset = device_getenv_int(sc->bnx_dev,
5918                             "msix.offset", offset_def);
5919                         if (offset >= ncpus2 ||
5920                             offset % sc->bnx_rx_retcnt != 0) {
5921                                 device_printf(sc->bnx_dev,
5922                                     "invalid msix.offset %d, use %d\n",
5923                                     offset, offset_def);
5924                                 offset = offset_def;
5925                         }
5926                 }
5927
5928                 for (i = 1; i < sc->bnx_intr_cnt; ++i) {
5929                         int idx = i - 1;
5930
5931                         intr = &sc->bnx_intr_data[i];
5932
5933                         KKASSERT(idx < sc->bnx_rx_retcnt);
5934                         intr->bnx_ret = &sc->bnx_rx_ret_ring[idx];
5935                         if (idx < sc->bnx_tx_ringcnt) {
5936                                 intr->bnx_txr = &sc->bnx_tx_ring[idx];
5937                                 intr->bnx_ret->bnx_txr = intr->bnx_txr;
5938                         }
5939
5940                         intr->bnx_intr_serialize =
5941                             &intr->bnx_ret->bnx_rx_ret_serialize;
5942                         intr->bnx_saved_status_tag =
5943                             &intr->bnx_ret->bnx_saved_status_tag;
5944
5945                         intr->bnx_intr_arg = intr->bnx_ret;
5946                         KKASSERT(idx + offset < ncpus2);
5947                         intr->bnx_intr_cpuid = idx + offset;
5948
5949                         if (intr->bnx_txr == NULL) {
5950                                 intr->bnx_intr_check = bnx_check_intr_rx;
5951                                 intr->bnx_intr_func = bnx_msix_rx;
5952                                 ksnprintf(intr->bnx_intr_desc0,
5953                                     sizeof(intr->bnx_intr_desc0), "%s rx%d",
5954                                     device_get_nameunit(sc->bnx_dev), idx);
5955                         } else {
5956                                 intr->bnx_intr_check = bnx_check_intr_rxtx;
5957                                 intr->bnx_intr_func = bnx_msix_rxtx;
5958                                 ksnprintf(intr->bnx_intr_desc0,
5959                                     sizeof(intr->bnx_intr_desc0), "%s rxtx%d",
5960                                     device_get_nameunit(sc->bnx_dev), idx);
5961
5962                                 intr->bnx_txr->bnx_tx_cpuid =
5963                                     intr->bnx_intr_cpuid;
5964                         }
5965                         intr->bnx_intr_desc = intr->bnx_intr_desc0;
5966
5967                         intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx;
5968                 }
5969         } else {
5970                 /*
5971                  * TX ring and link status
5972                  */
5973                 offset_def = device_get_unit(sc->bnx_dev) % ncpus2;
5974                 offset = device_getenv_int(sc->bnx_dev, "msix.txoff",
5975                     offset_def);
5976                 if (offset >= ncpus2) {
5977                         device_printf(sc->bnx_dev,
5978                             "invalid msix.txoff %d, use %d\n",
5979                             offset, offset_def);
5980                         offset = offset_def;
5981                 }
5982
5983                 intr = &sc->bnx_intr_data[0];
5984
5985                 intr->bnx_txr = &sc->bnx_tx_ring[0];
5986                 intr->bnx_intr_serialize = &sc->bnx_main_serialize;
5987                 intr->bnx_intr_check = bnx_check_intr_tx;
5988                 intr->bnx_saved_status_tag =
5989                     &intr->bnx_txr->bnx_saved_status_tag;
5990
5991                 intr->bnx_intr_func = bnx_msix_tx_status;
5992                 intr->bnx_intr_arg = intr->bnx_txr;
5993                 intr->bnx_intr_cpuid = offset;
5994
5995                 ksnprintf(intr->bnx_intr_desc0, sizeof(intr->bnx_intr_desc0),
5996                     "%s ststx", device_get_nameunit(sc->bnx_dev));
5997                 intr->bnx_intr_desc = intr->bnx_intr_desc0;
5998
5999                 intr->bnx_txr->bnx_tx_cpuid = intr->bnx_intr_cpuid;
6000
6001                 /*
6002                  * RX rings
6003                  */
6004                 if (sc->bnx_rx_retcnt == ncpus2) {
6005                         offset = 0;
6006                 } else {
6007                         offset_def = (sc->bnx_rx_retcnt *
6008                             device_get_unit(sc->bnx_dev)) % ncpus2;
6009
6010                         offset = device_getenv_int(sc->bnx_dev,
6011                             "msix.rxoff", offset_def);
6012                         if (offset >= ncpus2 ||
6013                             offset % sc->bnx_rx_retcnt != 0) {
6014                                 device_printf(sc->bnx_dev,
6015                                     "invalid msix.rxoff %d, use %d\n",
6016                                     offset, offset_def);
6017                                 offset = offset_def;
6018                         }
6019                 }
6020
6021                 for (i = 1; i < sc->bnx_intr_cnt; ++i) {
6022                         int idx = i - 1;
6023
6024                         intr = &sc->bnx_intr_data[i];
6025
6026                         KKASSERT(idx < sc->bnx_rx_retcnt);
6027                         intr->bnx_ret = &sc->bnx_rx_ret_ring[idx];
6028                         intr->bnx_intr_serialize =
6029                             &intr->bnx_ret->bnx_rx_ret_serialize;
6030                         intr->bnx_intr_check = bnx_check_intr_rx;
6031                         intr->bnx_saved_status_tag =
6032                             &intr->bnx_ret->bnx_saved_status_tag;
6033
6034                         intr->bnx_intr_func = bnx_msix_rx;
6035                         intr->bnx_intr_arg = intr->bnx_ret;
6036                         KKASSERT(idx + offset < ncpus2);
6037                         intr->bnx_intr_cpuid = idx + offset;
6038
6039                         ksnprintf(intr->bnx_intr_desc0,
6040                             sizeof(intr->bnx_intr_desc0), "%s rx%d",
6041                             device_get_nameunit(sc->bnx_dev), idx);
6042                         intr->bnx_intr_desc = intr->bnx_intr_desc0;
6043
6044                         intr->bnx_ret->bnx_msix_mbx = intr->bnx_intr_mbx;
6045                 }
6046         }
6047
6048         if (BNX_IS_5717_PLUS(sc)) {
6049                 sc->bnx_msix_mem_rid = PCIR_BAR(4);
6050         } else {
6051                 if (sc->bnx_res2 == NULL)
6052                         sc->bnx_msix_mem_rid = PCIR_BAR(2);
6053         }
6054         if (sc->bnx_msix_mem_rid != 0) {
6055                 sc->bnx_msix_mem_res = bus_alloc_resource_any(sc->bnx_dev,
6056                     SYS_RES_MEMORY, &sc->bnx_msix_mem_rid, RF_ACTIVE);
6057                 if (sc->bnx_msix_mem_res == NULL) {
6058                         device_printf(sc->bnx_dev,
6059                             "could not alloc MSI-X table\n");
6060                         return ENXIO;
6061                 }
6062         }
6063
6064         bnx_enable_msi(sc, TRUE);
6065
6066         error = pci_setup_msix(sc->bnx_dev);
6067         if (error) {
6068                 device_printf(sc->bnx_dev, "could not setup MSI-X\n");
6069                 goto back;
6070         }
6071         setup = TRUE;
6072
6073         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
6074                 intr = &sc->bnx_intr_data[i];
6075
6076                 error = pci_alloc_msix_vector(sc->bnx_dev, i,
6077                     &intr->bnx_intr_rid, intr->bnx_intr_cpuid);
6078                 if (error) {
6079                         device_printf(sc->bnx_dev,
6080                             "could not alloc MSI-X %d on cpu%d\n",
6081                             i, intr->bnx_intr_cpuid);
6082                         goto back;
6083                 }
6084
6085                 intr->bnx_intr_res = bus_alloc_resource_any(sc->bnx_dev,
6086                     SYS_RES_IRQ, &intr->bnx_intr_rid, RF_ACTIVE);
6087                 if (intr->bnx_intr_res == NULL) {
6088                         device_printf(sc->bnx_dev,
6089                             "could not alloc MSI-X %d resource\n", i);
6090                         error = ENXIO;
6091                         goto back;
6092                 }
6093         }
6094
6095         pci_enable_msix(sc->bnx_dev);
6096         sc->bnx_intr_type = PCI_INTR_TYPE_MSIX;
6097 back:
6098         if (error)
6099                 bnx_free_msix(sc, setup);
6100         return error;
6101 }
6102
6103 static void
6104 bnx_free_msix(struct bnx_softc *sc, boolean_t setup)
6105 {
6106         int i;
6107
6108         KKASSERT(sc->bnx_intr_cnt > 1);
6109
6110         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
6111                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
6112
6113                 if (intr->bnx_intr_res != NULL) {
6114                         bus_release_resource(sc->bnx_dev, SYS_RES_IRQ,
6115                             intr->bnx_intr_rid, intr->bnx_intr_res);
6116                 }
6117                 if (intr->bnx_intr_rid >= 0) {
6118                         pci_release_msix_vector(sc->bnx_dev,
6119                             intr->bnx_intr_rid);
6120                 }
6121         }
6122         if (setup)
6123                 pci_teardown_msix(sc->bnx_dev);
6124 }
6125
6126 static void
6127 bnx_rx_std_refill_sched_ipi(void *xret)
6128 {
6129         struct bnx_rx_ret_ring *ret = xret;
6130         struct bnx_rx_std_ring *std = ret->bnx_std;
6131         struct globaldata *gd = mycpu;
6132
6133         crit_enter_gd(gd);
6134
6135         atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask);
6136         cpu_sfence();
6137
6138         KKASSERT(std->bnx_rx_std_ithread.td_gd == gd);
6139         lwkt_schedule(&std->bnx_rx_std_ithread);
6140
6141         crit_exit_gd(gd);
6142 }
6143
6144 static void
6145 bnx_rx_std_refill_stop(void *xstd)
6146 {
6147         struct bnx_rx_std_ring *std = xstd;
6148         struct globaldata *gd = mycpu;
6149
6150         crit_enter_gd(gd);
6151
6152         std->bnx_rx_std_stop = 1;
6153         cpu_sfence();
6154
6155         KKASSERT(std->bnx_rx_std_ithread.td_gd == gd);
6156         lwkt_schedule(&std->bnx_rx_std_ithread);
6157
6158         crit_exit_gd(gd);
6159 }
6160
6161 static void
6162 bnx_serialize_skipmain(struct bnx_softc *sc)
6163 {
6164         lwkt_serialize_array_enter(sc->bnx_serialize,
6165             sc->bnx_serialize_cnt, 1);
6166 }
6167
6168 static void
6169 bnx_deserialize_skipmain(struct bnx_softc *sc)
6170 {
6171         lwkt_serialize_array_exit(sc->bnx_serialize,
6172             sc->bnx_serialize_cnt, 1);
6173 }
6174
6175 static void
6176 bnx_rx_std_refill_sched(struct bnx_rx_ret_ring *ret,
6177     struct bnx_rx_std_ring *std)
6178 {
6179         struct globaldata *gd = mycpu;
6180
6181         ret->bnx_rx_cnt = 0;
6182         cpu_sfence();
6183
6184         crit_enter_gd(gd);
6185
6186         atomic_set_int(&std->bnx_rx_std_refill, ret->bnx_rx_mask);
6187         cpu_sfence();
6188         if (atomic_poll_acquire_int(&std->bnx_rx_std_running)) {
6189                 if (std->bnx_rx_std_ithread.td_gd == gd) {
6190                         lwkt_schedule(&std->bnx_rx_std_ithread);
6191                 } else {
6192                         lwkt_send_ipiq(
6193                             std->bnx_rx_std_ithread.td_gd,
6194                             bnx_rx_std_refill_sched_ipi, ret);
6195                 }
6196         }
6197
6198         crit_exit_gd(gd);
6199 }
6200
6201 static struct pktinfo *
6202 bnx_rss_info(struct pktinfo *pi, const struct bge_rx_bd *cur_rx)
6203 {
6204         /* Don't pick up IPv6 packet */
6205         if (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6)
6206                 return NULL;
6207
6208         /* Don't pick up IP packet w/o IP checksum */
6209         if ((cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) == 0 ||
6210             (cur_rx->bge_error_flag & BGE_RXERRFLAG_IP_CSUM_NOK))
6211                 return NULL;
6212
6213         /* Don't pick up IP packet w/o TCP/UDP checksum */
6214         if ((cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) == 0)
6215                 return NULL;
6216
6217         /* May be IP fragment */
6218         if (cur_rx->bge_tcp_udp_csum != 0xffff)
6219                 return NULL;
6220
6221         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_IS_TCP)
6222                 pi->pi_l3proto = IPPROTO_TCP;
6223         else
6224                 pi->pi_l3proto = IPPROTO_UDP;
6225         pi->pi_netisr = NETISR_IP;
6226         pi->pi_flags = 0;
6227
6228         return pi;
6229 }
6230
6231 static void
6232 bnx_sig_pre_reset(struct bnx_softc *sc, int type)
6233 {
6234         if (type == BNX_RESET_START || type == BNX_RESET_SUSPEND)
6235                 bnx_ape_driver_state_change(sc, type);
6236 }
6237
6238 static void
6239 bnx_sig_post_reset(struct bnx_softc *sc, int type)
6240 {
6241         if (type == BNX_RESET_SHUTDOWN)
6242                 bnx_ape_driver_state_change(sc, type);
6243 }
6244
6245 /*
6246  * Clear all stale locks and select the lock for this driver instance.
6247  */
6248 static void
6249 bnx_ape_lock_init(struct bnx_softc *sc)
6250 {
6251         uint32_t bit, regbase;
6252         int i;
6253
6254         regbase = BGE_APE_PER_LOCK_GRANT;
6255
6256         /* Clear any stale locks. */
6257         for (i = BGE_APE_LOCK_PHY0; i <= BGE_APE_LOCK_GPIO; i++) {
6258                 switch (i) {
6259                 case BGE_APE_LOCK_PHY0:
6260                 case BGE_APE_LOCK_PHY1:
6261                 case BGE_APE_LOCK_PHY2:
6262                 case BGE_APE_LOCK_PHY3:
6263                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
6264                         break;
6265
6266                 default:
6267                         if (sc->bnx_func_addr == 0)
6268                                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
6269                         else
6270                                 bit = 1 << sc->bnx_func_addr;
6271                         break;
6272                 }
6273                 APE_WRITE_4(sc, regbase + 4 * i, bit);
6274         }
6275
6276         /* Select the PHY lock based on the device's function number. */
6277         switch (sc->bnx_func_addr) {
6278         case 0:
6279                 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY0;
6280                 break;
6281
6282         case 1:
6283                 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY1;
6284                 break;
6285
6286         case 2:
6287                 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY2;
6288                 break;
6289
6290         case 3:
6291                 sc->bnx_phy_ape_lock = BGE_APE_LOCK_PHY3;
6292                 break;
6293
6294         default:
6295                 device_printf(sc->bnx_dev,
6296                     "PHY lock not supported on this function\n");
6297                 break;
6298         }
6299 }
6300
6301 /*
6302  * Check for APE firmware, set flags, and print version info.
6303  */
6304 static void
6305 bnx_ape_read_fw_ver(struct bnx_softc *sc)
6306 {
6307         const char *fwtype;
6308         uint32_t apedata, features;
6309
6310         /* Check for a valid APE signature in shared memory. */
6311         apedata = APE_READ_4(sc, BGE_APE_SEG_SIG);
6312         if (apedata != BGE_APE_SEG_SIG_MAGIC) {
6313                 device_printf(sc->bnx_dev, "no APE signature\n");
6314                 sc->bnx_mfw_flags &= ~BNX_MFW_ON_APE;
6315                 return;
6316         }
6317
6318         /* Check if APE firmware is running. */
6319         apedata = APE_READ_4(sc, BGE_APE_FW_STATUS);
6320         if ((apedata & BGE_APE_FW_STATUS_READY) == 0) {
6321                 device_printf(sc->bnx_dev, "APE signature found "
6322                     "but FW status not ready! 0x%08x\n", apedata);
6323                 return;
6324         }
6325
6326         sc->bnx_mfw_flags |= BNX_MFW_ON_APE;
6327
6328         /* Fetch the APE firwmare type and version. */
6329         apedata = APE_READ_4(sc, BGE_APE_FW_VERSION);
6330         features = APE_READ_4(sc, BGE_APE_FW_FEATURES);
6331         if (features & BGE_APE_FW_FEATURE_NCSI) {
6332                 sc->bnx_mfw_flags |= BNX_MFW_TYPE_NCSI;
6333                 fwtype = "NCSI";
6334         } else if (features & BGE_APE_FW_FEATURE_DASH) {
6335                 sc->bnx_mfw_flags |= BNX_MFW_TYPE_DASH;
6336                 fwtype = "DASH";
6337         } else {
6338                 fwtype = "UNKN";
6339         }
6340
6341         /* Print the APE firmware version. */
6342         device_printf(sc->bnx_dev, "APE FW version: %s v%d.%d.%d.%d\n",
6343             fwtype,
6344             (apedata & BGE_APE_FW_VERSION_MAJMSK) >> BGE_APE_FW_VERSION_MAJSFT,
6345             (apedata & BGE_APE_FW_VERSION_MINMSK) >> BGE_APE_FW_VERSION_MINSFT,
6346             (apedata & BGE_APE_FW_VERSION_REVMSK) >> BGE_APE_FW_VERSION_REVSFT,
6347             (apedata & BGE_APE_FW_VERSION_BLDMSK));
6348 }
6349
6350 static int
6351 bnx_ape_lock(struct bnx_softc *sc, int locknum)
6352 {
6353         uint32_t bit, gnt, req, status;
6354         int i, off;
6355
6356         if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0)
6357                 return 0;
6358
6359         /* Lock request/grant registers have different bases. */
6360         req = BGE_APE_PER_LOCK_REQ;
6361         gnt = BGE_APE_PER_LOCK_GRANT;
6362
6363         off = 4 * locknum;
6364
6365         switch (locknum) {
6366         case BGE_APE_LOCK_GPIO:
6367                 /* Lock required when using GPIO. */
6368                 if (sc->bnx_func_addr == 0)
6369                         bit = BGE_APE_LOCK_REQ_DRIVER0;
6370                 else
6371                         bit = 1 << sc->bnx_func_addr;
6372                 break;
6373
6374         case BGE_APE_LOCK_GRC:
6375                 /* Lock required to reset the device. */
6376                 if (sc->bnx_func_addr == 0)
6377                         bit = BGE_APE_LOCK_REQ_DRIVER0;
6378                 else
6379                         bit = 1 << sc->bnx_func_addr;
6380                 break;
6381
6382         case BGE_APE_LOCK_MEM:
6383                 /* Lock required when accessing certain APE memory. */
6384                 if (sc->bnx_func_addr == 0)
6385                         bit = BGE_APE_LOCK_REQ_DRIVER0;
6386                 else
6387                         bit = 1 << sc->bnx_func_addr;
6388                 break;
6389
6390         case BGE_APE_LOCK_PHY0:
6391         case BGE_APE_LOCK_PHY1:
6392         case BGE_APE_LOCK_PHY2:
6393         case BGE_APE_LOCK_PHY3:
6394                 /* Lock required when accessing PHYs. */
6395                 bit = BGE_APE_LOCK_REQ_DRIVER0;
6396                 break;
6397
6398         default:
6399                 return EINVAL;
6400         }
6401
6402         /* Request a lock. */
6403         APE_WRITE_4(sc, req + off, bit);
6404
6405         /* Wait up to 1 second to acquire lock. */
6406         for (i = 0; i < 20000; i++) {
6407                 status = APE_READ_4(sc, gnt + off);
6408                 if (status == bit)
6409                         break;
6410                 DELAY(50);
6411         }
6412
6413         /* Handle any errors. */
6414         if (status != bit) {
6415                 if_printf(&sc->arpcom.ac_if, "APE lock %d request failed! "
6416                     "request = 0x%04x[0x%04x], status = 0x%04x[0x%04x]\n",
6417                     locknum, req + off, bit & 0xFFFF, gnt + off,
6418                     status & 0xFFFF);
6419                 /* Revoke the lock request. */
6420                 APE_WRITE_4(sc, gnt + off, bit);
6421                 return EBUSY;
6422         }
6423
6424         return 0;
6425 }
6426
6427 static void
6428 bnx_ape_unlock(struct bnx_softc *sc, int locknum)
6429 {
6430         uint32_t bit, gnt;
6431         int off;
6432
6433         if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0)
6434                 return;
6435
6436         gnt = BGE_APE_PER_LOCK_GRANT;
6437
6438         off = 4 * locknum;
6439
6440         switch (locknum) {
6441         case BGE_APE_LOCK_GPIO:
6442                 if (sc->bnx_func_addr == 0)
6443                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
6444                 else
6445                         bit = 1 << sc->bnx_func_addr;
6446                 break;
6447
6448         case BGE_APE_LOCK_GRC:
6449                 if (sc->bnx_func_addr == 0)
6450                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
6451                 else
6452                         bit = 1 << sc->bnx_func_addr;
6453                 break;
6454
6455         case BGE_APE_LOCK_MEM:
6456                 if (sc->bnx_func_addr == 0)
6457                         bit = BGE_APE_LOCK_GRANT_DRIVER0;
6458                 else
6459                         bit = 1 << sc->bnx_func_addr;
6460                 break;
6461
6462         case BGE_APE_LOCK_PHY0:
6463         case BGE_APE_LOCK_PHY1:
6464         case BGE_APE_LOCK_PHY2:
6465         case BGE_APE_LOCK_PHY3:
6466                 bit = BGE_APE_LOCK_GRANT_DRIVER0;
6467                 break;
6468
6469         default:
6470                 return;
6471         }
6472
6473         APE_WRITE_4(sc, gnt + off, bit);
6474 }
6475
6476 /*
6477  * Send an event to the APE firmware.
6478  */
6479 static void
6480 bnx_ape_send_event(struct bnx_softc *sc, uint32_t event)
6481 {
6482         uint32_t apedata;
6483         int i;
6484
6485         /* NCSI does not support APE events. */
6486         if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0)
6487                 return;
6488
6489         /* Wait up to 1ms for APE to service previous event. */
6490         for (i = 10; i > 0; i--) {
6491                 if (bnx_ape_lock(sc, BGE_APE_LOCK_MEM) != 0)
6492                         break;
6493                 apedata = APE_READ_4(sc, BGE_APE_EVENT_STATUS);
6494                 if ((apedata & BGE_APE_EVENT_STATUS_EVENT_PENDING) == 0) {
6495                         APE_WRITE_4(sc, BGE_APE_EVENT_STATUS, event |
6496                             BGE_APE_EVENT_STATUS_EVENT_PENDING);
6497                         bnx_ape_unlock(sc, BGE_APE_LOCK_MEM);
6498                         APE_WRITE_4(sc, BGE_APE_EVENT, BGE_APE_EVENT_1);
6499                         break;
6500                 }
6501                 bnx_ape_unlock(sc, BGE_APE_LOCK_MEM);
6502                 DELAY(100);
6503         }
6504         if (i == 0) {
6505                 if_printf(&sc->arpcom.ac_if,
6506                     "APE event 0x%08x send timed out\n", event);
6507         }
6508 }
6509
6510 static void
6511 bnx_ape_driver_state_change(struct bnx_softc *sc, int kind)
6512 {
6513         uint32_t apedata, event;
6514
6515         if ((sc->bnx_mfw_flags & BNX_MFW_ON_APE) == 0)
6516                 return;
6517
6518         switch (kind) {
6519         case BNX_RESET_START:
6520                 /* If this is the first load, clear the load counter. */
6521                 apedata = APE_READ_4(sc, BGE_APE_HOST_SEG_SIG);
6522                 if (apedata != BGE_APE_HOST_SEG_SIG_MAGIC) {
6523                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, 0);
6524                 } else {
6525                         apedata = APE_READ_4(sc, BGE_APE_HOST_INIT_COUNT);
6526                         APE_WRITE_4(sc, BGE_APE_HOST_INIT_COUNT, ++apedata);
6527                 }
6528                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_SIG,
6529                     BGE_APE_HOST_SEG_SIG_MAGIC);
6530                 APE_WRITE_4(sc, BGE_APE_HOST_SEG_LEN,
6531                     BGE_APE_HOST_SEG_LEN_MAGIC);
6532
6533                 /* Add some version info if bnx(4) supports it. */
6534                 APE_WRITE_4(sc, BGE_APE_HOST_DRIVER_ID,
6535                     BGE_APE_HOST_DRIVER_ID_MAGIC(1, 0));
6536                 APE_WRITE_4(sc, BGE_APE_HOST_BEHAVIOR,
6537                     BGE_APE_HOST_BEHAV_NO_PHYLOCK);
6538                 APE_WRITE_4(sc, BGE_APE_HOST_HEARTBEAT_INT_MS,
6539                     BGE_APE_HOST_HEARTBEAT_INT_DISABLE);
6540                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
6541                     BGE_APE_HOST_DRVR_STATE_START);
6542                 event = BGE_APE_EVENT_STATUS_STATE_START;
6543                 break;
6544
6545         case BNX_RESET_SHUTDOWN:
6546                 APE_WRITE_4(sc, BGE_APE_HOST_DRVR_STATE,
6547                     BGE_APE_HOST_DRVR_STATE_UNLOAD);
6548                 event = BGE_APE_EVENT_STATUS_STATE_UNLOAD;
6549                 break;
6550
6551         case BNX_RESET_SUSPEND:
6552                 event = BGE_APE_EVENT_STATUS_STATE_SUSPEND;
6553                 break;
6554
6555         default:
6556                 return;
6557         }
6558
6559         bnx_ape_send_event(sc, event | BGE_APE_EVENT_STATUS_DRIVER_EVNT |
6560             BGE_APE_EVENT_STATUS_STATE_CHNGE);
6561 }