24e8a1bf24c3cfcff96de38fb21c0928b0fae72e
[dragonfly.git] / sys / dev / netif / bnx / if_bnx.c
1 /*
2  * Copyright (c) 2001 Wind River Systems
3  * Copyright (c) 1997, 1998, 1999, 2001
4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. All advertising materials mentioning features or use of this software
15  *    must display the following acknowledgement:
16  *      This product includes software developed by Bill Paul.
17  * 4. Neither the name of the author nor the names of any co-contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
31  * THE POSSIBILITY OF SUCH DAMAGE.
32  *
33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
34  */
35
36 #include "opt_bnx.h"
37 #include "opt_ifpoll.h"
38
39 #include <sys/param.h>
40 #include <sys/bus.h>
41 #include <sys/endian.h>
42 #include <sys/kernel.h>
43 #include <sys/interrupt.h>
44 #include <sys/mbuf.h>
45 #include <sys/malloc.h>
46 #include <sys/queue.h>
47 #include <sys/rman.h>
48 #include <sys/serialize.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
51 #include <sys/sysctl.h>
52
53 #include <netinet/ip.h>
54 #include <netinet/tcp.h>
55
56 #include <net/bpf.h>
57 #include <net/ethernet.h>
58 #include <net/if.h>
59 #include <net/if_arp.h>
60 #include <net/if_dl.h>
61 #include <net/if_media.h>
62 #include <net/if_poll.h>
63 #include <net/if_types.h>
64 #include <net/ifq_var.h>
65 #include <net/vlan/if_vlan_var.h>
66 #include <net/vlan/if_vlan_ether.h>
67
68 #include <dev/netif/mii_layer/mii.h>
69 #include <dev/netif/mii_layer/miivar.h>
70 #include <dev/netif/mii_layer/brgphyreg.h>
71
72 #include <bus/pci/pcidevs.h>
73 #include <bus/pci/pcireg.h>
74 #include <bus/pci/pcivar.h>
75
76 #include <dev/netif/bge/if_bgereg.h>
77 #include <dev/netif/bnx/if_bnxvar.h>
78
79 /* "device miibus" required.  See GENERIC if you get errors here. */
80 #include "miibus_if.h"
81
82 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
83
84 #define BNX_INTR_CKINTVL        ((10 * hz) / 1000)      /* 10ms */
85
86 static const struct bnx_type {
87         uint16_t                bnx_vid;
88         uint16_t                bnx_did;
89         char                    *bnx_name;
90 } bnx_devs[] = {
91         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
92                 "Broadcom BCM5717 Gigabit Ethernet" },
93         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717C,
94                 "Broadcom BCM5717C Gigabit Ethernet" },
95         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
96                 "Broadcom BCM5718 Gigabit Ethernet" },
97         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
98                 "Broadcom BCM5719 Gigabit Ethernet" },
99         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
100                 "Broadcom BCM5720 Gigabit Ethernet" },
101
102         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5725,
103                 "Broadcom BCM5725 Gigabit Ethernet" },
104         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5727,
105                 "Broadcom BCM5727 Gigabit Ethernet" },
106         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5762,
107                 "Broadcom BCM5762 Gigabit Ethernet" },
108
109         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
110                 "Broadcom BCM57761 Gigabit Ethernet" },
111         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762,
112                 "Broadcom BCM57762 Gigabit Ethernet" },
113         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
114                 "Broadcom BCM57765 Gigabit Ethernet" },
115         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766,
116                 "Broadcom BCM57766 Gigabit Ethernet" },
117         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
118                 "Broadcom BCM57781 Gigabit Ethernet" },
119         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782,
120                 "Broadcom BCM57782 Gigabit Ethernet" },
121         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
122                 "Broadcom BCM57785 Gigabit Ethernet" },
123         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786,
124                 "Broadcom BCM57786 Gigabit Ethernet" },
125         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
126                 "Broadcom BCM57791 Fast Ethernet" },
127         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
128                 "Broadcom BCM57795 Fast Ethernet" },
129
130         { 0, 0, NULL }
131 };
132
133 static const int bnx_tx_mailbox[BNX_TX_RING_MAX] = {
134         BGE_MBX_TX_HOST_PROD0_LO,
135         BGE_MBX_TX_HOST_PROD0_HI,
136         BGE_MBX_TX_HOST_PROD1_LO,
137         BGE_MBX_TX_HOST_PROD1_HI
138 };
139
140 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
141 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
142 #define BNX_IS_57765_PLUS(sc)           ((sc)->bnx_flags & BNX_FLAG_57765_PLUS)
143 #define BNX_IS_57765_FAMILY(sc)  \
144         ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY)
145
146 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
147
148 static int      bnx_probe(device_t);
149 static int      bnx_attach(device_t);
150 static int      bnx_detach(device_t);
151 static void     bnx_shutdown(device_t);
152 static int      bnx_suspend(device_t);
153 static int      bnx_resume(device_t);
154 static int      bnx_miibus_readreg(device_t, int, int);
155 static int      bnx_miibus_writereg(device_t, int, int, int);
156 static void     bnx_miibus_statchg(device_t);
157
158 #ifdef IFPOLL_ENABLE
159 static void     bnx_npoll(struct ifnet *, struct ifpoll_info *);
160 static void     bnx_npoll_rx(struct ifnet *, void *, int);
161 static void     bnx_npoll_tx(struct ifnet *, void *, int);
162 static void     bnx_npoll_status(struct ifnet *);
163 #endif
164 static void     bnx_intr_legacy(void *);
165 static void     bnx_msi(void *);
166 static void     bnx_intr(struct bnx_softc *);
167 static void     bnx_enable_intr(struct bnx_softc *);
168 static void     bnx_disable_intr(struct bnx_softc *);
169 static void     bnx_txeof(struct bnx_tx_ring *, uint16_t);
170 static void     bnx_rxeof(struct bnx_rx_ret_ring *, uint16_t, int);
171 static int      bnx_alloc_intr(struct bnx_softc *);
172 static int      bnx_setup_intr(struct bnx_softc *);
173 static void     bnx_free_intr(struct bnx_softc *);
174 static void     bnx_teardown_intr(struct bnx_softc *, int);
175 static void     bnx_check_intr(void *);
176 static void     bnx_rx_std_refill_ithread(void *);
177 static void     bnx_rx_std_refill(void *, void *);
178
179 static void     bnx_start(struct ifnet *, struct ifaltq_subque *);
180 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
181 static void     bnx_init(void *);
182 static void     bnx_stop(struct bnx_softc *);
183 static void     bnx_watchdog(struct ifaltq_subque *);
184 static int      bnx_ifmedia_upd(struct ifnet *);
185 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
186 static void     bnx_tick(void *);
187 static void     bnx_serialize(struct ifnet *, enum ifnet_serialize);
188 static void     bnx_deserialize(struct ifnet *, enum ifnet_serialize);
189 static int      bnx_tryserialize(struct ifnet *, enum ifnet_serialize);
190 #ifdef INVARIANTS
191 static void     bnx_serialize_assert(struct ifnet *, enum ifnet_serialize,
192                     boolean_t);
193 #endif
194
195 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
196 static void     bnx_free_jumbo_mem(struct bnx_softc *);
197 static struct bnx_jslot
198                 *bnx_jalloc(struct bnx_softc *);
199 static void     bnx_jfree(void *);
200 static void     bnx_jref(void *);
201 static int      bnx_newbuf_std(struct bnx_rx_ret_ring *, int, int);
202 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
203 static void     bnx_setup_rxdesc_std(struct bnx_rx_std_ring *, int);
204 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
205 static int      bnx_init_rx_ring_std(struct bnx_rx_std_ring *);
206 static void     bnx_free_rx_ring_std(struct bnx_rx_std_ring *);
207 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
208 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
209 static void     bnx_free_tx_ring(struct bnx_tx_ring *);
210 static int      bnx_init_tx_ring(struct bnx_tx_ring *);
211 static int      bnx_create_tx_ring(struct bnx_tx_ring *);
212 static void     bnx_destroy_tx_ring(struct bnx_tx_ring *);
213 static int      bnx_create_rx_ret_ring(struct bnx_rx_ret_ring *);
214 static void     bnx_destroy_rx_ret_ring(struct bnx_rx_ret_ring *);
215 static int      bnx_dma_alloc(device_t);
216 static void     bnx_dma_free(struct bnx_softc *);
217 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
218                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
219 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
220 static struct mbuf *
221                 bnx_defrag_shortdma(struct mbuf *);
222 static int      bnx_encap(struct bnx_tx_ring *, struct mbuf **,
223                     uint32_t *, int *);
224 static int      bnx_setup_tso(struct bnx_tx_ring *, struct mbuf **,
225                     uint16_t *, uint16_t *);
226 static void     bnx_setup_serialize(struct bnx_softc *);
227 static void     bnx_set_tick_cpuid(struct bnx_softc *, boolean_t);
228
229 static void     bnx_reset(struct bnx_softc *);
230 static int      bnx_chipinit(struct bnx_softc *);
231 static int      bnx_blockinit(struct bnx_softc *);
232 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
233 static void     bnx_enable_msi(struct bnx_softc *sc);
234 static void     bnx_setmulti(struct bnx_softc *);
235 static void     bnx_setpromisc(struct bnx_softc *);
236 static void     bnx_stats_update_regs(struct bnx_softc *);
237 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
238
239 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
240 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
241 #ifdef notdef
242 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
243 #endif
244 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
245 static void     bnx_writembx(struct bnx_softc *, int, int);
246 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
247 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
248 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
249
250 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
251 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
252 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
253 static void     bnx_link_poll(struct bnx_softc *);
254
255 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
256 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
257 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
258 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
259
260 static void     bnx_coal_change(struct bnx_softc *);
261 static int      bnx_sysctl_force_defrag(SYSCTL_HANDLER_ARGS);
262 static int      bnx_sysctl_tx_wreg(SYSCTL_HANDLER_ARGS);
263 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
264 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
265 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
266 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
267 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
268 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
269 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
270                     int, int, uint32_t);
271 #ifdef IFPOLL_ENABLE
272 static int      bnx_sysctl_npoll_offset(SYSCTL_HANDLER_ARGS);
273 static int      bnx_sysctl_npoll_rxoff(SYSCTL_HANDLER_ARGS);
274 static int      bnx_sysctl_npoll_txoff(SYSCTL_HANDLER_ARGS);
275 #endif
276 static int      bnx_sysctl_std_refill(SYSCTL_HANDLER_ARGS);
277
278 static int      bnx_msi_enable = 1;
279 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
280
281 static device_method_t bnx_methods[] = {
282         /* Device interface */
283         DEVMETHOD(device_probe,         bnx_probe),
284         DEVMETHOD(device_attach,        bnx_attach),
285         DEVMETHOD(device_detach,        bnx_detach),
286         DEVMETHOD(device_shutdown,      bnx_shutdown),
287         DEVMETHOD(device_suspend,       bnx_suspend),
288         DEVMETHOD(device_resume,        bnx_resume),
289
290         /* bus interface */
291         DEVMETHOD(bus_print_child,      bus_generic_print_child),
292         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
293
294         /* MII interface */
295         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
296         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
297         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
298
299         DEVMETHOD_END
300 };
301
302 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
303 static devclass_t bnx_devclass;
304
305 DECLARE_DUMMY_MODULE(if_bnx);
306 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
307 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
308
309 static uint32_t
310 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
311 {
312         device_t dev = sc->bnx_dev;
313         uint32_t val;
314
315         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
316         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
317         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
318         return (val);
319 }
320
321 static void
322 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
323 {
324         device_t dev = sc->bnx_dev;
325
326         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
327         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
328         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
329 }
330
331 static void
332 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
333 {
334         CSR_WRITE_4(sc, off, val);
335 }
336
337 static void
338 bnx_writembx(struct bnx_softc *sc, int off, int val)
339 {
340         CSR_WRITE_4(sc, off, val);
341 }
342
343 /*
344  * Read a sequence of bytes from NVRAM.
345  */
346 static int
347 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
348 {
349         return (1);
350 }
351
352 /*
353  * Read a byte of data stored in the EEPROM at address 'addr.' The
354  * BCM570x supports both the traditional bitbang interface and an
355  * auto access interface for reading the EEPROM. We use the auto
356  * access method.
357  */
358 static uint8_t
359 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
360 {
361         int i;
362         uint32_t byte = 0;
363
364         /*
365          * Enable use of auto EEPROM access so we can avoid
366          * having to use the bitbang method.
367          */
368         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
369
370         /* Reset the EEPROM, load the clock period. */
371         CSR_WRITE_4(sc, BGE_EE_ADDR,
372             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
373         DELAY(20);
374
375         /* Issue the read EEPROM command. */
376         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
377
378         /* Wait for completion */
379         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
380                 DELAY(10);
381                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
382                         break;
383         }
384
385         if (i == BNX_TIMEOUT) {
386                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
387                 return(1);
388         }
389
390         /* Get result. */
391         byte = CSR_READ_4(sc, BGE_EE_DATA);
392
393         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
394
395         return(0);
396 }
397
398 /*
399  * Read a sequence of bytes from the EEPROM.
400  */
401 static int
402 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
403 {
404         size_t i;
405         int err;
406         uint8_t byte;
407
408         for (byte = 0, err = 0, i = 0; i < len; i++) {
409                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
410                 if (err)
411                         break;
412                 *(dest + i) = byte;
413         }
414
415         return(err ? 1 : 0);
416 }
417
418 static int
419 bnx_miibus_readreg(device_t dev, int phy, int reg)
420 {
421         struct bnx_softc *sc = device_get_softc(dev);
422         uint32_t val;
423         int i;
424
425         KASSERT(phy == sc->bnx_phyno,
426             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
427
428         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
429         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
430                 CSR_WRITE_4(sc, BGE_MI_MODE,
431                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
432                 DELAY(80);
433         }
434
435         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
436             BGE_MIPHY(phy) | BGE_MIREG(reg));
437
438         /* Poll for the PHY register access to complete. */
439         for (i = 0; i < BNX_TIMEOUT; i++) {
440                 DELAY(10);
441                 val = CSR_READ_4(sc, BGE_MI_COMM);
442                 if ((val & BGE_MICOMM_BUSY) == 0) {
443                         DELAY(5);
444                         val = CSR_READ_4(sc, BGE_MI_COMM);
445                         break;
446                 }
447         }
448         if (i == BNX_TIMEOUT) {
449                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
450                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
451                 val = 0;
452         }
453
454         /* Restore the autopoll bit if necessary. */
455         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
456                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
457                 DELAY(80);
458         }
459
460         if (val & BGE_MICOMM_READFAIL)
461                 return 0;
462
463         return (val & 0xFFFF);
464 }
465
466 static int
467 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
468 {
469         struct bnx_softc *sc = device_get_softc(dev);
470         int i;
471
472         KASSERT(phy == sc->bnx_phyno,
473             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
474
475         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
476         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
477                 CSR_WRITE_4(sc, BGE_MI_MODE,
478                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
479                 DELAY(80);
480         }
481
482         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
483             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
484
485         for (i = 0; i < BNX_TIMEOUT; i++) {
486                 DELAY(10);
487                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
488                         DELAY(5);
489                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
490                         break;
491                 }
492         }
493         if (i == BNX_TIMEOUT) {
494                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
495                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
496         }
497
498         /* Restore the autopoll bit if necessary. */
499         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
500                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
501                 DELAY(80);
502         }
503
504         return 0;
505 }
506
507 static void
508 bnx_miibus_statchg(device_t dev)
509 {
510         struct bnx_softc *sc;
511         struct mii_data *mii;
512
513         sc = device_get_softc(dev);
514         mii = device_get_softc(sc->bnx_miibus);
515
516         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
517             (IFM_ACTIVE | IFM_AVALID)) {
518                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
519                 case IFM_10_T:
520                 case IFM_100_TX:
521                         sc->bnx_link = 1;
522                         break;
523                 case IFM_1000_T:
524                 case IFM_1000_SX:
525                 case IFM_2500_SX:
526                         sc->bnx_link = 1;
527                         break;
528                 default:
529                         sc->bnx_link = 0;
530                         break;
531                 }
532         } else {
533                 sc->bnx_link = 0;
534         }
535         if (sc->bnx_link == 0)
536                 return;
537
538         BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
539         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
540             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
541                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
542         } else {
543                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
544         }
545
546         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
547                 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
548         } else {
549                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
550         }
551 }
552
553 /*
554  * Memory management for jumbo frames.
555  */
556 static int
557 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
558 {
559         struct ifnet *ifp = &sc->arpcom.ac_if;
560         struct bnx_jslot *entry;
561         uint8_t *ptr;
562         bus_addr_t paddr;
563         int i, error;
564
565         /*
566          * Create tag for jumbo mbufs.
567          * This is really a bit of a kludge. We allocate a special
568          * jumbo buffer pool which (thanks to the way our DMA
569          * memory allocation works) will consist of contiguous
570          * pages. This means that even though a jumbo buffer might
571          * be larger than a page size, we don't really need to
572          * map it into more than one DMA segment. However, the
573          * default mbuf tag will result in multi-segment mappings,
574          * so we have to create a special jumbo mbuf tag that
575          * lets us get away with mapping the jumbo buffers as
576          * a single segment. I think eventually the driver should
577          * be changed so that it uses ordinary mbufs and cluster
578          * buffers, i.e. jumbo frames can span multiple DMA
579          * descriptors. But that's a project for another day.
580          */
581
582         /*
583          * Create DMA stuffs for jumbo RX ring.
584          */
585         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
586                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
587                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
588                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
589                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
590         if (error) {
591                 if_printf(ifp, "could not create jumbo RX ring\n");
592                 return error;
593         }
594
595         /*
596          * Create DMA stuffs for jumbo buffer block.
597          */
598         error = bnx_dma_block_alloc(sc, BNX_JMEM,
599                                     &sc->bnx_cdata.bnx_jumbo_tag,
600                                     &sc->bnx_cdata.bnx_jumbo_map,
601                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
602                                     &paddr);
603         if (error) {
604                 if_printf(ifp, "could not create jumbo buffer\n");
605                 return error;
606         }
607
608         SLIST_INIT(&sc->bnx_jfree_listhead);
609
610         /*
611          * Now divide it up into 9K pieces and save the addresses
612          * in an array. Note that we play an evil trick here by using
613          * the first few bytes in the buffer to hold the the address
614          * of the softc structure for this interface. This is because
615          * bnx_jfree() needs it, but it is called by the mbuf management
616          * code which will not pass it to us explicitly.
617          */
618         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
619                 entry = &sc->bnx_cdata.bnx_jslots[i];
620                 entry->bnx_sc = sc;
621                 entry->bnx_buf = ptr;
622                 entry->bnx_paddr = paddr;
623                 entry->bnx_inuse = 0;
624                 entry->bnx_slot = i;
625                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
626
627                 ptr += BNX_JLEN;
628                 paddr += BNX_JLEN;
629         }
630         return 0;
631 }
632
633 static void
634 bnx_free_jumbo_mem(struct bnx_softc *sc)
635 {
636         /* Destroy jumbo RX ring. */
637         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
638                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
639                            sc->bnx_ldata.bnx_rx_jumbo_ring);
640
641         /* Destroy jumbo buffer block. */
642         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
643                            sc->bnx_cdata.bnx_jumbo_map,
644                            sc->bnx_ldata.bnx_jumbo_buf);
645 }
646
647 /*
648  * Allocate a jumbo buffer.
649  */
650 static struct bnx_jslot *
651 bnx_jalloc(struct bnx_softc *sc)
652 {
653         struct bnx_jslot *entry;
654
655         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
656         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
657         if (entry) {
658                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
659                 entry->bnx_inuse = 1;
660         } else {
661                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
662         }
663         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
664         return(entry);
665 }
666
667 /*
668  * Adjust usage count on a jumbo buffer.
669  */
670 static void
671 bnx_jref(void *arg)
672 {
673         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
674         struct bnx_softc *sc = entry->bnx_sc;
675
676         if (sc == NULL)
677                 panic("bnx_jref: can't find softc pointer!");
678
679         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
680                 panic("bnx_jref: asked to reference buffer "
681                     "that we don't manage!");
682         } else if (entry->bnx_inuse == 0) {
683                 panic("bnx_jref: buffer already free!");
684         } else {
685                 atomic_add_int(&entry->bnx_inuse, 1);
686         }
687 }
688
689 /*
690  * Release a jumbo buffer.
691  */
692 static void
693 bnx_jfree(void *arg)
694 {
695         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
696         struct bnx_softc *sc = entry->bnx_sc;
697
698         if (sc == NULL)
699                 panic("bnx_jfree: can't find softc pointer!");
700
701         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
702                 panic("bnx_jfree: asked to free buffer that we don't manage!");
703         } else if (entry->bnx_inuse == 0) {
704                 panic("bnx_jfree: buffer already free!");
705         } else {
706                 /*
707                  * Possible MP race to 0, use the serializer.  The atomic insn
708                  * is still needed for races against bnx_jref().
709                  */
710                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
711                 atomic_subtract_int(&entry->bnx_inuse, 1);
712                 if (entry->bnx_inuse == 0) {
713                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, 
714                                           entry, jslot_link);
715                 }
716                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
717         }
718 }
719
720
721 /*
722  * Intialize a standard receive ring descriptor.
723  */
724 static int
725 bnx_newbuf_std(struct bnx_rx_ret_ring *ret, int i, int init)
726 {
727         struct mbuf *m_new = NULL;
728         bus_dma_segment_t seg;
729         bus_dmamap_t map;
730         int error, nsegs;
731         struct bnx_rx_buf *rb;
732
733         rb = &ret->bnx_std->bnx_rx_std_buf[i];
734         KASSERT(!rb->bnx_rx_refilled, ("RX buf %dth has been refilled", i));
735
736         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
737         if (m_new == NULL) {
738                 error = ENOBUFS;
739                 goto back;
740         }
741         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
742         m_adj(m_new, ETHER_ALIGN);
743
744         error = bus_dmamap_load_mbuf_segment(ret->bnx_rx_mtag,
745             ret->bnx_rx_tmpmap, m_new, &seg, 1, &nsegs, BUS_DMA_NOWAIT);
746         if (error) {
747                 m_freem(m_new);
748                 goto back;
749         }
750
751         if (!init) {
752                 bus_dmamap_sync(ret->bnx_rx_mtag, rb->bnx_rx_dmamap,
753                     BUS_DMASYNC_POSTREAD);
754                 bus_dmamap_unload(ret->bnx_rx_mtag, rb->bnx_rx_dmamap);
755         }
756
757         map = ret->bnx_rx_tmpmap;
758         ret->bnx_rx_tmpmap = rb->bnx_rx_dmamap;
759
760         rb->bnx_rx_dmamap = map;
761         rb->bnx_rx_mbuf = m_new;
762         rb->bnx_rx_paddr = seg.ds_addr;
763 back:
764         cpu_sfence();
765         rb->bnx_rx_refilled = 1;
766         return error;
767 }
768
769 static void
770 bnx_setup_rxdesc_std(struct bnx_rx_std_ring *std, int i)
771 {
772         struct bnx_rx_buf *rb;
773         struct bge_rx_bd *r;
774
775         rb = &std->bnx_rx_std_buf[i];
776         KASSERT(rb->bnx_rx_refilled, ("RX buf %dth is not refilled", i));
777         rb->bnx_rx_refilled = 0;
778
779         r = &std->bnx_rx_std_ring[i];
780         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rb->bnx_rx_paddr);
781         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rb->bnx_rx_paddr);
782         r->bge_len = rb->bnx_rx_mbuf->m_len;
783         r->bge_idx = i;
784         r->bge_flags = BGE_RXBDFLAG_END;
785 }
786
787 /*
788  * Initialize a jumbo receive ring descriptor. This allocates
789  * a jumbo buffer from the pool managed internally by the driver.
790  */
791 static int
792 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
793 {
794         struct mbuf *m_new = NULL;
795         struct bnx_jslot *buf;
796         bus_addr_t paddr;
797
798         /* Allocate the mbuf. */
799         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
800         if (m_new == NULL)
801                 return ENOBUFS;
802
803         /* Allocate the jumbo buffer */
804         buf = bnx_jalloc(sc);
805         if (buf == NULL) {
806                 m_freem(m_new);
807                 return ENOBUFS;
808         }
809
810         /* Attach the buffer to the mbuf. */
811         m_new->m_ext.ext_arg = buf;
812         m_new->m_ext.ext_buf = buf->bnx_buf;
813         m_new->m_ext.ext_free = bnx_jfree;
814         m_new->m_ext.ext_ref = bnx_jref;
815         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
816
817         m_new->m_flags |= M_EXT;
818
819         m_new->m_data = m_new->m_ext.ext_buf;
820         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
821
822         paddr = buf->bnx_paddr;
823         m_adj(m_new, ETHER_ALIGN);
824         paddr += ETHER_ALIGN;
825
826         /* Save necessary information */
827         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_mbuf = m_new;
828         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_rx_paddr = paddr;
829
830         /* Set up the descriptor. */
831         bnx_setup_rxdesc_jumbo(sc, i);
832         return 0;
833 }
834
835 static void
836 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
837 {
838         struct bge_rx_bd *r;
839         struct bnx_rx_buf *rc;
840
841         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
842         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
843
844         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_rx_paddr);
845         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_rx_paddr);
846         r->bge_len = rc->bnx_rx_mbuf->m_len;
847         r->bge_idx = i;
848         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
849 }
850
851 static int
852 bnx_init_rx_ring_std(struct bnx_rx_std_ring *std)
853 {
854         int i, error;
855
856         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
857                 /* Use the first RX return ring's tmp RX mbuf DMA map */
858                 error = bnx_newbuf_std(&std->bnx_sc->bnx_rx_ret_ring[0], i, 1);
859                 if (error)
860                         return error;
861                 bnx_setup_rxdesc_std(std, i);
862         }
863
864         std->bnx_rx_std_refill = 0;
865         std->bnx_rx_std_running = 0;
866         cpu_sfence();
867         lwkt_serialize_handler_enable(&std->bnx_rx_std_serialize);
868
869         std->bnx_rx_std = BGE_STD_RX_RING_CNT - 1;
870         bnx_writembx(std->bnx_sc, BGE_MBX_RX_STD_PROD_LO, std->bnx_rx_std);
871
872         return(0);
873 }
874
875 static void
876 bnx_free_rx_ring_std(struct bnx_rx_std_ring *std)
877 {
878         int i;
879
880         lwkt_serialize_handler_disable(&std->bnx_rx_std_serialize);
881
882         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
883                 struct bnx_rx_buf *rb = &std->bnx_rx_std_buf[i];
884
885                 rb->bnx_rx_refilled = 0;
886                 if (rb->bnx_rx_mbuf != NULL) {
887                         bus_dmamap_unload(std->bnx_rx_mtag, rb->bnx_rx_dmamap);
888                         m_freem(rb->bnx_rx_mbuf);
889                         rb->bnx_rx_mbuf = NULL;
890                 }
891                 bzero(&std->bnx_rx_std_ring[i], sizeof(struct bge_rx_bd));
892         }
893 }
894
895 static int
896 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
897 {
898         struct bge_rcb *rcb;
899         int i, error;
900
901         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
902                 error = bnx_newbuf_jumbo(sc, i, 1);
903                 if (error)
904                         return error;
905         }
906
907         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
908
909         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
910         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
911         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
912
913         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
914
915         return(0);
916 }
917
918 static void
919 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
920 {
921         int i;
922
923         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
924                 struct bnx_rx_buf *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
925
926                 if (rc->bnx_rx_mbuf != NULL) {
927                         m_freem(rc->bnx_rx_mbuf);
928                         rc->bnx_rx_mbuf = NULL;
929                 }
930                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
931                     sizeof(struct bge_rx_bd));
932         }
933 }
934
935 static void
936 bnx_free_tx_ring(struct bnx_tx_ring *txr)
937 {
938         int i;
939
940         for (i = 0; i < BGE_TX_RING_CNT; i++) {
941                 struct bnx_tx_buf *buf = &txr->bnx_tx_buf[i];
942
943                 if (buf->bnx_tx_mbuf != NULL) {
944                         bus_dmamap_unload(txr->bnx_tx_mtag,
945                             buf->bnx_tx_dmamap);
946                         m_freem(buf->bnx_tx_mbuf);
947                         buf->bnx_tx_mbuf = NULL;
948                 }
949                 bzero(&txr->bnx_tx_ring[i], sizeof(struct bge_tx_bd));
950         }
951         txr->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
952 }
953
954 static int
955 bnx_init_tx_ring(struct bnx_tx_ring *txr)
956 {
957         txr->bnx_tx_cnt = 0;
958         txr->bnx_tx_saved_considx = 0;
959         txr->bnx_tx_prodidx = 0;
960
961         /* Initialize transmit producer index for host-memory send ring. */
962         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, txr->bnx_tx_prodidx);
963
964         return(0);
965 }
966
967 static void
968 bnx_setmulti(struct bnx_softc *sc)
969 {
970         struct ifnet *ifp;
971         struct ifmultiaddr *ifma;
972         uint32_t hashes[4] = { 0, 0, 0, 0 };
973         int h, i;
974
975         ifp = &sc->arpcom.ac_if;
976
977         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
978                 for (i = 0; i < 4; i++)
979                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
980                 return;
981         }
982
983         /* First, zot all the existing filters. */
984         for (i = 0; i < 4; i++)
985                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
986
987         /* Now program new ones. */
988         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
989                 if (ifma->ifma_addr->sa_family != AF_LINK)
990                         continue;
991                 h = ether_crc32_le(
992                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
993                     ETHER_ADDR_LEN) & 0x7f;
994                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
995         }
996
997         for (i = 0; i < 4; i++)
998                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
999 }
1000
1001 /*
1002  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1003  * self-test results.
1004  */
1005 static int
1006 bnx_chipinit(struct bnx_softc *sc)
1007 {
1008         uint32_t dma_rw_ctl, mode_ctl;
1009         int i;
1010
1011         /* Set endian type before we access any non-PCI registers. */
1012         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
1013             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
1014
1015         /* Clear the MAC control register */
1016         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
1017
1018         /*
1019          * Clear the MAC statistics block in the NIC's
1020          * internal memory.
1021          */
1022         for (i = BGE_STATS_BLOCK;
1023             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1024                 BNX_MEMWIN_WRITE(sc, i, 0);
1025
1026         for (i = BGE_STATUS_BLOCK;
1027             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1028                 BNX_MEMWIN_WRITE(sc, i, 0);
1029
1030         if (BNX_IS_57765_FAMILY(sc)) {
1031                 uint32_t val;
1032
1033                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) {
1034                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1035                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1036
1037                         /* Access the lower 1K of PL PCI-E block registers. */
1038                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1039                             val | BGE_MODECTL_PCIE_PL_SEL);
1040
1041                         val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5);
1042                         val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ;
1043                         CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val);
1044
1045                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1046                 }
1047                 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) {
1048                         /* Fix transmit hangs */
1049                         val = CSR_READ_4(sc, BGE_CPMU_PADRNG_CTL);
1050                         val |= BGE_CPMU_PADRNG_CTL_RDIV2;
1051                         CSR_WRITE_4(sc, BGE_CPMU_PADRNG_CTL, val);
1052
1053                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1054                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1055
1056                         /* Access the lower 1K of DL PCI-E block registers. */
1057                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1058                             val | BGE_MODECTL_PCIE_DL_SEL);
1059
1060                         val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX);
1061                         val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK;
1062                         val |= BGE_PCIE_DL_LO_FTSMAX_VAL;
1063                         CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val);
1064
1065                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1066                 }
1067
1068                 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK);
1069                 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK;
1070                 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25;
1071                 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val);
1072         }
1073
1074         /*
1075          * Set up the PCI DMA control register.
1076          */
1077         dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4);
1078         /*
1079          * Disable 32bytes cache alignment for DMA write to host memory
1080          *
1081          * NOTE:
1082          * 64bytes cache alignment for DMA write to host memory is still
1083          * enabled.
1084          */
1085         dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1086         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1087                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1088         /*
1089          * Enable HW workaround for controllers that misinterpret
1090          * a status tag update and leave interrupts permanently
1091          * disabled.
1092          */
1093         if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1094             sc->bnx_asicrev != BGE_ASICREV_BCM5762 &&
1095             !BNX_IS_57765_FAMILY(sc))
1096                 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1097         if (bootverbose) {
1098                 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n",
1099                     dma_rw_ctl);
1100         }
1101         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1102
1103         /*
1104          * Set up general mode register.
1105          */
1106         mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
1107             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1108         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1109
1110         /*
1111          * Disable memory write invalidate.  Apparently it is not supported
1112          * properly by these devices.  Also ensure that INTx isn't disabled,
1113          * as these chips need it even when using MSI.
1114          */
1115         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1116             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1117
1118         /* Set the timer prescaler (always 66Mhz) */
1119         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1120
1121         return(0);
1122 }
1123
1124 static int
1125 bnx_blockinit(struct bnx_softc *sc)
1126 {
1127         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
1128         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
1129         struct bnx_intr_data *intr = &sc->bnx_intr_data[0];
1130         struct bge_rcb *rcb;
1131         bus_size_t vrcb;
1132         bge_hostaddr taddr;
1133         uint32_t val;
1134         int i, limit;
1135
1136         /*
1137          * Initialize the memory window pointer register so that
1138          * we can access the first 32K of internal NIC RAM. This will
1139          * allow us to set up the TX send ring RCBs and the RX return
1140          * ring RCBs, plus other things which live in NIC memory.
1141          */
1142         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1143
1144         /* Configure mbuf pool watermarks */
1145         if (BNX_IS_57765_PLUS(sc)) {
1146                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1147                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1148                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1149                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1150                 } else {
1151                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1152                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1153                 }
1154         } else {
1155                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1156                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1157                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1158         }
1159
1160         /* Configure DMA resource watermarks */
1161         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1162         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1163
1164         /* Enable buffer manager */
1165         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1166         /*
1167          * Change the arbitration algorithm of TXMBUF read request to
1168          * round-robin instead of priority based for BCM5719.  When
1169          * TXFIFO is almost empty, RDMA will hold its request until
1170          * TXFIFO is not almost empty.
1171          */
1172         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1173                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1174         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1175             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 ||
1176             sc->bnx_chipid == BGE_CHIPID_BCM5720_A0)
1177                 val |= BGE_BMANMODE_LOMBUF_ATTN;
1178         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1179
1180         /* Poll for buffer manager start indication */
1181         for (i = 0; i < BNX_TIMEOUT; i++) {
1182                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1183                         break;
1184                 DELAY(10);
1185         }
1186
1187         if (i == BNX_TIMEOUT) {
1188                 if_printf(&sc->arpcom.ac_if,
1189                           "buffer manager failed to start\n");
1190                 return(ENXIO);
1191         }
1192
1193         /* Enable flow-through queues */
1194         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1195         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1196
1197         /* Wait until queue initialization is complete */
1198         for (i = 0; i < BNX_TIMEOUT; i++) {
1199                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1200                         break;
1201                 DELAY(10);
1202         }
1203
1204         if (i == BNX_TIMEOUT) {
1205                 if_printf(&sc->arpcom.ac_if,
1206                           "flow-through queue init failed\n");
1207                 return(ENXIO);
1208         }
1209
1210         /*
1211          * Summary of rings supported by the controller:
1212          *
1213          * Standard Receive Producer Ring
1214          * - This ring is used to feed receive buffers for "standard"
1215          *   sized frames (typically 1536 bytes) to the controller.
1216          *
1217          * Jumbo Receive Producer Ring
1218          * - This ring is used to feed receive buffers for jumbo sized
1219          *   frames (i.e. anything bigger than the "standard" frames)
1220          *   to the controller.
1221          *
1222          * Mini Receive Producer Ring
1223          * - This ring is used to feed receive buffers for "mini"
1224          *   sized frames to the controller.
1225          * - This feature required external memory for the controller
1226          *   but was never used in a production system.  Should always
1227          *   be disabled.
1228          *
1229          * Receive Return Ring
1230          * - After the controller has placed an incoming frame into a
1231          *   receive buffer that buffer is moved into a receive return
1232          *   ring.  The driver is then responsible to passing the
1233          *   buffer up to the stack.  Many versions of the controller
1234          *   support multiple RR rings.
1235          *
1236          * Send Ring
1237          * - This ring is used for outgoing frames.  Many versions of
1238          *   the controller support multiple send rings.
1239          */
1240
1241         /* Initialize the standard receive producer ring control block. */
1242         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1243         rcb->bge_hostaddr.bge_addr_lo =
1244             BGE_ADDR_LO(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1245         rcb->bge_hostaddr.bge_addr_hi =
1246             BGE_ADDR_HI(sc->bnx_rx_std_ring.bnx_rx_std_ring_paddr);
1247         if (BNX_IS_57765_PLUS(sc)) {
1248                 /*
1249                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1250                  * Bits 15-2 : Maximum RX frame size
1251                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1252                  * Bit 0     : Reserved
1253                  */
1254                 rcb->bge_maxlen_flags =
1255                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1256         } else {
1257                 /*
1258                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1259                  * Bits 15-2 : Reserved (should be 0)
1260                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1261                  * Bit 0     : Reserved
1262                  */
1263                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1264         }
1265         if (BNX_IS_5717_PLUS(sc))
1266                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1267         else
1268                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1269         /* Write the standard receive producer ring control block. */
1270         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1271         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1272         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1273         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1274         /* Reset the standard receive producer ring producer index. */
1275         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1276
1277         /*
1278          * Initialize the jumbo RX producer ring control
1279          * block.  We set the 'ring disabled' bit in the
1280          * flags field until we're actually ready to start
1281          * using this ring (i.e. once we set the MTU
1282          * high enough to require it).
1283          */
1284         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1285                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1286                 /* Get the jumbo receive producer ring RCB parameters. */
1287                 rcb->bge_hostaddr.bge_addr_lo =
1288                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1289                 rcb->bge_hostaddr.bge_addr_hi =
1290                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1291                 rcb->bge_maxlen_flags =
1292                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1293                     BGE_RCB_FLAG_RING_DISABLED);
1294                 if (BNX_IS_5717_PLUS(sc))
1295                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1296                 else
1297                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1298                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1299                     rcb->bge_hostaddr.bge_addr_hi);
1300                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1301                     rcb->bge_hostaddr.bge_addr_lo);
1302                 /* Program the jumbo receive producer ring RCB parameters. */
1303                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1304                     rcb->bge_maxlen_flags);
1305                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1306                 /* Reset the jumbo receive producer ring producer index. */
1307                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1308         }
1309
1310         /*
1311          * The BD ring replenish thresholds control how often the
1312          * hardware fetches new BD's from the producer rings in host
1313          * memory.  Setting the value too low on a busy system can
1314          * starve the hardware and recue the throughpout.
1315          *
1316          * Set the BD ring replentish thresholds. The recommended
1317          * values are 1/8th the number of descriptors allocated to
1318          * each ring.
1319          */
1320         val = 8;
1321         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1322         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1323                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1324                     BGE_JUMBO_RX_RING_CNT/8);
1325         }
1326         if (BNX_IS_57765_PLUS(sc)) {
1327                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1328                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1329         }
1330
1331         /*
1332          * Disable all send rings by setting the 'ring disabled' bit
1333          * in the flags field of all the TX send ring control blocks,
1334          * located in NIC memory.
1335          */
1336         if (BNX_IS_5717_PLUS(sc))
1337                 limit = 4;
1338         else if (BNX_IS_57765_FAMILY(sc) ||
1339             sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1340                 limit = 2;
1341         else
1342                 limit = 1;
1343         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1344         for (i = 0; i < limit; i++) {
1345                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1346                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1347                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1348                 vrcb += sizeof(struct bge_rcb);
1349         }
1350
1351         /* Configure send ring RCB 0 (we use only the first ring) */
1352         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1353         BGE_HOSTADDR(taddr, txr->bnx_tx_ring_paddr);
1354         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1355         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1356         if (BNX_IS_5717_PLUS(sc)) {
1357                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
1358         } else {
1359                 RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1360                     BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1361         }
1362         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1363             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1364
1365         /*
1366          * Disable all receive return rings by setting the
1367          * 'ring disabled' bit in the flags field of all the receive
1368          * return ring control blocks, located in NIC memory.
1369          */
1370         if (BNX_IS_5717_PLUS(sc)) {
1371                 /* Should be 17, use 16 until we get an SRAM map. */
1372                 limit = 16;
1373         } else if (BNX_IS_57765_FAMILY(sc) ||
1374             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1375                 limit = 4;
1376         } else {
1377                 limit = 1;
1378         }
1379         /* Disable all receive return rings. */
1380         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1381         for (i = 0; i < limit; i++) {
1382                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1383                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1384                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1385                     BGE_RCB_FLAG_RING_DISABLED);
1386                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1387                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1388                     (i * (sizeof(uint64_t))), 0);
1389                 vrcb += sizeof(struct bge_rcb);
1390         }
1391
1392         /*
1393          * Set up receive return ring 0.  Note that the NIC address
1394          * for RX return rings is 0x0.  The return rings live entirely
1395          * within the host, so the nicaddr field in the RCB isn't used.
1396          */
1397         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1398         BGE_HOSTADDR(taddr, ret->bnx_rx_ret_ring_paddr);
1399         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1400         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1401         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1402         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1403             BGE_RCB_MAXLEN_FLAGS(BNX_RETURN_RING_CNT, 0));
1404
1405         /* Set random backoff seed for TX */
1406         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1407             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1408             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1409             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1410             BGE_TX_BACKOFF_SEED_MASK);
1411
1412         /* Set inter-packet gap */
1413         val = 0x2620;
1414         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1415             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1416                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1417                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1418         }
1419         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1420
1421         /*
1422          * Specify which ring to use for packets that don't match
1423          * any RX rules.
1424          */
1425         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1426
1427         /*
1428          * Configure number of RX lists. One interrupt distribution
1429          * list, sixteen active lists, one bad frames class.
1430          */
1431         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1432
1433         /* Inialize RX list placement stats mask. */
1434         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1435         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1436
1437         /* Disable host coalescing until we get it set up */
1438         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1439
1440         /* Poll to make sure it's shut down. */
1441         for (i = 0; i < BNX_TIMEOUT; i++) {
1442                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1443                         break;
1444                 DELAY(10);
1445         }
1446
1447         if (i == BNX_TIMEOUT) {
1448                 if_printf(&sc->arpcom.ac_if,
1449                           "host coalescing engine failed to idle\n");
1450                 return(ENXIO);
1451         }
1452
1453         /* Set up host coalescing defaults */
1454         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bnx_rx_coal_ticks);
1455         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bnx_tx_coal_ticks);
1456         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bnx_rx_coal_bds);
1457         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bnx_tx_coal_bds);
1458         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, sc->bnx_rx_coal_bds_int);
1459         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, sc->bnx_tx_coal_bds_int);
1460
1461         /* Set up address of status block */
1462         bzero(intr->bnx_status_block, BGE_STATUS_BLK_SZ);
1463         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1464             BGE_ADDR_HI(intr->bnx_status_block_paddr));
1465         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1466             BGE_ADDR_LO(intr->bnx_status_block_paddr));
1467
1468         /* Set up status block partail update size. */
1469         val = BGE_STATBLKSZ_32BYTE;
1470 #if 0
1471         /*
1472          * Does not seem to have visible effect in both
1473          * bulk data (1472B UDP datagram) and tiny data
1474          * (18B UDP datagram) TX tests.
1475          */
1476         val |= BGE_HCCMODE_CLRTICK_TX;
1477 #endif
1478         /* Turn on host coalescing state machine */
1479         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1480
1481         /* Turn on RX BD completion state machine and enable attentions */
1482         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1483             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1484
1485         /* Turn on RX list placement state machine */
1486         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1487
1488         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1489             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1490             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1491             BGE_MACMODE_FRMHDR_DMA_ENB;
1492
1493         if (sc->bnx_flags & BNX_FLAG_TBI)
1494                 val |= BGE_PORTMODE_TBI;
1495         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1496                 val |= BGE_PORTMODE_GMII;
1497         else
1498                 val |= BGE_PORTMODE_MII;
1499
1500         /* Turn on DMA, clear stats */
1501         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1502
1503         /* Set misc. local control, enable interrupts on attentions */
1504         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1505
1506 #ifdef notdef
1507         /* Assert GPIO pins for PHY reset */
1508         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1509             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1510         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1511             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1512 #endif
1513
1514         /* Turn on write DMA state machine */
1515         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1516         /* Enable host coalescing bug fix. */
1517         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1518         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1519                 /* Request larger DMA burst size to get better performance. */
1520                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1521         }
1522         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1523         DELAY(40);
1524
1525         if (BNX_IS_57765_PLUS(sc)) {
1526                 uint32_t dmactl, dmactl_reg;
1527
1528                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1529                         dmactl_reg = BGE_RDMA_RSRVCTRL2;
1530                 else
1531                         dmactl_reg = BGE_RDMA_RSRVCTRL;
1532
1533                 dmactl = CSR_READ_4(sc, dmactl_reg);
1534                 /*
1535                  * Adjust tx margin to prevent TX data corruption and
1536                  * fix internal FIFO overflow.
1537                  */
1538                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1539                     sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1540                     sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1541                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1542                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1543                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1544                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1545                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1546                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1547                 }
1548                 /*
1549                  * Enable fix for read DMA FIFO overruns.
1550                  * The fix is to limit the number of RX BDs
1551                  * the hardware would fetch at a fime.
1552                  */
1553                 CSR_WRITE_4(sc, dmactl_reg,
1554                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1555         }
1556
1557         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1558                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1559                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1560                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1561                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1562         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1563             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1564                 uint32_t ctrl_reg;
1565
1566                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5762)
1567                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL2;
1568                 else
1569                         ctrl_reg = BGE_RDMA_LSO_CRPTEN_CTRL;
1570
1571                 /*
1572                  * Allow 4KB burst length reads for non-LSO frames.
1573                  * Enable 512B burst length reads for buffer descriptors.
1574                  */
1575                 CSR_WRITE_4(sc, ctrl_reg,
1576                     CSR_READ_4(sc, ctrl_reg) |
1577                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1578                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1579         }
1580
1581         /* Turn on read DMA state machine */
1582         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1583         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1584                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1585         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1586             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1587             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1588                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1589                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1590                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1591         }
1592         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
1593             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
1594                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1595                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1596                 /*
1597                  * Allow multiple outstanding read requests from
1598                  * non-LSO read DMA engine.
1599                  */
1600                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1601         }
1602         if (sc->bnx_asicrev == BGE_ASICREV_BCM57766)
1603                 val |= BGE_RDMAMODE_JMB_2K_MMRR;
1604         if (sc->bnx_flags & BNX_FLAG_TSO)
1605                 val |= BGE_RDMAMODE_TSO4_ENABLE;
1606         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1607         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1608         DELAY(40);
1609
1610         /* Turn on RX data completion state machine */
1611         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1612
1613         /* Turn on RX BD initiator state machine */
1614         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1615
1616         /* Turn on RX data and RX BD initiator state machine */
1617         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1618
1619         /* Turn on send BD completion state machine */
1620         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1621
1622         /* Turn on send data completion state machine */
1623         val = BGE_SDCMODE_ENABLE;
1624         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1625                 val |= BGE_SDCMODE_CDELAY; 
1626         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1627
1628         /* Turn on send data initiator state machine */
1629         if (sc->bnx_flags & BNX_FLAG_TSO) {
1630                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE |
1631                     BGE_SDIMODE_HW_LSO_PRE_DMA);
1632         } else {
1633                 CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1634         }
1635
1636         /* Turn on send BD initiator state machine */
1637         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1638
1639         /* Turn on send BD selector state machine */
1640         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1641
1642         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1643         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1644             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1645
1646         /* ack/clear link change events */
1647         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1648             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1649             BGE_MACSTAT_LINK_CHANGED);
1650         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1651
1652         /*
1653          * Enable attention when the link has changed state for
1654          * devices that use auto polling.
1655          */
1656         if (sc->bnx_flags & BNX_FLAG_TBI) {
1657                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1658         } else {
1659                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1660                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1661                         DELAY(80);
1662                 }
1663         }
1664
1665         /*
1666          * Clear any pending link state attention.
1667          * Otherwise some link state change events may be lost until attention
1668          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1669          * It's not necessary on newer BCM chips - perhaps enabling link
1670          * state change attentions implies clearing pending attention.
1671          */
1672         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1673             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1674             BGE_MACSTAT_LINK_CHANGED);
1675
1676         /* Enable link state change attentions. */
1677         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1678
1679         return(0);
1680 }
1681
1682 /*
1683  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1684  * against our list and return its name if we find a match. Note
1685  * that since the Broadcom controller contains VPD support, we
1686  * can get the device name string from the controller itself instead
1687  * of the compiled-in string. This is a little slow, but it guarantees
1688  * we'll always announce the right product name.
1689  */
1690 static int
1691 bnx_probe(device_t dev)
1692 {
1693         const struct bnx_type *t;
1694         uint16_t product, vendor;
1695
1696         if (!pci_is_pcie(dev))
1697                 return ENXIO;
1698
1699         product = pci_get_device(dev);
1700         vendor = pci_get_vendor(dev);
1701
1702         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1703                 if (vendor == t->bnx_vid && product == t->bnx_did)
1704                         break;
1705         }
1706         if (t->bnx_name == NULL)
1707                 return ENXIO;
1708
1709         device_set_desc(dev, t->bnx_name);
1710         return 0;
1711 }
1712
1713 static int
1714 bnx_attach(device_t dev)
1715 {
1716         struct ifnet *ifp;
1717         struct bnx_softc *sc;
1718         struct bnx_rx_std_ring *std;
1719         uint32_t hwcfg = 0;
1720         int error = 0, rid, capmask, i, std_cpuid, std_cpuid_def;
1721         uint8_t ether_addr[ETHER_ADDR_LEN];
1722         uint16_t product;
1723         uintptr_t mii_priv = 0;
1724 #ifdef BNX_TSO_DEBUG
1725         char desc[32];
1726 #endif
1727 #ifdef IFPOLL_ENABLE
1728         int offset, offset_def;
1729 #endif
1730
1731         sc = device_get_softc(dev);
1732         sc->bnx_dev = dev;
1733         callout_init_mp(&sc->bnx_tick_timer);
1734         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1735         lwkt_serialize_init(&sc->bnx_main_serialize);
1736
1737         product = pci_get_device(dev);
1738
1739 #ifndef BURN_BRIDGES
1740         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1741                 uint32_t irq, mem;
1742
1743                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1744                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1745
1746                 device_printf(dev, "chip is in D%d power mode "
1747                     "-- setting to D0\n", pci_get_powerstate(dev));
1748
1749                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1750
1751                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1752                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1753         }
1754 #endif  /* !BURN_BRIDGE */
1755
1756         /*
1757          * Map control/status registers.
1758          */
1759         pci_enable_busmaster(dev);
1760
1761         rid = BGE_PCI_BAR0;
1762         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1763             RF_ACTIVE);
1764
1765         if (sc->bnx_res == NULL) {
1766                 device_printf(dev, "couldn't map memory\n");
1767                 return ENXIO;
1768         }
1769
1770         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1771         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1772
1773         /* Save various chip information */
1774         sc->bnx_chipid =
1775             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1776             BGE_PCIMISCCTL_ASICREV_SHIFT;
1777         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1778                 /* All chips having dedicated ASICREV register have CPMU */
1779                 sc->bnx_flags |= BNX_FLAG_CPMU;
1780
1781                 switch (product) {
1782                 case PCI_PRODUCT_BROADCOM_BCM5717:
1783                 case PCI_PRODUCT_BROADCOM_BCM5717C:
1784                 case PCI_PRODUCT_BROADCOM_BCM5718:
1785                 case PCI_PRODUCT_BROADCOM_BCM5719:
1786                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1787                 case PCI_PRODUCT_BROADCOM_BCM5725:
1788                 case PCI_PRODUCT_BROADCOM_BCM5727:
1789                 case PCI_PRODUCT_BROADCOM_BCM5762:
1790                         sc->bnx_chipid = pci_read_config(dev,
1791                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1792                         break;
1793
1794                 case PCI_PRODUCT_BROADCOM_BCM57761:
1795                 case PCI_PRODUCT_BROADCOM_BCM57762:
1796                 case PCI_PRODUCT_BROADCOM_BCM57765:
1797                 case PCI_PRODUCT_BROADCOM_BCM57766:
1798                 case PCI_PRODUCT_BROADCOM_BCM57781:
1799                 case PCI_PRODUCT_BROADCOM_BCM57782:
1800                 case PCI_PRODUCT_BROADCOM_BCM57785:
1801                 case PCI_PRODUCT_BROADCOM_BCM57786:
1802                 case PCI_PRODUCT_BROADCOM_BCM57791:
1803                 case PCI_PRODUCT_BROADCOM_BCM57795:
1804                         sc->bnx_chipid = pci_read_config(dev,
1805                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1806                         break;
1807
1808                 default:
1809                         sc->bnx_chipid = pci_read_config(dev,
1810                             BGE_PCI_PRODID_ASICREV, 4);
1811                         break;
1812                 }
1813         }
1814         if (sc->bnx_chipid == BGE_CHIPID_BCM5717_C0)
1815                 sc->bnx_chipid = BGE_CHIPID_BCM5720_A0;
1816
1817         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1818         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1819
1820         switch (sc->bnx_asicrev) {
1821         case BGE_ASICREV_BCM5717:
1822         case BGE_ASICREV_BCM5719:
1823         case BGE_ASICREV_BCM5720:
1824                 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS;
1825                 break;
1826
1827         case BGE_ASICREV_BCM5762:
1828                 sc->bnx_flags |= BNX_FLAG_57765_PLUS;
1829                 break;
1830
1831         case BGE_ASICREV_BCM57765:
1832         case BGE_ASICREV_BCM57766:
1833                 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS;
1834                 break;
1835         }
1836
1837         sc->bnx_flags |= BNX_FLAG_TSO;
1838         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 &&
1839             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0)
1840                 sc->bnx_flags &= ~BNX_FLAG_TSO;
1841
1842         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1843             BNX_IS_57765_FAMILY(sc)) {
1844                 /*
1845                  * All BCM57785 and BCM5718 families chips have a bug that
1846                  * under certain situation interrupt will not be enabled
1847                  * even if status tag is written to interrupt mailbox.
1848                  *
1849                  * While BCM5719 and BCM5720 have a hardware workaround
1850                  * which could fix the above bug.
1851                  * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in
1852                  * bnx_chipinit().
1853                  *
1854                  * For the rest of the chips in these two families, we will
1855                  * have to poll the status block at high rate (10ms currently)
1856                  * to check whether the interrupt is hosed or not.
1857                  * See bnx_check_intr() for details.
1858                  */
1859                 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
1860         }
1861
1862         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
1863         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1864             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1865                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
1866         else
1867                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
1868         device_printf(dev, "CHIP ID 0x%08x; "
1869                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
1870                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
1871
1872         /*
1873          * Set various PHY quirk flags.
1874          */
1875
1876         capmask = MII_CAPMASK_DEFAULT;
1877         if (product == PCI_PRODUCT_BROADCOM_BCM57791 ||
1878             product == PCI_PRODUCT_BROADCOM_BCM57795) {
1879                 /* 10/100 only */
1880                 capmask &= ~BMSR_EXTSTAT;
1881         }
1882
1883         mii_priv |= BRGPHY_FLAG_WIRESPEED;
1884         if (sc->bnx_chipid == BGE_CHIPID_BCM5762_A0)
1885                 mii_priv |= BRGPHY_FLAG_5762_A0;
1886
1887         /* Initialize if_name earlier, so if_printf could be used */
1888         ifp = &sc->arpcom.ac_if;
1889         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1890
1891         /* Try to reset the chip. */
1892         bnx_reset(sc);
1893
1894         if (bnx_chipinit(sc)) {
1895                 device_printf(dev, "chip initialization failed\n");
1896                 error = ENXIO;
1897                 goto fail;
1898         }
1899
1900         /*
1901          * Get station address
1902          */
1903         error = bnx_get_eaddr(sc, ether_addr);
1904         if (error) {
1905                 device_printf(dev, "failed to read station address\n");
1906                 goto fail;
1907         }
1908
1909         /* XXX */
1910         sc->bnx_tx_ringcnt = 1;
1911         sc->bnx_rx_retcnt = 1;
1912         sc->bnx_intr_cnt = 1;
1913
1914         if ((sc->bnx_rx_retcnt == 1 && sc->bnx_tx_ringcnt == 1) ||
1915             (sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt > 1)) {
1916                 /*
1917                  * The RX ring and the corresponding TX ring processing
1918                  * should be on the same CPU, since they share the same
1919                  * status block.
1920                  */
1921                 sc->bnx_flags |= BNX_FLAG_RXTX_BUNDLE;
1922                 if (bootverbose)
1923                         device_printf(dev, "RX/TX bundle\n");
1924         } else {
1925                 KKASSERT(sc->bnx_rx_retcnt > 1 && sc->bnx_tx_ringcnt == 1);
1926         }
1927
1928         error = bnx_dma_alloc(dev);
1929         if (error)
1930                 goto fail;
1931
1932 #ifdef IFPOLL_ENABLE
1933         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
1934                 /*
1935                  * NPOLLING RX/TX CPU offset
1936                  */
1937                 if (sc->bnx_rx_retcnt == ncpus2) {
1938                         offset = 0;
1939                 } else {
1940                         offset_def =
1941                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
1942                         offset = device_getenv_int(dev, "npoll.offset",
1943                             offset_def);
1944                         if (offset >= ncpus2 ||
1945                             offset % sc->bnx_rx_retcnt != 0) {
1946                                 device_printf(dev, "invalid npoll.offset %d, "
1947                                     "use %d\n", offset, offset_def);
1948                                 offset = offset_def;
1949                         }
1950                 }
1951                 sc->bnx_npoll_rxoff = offset;
1952                 sc->bnx_npoll_txoff = offset;
1953         } else {
1954                 /*
1955                  * NPOLLING RX CPU offset
1956                  */
1957                 if (sc->bnx_rx_retcnt == ncpus2) {
1958                         offset = 0;
1959                 } else {
1960                         offset_def =
1961                         (sc->bnx_rx_retcnt * device_get_unit(dev)) % ncpus2;
1962                         offset = device_getenv_int(dev, "npoll.rxoff",
1963                             offset_def);
1964                         if (offset >= ncpus2 ||
1965                             offset % sc->bnx_rx_retcnt != 0) {
1966                                 device_printf(dev, "invalid npoll.rxoff %d, "
1967                                     "use %d\n", offset, offset_def);
1968                                 offset = offset_def;
1969                         }
1970                 }
1971                 sc->bnx_npoll_rxoff = offset;
1972
1973                 /*
1974                  * NPOLLING TX CPU offset
1975                  */
1976                 offset_def = device_get_unit(dev) % ncpus2;
1977                 offset = device_getenv_int(dev, "npoll.txoff", offset_def);
1978                 if (offset >= ncpus2) {
1979                         device_printf(dev, "invalid npoll.txoff %d, use %d\n",
1980                             offset, offset_def);
1981                         offset = offset_def;
1982                 }
1983                 sc->bnx_npoll_txoff = offset;
1984         }
1985 #endif  /* IFPOLL_ENABLE */
1986
1987         /*
1988          * Allocate interrupt
1989          */
1990         error = bnx_alloc_intr(sc);
1991         if (error)
1992                 goto fail;
1993
1994         /* Setup serializers */
1995         bnx_setup_serialize(sc);
1996
1997         /* Set default tuneable values. */
1998         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
1999         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
2000         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
2001         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
2002         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_INT_DEF;
2003         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_INT_DEF;
2004
2005         /* Set up ifnet structure */
2006         ifp->if_softc = sc;
2007         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2008         ifp->if_ioctl = bnx_ioctl;
2009         ifp->if_start = bnx_start;
2010 #ifdef IFPOLL_ENABLE
2011         ifp->if_npoll = bnx_npoll;
2012 #endif
2013         ifp->if_init = bnx_init;
2014         ifp->if_serialize = bnx_serialize;
2015         ifp->if_deserialize = bnx_deserialize;
2016         ifp->if_tryserialize = bnx_tryserialize;
2017 #ifdef INVARIANTS
2018         ifp->if_serialize_assert = bnx_serialize_assert;
2019 #endif
2020         ifp->if_mtu = ETHERMTU;
2021         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2022
2023         ifp->if_capabilities |= IFCAP_HWCSUM;
2024         ifp->if_hwassist = BNX_CSUM_FEATURES;
2025         if (sc->bnx_flags & BNX_FLAG_TSO) {
2026                 ifp->if_capabilities |= IFCAP_TSO;
2027                 ifp->if_hwassist |= CSUM_TSO;
2028         }
2029         ifp->if_capenable = ifp->if_capabilities;
2030
2031         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
2032         ifq_set_ready(&ifp->if_snd);
2033         ifq_set_subq_cnt(&ifp->if_snd, sc->bnx_tx_ringcnt);
2034
2035         /*
2036          * Figure out what sort of media we have by checking the
2037          * hardware config word in the first 32k of NIC internal memory,
2038          * or fall back to examining the EEPROM if necessary.
2039          * Note: on some BCM5700 cards, this value appears to be unset.
2040          * If that's the case, we have to rely on identifying the NIC
2041          * by its PCI subsystem ID, as we do below for the SysKonnect
2042          * SK-9D41.
2043          */
2044         if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) {
2045                 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
2046         } else {
2047                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
2048                                     sizeof(hwcfg))) {
2049                         device_printf(dev, "failed to read EEPROM\n");
2050                         error = ENXIO;
2051                         goto fail;
2052                 }
2053                 hwcfg = ntohl(hwcfg);
2054         }
2055
2056         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
2057         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
2058             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
2059                 sc->bnx_flags |= BNX_FLAG_TBI;
2060
2061         /* Setup MI MODE */
2062         if (sc->bnx_flags & BNX_FLAG_CPMU)
2063                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
2064         else
2065                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
2066
2067         /* Setup link status update stuffs */
2068         if (sc->bnx_flags & BNX_FLAG_TBI) {
2069                 sc->bnx_link_upd = bnx_tbi_link_upd;
2070                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2071         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
2072                 sc->bnx_link_upd = bnx_autopoll_link_upd;
2073                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2074         } else {
2075                 sc->bnx_link_upd = bnx_copper_link_upd;
2076                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2077         }
2078
2079         /* Set default PHY address */
2080         sc->bnx_phyno = 1;
2081
2082         /*
2083          * PHY address mapping for various devices.
2084          *
2085          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
2086          * ---------+-------+-------+-------+-------+
2087          * BCM57XX  |   1   |   X   |   X   |   X   |
2088          * BCM5704  |   1   |   X   |   1   |   X   |
2089          * BCM5717  |   1   |   8   |   2   |   9   |
2090          * BCM5719  |   1   |   8   |   2   |   9   |
2091          * BCM5720  |   1   |   8   |   2   |   9   |
2092          *
2093          * Other addresses may respond but they are not
2094          * IEEE compliant PHYs and should be ignored.
2095          */
2096         if (BNX_IS_5717_PLUS(sc)) {
2097                 int f;
2098
2099                 f = pci_get_function(dev);
2100                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
2101                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
2102                             BGE_SGDIGSTS_IS_SERDES)
2103                                 sc->bnx_phyno = f + 8;
2104                         else
2105                                 sc->bnx_phyno = f + 1;
2106                 } else {
2107                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
2108                             BGE_CPMU_PHY_STRAP_IS_SERDES)
2109                                 sc->bnx_phyno = f + 8;
2110                         else
2111                                 sc->bnx_phyno = f + 1;
2112                 }
2113         }
2114
2115         if (sc->bnx_flags & BNX_FLAG_TBI) {
2116                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
2117                     bnx_ifmedia_upd, bnx_ifmedia_sts);
2118                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2119                 ifmedia_add(&sc->bnx_ifmedia,
2120                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2121                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2122                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2123                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2124         } else {
2125                 struct mii_probe_args mii_args;
2126
2127                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2128                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2129                 mii_args.mii_capmask = capmask;
2130                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2131                 mii_args.mii_priv = mii_priv;
2132
2133                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2134                 if (error) {
2135                         device_printf(dev, "MII without any PHY!\n");
2136                         goto fail;
2137                 }
2138         }
2139
2140         /*
2141          * Create sysctl nodes.
2142          */
2143         sysctl_ctx_init(&sc->bnx_sysctl_ctx);
2144         sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx,
2145                                               SYSCTL_STATIC_CHILDREN(_hw),
2146                                               OID_AUTO,
2147                                               device_get_nameunit(dev),
2148                                               CTLFLAG_RD, 0, "");
2149         if (sc->bnx_sysctl_tree == NULL) {
2150                 device_printf(dev, "can't add sysctl node\n");
2151                 error = ENXIO;
2152                 goto fail;
2153         }
2154
2155         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2156             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2157             "rx_rings", CTLFLAG_RD, &sc->bnx_rx_retcnt, 0, "# of RX rings");
2158         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2159             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2160             "tx_rings", CTLFLAG_RD, &sc->bnx_tx_ringcnt, 0, "# of TX rings");
2161
2162         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2163                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2164                         OID_AUTO, "rx_coal_ticks",
2165                         CTLTYPE_INT | CTLFLAG_RW,
2166                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2167                         "Receive coalescing ticks (usec).");
2168         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2169                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2170                         OID_AUTO, "tx_coal_ticks",
2171                         CTLTYPE_INT | CTLFLAG_RW,
2172                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2173                         "Transmit coalescing ticks (usec).");
2174         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2175                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2176                         OID_AUTO, "rx_coal_bds",
2177                         CTLTYPE_INT | CTLFLAG_RW,
2178                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2179                         "Receive max coalesced BD count.");
2180         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2181                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2182                         OID_AUTO, "tx_coal_bds",
2183                         CTLTYPE_INT | CTLFLAG_RW,
2184                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2185                         "Transmit max coalesced BD count.");
2186         /*
2187          * A common design characteristic for many Broadcom
2188          * client controllers is that they only support a
2189          * single outstanding DMA read operation on the PCIe
2190          * bus. This means that it will take twice as long to
2191          * fetch a TX frame that is split into header and
2192          * payload buffers as it does to fetch a single,
2193          * contiguous TX frame (2 reads vs. 1 read). For these
2194          * controllers, coalescing buffers to reduce the number
2195          * of memory reads is effective way to get maximum
2196          * performance(about 940Mbps).  Without collapsing TX
2197          * buffers the maximum TCP bulk transfer performance
2198          * is about 850Mbps. However forcing coalescing mbufs
2199          * consumes a lot of CPU cycles, so leave it off by
2200          * default.
2201          */
2202         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2203             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2204             "force_defrag", CTLTYPE_INT | CTLFLAG_RW,
2205             sc, 0, bnx_sysctl_force_defrag, "I",
2206             "Force defragment on TX path");
2207
2208         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2209             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2210             "tx_wreg", CTLTYPE_INT | CTLFLAG_RW,
2211             sc, 0, bnx_sysctl_tx_wreg, "I",
2212             "# of segments before writing to hardware register");
2213
2214         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2215             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2216             "std_refill", CTLTYPE_INT | CTLFLAG_RW,
2217             sc, 0, bnx_sysctl_std_refill, "I",
2218             "# of packets received before scheduling standard refilling");
2219
2220         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2221             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2222             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2223             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2224             "Receive max coalesced BD count during interrupt.");
2225         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2226             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2227             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2228             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2229             "Transmit max coalesced BD count during interrupt.");
2230
2231 #ifdef IFPOLL_ENABLE
2232         if (sc->bnx_flags & BNX_FLAG_RXTX_BUNDLE) {
2233                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2234                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2235                     "npoll_offset", CTLTYPE_INT | CTLFLAG_RW,
2236                     sc, 0, bnx_sysctl_npoll_offset, "I",
2237                     "NPOLLING cpu offset");
2238         } else {
2239                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2240                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2241                     "npoll_rxoff", CTLTYPE_INT | CTLFLAG_RW,
2242                     sc, 0, bnx_sysctl_npoll_rxoff, "I",
2243                     "NPOLLING RX cpu offset");
2244                 SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2245                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2246                     "npoll_txoff", CTLTYPE_INT | CTLFLAG_RW,
2247                     sc, 0, bnx_sysctl_npoll_txoff, "I",
2248                     "NPOLLING TX cpu offset");
2249         }
2250 #endif
2251
2252 #ifdef BNX_TSO_DEBUG
2253         for (i = 0; i < BNX_TSO_NSTATS; ++i) {
2254                 ksnprintf(desc, sizeof(desc), "tso%d", i + 1);
2255                 SYSCTL_ADD_ULONG(&sc->bnx_sysctl_ctx,
2256                     SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2257                     desc, CTLFLAG_RW, &sc->bnx_tsosegs[i], "");
2258         }
2259 #endif
2260
2261         /*
2262          * Call MI attach routine.
2263          */
2264         ether_ifattach(ifp, ether_addr, NULL);
2265
2266         /* Setup TX rings and subqueues */
2267         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2268                 struct ifaltq_subque *ifsq = ifq_get_subq(&ifp->if_snd, i);
2269                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
2270
2271                 ifsq_set_cpuid(ifsq, txr->bnx_tx_cpuid);
2272                 ifsq_set_hw_serialize(ifsq, &txr->bnx_tx_serialize);
2273                 ifsq_set_priv(ifsq, txr);
2274                 txr->bnx_ifsq = ifsq;
2275
2276                 ifsq_watchdog_init(&txr->bnx_tx_watchdog, ifsq, bnx_watchdog);
2277         }
2278
2279         error = bnx_setup_intr(sc);
2280         if (error) {
2281                 ether_ifdetach(ifp);
2282                 goto fail;
2283         }
2284         bnx_set_tick_cpuid(sc, FALSE);
2285
2286         /*
2287          * Create RX standard ring refilling thread
2288          */
2289         std_cpuid_def = device_get_unit(dev) % ncpus;
2290         std_cpuid = device_getenv_int(dev, "std.cpuid", std_cpuid_def);
2291         if (std_cpuid < 0 || std_cpuid >= ncpus) {
2292                 device_printf(dev, "invalid std.cpuid %d, use %d\n",
2293                     std_cpuid, std_cpuid_def);
2294                 std_cpuid = std_cpuid_def;
2295         }
2296
2297         std = &sc->bnx_rx_std_ring;
2298         lwkt_create(bnx_rx_std_refill_ithread, std, NULL,
2299             &std->bnx_rx_std_ithread, TDF_NOSTART | TDF_INTTHREAD, std_cpuid,
2300             "%s std", device_get_nameunit(dev));
2301         lwkt_setpri(&std->bnx_rx_std_ithread, TDPRI_INT_MED);
2302         std->bnx_rx_std_ithread.td_preemptable = lwkt_preempt;
2303         sc->bnx_flags |= BNX_FLAG_STD_THREAD;
2304
2305         return(0);
2306 fail:
2307         bnx_detach(dev);
2308         return(error);
2309 }
2310
2311 static int
2312 bnx_detach(device_t dev)
2313 {
2314         struct bnx_softc *sc = device_get_softc(dev);
2315
2316         if (device_is_attached(dev)) {
2317                 struct ifnet *ifp = &sc->arpcom.ac_if;
2318
2319                 ifnet_serialize_all(ifp);
2320                 bnx_stop(sc);
2321                 bnx_reset(sc);
2322                 bnx_teardown_intr(sc, sc->bnx_intr_cnt);
2323                 ifnet_deserialize_all(ifp);
2324
2325                 ether_ifdetach(ifp);
2326         }
2327
2328         if (sc->bnx_flags & BNX_FLAG_STD_THREAD) {
2329                 struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
2330
2331                 tsleep_interlock(std, 0);
2332                 std->bnx_rx_std_stop = 1;
2333                 cpu_sfence();
2334                 lwkt_schedule(&std->bnx_rx_std_ithread);
2335                 tsleep(std, PINTERLOCKED, "bnx_detach", 0);
2336                 if (bootverbose)
2337                         device_printf(dev, "RX std ithread exited\n");
2338         }
2339
2340         if (sc->bnx_flags & BNX_FLAG_TBI)
2341                 ifmedia_removeall(&sc->bnx_ifmedia);
2342         if (sc->bnx_miibus)
2343                 device_delete_child(dev, sc->bnx_miibus);
2344         bus_generic_detach(dev);
2345
2346         bnx_free_intr(sc);
2347
2348         if (sc->bnx_res != NULL) {
2349                 bus_release_resource(dev, SYS_RES_MEMORY,
2350                     BGE_PCI_BAR0, sc->bnx_res);
2351         }
2352
2353         if (sc->bnx_sysctl_tree != NULL)
2354                 sysctl_ctx_free(&sc->bnx_sysctl_ctx);
2355
2356         bnx_dma_free(sc);
2357
2358         if (sc->bnx_serialize != NULL)
2359                 kfree(sc->bnx_serialize, M_DEVBUF);
2360
2361         return 0;
2362 }
2363
2364 static void
2365 bnx_reset(struct bnx_softc *sc)
2366 {
2367         device_t dev;
2368         uint32_t cachesize, command, pcistate, reset;
2369         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2370         int i, val = 0;
2371         uint16_t devctl;
2372
2373         dev = sc->bnx_dev;
2374
2375         write_op = bnx_writemem_direct;
2376
2377         /* Save some important PCI state. */
2378         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2379         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2380         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2381
2382         pci_write_config(dev, BGE_PCI_MISC_CTL,
2383             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2384             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2385             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2386
2387         /* Disable fastboot on controllers that support it. */
2388         if (bootverbose)
2389                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2390         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2391
2392         /*
2393          * Write the magic number to SRAM at offset 0xB50.
2394          * When firmware finishes its initialization it will
2395          * write ~BGE_MAGIC_NUMBER to the same location.
2396          */
2397         bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2398
2399         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2400
2401         /* XXX: Broadcom Linux driver. */
2402         /* Force PCI-E 1.0a mode */
2403         if (!BNX_IS_57765_PLUS(sc) &&
2404             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2405             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2406              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2407                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2408                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2409         }
2410         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2411                 /* Prevent PCIE link training during global reset */
2412                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2413                 reset |= (1<<29);
2414         }
2415
2416         /* 
2417          * Set GPHY Power Down Override to leave GPHY
2418          * powered up in D0 uninitialized.
2419          */
2420         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2421                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2422
2423         /* Issue global reset */
2424         write_op(sc, BGE_MISC_CFG, reset);
2425
2426         DELAY(1000);
2427
2428         /* XXX: Broadcom Linux driver. */
2429         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2430                 uint32_t v;
2431
2432                 DELAY(500000); /* wait for link training to complete */
2433                 v = pci_read_config(dev, 0xc4, 4);
2434                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2435         }
2436
2437         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2438
2439         /* Disable no snoop and disable relaxed ordering. */
2440         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2441
2442         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2443         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2444                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2445                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2446         }
2447
2448         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2449             devctl, 2);
2450
2451         /* Clear error status. */
2452         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2453             PCIEM_DEVSTS_CORR_ERR |
2454             PCIEM_DEVSTS_NFATAL_ERR |
2455             PCIEM_DEVSTS_FATAL_ERR |
2456             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2457
2458         /* Reset some of the PCI state that got zapped by reset */
2459         pci_write_config(dev, BGE_PCI_MISC_CTL,
2460             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2461             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2462             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2463         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2464         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2465         write_op(sc, BGE_MISC_CFG, (65 << 1));
2466
2467         /* Enable memory arbiter */
2468         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2469
2470         /*
2471          * Poll until we see the 1's complement of the magic number.
2472          * This indicates that the firmware initialization is complete.
2473          */
2474         for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2475                 val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2476                 if (val == ~BGE_MAGIC_NUMBER)
2477                         break;
2478                 DELAY(10);
2479         }
2480         if (i == BNX_FIRMWARE_TIMEOUT) {
2481                 if_printf(&sc->arpcom.ac_if, "firmware handshake "
2482                           "timed out, found 0x%08x\n", val);
2483         }
2484
2485         /* BCM57765 A0 needs additional time before accessing. */
2486         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2487                 DELAY(10 * 1000);
2488
2489         /*
2490          * XXX Wait for the value of the PCISTATE register to
2491          * return to its original pre-reset state. This is a
2492          * fairly good indicator of reset completion. If we don't
2493          * wait for the reset to fully complete, trying to read
2494          * from the device's non-PCI registers may yield garbage
2495          * results.
2496          */
2497         for (i = 0; i < BNX_TIMEOUT; i++) {
2498                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2499                         break;
2500                 DELAY(10);
2501         }
2502
2503         /* Fix up byte swapping */
2504         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2505
2506         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2507
2508         /*
2509          * The 5704 in TBI mode apparently needs some special
2510          * adjustment to insure the SERDES drive level is set
2511          * to 1.2V.
2512          */
2513         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2514             (sc->bnx_flags & BNX_FLAG_TBI)) {
2515                 uint32_t serdescfg;
2516
2517                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2518                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2519                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2520         }
2521
2522         CSR_WRITE_4(sc, BGE_MI_MODE,
2523             sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
2524         DELAY(80);
2525
2526         /* XXX: Broadcom Linux driver. */
2527         if (!BNX_IS_57765_PLUS(sc)) {
2528                 uint32_t v;
2529
2530                 /* Enable Data FIFO protection. */
2531                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
2532                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
2533         }
2534
2535         DELAY(10000);
2536
2537         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2538                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2539                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2540         }
2541 }
2542
2543 /*
2544  * Frame reception handling. This is called if there's a frame
2545  * on the receive return list.
2546  *
2547  * Note: we have to be able to handle two possibilities here:
2548  * 1) the frame is from the jumbo recieve ring
2549  * 2) the frame is from the standard receive ring
2550  */
2551
2552 static void
2553 bnx_rxeof(struct bnx_rx_ret_ring *ret, uint16_t rx_prod, int count)
2554 {
2555         struct bnx_softc *sc = ret->bnx_sc;
2556         struct bnx_rx_std_ring *std = ret->bnx_std;
2557         struct ifnet *ifp = &sc->arpcom.ac_if;
2558
2559         while (ret->bnx_rx_saved_considx != rx_prod && count != 0) {
2560                 struct bge_rx_bd *cur_rx;
2561                 struct bnx_rx_buf *rb;
2562                 uint32_t rxidx;
2563                 struct mbuf *m = NULL;
2564                 uint16_t vlan_tag = 0;
2565                 int have_tag = 0;
2566
2567                 --count;
2568
2569                 cur_rx = &ret->bnx_rx_ret_ring[ret->bnx_rx_saved_considx];
2570
2571                 rxidx = cur_rx->bge_idx;
2572                 BNX_INC(ret->bnx_rx_saved_considx, BNX_RETURN_RING_CNT);
2573
2574                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2575                         have_tag = 1;
2576                         vlan_tag = cur_rx->bge_vlan_tag;
2577                 }
2578
2579                 if (ret->bnx_rx_cnt >= ret->bnx_rx_cntmax) {
2580                         ret->bnx_rx_cnt = 0;
2581                         cpu_sfence();
2582                         atomic_set_int(&std->bnx_rx_std_refill,
2583                             ret->bnx_rx_mask);
2584                         if (atomic_poll_acquire_int(&std->bnx_rx_std_running))
2585                                 lwkt_schedule(&std->bnx_rx_std_ithread);
2586                 }
2587                 ret->bnx_rx_cnt++;
2588
2589                 rb = &std->bnx_rx_std_buf[rxidx];
2590                 m = rb->bnx_rx_mbuf;
2591                 if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2592                         IFNET_STAT_INC(ifp, ierrors, 1);
2593                         cpu_sfence();
2594                         rb->bnx_rx_refilled = 1;
2595                         continue;
2596                 }
2597                 if (bnx_newbuf_std(ret, rxidx, 0)) {
2598                         IFNET_STAT_INC(ifp, ierrors, 1);
2599                         continue;
2600                 }
2601
2602                 IFNET_STAT_INC(ifp, ipackets, 1);
2603                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2604                 m->m_pkthdr.rcvif = ifp;
2605
2606                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2607                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2608                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2609                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2610                                 if ((cur_rx->bge_error_flag &
2611                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2612                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2613                         }
2614                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2615                                 m->m_pkthdr.csum_data =
2616                                     cur_rx->bge_tcp_udp_csum;
2617                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2618                                     CSUM_PSEUDO_HDR;
2619                         }
2620                 }
2621
2622                 /*
2623                  * If we received a packet with a vlan tag, pass it
2624                  * to vlan_input() instead of ether_input().
2625                  */
2626                 if (have_tag) {
2627                         m->m_flags |= M_VLANTAG;
2628                         m->m_pkthdr.ether_vlantag = vlan_tag;
2629                 }
2630                 ifp->if_input(ifp, m);
2631         }
2632         bnx_writembx(sc, ret->bnx_rx_mbx, ret->bnx_rx_saved_considx);
2633 }
2634
2635 static void
2636 bnx_txeof(struct bnx_tx_ring *txr, uint16_t tx_cons)
2637 {
2638         struct ifnet *ifp = &txr->bnx_sc->arpcom.ac_if;
2639
2640         /*
2641          * Go through our tx ring and free mbufs for those
2642          * frames that have been sent.
2643          */
2644         while (txr->bnx_tx_saved_considx != tx_cons) {
2645                 struct bnx_tx_buf *buf;
2646                 uint32_t idx = 0;
2647
2648                 idx = txr->bnx_tx_saved_considx;
2649                 buf = &txr->bnx_tx_buf[idx];
2650                 if (buf->bnx_tx_mbuf != NULL) {
2651                         IFNET_STAT_INC(ifp, opackets, 1);
2652                         bus_dmamap_unload(txr->bnx_tx_mtag,
2653                             buf->bnx_tx_dmamap);
2654                         m_freem(buf->bnx_tx_mbuf);
2655                         buf->bnx_tx_mbuf = NULL;
2656                 }
2657                 txr->bnx_tx_cnt--;
2658                 BNX_INC(txr->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2659         }
2660
2661         if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) >=
2662             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2663                 ifsq_clr_oactive(txr->bnx_ifsq);
2664
2665         if (txr->bnx_tx_cnt == 0)
2666                 txr->bnx_tx_watchdog.wd_timer = 0;
2667
2668         if (!ifsq_is_empty(txr->bnx_ifsq))
2669                 ifsq_devstart(txr->bnx_ifsq);
2670 }
2671
2672 #ifdef IFPOLL_ENABLE
2673
2674 static void
2675 bnx_npoll_rx(struct ifnet *ifp __unused, void *xret, int cycle)
2676 {
2677         struct bnx_rx_ret_ring *ret = xret;
2678         uint16_t rx_prod;
2679
2680         ASSERT_SERIALIZED(&ret->bnx_rx_ret_serialize);
2681
2682         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
2683         cpu_lfence();
2684
2685         rx_prod = *ret->bnx_rx_considx;
2686         if (ret->bnx_rx_saved_considx != rx_prod)
2687                 bnx_rxeof(ret, rx_prod, cycle);
2688 }
2689
2690 static void
2691 bnx_npoll_tx(struct ifnet *ifp __unused, void *xtxr, int cycle __unused)
2692 {
2693         struct bnx_tx_ring *txr = xtxr;
2694         uint16_t tx_cons;
2695
2696         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
2697
2698         tx_cons = *txr->bnx_tx_considx;
2699         if (txr->bnx_tx_saved_considx != tx_cons)
2700                 bnx_txeof(txr, tx_cons);
2701 }
2702
2703 static void
2704 bnx_npoll_status(struct ifnet *ifp)
2705 {
2706         struct bnx_softc *sc = ifp->if_softc;
2707         struct bge_status_block *sblk = sc->bnx_intr_data[0].bnx_status_block;
2708
2709         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
2710
2711         if ((sblk->bge_status & BGE_STATFLAG_LINKSTATE_CHANGED) ||
2712             sc->bnx_link_evt)
2713                 bnx_link_poll(sc);
2714 }
2715
2716 static void
2717 bnx_npoll(struct ifnet *ifp, struct ifpoll_info *info)
2718 {
2719         struct bnx_softc *sc = ifp->if_softc;
2720         int i;
2721
2722         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2723
2724         if (info != NULL) {
2725                 /*
2726                  * TODO handle RXTX bundle and non-bundle
2727                  */
2728                 info->ifpi_status.status_func = bnx_npoll_status;
2729                 info->ifpi_status.serializer = &sc->bnx_main_serialize;
2730
2731                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2732                         struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
2733                         int idx = i + sc->bnx_npoll_txoff;
2734
2735                         KKASSERT(idx < ncpus2);
2736                         info->ifpi_tx[idx].poll_func = bnx_npoll_tx;
2737                         info->ifpi_tx[idx].arg = txr;
2738                         info->ifpi_tx[idx].serializer = &txr->bnx_tx_serialize;
2739                         ifsq_set_cpuid(txr->bnx_ifsq, idx);
2740                 }
2741
2742                 for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
2743                         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
2744                         int idx = i + sc->bnx_npoll_rxoff;
2745
2746                         KKASSERT(idx < ncpus2);
2747                         info->ifpi_rx[idx].poll_func = bnx_npoll_rx;
2748                         info->ifpi_rx[idx].arg = ret;
2749                         info->ifpi_rx[idx].serializer =
2750                             &ret->bnx_rx_ret_serialize;
2751                 }
2752
2753                 if (ifp->if_flags & IFF_RUNNING) {
2754                         bnx_disable_intr(sc);
2755                         bnx_set_tick_cpuid(sc, TRUE);
2756                 }
2757         } else {
2758                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
2759                         ifsq_set_cpuid(sc->bnx_tx_ring[i].bnx_ifsq,
2760                             sc->bnx_tx_ring[i].bnx_tx_cpuid);
2761                 }
2762                 if (ifp->if_flags & IFF_RUNNING) {
2763                         bnx_enable_intr(sc);
2764                         bnx_set_tick_cpuid(sc, FALSE);
2765                 }
2766         }
2767 }
2768
2769 #endif  /* IFPOLL_ENABLE */
2770
2771 static void
2772 bnx_intr_legacy(void *xsc)
2773 {
2774         struct bnx_softc *sc = xsc;
2775         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
2776
2777         if (ret->bnx_saved_status_tag == *ret->bnx_hw_status_tag) {
2778                 uint32_t val;
2779
2780                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
2781                 if (val & BGE_PCISTAT_INTR_NOTACT)
2782                         return;
2783         }
2784
2785         /*
2786          * NOTE:
2787          * Interrupt will have to be disabled if tagged status
2788          * is used, else interrupt will always be asserted on
2789          * certain chips (at least on BCM5750 AX/BX).
2790          */
2791         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2792
2793         bnx_intr(sc);
2794 }
2795
2796 static void
2797 bnx_msi(void *xsc)
2798 {
2799         bnx_intr(xsc);
2800 }
2801
2802 static void
2803 bnx_intr(struct bnx_softc *sc)
2804 {
2805         struct ifnet *ifp = &sc->arpcom.ac_if;
2806         struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[0];
2807         struct bge_status_block *sblk = sc->bnx_intr_data[0].bnx_status_block;
2808         uint32_t status;
2809
2810         ASSERT_SERIALIZED(&sc->bnx_main_serialize);
2811
2812         ret->bnx_saved_status_tag = *ret->bnx_hw_status_tag;
2813         /*
2814          * Use a load fence to ensure that status_tag is saved 
2815          * before rx_prod, tx_cons and status.
2816          */
2817         cpu_lfence();
2818
2819         status = sblk->bge_status;
2820
2821         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
2822                 bnx_link_poll(sc);
2823
2824         if (ifp->if_flags & IFF_RUNNING) {
2825                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[0];
2826                 uint16_t rx_prod, tx_cons;
2827
2828                 lwkt_serialize_enter(&ret->bnx_rx_ret_serialize);
2829                 rx_prod = *ret->bnx_rx_considx;
2830                 if (ret->bnx_rx_saved_considx != rx_prod)
2831                         bnx_rxeof(ret, rx_prod, -1);
2832                 lwkt_serialize_exit(&ret->bnx_rx_ret_serialize);
2833
2834                 lwkt_serialize_enter(&txr->bnx_tx_serialize);
2835                 tx_cons = *txr->bnx_tx_considx;
2836                 if (txr->bnx_tx_saved_considx != tx_cons)
2837                         bnx_txeof(txr, tx_cons);
2838                 lwkt_serialize_exit(&txr->bnx_tx_serialize);
2839         }
2840
2841         bnx_writembx(sc, BGE_MBX_IRQ0_LO, ret->bnx_saved_status_tag << 24);
2842 }
2843
2844 static void
2845 bnx_tick(void *xsc)
2846 {
2847         struct bnx_softc *sc = xsc;
2848
2849         lwkt_serialize_enter(&sc->bnx_main_serialize);
2850
2851         bnx_stats_update_regs(sc);
2852
2853         if (sc->bnx_flags & BNX_FLAG_TBI) {
2854                 /*
2855                  * Since in TBI mode auto-polling can't be used we should poll
2856                  * link status manually. Here we register pending link event
2857                  * and trigger interrupt.
2858                  */
2859                 sc->bnx_link_evt++;
2860                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
2861         } else if (!sc->bnx_link) {
2862                 mii_tick(device_get_softc(sc->bnx_miibus));
2863         }
2864
2865         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
2866             sc->bnx_tick_cpuid);
2867
2868         lwkt_serialize_exit(&sc->bnx_main_serialize);
2869 }
2870
2871 static void
2872 bnx_stats_update_regs(struct bnx_softc *sc)
2873 {
2874         struct ifnet *ifp = &sc->arpcom.ac_if;
2875         struct bge_mac_stats_regs stats;
2876         uint32_t *s;
2877         int i;
2878
2879         s = (uint32_t *)&stats;
2880         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
2881                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
2882                 s++;
2883         }
2884
2885         IFNET_STAT_SET(ifp, collisions,
2886            (stats.dot3StatsSingleCollisionFrames +
2887            stats.dot3StatsMultipleCollisionFrames +
2888            stats.dot3StatsExcessiveCollisions +
2889            stats.dot3StatsLateCollisions));
2890 }
2891
2892 /*
2893  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
2894  * pointers to descriptors.
2895  */
2896 static int
2897 bnx_encap(struct bnx_tx_ring *txr, struct mbuf **m_head0, uint32_t *txidx,
2898     int *segs_used)
2899 {
2900         struct bge_tx_bd *d = NULL;
2901         uint16_t csum_flags = 0, vlan_tag = 0, mss = 0;
2902         bus_dma_segment_t segs[BNX_NSEG_NEW];
2903         bus_dmamap_t map;
2904         int error, maxsegs, nsegs, idx, i;
2905         struct mbuf *m_head = *m_head0, *m_new;
2906
2907         if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2908 #ifdef BNX_TSO_DEBUG
2909                 int tso_nsegs;
2910 #endif
2911
2912                 error = bnx_setup_tso(txr, m_head0, &mss, &csum_flags);
2913                 if (error)
2914                         return error;
2915                 m_head = *m_head0;
2916
2917 #ifdef BNX_TSO_DEBUG
2918                 tso_nsegs = (m_head->m_pkthdr.len /
2919                     m_head->m_pkthdr.tso_segsz) - 1;
2920                 if (tso_nsegs > (BNX_TSO_NSTATS - 1))
2921                         tso_nsegs = BNX_TSO_NSTATS - 1;
2922                 else if (tso_nsegs < 0)
2923                         tso_nsegs = 0;
2924                 txr->bnx_sc->bnx_tsosegs[tso_nsegs]++;
2925 #endif
2926         } else if (m_head->m_pkthdr.csum_flags & BNX_CSUM_FEATURES) {
2927                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
2928                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
2929                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2930                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
2931                 if (m_head->m_flags & M_LASTFRAG)
2932                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
2933                 else if (m_head->m_flags & M_FRAG)
2934                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
2935         }
2936         if (m_head->m_flags & M_VLANTAG) {
2937                 csum_flags |= BGE_TXBDFLAG_VLAN_TAG;
2938                 vlan_tag = m_head->m_pkthdr.ether_vlantag;
2939         }
2940
2941         idx = *txidx;
2942         map = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
2943
2944         maxsegs = (BGE_TX_RING_CNT - txr->bnx_tx_cnt) - BNX_NSEG_RSVD;
2945         KASSERT(maxsegs >= BNX_NSEG_SPARE,
2946                 ("not enough segments %d", maxsegs));
2947
2948         if (maxsegs > BNX_NSEG_NEW)
2949                 maxsegs = BNX_NSEG_NEW;
2950
2951         /*
2952          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
2953          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
2954          * but when such padded frames employ the bge IP/TCP checksum
2955          * offload, the hardware checksum assist gives incorrect results
2956          * (possibly from incorporating its own padding into the UDP/TCP
2957          * checksum; who knows).  If we pad such runts with zeros, the
2958          * onboard checksum comes out correct.
2959          */
2960         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
2961             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
2962                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
2963                 if (error)
2964                         goto back;
2965         }
2966
2967         if ((txr->bnx_tx_flags & BNX_TX_FLAG_SHORTDMA) &&
2968             m_head->m_next != NULL) {
2969                 m_new = bnx_defrag_shortdma(m_head);
2970                 if (m_new == NULL) {
2971                         error = ENOBUFS;
2972                         goto back;
2973                 }
2974                 *m_head0 = m_head = m_new;
2975         }
2976         if ((m_head->m_pkthdr.csum_flags & CSUM_TSO) == 0 &&
2977             (txr->bnx_tx_flags & BNX_TX_FLAG_FORCE_DEFRAG) &&
2978             m_head->m_next != NULL) {
2979                 /*
2980                  * Forcefully defragment mbuf chain to overcome hardware
2981                  * limitation which only support a single outstanding
2982                  * DMA read operation.  If it fails, keep moving on using
2983                  * the original mbuf chain.
2984                  */
2985                 m_new = m_defrag(m_head, MB_DONTWAIT);
2986                 if (m_new != NULL)
2987                         *m_head0 = m_head = m_new;
2988         }
2989
2990         error = bus_dmamap_load_mbuf_defrag(txr->bnx_tx_mtag, map,
2991             m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
2992         if (error)
2993                 goto back;
2994         *segs_used += nsegs;
2995
2996         m_head = *m_head0;
2997         bus_dmamap_sync(txr->bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
2998
2999         for (i = 0; ; i++) {
3000                 d = &txr->bnx_tx_ring[idx];
3001
3002                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
3003                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
3004                 d->bge_len = segs[i].ds_len;
3005                 d->bge_flags = csum_flags;
3006                 d->bge_vlan_tag = vlan_tag;
3007                 d->bge_mss = mss;
3008
3009                 if (i == nsegs - 1)
3010                         break;
3011                 BNX_INC(idx, BGE_TX_RING_CNT);
3012         }
3013         /* Mark the last segment as end of packet... */
3014         d->bge_flags |= BGE_TXBDFLAG_END;
3015
3016         /*
3017          * Insure that the map for this transmission is placed at
3018          * the array index of the last descriptor in this chain.
3019          */
3020         txr->bnx_tx_buf[*txidx].bnx_tx_dmamap = txr->bnx_tx_buf[idx].bnx_tx_dmamap;
3021         txr->bnx_tx_buf[idx].bnx_tx_dmamap = map;
3022         txr->bnx_tx_buf[idx].bnx_tx_mbuf = m_head;
3023         txr->bnx_tx_cnt += nsegs;
3024
3025         BNX_INC(idx, BGE_TX_RING_CNT);
3026         *txidx = idx;
3027 back:
3028         if (error) {
3029                 m_freem(*m_head0);
3030                 *m_head0 = NULL;
3031         }
3032         return error;
3033 }
3034
3035 /*
3036  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
3037  * to the mbuf data regions directly in the transmit descriptors.
3038  */
3039 static void
3040 bnx_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
3041 {
3042         struct bnx_tx_ring *txr = ifsq_get_priv(ifsq);
3043         struct mbuf *m_head = NULL;
3044         uint32_t prodidx;
3045         int nsegs = 0;
3046
3047         KKASSERT(txr->bnx_ifsq == ifsq);
3048         ASSERT_SERIALIZED(&txr->bnx_tx_serialize);
3049
3050         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifsq_is_oactive(ifsq))
3051                 return;
3052
3053         prodidx = txr->bnx_tx_prodidx;
3054
3055         while (txr->bnx_tx_buf[prodidx].bnx_tx_mbuf == NULL) {
3056                 /*
3057                  * Sanity check: avoid coming within BGE_NSEG_RSVD
3058                  * descriptors of the end of the ring.  Also make
3059                  * sure there are BGE_NSEG_SPARE descriptors for
3060                  * jumbo buffers' or TSO segments' defragmentation.
3061                  */
3062                 if ((BGE_TX_RING_CNT - txr->bnx_tx_cnt) <
3063                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
3064                         ifsq_set_oactive(ifsq);
3065                         break;
3066                 }
3067
3068                 m_head = ifsq_dequeue(ifsq, NULL);
3069                 if (m_head == NULL)
3070                         break;
3071
3072                 /*
3073                  * Pack the data into the transmit ring. If we
3074                  * don't have room, set the OACTIVE flag and wait
3075                  * for the NIC to drain the ring.
3076                  */
3077                 if (bnx_encap(txr, &m_head, &prodidx, &nsegs)) {
3078                         ifsq_set_oactive(ifsq);
3079                         IFNET_STAT_INC(ifp, oerrors, 1);
3080                         break;
3081                 }
3082
3083                 if (nsegs >= txr->bnx_tx_wreg) {
3084                         /* Transmit */
3085                         bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3086                         nsegs = 0;
3087                 }
3088
3089                 ETHER_BPF_MTAP(ifp, m_head);
3090
3091                 /*
3092                  * Set a timeout in case the chip goes out to lunch.
3093                  */
3094                 txr->bnx_tx_watchdog.wd_timer = 5;
3095         }
3096
3097         if (nsegs > 0) {
3098                 /* Transmit */
3099                 bnx_writembx(txr->bnx_sc, txr->bnx_tx_mbx, prodidx);
3100         }
3101         txr->bnx_tx_prodidx = prodidx;
3102 }
3103
3104 static void
3105 bnx_init(void *xsc)
3106 {
3107         struct bnx_softc *sc = xsc;
3108         struct ifnet *ifp = &sc->arpcom.ac_if;
3109         uint16_t *m;
3110         uint32_t mode;
3111         int i;
3112         boolean_t polling;
3113
3114         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3115
3116         /* Cancel pending I/O and flush buffers. */
3117         bnx_stop(sc);
3118         bnx_reset(sc);
3119         bnx_chipinit(sc);
3120
3121         /*
3122          * Init the various state machines, ring
3123          * control blocks and firmware.
3124          */
3125         if (bnx_blockinit(sc)) {
3126                 if_printf(ifp, "initialization failure\n");
3127                 bnx_stop(sc);
3128                 return;
3129         }
3130
3131         /* Specify MTU. */
3132         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
3133             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
3134
3135         /* Load our MAC address. */
3136         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
3137         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
3138         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
3139
3140         /* Enable or disable promiscuous mode as needed. */
3141         bnx_setpromisc(sc);
3142
3143         /* Program multicast filter. */
3144         bnx_setmulti(sc);
3145
3146         /* Init RX ring. */
3147         if (bnx_init_rx_ring_std(&sc->bnx_rx_std_ring)) {
3148                 if_printf(ifp, "RX ring initialization failed\n");
3149                 bnx_stop(sc);
3150                 return;
3151         }
3152
3153         /* Init jumbo RX ring. */
3154         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
3155                 if (bnx_init_rx_ring_jumbo(sc)) {
3156                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
3157                         bnx_stop(sc);
3158                         return;
3159                 }
3160         }
3161
3162         /* Init our RX return ring index */
3163         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
3164                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
3165
3166                 ret->bnx_rx_saved_considx = 0;
3167                 ret->bnx_rx_cnt = 0;
3168         }
3169
3170         /* Init TX ring. */
3171         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3172                 bnx_init_tx_ring(&sc->bnx_tx_ring[i]);
3173
3174         /* Enable TX MAC state machine lockup fix. */
3175         mode = CSR_READ_4(sc, BGE_TX_MODE);
3176         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3177         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720 ||
3178             sc->bnx_asicrev == BGE_ASICREV_BCM5762) {
3179                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3180                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
3181                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3182         }
3183         /* Turn on transmitter */
3184         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3185
3186         /* Turn on receiver */
3187         BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3188
3189         /*
3190          * Set the number of good frames to receive after RX MBUF
3191          * Low Watermark has been reached.  After the RX MAC receives
3192          * this number of frames, it will drop subsequent incoming
3193          * frames until the MBUF High Watermark is reached.
3194          */
3195         if (BNX_IS_57765_FAMILY(sc))
3196                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3197         else
3198                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3199
3200         if (sc->bnx_intr_type == PCI_INTR_TYPE_MSI) {
3201                 if (bootverbose) {
3202                         if_printf(ifp, "MSI_MODE: %#x\n",
3203                             CSR_READ_4(sc, BGE_MSI_MODE));
3204                 }
3205         }
3206
3207         /* Tell firmware we're alive. */
3208         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3209
3210         /* Enable host interrupts if polling(4) is not enabled. */
3211         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3212
3213         polling = FALSE;
3214 #ifdef IFPOLL_ENABLE
3215         if (ifp->if_flags & IFF_NPOLLING)
3216                 polling = TRUE;
3217 #endif
3218         if (polling)
3219                 bnx_disable_intr(sc);
3220         else
3221                 bnx_enable_intr(sc);
3222         bnx_set_tick_cpuid(sc, polling);
3223
3224         bnx_ifmedia_upd(ifp);
3225
3226         ifp->if_flags |= IFF_RUNNING;
3227         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3228                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3229
3230                 ifsq_clr_oactive(txr->bnx_ifsq);
3231                 ifsq_watchdog_start(&txr->bnx_tx_watchdog);
3232         }
3233
3234         callout_reset_bycpu(&sc->bnx_tick_timer, hz, bnx_tick, sc,
3235             sc->bnx_tick_cpuid);
3236 }
3237
3238 /*
3239  * Set media options.
3240  */
3241 static int
3242 bnx_ifmedia_upd(struct ifnet *ifp)
3243 {
3244         struct bnx_softc *sc = ifp->if_softc;
3245
3246         /* If this is a 1000baseX NIC, enable the TBI port. */
3247         if (sc->bnx_flags & BNX_FLAG_TBI) {
3248                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3249
3250                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3251                         return(EINVAL);
3252
3253                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3254                 case IFM_AUTO:
3255                         break;
3256
3257                 case IFM_1000_SX:
3258                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3259                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3260                                     BGE_MACMODE_HALF_DUPLEX);
3261                         } else {
3262                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3263                                     BGE_MACMODE_HALF_DUPLEX);
3264                         }
3265                         break;
3266                 default:
3267                         return(EINVAL);
3268                 }
3269         } else {
3270                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3271
3272                 sc->bnx_link_evt++;
3273                 sc->bnx_link = 0;
3274                 if (mii->mii_instance) {
3275                         struct mii_softc *miisc;
3276
3277                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3278                                 mii_phy_reset(miisc);
3279                 }
3280                 mii_mediachg(mii);
3281
3282                 /*
3283                  * Force an interrupt so that we will call bnx_link_upd
3284                  * if needed and clear any pending link state attention.
3285                  * Without this we are not getting any further interrupts
3286                  * for link state changes and thus will not UP the link and
3287                  * not be able to send in bnx_start.  The only way to get
3288                  * things working was to receive a packet and get an RX
3289                  * intr.
3290                  *
3291                  * bnx_tick should help for fiber cards and we might not
3292                  * need to do this here if BNX_FLAG_TBI is set but as
3293                  * we poll for fiber anyway it should not harm.
3294                  */
3295                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3296         }
3297         return(0);
3298 }
3299
3300 /*
3301  * Report current media status.
3302  */
3303 static void
3304 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3305 {
3306         struct bnx_softc *sc = ifp->if_softc;
3307
3308         if (sc->bnx_flags & BNX_FLAG_TBI) {
3309                 ifmr->ifm_status = IFM_AVALID;
3310                 ifmr->ifm_active = IFM_ETHER;
3311                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3312                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3313                         ifmr->ifm_status |= IFM_ACTIVE;
3314                 } else {
3315                         ifmr->ifm_active |= IFM_NONE;
3316                         return;
3317                 }
3318
3319                 ifmr->ifm_active |= IFM_1000_SX;
3320                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3321                         ifmr->ifm_active |= IFM_HDX;    
3322                 else
3323                         ifmr->ifm_active |= IFM_FDX;
3324         } else {
3325                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3326
3327                 mii_pollstat(mii);
3328                 ifmr->ifm_active = mii->mii_media_active;
3329                 ifmr->ifm_status = mii->mii_media_status;
3330         }
3331 }
3332
3333 static int
3334 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3335 {
3336         struct bnx_softc *sc = ifp->if_softc;
3337         struct ifreq *ifr = (struct ifreq *)data;
3338         int mask, error = 0;
3339
3340         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3341
3342         switch (command) {
3343         case SIOCSIFMTU:
3344                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3345                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3346                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3347                         error = EINVAL;
3348                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3349                         ifp->if_mtu = ifr->ifr_mtu;
3350                         if (ifp->if_flags & IFF_RUNNING)
3351                                 bnx_init(sc);
3352                 }
3353                 break;
3354         case SIOCSIFFLAGS:
3355                 if (ifp->if_flags & IFF_UP) {
3356                         if (ifp->if_flags & IFF_RUNNING) {
3357                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3358
3359                                 /*
3360                                  * If only the state of the PROMISC flag
3361                                  * changed, then just use the 'set promisc
3362                                  * mode' command instead of reinitializing
3363                                  * the entire NIC. Doing a full re-init
3364                                  * means reloading the firmware and waiting
3365                                  * for it to start up, which may take a
3366                                  * second or two.  Similarly for ALLMULTI.
3367                                  */
3368                                 if (mask & IFF_PROMISC)
3369                                         bnx_setpromisc(sc);
3370                                 if (mask & IFF_ALLMULTI)
3371                                         bnx_setmulti(sc);
3372                         } else {
3373                                 bnx_init(sc);
3374                         }
3375                 } else if (ifp->if_flags & IFF_RUNNING) {
3376                         bnx_stop(sc);
3377                 }
3378                 sc->bnx_if_flags = ifp->if_flags;
3379                 break;
3380         case SIOCADDMULTI:
3381         case SIOCDELMULTI:
3382                 if (ifp->if_flags & IFF_RUNNING)
3383                         bnx_setmulti(sc);
3384                 break;
3385         case SIOCSIFMEDIA:
3386         case SIOCGIFMEDIA:
3387                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3388                         error = ifmedia_ioctl(ifp, ifr,
3389                             &sc->bnx_ifmedia, command);
3390                 } else {
3391                         struct mii_data *mii;
3392
3393                         mii = device_get_softc(sc->bnx_miibus);
3394                         error = ifmedia_ioctl(ifp, ifr,
3395                                               &mii->mii_media, command);
3396                 }
3397                 break;
3398         case SIOCSIFCAP:
3399                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3400                 if (mask & IFCAP_HWCSUM) {
3401                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3402                         if (ifp->if_capenable & IFCAP_TXCSUM)
3403                                 ifp->if_hwassist |= BNX_CSUM_FEATURES;
3404                         else
3405                                 ifp->if_hwassist &= ~BNX_CSUM_FEATURES;
3406                 }
3407                 if (mask & IFCAP_TSO) {
3408                         ifp->if_capenable ^= (mask & IFCAP_TSO);
3409                         if (ifp->if_capenable & IFCAP_TSO)
3410                                 ifp->if_hwassist |= CSUM_TSO;
3411                         else
3412                                 ifp->if_hwassist &= ~CSUM_TSO;
3413                 }
3414                 break;
3415         default:
3416                 error = ether_ioctl(ifp, command, data);
3417                 break;
3418         }
3419         return error;
3420 }
3421
3422 static void
3423 bnx_watchdog(struct ifaltq_subque *ifsq)
3424 {
3425         struct ifnet *ifp = ifsq_get_ifp(ifsq);
3426         struct bnx_softc *sc = ifp->if_softc;
3427         int i;
3428
3429         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3430
3431         if_printf(ifp, "watchdog timeout -- resetting\n");
3432
3433         bnx_init(sc);
3434
3435         IFNET_STAT_INC(ifp, oerrors, 1);
3436
3437         for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3438                 ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
3439 }
3440
3441 /*
3442  * Stop the adapter and free any mbufs allocated to the
3443  * RX and TX lists.
3444  */
3445 static void
3446 bnx_stop(struct bnx_softc *sc)
3447 {
3448         struct ifnet *ifp = &sc->arpcom.ac_if;
3449         int i;
3450
3451         ASSERT_IFNET_SERIALIZED_ALL(ifp);
3452
3453         callout_stop(&sc->bnx_tick_timer);
3454
3455         /*
3456          * Disable all of the receiver blocks
3457          */
3458         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3459         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3460         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3461         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3462         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3463         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3464
3465         /*
3466          * Disable all of the transmit blocks
3467          */
3468         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3469         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3470         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3471         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3472         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3473         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3474
3475         /*
3476          * Shut down all of the memory managers and related
3477          * state machines.
3478          */
3479         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3480         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3481         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3482         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3483
3484         /* Disable host interrupts. */
3485         bnx_disable_intr(sc);
3486
3487         /*
3488          * Tell firmware we're shutting down.
3489          */
3490         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3491
3492         /* Free the RX lists. */
3493         bnx_free_rx_ring_std(&sc->bnx_rx_std_ring);
3494
3495         /* Free jumbo RX list. */
3496         if (BNX_IS_JUMBO_CAPABLE(sc))
3497                 bnx_free_rx_ring_jumbo(sc);
3498
3499         /* Free TX buffers. */
3500         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3501                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3502
3503                 txr->bnx_saved_status_tag = 0;
3504                 bnx_free_tx_ring(txr);
3505         }
3506
3507         /* Clear saved status tag */
3508         for (i = 0; i < sc->bnx_rx_retcnt; ++i)
3509                 sc->bnx_rx_ret_ring[i].bnx_saved_status_tag = 0;
3510
3511         sc->bnx_link = 0;
3512         sc->bnx_coal_chg = 0;
3513
3514         ifp->if_flags &= ~IFF_RUNNING;
3515         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3516                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3517
3518                 ifsq_clr_oactive(txr->bnx_ifsq);
3519                 ifsq_watchdog_stop(&txr->bnx_tx_watchdog);
3520         }
3521 }
3522
3523 /*
3524  * Stop all chip I/O so that the kernel's probe routines don't
3525  * get confused by errant DMAs when rebooting.
3526  */
3527 static void
3528 bnx_shutdown(device_t dev)
3529 {
3530         struct bnx_softc *sc = device_get_softc(dev);
3531         struct ifnet *ifp = &sc->arpcom.ac_if;
3532
3533         ifnet_serialize_all(ifp);
3534         bnx_stop(sc);
3535         bnx_reset(sc);
3536         ifnet_deserialize_all(ifp);
3537 }
3538
3539 static int
3540 bnx_suspend(device_t dev)
3541 {
3542         struct bnx_softc *sc = device_get_softc(dev);
3543         struct ifnet *ifp = &sc->arpcom.ac_if;
3544
3545         ifnet_serialize_all(ifp);
3546         bnx_stop(sc);
3547         ifnet_deserialize_all(ifp);
3548
3549         return 0;
3550 }
3551
3552 static int
3553 bnx_resume(device_t dev)
3554 {
3555         struct bnx_softc *sc = device_get_softc(dev);
3556         struct ifnet *ifp = &sc->arpcom.ac_if;
3557
3558         ifnet_serialize_all(ifp);
3559
3560         if (ifp->if_flags & IFF_UP) {
3561                 int i;
3562
3563                 bnx_init(sc);
3564                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3565                         ifsq_devstart_sched(sc->bnx_tx_ring[i].bnx_ifsq);
3566         }
3567
3568         ifnet_deserialize_all(ifp);
3569
3570         return 0;
3571 }
3572
3573 static void
3574 bnx_setpromisc(struct bnx_softc *sc)
3575 {
3576         struct ifnet *ifp = &sc->arpcom.ac_if;
3577
3578         if (ifp->if_flags & IFF_PROMISC)
3579                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3580         else
3581                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3582 }
3583
3584 static void
3585 bnx_dma_free(struct bnx_softc *sc)
3586 {
3587         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
3588         int i;
3589
3590         /* Destroy RX return rings */
3591         if (sc->bnx_rx_ret_ring != NULL) {
3592                 for (i = 0; i < sc->bnx_rx_retcnt; ++i)
3593                         bnx_destroy_rx_ret_ring(&sc->bnx_rx_ret_ring[i]);
3594                 kfree(sc->bnx_rx_ret_ring, M_DEVBUF);
3595         }
3596
3597         /* Destroy RX mbuf DMA stuffs. */
3598         if (std->bnx_rx_mtag != NULL) {
3599                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3600                         KKASSERT(std->bnx_rx_std_buf[i].bnx_rx_mbuf == NULL);
3601                         bus_dmamap_destroy(std->bnx_rx_mtag,
3602                             std->bnx_rx_std_buf[i].bnx_rx_dmamap);
3603                 }
3604                 bus_dma_tag_destroy(std->bnx_rx_mtag);
3605         }
3606
3607         /* Destroy standard RX ring */
3608         bnx_dma_block_free(std->bnx_rx_std_ring_tag,
3609             std->bnx_rx_std_ring_map, std->bnx_rx_std_ring);
3610
3611         /* Destroy TX rings */
3612         if (sc->bnx_tx_ring != NULL) {
3613                 for (i = 0; i < sc->bnx_tx_ringcnt; ++i)
3614                         bnx_destroy_tx_ring(&sc->bnx_tx_ring[i]);
3615                 kfree(sc->bnx_tx_ring, M_DEVBUF);
3616         }
3617
3618         if (BNX_IS_JUMBO_CAPABLE(sc))
3619                 bnx_free_jumbo_mem(sc);
3620
3621         /* Destroy status blocks */
3622         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
3623                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
3624
3625                 bnx_dma_block_free(intr->bnx_status_tag,
3626                     intr->bnx_status_map, intr->bnx_status_block);
3627         }
3628
3629         /* Destroy the parent tag */
3630         if (sc->bnx_cdata.bnx_parent_tag != NULL)
3631                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
3632 }
3633
3634 static int
3635 bnx_dma_alloc(device_t dev)
3636 {
3637         struct bnx_softc *sc = device_get_softc(dev);
3638         struct bnx_rx_std_ring *std = &sc->bnx_rx_std_ring;
3639         int i, error, mbx;
3640
3641         /*
3642          * Allocate the parent bus DMA tag appropriate for PCI.
3643          *
3644          * All of the NetExtreme/NetLink controllers have 4GB boundary
3645          * DMA bug.
3646          * Whenever an address crosses a multiple of the 4GB boundary
3647          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
3648          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
3649          * state machine will lockup and cause the device to hang.
3650          */
3651         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
3652             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
3653             BUS_SPACE_MAXSIZE_32BIT, 0, BUS_SPACE_MAXSIZE_32BIT,
3654             0, &sc->bnx_cdata.bnx_parent_tag);
3655         if (error) {
3656                 device_printf(dev, "could not create parent DMA tag\n");
3657                 return error;
3658         }
3659
3660         /*
3661          * Create DMA stuffs for status blocks.
3662          */
3663         for (i = 0; i < sc->bnx_intr_cnt; ++i) {
3664                 struct bnx_intr_data *intr = &sc->bnx_intr_data[i];
3665
3666                 error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
3667                     &intr->bnx_status_tag, &intr->bnx_status_map,
3668                     (void *)&intr->bnx_status_block,
3669                     &intr->bnx_status_block_paddr);
3670                 if (error) {
3671                         device_printf(dev,
3672                             "could not create %dth status block\n", i);
3673                         return error;
3674                 }
3675         }
3676
3677         /*
3678          * Create DMA tag and maps for RX mbufs.
3679          */
3680         std->bnx_sc = sc;
3681         lwkt_serialize_init(&std->bnx_rx_std_serialize);
3682         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3683             BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3684             NULL, NULL, MCLBYTES, 1, MCLBYTES,
3685             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK, &std->bnx_rx_mtag);
3686         if (error) {
3687                 device_printf(dev, "could not create RX mbuf DMA tag\n");
3688                 return error;
3689         }
3690
3691         for (i = 0; i < BGE_STD_RX_RING_CNT; ++i) {
3692                 error = bus_dmamap_create(std->bnx_rx_mtag, BUS_DMA_WAITOK,
3693                     &std->bnx_rx_std_buf[i].bnx_rx_dmamap);
3694                 if (error) {
3695                         int j;
3696
3697                         for (j = 0; j < i; ++j) {
3698                                 bus_dmamap_destroy(std->bnx_rx_mtag,
3699                                     std->bnx_rx_std_buf[j].bnx_rx_dmamap);
3700                         }
3701                         bus_dma_tag_destroy(std->bnx_rx_mtag);
3702                         std->bnx_rx_mtag = NULL;
3703
3704                         device_printf(dev,
3705                             "could not create %dth RX mbuf DMA map\n", i);
3706                         return error;
3707                 }
3708         }
3709
3710         /*
3711          * Create DMA stuffs for standard RX ring.
3712          */
3713         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
3714             &std->bnx_rx_std_ring_tag,
3715             &std->bnx_rx_std_ring_map,
3716             (void *)&std->bnx_rx_std_ring,
3717             &std->bnx_rx_std_ring_paddr);
3718         if (error) {
3719                 device_printf(dev, "could not create std RX ring\n");
3720                 return error;
3721         }
3722
3723         /*
3724          * Create RX return rings
3725          */
3726         mbx = BGE_MBX_RX_CONS0_LO;
3727         sc->bnx_rx_ret_ring = kmalloc_cachealign(
3728             sizeof(struct bnx_rx_ret_ring) * sc->bnx_rx_retcnt, M_DEVBUF,
3729             M_WAITOK | M_ZERO);
3730         for (i = 0; i < sc->bnx_rx_retcnt; ++i) {
3731                 struct bnx_rx_ret_ring *ret = &sc->bnx_rx_ret_ring[i];
3732                 struct bnx_intr_data *intr;
3733
3734                 ret->bnx_sc = sc;
3735                 ret->bnx_std = std;
3736                 ret->bnx_rx_mbx = mbx;
3737                 ret->bnx_rx_cntmax = (BGE_STD_RX_RING_CNT / 4) /
3738                     sc->bnx_rx_retcnt;
3739                 ret->bnx_rx_mask = 1 << i;
3740
3741                 if (sc->bnx_rx_retcnt == 1) {
3742                         intr = &sc->bnx_intr_data[0];
3743                 } else {
3744                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
3745                         intr = &sc->bnx_intr_data[i + 1];
3746                 }
3747
3748                 ret->bnx_rx_considx =
3749                     &intr->bnx_status_block->bge_idx[0].bge_rx_prod_idx;
3750                 ret->bnx_hw_status_tag =
3751                     &intr->bnx_status_block->bge_status_tag;
3752
3753                 error = bnx_create_rx_ret_ring(ret);
3754                 if (error) {
3755                         device_printf(dev,
3756                             "could not create %dth RX ret ring\n", i);
3757                         return error;
3758                 }
3759                 mbx += 8;
3760         }
3761
3762         /*
3763          * Create TX rings
3764          */
3765         sc->bnx_tx_ring = kmalloc_cachealign(
3766             sizeof(struct bnx_tx_ring) * sc->bnx_tx_ringcnt, M_DEVBUF,
3767             M_WAITOK | M_ZERO);
3768         for (i = 0; i < sc->bnx_tx_ringcnt; ++i) {
3769                 struct bnx_tx_ring *txr = &sc->bnx_tx_ring[i];
3770                 struct bnx_intr_data *intr;
3771
3772                 txr->bnx_sc = sc;
3773                 txr->bnx_tx_mbx = bnx_tx_mailbox[i];
3774
3775                 if (sc->bnx_tx_ringcnt == 1) {
3776                         intr = &sc->bnx_intr_data[0];
3777                 } else {
3778                         KKASSERT(i + 1 < sc->bnx_intr_cnt);
3779                         intr = &sc->bnx_intr_data[i + 1];
3780                 }
3781
3782                 txr->bnx_tx_considx =
3783                     &intr->bnx_status_block->bge_idx[0].bge_tx_cons_idx;
3784
3785                 error = bnx_create_tx_ring(txr);
3786                 if (error) {
3787                         device_printf(dev,
3788                             "could not create %dth TX ring\n", i);
3789                         return error;
3790                 }
3791         }
3792
3793         /*
3794          * Create jumbo buffer pool.
3795          */
3796         if (BNX_IS_JUMBO_CAPABLE(sc)) {
3797                 error = bnx_alloc_jumbo_mem(sc);
3798                 if (error) {
3799                         device_printf(dev,
3800                             "could not create jumbo buffer pool\n");
3801                         return error;
3802                 }
3803         }
3804
3805         return 0;
3806 }
3807
3808 static int
3809 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
3810                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
3811 {
3812         bus_dmamem_t dmem;
3813         int error;
3814
3815         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
3816                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3817                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
3818         if (error)
3819                 return error;
3820
3821         *tag = dmem.dmem_tag;
3822         *map = dmem.dmem_map;
3823         *addr = dmem.dmem_addr;
3824         *paddr = dmem.dmem_busaddr;
3825
3826         return 0;
3827 }
3828
3829 static void
3830 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
3831 {
3832         if (tag != NULL) {
3833                 bus_dmamap_unload(tag, map);
3834                 bus_dmamem_free(tag, addr, map);
3835                 bus_dma_tag_destroy(tag);
3836         }
3837 }
3838
3839 static void
3840 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
3841 {
3842         struct ifnet *ifp = &sc->arpcom.ac_if;
3843
3844 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
3845
3846         /*
3847          * Sometimes PCS encoding errors are detected in
3848          * TBI mode (on fiber NICs), and for some reason
3849          * the chip will signal them as link changes.
3850          * If we get a link change event, but the 'PCS
3851          * encoding error' bit in the MAC status register
3852          * is set, don't bother doing a link check.
3853          * This avoids spurious "gigabit link up" messages
3854          * that sometimes appear on fiber NICs during
3855          * periods of heavy traffic.
3856          */
3857         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
3858                 if (!sc->bnx_link) {
3859                         sc->bnx_link++;
3860                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
3861                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3862                                     BGE_MACMODE_TBI_SEND_CFGS);
3863                         }
3864                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
3865
3866                         if (bootverbose)
3867                                 if_printf(ifp, "link UP\n");
3868
3869                         ifp->if_link_state = LINK_STATE_UP;
3870                         if_link_state_change(ifp);
3871                 }
3872         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
3873                 if (sc->bnx_link) {
3874                         sc->bnx_link = 0;
3875
3876                         if (bootverbose)
3877                                 if_printf(ifp, "link DOWN\n");
3878
3879                         ifp->if_link_state = LINK_STATE_DOWN;
3880                         if_link_state_change(ifp);
3881                 }
3882         }
3883
3884 #undef PCS_ENCODE_ERR
3885
3886         /* Clear the attention. */
3887         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3888             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3889             BGE_MACSTAT_LINK_CHANGED);
3890 }
3891
3892 static void
3893 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3894 {
3895         struct ifnet *ifp = &sc->arpcom.ac_if;
3896         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3897
3898         mii_pollstat(mii);
3899         bnx_miibus_statchg(sc->bnx_dev);
3900
3901         if (bootverbose) {
3902                 if (sc->bnx_link)
3903                         if_printf(ifp, "link UP\n");
3904                 else
3905                         if_printf(ifp, "link DOWN\n");
3906         }
3907
3908         /* Clear the attention. */
3909         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3910             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3911             BGE_MACSTAT_LINK_CHANGED);
3912 }
3913
3914 static void
3915 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3916 {
3917         struct ifnet *ifp = &sc->arpcom.ac_if;
3918         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3919
3920         mii_pollstat(mii);
3921
3922         if (!sc->bnx_link &&
3923             (mii->mii_media_status & IFM_ACTIVE) &&
3924             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3925                 sc->bnx_link++;
3926                 if (bootverbose)
3927                         if_printf(ifp, "link UP\n");
3928         } else if (sc->bnx_link &&
3929             (!(mii->mii_media_status & IFM_ACTIVE) ||
3930             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3931                 sc->bnx_link = 0;
3932                 if (bootverbose)
3933                         if_printf(ifp, "link DOWN\n");
3934         }
3935
3936         /* Clear the attention. */
3937         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3938             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3939             BGE_MACSTAT_LINK_CHANGED);
3940 }
3941
3942 static int
3943 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
3944 {
3945         struct bnx_softc *sc = arg1;
3946
3947         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3948             &sc->bnx_rx_coal_ticks,
3949             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
3950             BNX_RX_COAL_TICKS_CHG);
3951 }
3952
3953 static int
3954 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
3955 {
3956         struct bnx_softc *sc = arg1;
3957
3958         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3959             &sc->bnx_tx_coal_ticks,
3960             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
3961             BNX_TX_COAL_TICKS_CHG);
3962 }
3963
3964 static int
3965 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
3966 {
3967         struct bnx_softc *sc = arg1;
3968
3969         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3970             &sc->bnx_rx_coal_bds,
3971             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3972             BNX_RX_COAL_BDS_CHG);
3973 }
3974
3975 static int
3976 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
3977 {
3978         struct bnx_softc *sc = arg1;
3979
3980         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3981             &sc->bnx_tx_coal_bds,
3982             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3983             BNX_TX_COAL_BDS_CHG);
3984 }
3985
3986 static int
3987 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3988 {
3989         struct bnx_softc *sc = arg1;
3990
3991         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3992             &sc->bnx_rx_coal_bds_int,
3993             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3994             BNX_RX_COAL_BDS_INT_CHG);
3995 }
3996
3997 static int
3998 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3999 {
4000         struct bnx_softc *sc = arg1;
4001
4002         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
4003             &sc->bnx_tx_coal_bds_int,
4004             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,