bnx: Enable software workaround for possiblely hosed interrupt
[dragonfly.git] / sys / dev / netif / bnx / if_bnx.c
1 /*
2  * Copyright (c) 2001 Wind River Systems
3  * Copyright (c) 1997, 1998, 1999, 2001
4  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  * 3. All advertising materials mentioning features or use of this software
15  *    must display the following acknowledgement:
16  *      This product includes software developed by Bill Paul.
17  * 4. Neither the name of the author nor the names of any co-contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
25  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
31  * THE POSSIBILITY OF SUCH DAMAGE.
32  *
33  * $FreeBSD: src/sys/dev/bge/if_bge.c,v 1.3.2.39 2005/07/03 03:41:18 silby Exp $
34  */
35
36
37 #include "opt_polling.h"
38
39 #include <sys/param.h>
40 #include <sys/bus.h>
41 #include <sys/endian.h>
42 #include <sys/kernel.h>
43 #include <sys/interrupt.h>
44 #include <sys/mbuf.h>
45 #include <sys/malloc.h>
46 #include <sys/queue.h>
47 #include <sys/rman.h>
48 #include <sys/serialize.h>
49 #include <sys/socket.h>
50 #include <sys/sockio.h>
51 #include <sys/sysctl.h>
52
53 #include <net/bpf.h>
54 #include <net/ethernet.h>
55 #include <net/if.h>
56 #include <net/if_arp.h>
57 #include <net/if_dl.h>
58 #include <net/if_media.h>
59 #include <net/if_types.h>
60 #include <net/ifq_var.h>
61 #include <net/vlan/if_vlan_var.h>
62 #include <net/vlan/if_vlan_ether.h>
63
64 #include <dev/netif/mii_layer/mii.h>
65 #include <dev/netif/mii_layer/miivar.h>
66 #include <dev/netif/mii_layer/brgphyreg.h>
67
68 #include <bus/pci/pcidevs.h>
69 #include <bus/pci/pcireg.h>
70 #include <bus/pci/pcivar.h>
71
72 #include <dev/netif/bge/if_bgereg.h>
73 #include <dev/netif/bnx/if_bnxvar.h>
74
75 /* "device miibus" required.  See GENERIC if you get errors here. */
76 #include "miibus_if.h"
77
78 #define BNX_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
79
80 #define BNX_INTR_CKINTVL        ((10 * hz) / 1000)      /* 10ms */
81
82 static const struct bnx_type {
83         uint16_t                bnx_vid;
84         uint16_t                bnx_did;
85         char                    *bnx_name;
86 } bnx_devs[] = {
87         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5717,
88                 "Broadcom BCM5717 Gigabit Ethernet" },
89         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5718,
90                 "Broadcom BCM5718 Gigabit Ethernet" },
91         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5719,
92                 "Broadcom BCM5719 Gigabit Ethernet" },
93         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM5720_ALT,
94                 "Broadcom BCM5720 Gigabit Ethernet" },
95
96         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57761,
97                 "Broadcom BCM57761 Gigabit Ethernet" },
98         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57762,
99                 "Broadcom BCM57762 Gigabit Ethernet" },
100         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57765,
101                 "Broadcom BCM57765 Gigabit Ethernet" },
102         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57766,
103                 "Broadcom BCM57766 Gigabit Ethernet" },
104         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57781,
105                 "Broadcom BCM57781 Gigabit Ethernet" },
106         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57782,
107                 "Broadcom BCM57782 Gigabit Ethernet" },
108         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57785,
109                 "Broadcom BCM57785 Gigabit Ethernet" },
110         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57786,
111                 "Broadcom BCM57786 Gigabit Ethernet" },
112         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57791,
113                 "Broadcom BCM57791 Fast Ethernet" },
114         { PCI_VENDOR_BROADCOM, PCI_PRODUCT_BROADCOM_BCM57795,
115                 "Broadcom BCM57795 Fast Ethernet" },
116
117         { 0, 0, NULL }
118 };
119
120 #define BNX_IS_JUMBO_CAPABLE(sc)        ((sc)->bnx_flags & BNX_FLAG_JUMBO)
121 #define BNX_IS_5717_PLUS(sc)            ((sc)->bnx_flags & BNX_FLAG_5717_PLUS)
122 #define BNX_IS_57765_PLUS(sc)           ((sc)->bnx_flags & BNX_FLAG_57765_PLUS)
123 #define BNX_IS_57765_FAMILY(sc)  \
124         ((sc)->bnx_flags & BNX_FLAG_57765_FAMILY)
125
126 typedef int     (*bnx_eaddr_fcn_t)(struct bnx_softc *, uint8_t[]);
127
128 static int      bnx_probe(device_t);
129 static int      bnx_attach(device_t);
130 static int      bnx_detach(device_t);
131 static void     bnx_shutdown(device_t);
132 static int      bnx_suspend(device_t);
133 static int      bnx_resume(device_t);
134 static int      bnx_miibus_readreg(device_t, int, int);
135 static int      bnx_miibus_writereg(device_t, int, int, int);
136 static void     bnx_miibus_statchg(device_t);
137
138 #ifdef DEVICE_POLLING
139 static void     bnx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count);
140 #endif
141 static void     bnx_intr_legacy(void *);
142 static void     bnx_msi(void *);
143 static void     bnx_msi_oneshot(void *);
144 static void     bnx_intr(struct bnx_softc *);
145 static void     bnx_enable_intr(struct bnx_softc *);
146 static void     bnx_disable_intr(struct bnx_softc *);
147 static void     bnx_txeof(struct bnx_softc *, uint16_t);
148 static void     bnx_rxeof(struct bnx_softc *, uint16_t);
149
150 static void     bnx_start(struct ifnet *);
151 static int      bnx_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
152 static void     bnx_init(void *);
153 static void     bnx_stop(struct bnx_softc *);
154 static void     bnx_watchdog(struct ifnet *);
155 static int      bnx_ifmedia_upd(struct ifnet *);
156 static void     bnx_ifmedia_sts(struct ifnet *, struct ifmediareq *);
157 static void     bnx_tick(void *);
158
159 static int      bnx_alloc_jumbo_mem(struct bnx_softc *);
160 static void     bnx_free_jumbo_mem(struct bnx_softc *);
161 static struct bnx_jslot
162                 *bnx_jalloc(struct bnx_softc *);
163 static void     bnx_jfree(void *);
164 static void     bnx_jref(void *);
165 static int      bnx_newbuf_std(struct bnx_softc *, int, int);
166 static int      bnx_newbuf_jumbo(struct bnx_softc *, int, int);
167 static void     bnx_setup_rxdesc_std(struct bnx_softc *, int);
168 static void     bnx_setup_rxdesc_jumbo(struct bnx_softc *, int);
169 static int      bnx_init_rx_ring_std(struct bnx_softc *);
170 static void     bnx_free_rx_ring_std(struct bnx_softc *);
171 static int      bnx_init_rx_ring_jumbo(struct bnx_softc *);
172 static void     bnx_free_rx_ring_jumbo(struct bnx_softc *);
173 static void     bnx_free_tx_ring(struct bnx_softc *);
174 static int      bnx_init_tx_ring(struct bnx_softc *);
175 static int      bnx_dma_alloc(struct bnx_softc *);
176 static void     bnx_dma_free(struct bnx_softc *);
177 static int      bnx_dma_block_alloc(struct bnx_softc *, bus_size_t,
178                     bus_dma_tag_t *, bus_dmamap_t *, void **, bus_addr_t *);
179 static void     bnx_dma_block_free(bus_dma_tag_t, bus_dmamap_t, void *);
180 static struct mbuf *
181                 bnx_defrag_shortdma(struct mbuf *);
182 static int      bnx_encap(struct bnx_softc *, struct mbuf **, uint32_t *);
183
184 static void     bnx_reset(struct bnx_softc *);
185 static int      bnx_chipinit(struct bnx_softc *);
186 static int      bnx_blockinit(struct bnx_softc *);
187 static void     bnx_stop_block(struct bnx_softc *, bus_size_t, uint32_t);
188 static void     bnx_enable_msi(struct bnx_softc *sc);
189 static void     bnx_setmulti(struct bnx_softc *);
190 static void     bnx_setpromisc(struct bnx_softc *);
191 static void     bnx_stats_update_regs(struct bnx_softc *);
192 static uint32_t bnx_dma_swap_options(struct bnx_softc *);
193
194 static uint32_t bnx_readmem_ind(struct bnx_softc *, uint32_t);
195 static void     bnx_writemem_ind(struct bnx_softc *, uint32_t, uint32_t);
196 #ifdef notdef
197 static uint32_t bnx_readreg_ind(struct bnx_softc *, uint32_t);
198 #endif
199 static void     bnx_writereg_ind(struct bnx_softc *, uint32_t, uint32_t);
200 static void     bnx_writemem_direct(struct bnx_softc *, uint32_t, uint32_t);
201 static void     bnx_writembx(struct bnx_softc *, int, int);
202 static uint8_t  bnx_nvram_getbyte(struct bnx_softc *, int, uint8_t *);
203 static int      bnx_read_nvram(struct bnx_softc *, caddr_t, int, int);
204 static uint8_t  bnx_eeprom_getbyte(struct bnx_softc *, uint32_t, uint8_t *);
205 static int      bnx_read_eeprom(struct bnx_softc *, caddr_t, uint32_t, size_t);
206
207 static void     bnx_tbi_link_upd(struct bnx_softc *, uint32_t);
208 static void     bnx_copper_link_upd(struct bnx_softc *, uint32_t);
209 static void     bnx_autopoll_link_upd(struct bnx_softc *, uint32_t);
210 static void     bnx_link_poll(struct bnx_softc *);
211
212 static int      bnx_get_eaddr_mem(struct bnx_softc *, uint8_t[]);
213 static int      bnx_get_eaddr_nvram(struct bnx_softc *, uint8_t[]);
214 static int      bnx_get_eaddr_eeprom(struct bnx_softc *, uint8_t[]);
215 static int      bnx_get_eaddr(struct bnx_softc *, uint8_t[]);
216
217 static void     bnx_coal_change(struct bnx_softc *);
218 static int      bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS);
219 static int      bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS);
220 static int      bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS);
221 static int      bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS);
222 static int      bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS);
223 static int      bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS);
224 static int      bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *,
225                     int, int, uint32_t);
226
227 static int      bnx_msi_enable = 1;
228 TUNABLE_INT("hw.bnx.msi.enable", &bnx_msi_enable);
229
230 static device_method_t bnx_methods[] = {
231         /* Device interface */
232         DEVMETHOD(device_probe,         bnx_probe),
233         DEVMETHOD(device_attach,        bnx_attach),
234         DEVMETHOD(device_detach,        bnx_detach),
235         DEVMETHOD(device_shutdown,      bnx_shutdown),
236         DEVMETHOD(device_suspend,       bnx_suspend),
237         DEVMETHOD(device_resume,        bnx_resume),
238
239         /* bus interface */
240         DEVMETHOD(bus_print_child,      bus_generic_print_child),
241         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
242
243         /* MII interface */
244         DEVMETHOD(miibus_readreg,       bnx_miibus_readreg),
245         DEVMETHOD(miibus_writereg,      bnx_miibus_writereg),
246         DEVMETHOD(miibus_statchg,       bnx_miibus_statchg),
247
248         { 0, 0 }
249 };
250
251 static DEFINE_CLASS_0(bnx, bnx_driver, bnx_methods, sizeof(struct bnx_softc));
252 static devclass_t bnx_devclass;
253
254 DECLARE_DUMMY_MODULE(if_bnx);
255 DRIVER_MODULE(if_bnx, pci, bnx_driver, bnx_devclass, NULL, NULL);
256 DRIVER_MODULE(miibus, bnx, miibus_driver, miibus_devclass, NULL, NULL);
257
258 static uint32_t
259 bnx_readmem_ind(struct bnx_softc *sc, uint32_t off)
260 {
261         device_t dev = sc->bnx_dev;
262         uint32_t val;
263
264         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
265             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
266                 return 0;
267
268         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
269         val = pci_read_config(dev, BGE_PCI_MEMWIN_DATA, 4);
270         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
271         return (val);
272 }
273
274 static void
275 bnx_writemem_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
276 {
277         device_t dev = sc->bnx_dev;
278
279         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
280             off >= BGE_STATS_BLOCK && off < BGE_SEND_RING_1_TO_4)
281                 return;
282
283         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, off, 4);
284         pci_write_config(dev, BGE_PCI_MEMWIN_DATA, val, 4);
285         pci_write_config(dev, BGE_PCI_MEMWIN_BASEADDR, 0, 4);
286 }
287
288 #ifdef notdef
289 static uint32_t
290 bnx_readreg_ind(struct bnx_softc *sc, uin32_t off)
291 {
292         device_t dev = sc->bnx_dev;
293
294         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
295         return(pci_read_config(dev, BGE_PCI_REG_DATA, 4));
296 }
297 #endif
298
299 static void
300 bnx_writereg_ind(struct bnx_softc *sc, uint32_t off, uint32_t val)
301 {
302         device_t dev = sc->bnx_dev;
303
304         pci_write_config(dev, BGE_PCI_REG_BASEADDR, off, 4);
305         pci_write_config(dev, BGE_PCI_REG_DATA, val, 4);
306 }
307
308 static void
309 bnx_writemem_direct(struct bnx_softc *sc, uint32_t off, uint32_t val)
310 {
311         CSR_WRITE_4(sc, off, val);
312 }
313
314 static void
315 bnx_writembx(struct bnx_softc *sc, int off, int val)
316 {
317         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906)
318                 off += BGE_LPMBX_IRQ0_HI - BGE_MBX_IRQ0_HI;
319
320         CSR_WRITE_4(sc, off, val);
321 }
322
323 static uint8_t
324 bnx_nvram_getbyte(struct bnx_softc *sc, int addr, uint8_t *dest)
325 {
326         uint32_t access, byte = 0;
327         int i;
328
329         /* Lock. */
330         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_SET1);
331         for (i = 0; i < 8000; i++) {
332                 if (CSR_READ_4(sc, BGE_NVRAM_SWARB) & BGE_NVRAMSWARB_GNT1)
333                         break;
334                 DELAY(20);
335         }
336         if (i == 8000)
337                 return (1);
338
339         /* Enable access. */
340         access = CSR_READ_4(sc, BGE_NVRAM_ACCESS);
341         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access | BGE_NVRAMACC_ENABLE);
342
343         CSR_WRITE_4(sc, BGE_NVRAM_ADDR, addr & 0xfffffffc);
344         CSR_WRITE_4(sc, BGE_NVRAM_CMD, BGE_NVRAM_READCMD);
345         for (i = 0; i < BNX_TIMEOUT * 10; i++) {
346                 DELAY(10);
347                 if (CSR_READ_4(sc, BGE_NVRAM_CMD) & BGE_NVRAMCMD_DONE) {
348                         DELAY(10);
349                         break;
350                 }
351         }
352
353         if (i == BNX_TIMEOUT * 10) {
354                 if_printf(&sc->arpcom.ac_if, "nvram read timed out\n");
355                 return (1);
356         }
357
358         /* Get result. */
359         byte = CSR_READ_4(sc, BGE_NVRAM_RDDATA);
360
361         *dest = (bswap32(byte) >> ((addr % 4) * 8)) & 0xFF;
362
363         /* Disable access. */
364         CSR_WRITE_4(sc, BGE_NVRAM_ACCESS, access);
365
366         /* Unlock. */
367         CSR_WRITE_4(sc, BGE_NVRAM_SWARB, BGE_NVRAMSWARB_CLR1);
368         CSR_READ_4(sc, BGE_NVRAM_SWARB);
369
370         return (0);
371 }
372
373 /*
374  * Read a sequence of bytes from NVRAM.
375  */
376 static int
377 bnx_read_nvram(struct bnx_softc *sc, caddr_t dest, int off, int cnt)
378 {
379         int err = 0, i;
380         uint8_t byte = 0;
381
382         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
383                 return (1);
384
385         for (i = 0; i < cnt; i++) {
386                 err = bnx_nvram_getbyte(sc, off + i, &byte);
387                 if (err)
388                         break;
389                 *(dest + i) = byte;
390         }
391
392         return (err ? 1 : 0);
393 }
394
395 /*
396  * Read a byte of data stored in the EEPROM at address 'addr.' The
397  * BCM570x supports both the traditional bitbang interface and an
398  * auto access interface for reading the EEPROM. We use the auto
399  * access method.
400  */
401 static uint8_t
402 bnx_eeprom_getbyte(struct bnx_softc *sc, uint32_t addr, uint8_t *dest)
403 {
404         int i;
405         uint32_t byte = 0;
406
407         /*
408          * Enable use of auto EEPROM access so we can avoid
409          * having to use the bitbang method.
410          */
411         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_AUTO_EEPROM);
412
413         /* Reset the EEPROM, load the clock period. */
414         CSR_WRITE_4(sc, BGE_EE_ADDR,
415             BGE_EEADDR_RESET|BGE_EEHALFCLK(BGE_HALFCLK_384SCL));
416         DELAY(20);
417
418         /* Issue the read EEPROM command. */
419         CSR_WRITE_4(sc, BGE_EE_ADDR, BGE_EE_READCMD | addr);
420
421         /* Wait for completion */
422         for(i = 0; i < BNX_TIMEOUT * 10; i++) {
423                 DELAY(10);
424                 if (CSR_READ_4(sc, BGE_EE_ADDR) & BGE_EEADDR_DONE)
425                         break;
426         }
427
428         if (i == BNX_TIMEOUT) {
429                 if_printf(&sc->arpcom.ac_if, "eeprom read timed out\n");
430                 return(1);
431         }
432
433         /* Get result. */
434         byte = CSR_READ_4(sc, BGE_EE_DATA);
435
436         *dest = (byte >> ((addr % 4) * 8)) & 0xFF;
437
438         return(0);
439 }
440
441 /*
442  * Read a sequence of bytes from the EEPROM.
443  */
444 static int
445 bnx_read_eeprom(struct bnx_softc *sc, caddr_t dest, uint32_t off, size_t len)
446 {
447         size_t i;
448         int err;
449         uint8_t byte;
450
451         for (byte = 0, err = 0, i = 0; i < len; i++) {
452                 err = bnx_eeprom_getbyte(sc, off + i, &byte);
453                 if (err)
454                         break;
455                 *(dest + i) = byte;
456         }
457
458         return(err ? 1 : 0);
459 }
460
461 static int
462 bnx_miibus_readreg(device_t dev, int phy, int reg)
463 {
464         struct bnx_softc *sc = device_get_softc(dev);
465         uint32_t val;
466         int i;
467
468         KASSERT(phy == sc->bnx_phyno,
469             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
470
471         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
472         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
473                 CSR_WRITE_4(sc, BGE_MI_MODE,
474                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
475                 DELAY(80);
476         }
477
478         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_READ | BGE_MICOMM_BUSY |
479             BGE_MIPHY(phy) | BGE_MIREG(reg));
480
481         /* Poll for the PHY register access to complete. */
482         for (i = 0; i < BNX_TIMEOUT; i++) {
483                 DELAY(10);
484                 val = CSR_READ_4(sc, BGE_MI_COMM);
485                 if ((val & BGE_MICOMM_BUSY) == 0) {
486                         DELAY(5);
487                         val = CSR_READ_4(sc, BGE_MI_COMM);
488                         break;
489                 }
490         }
491         if (i == BNX_TIMEOUT) {
492                 if_printf(&sc->arpcom.ac_if, "PHY read timed out "
493                     "(phy %d, reg %d, val 0x%08x)\n", phy, reg, val);
494                 val = 0;
495         }
496
497         /* Restore the autopoll bit if necessary. */
498         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
499                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
500                 DELAY(80);
501         }
502
503         if (val & BGE_MICOMM_READFAIL)
504                 return 0;
505
506         return (val & 0xFFFF);
507 }
508
509 static int
510 bnx_miibus_writereg(device_t dev, int phy, int reg, int val)
511 {
512         struct bnx_softc *sc = device_get_softc(dev);
513         int i;
514
515         KASSERT(phy == sc->bnx_phyno,
516             ("invalid phyno %d, should be %d", phy, sc->bnx_phyno));
517
518         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
519             (reg == BRGPHY_MII_1000CTL || reg == BRGPHY_MII_AUXCTL))
520                return 0;
521
522         /* Clear the autopoll bit if set, otherwise may trigger PCI errors. */
523         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
524                 CSR_WRITE_4(sc, BGE_MI_MODE,
525                     sc->bnx_mi_mode & ~BGE_MIMODE_AUTOPOLL);
526                 DELAY(80);
527         }
528
529         CSR_WRITE_4(sc, BGE_MI_COMM, BGE_MICMD_WRITE | BGE_MICOMM_BUSY |
530             BGE_MIPHY(phy) | BGE_MIREG(reg) | val);
531
532         for (i = 0; i < BNX_TIMEOUT; i++) {
533                 DELAY(10);
534                 if (!(CSR_READ_4(sc, BGE_MI_COMM) & BGE_MICOMM_BUSY)) {
535                         DELAY(5);
536                         CSR_READ_4(sc, BGE_MI_COMM); /* dummy read */
537                         break;
538                 }
539         }
540         if (i == BNX_TIMEOUT) {
541                 if_printf(&sc->arpcom.ac_if, "PHY write timed out "
542                     "(phy %d, reg %d, val %d)\n", phy, reg, val);
543         }
544
545         /* Restore the autopoll bit if necessary. */
546         if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
547                 CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
548                 DELAY(80);
549         }
550
551         return 0;
552 }
553
554 static void
555 bnx_miibus_statchg(device_t dev)
556 {
557         struct bnx_softc *sc;
558         struct mii_data *mii;
559
560         sc = device_get_softc(dev);
561         mii = device_get_softc(sc->bnx_miibus);
562
563         if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) ==
564             (IFM_ACTIVE | IFM_AVALID)) {
565                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
566                 case IFM_10_T:
567                 case IFM_100_TX:
568                         sc->bnx_link = 1;
569                         break;
570                 case IFM_1000_T:
571                 case IFM_1000_SX:
572                 case IFM_2500_SX:
573                         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
574                                 sc->bnx_link = 1;
575                         else
576                                 sc->bnx_link = 0;
577                         break;
578                 default:
579                         sc->bnx_link = 0;
580                         break;
581                 }
582         } else {
583                 sc->bnx_link = 0;
584         }
585         if (sc->bnx_link == 0)
586                 return;
587
588         BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_PORTMODE);
589         if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T ||
590             IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) {
591                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_GMII);
592         } else {
593                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_PORTMODE_MII);
594         }
595
596         if ((mii->mii_media_active & IFM_GMASK) == IFM_FDX) {
597                 BNX_CLRBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
598         } else {
599                 BNX_SETBIT(sc, BGE_MAC_MODE, BGE_MACMODE_HALF_DUPLEX);
600         }
601 }
602
603 /*
604  * Memory management for jumbo frames.
605  */
606 static int
607 bnx_alloc_jumbo_mem(struct bnx_softc *sc)
608 {
609         struct ifnet *ifp = &sc->arpcom.ac_if;
610         struct bnx_jslot *entry;
611         uint8_t *ptr;
612         bus_addr_t paddr;
613         int i, error;
614
615         /*
616          * Create tag for jumbo mbufs.
617          * This is really a bit of a kludge. We allocate a special
618          * jumbo buffer pool which (thanks to the way our DMA
619          * memory allocation works) will consist of contiguous
620          * pages. This means that even though a jumbo buffer might
621          * be larger than a page size, we don't really need to
622          * map it into more than one DMA segment. However, the
623          * default mbuf tag will result in multi-segment mappings,
624          * so we have to create a special jumbo mbuf tag that
625          * lets us get away with mapping the jumbo buffers as
626          * a single segment. I think eventually the driver should
627          * be changed so that it uses ordinary mbufs and cluster
628          * buffers, i.e. jumbo frames can span multiple DMA
629          * descriptors. But that's a project for another day.
630          */
631
632         /*
633          * Create DMA stuffs for jumbo RX ring.
634          */
635         error = bnx_dma_block_alloc(sc, BGE_JUMBO_RX_RING_SZ,
636                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
637                                     &sc->bnx_cdata.bnx_rx_jumbo_ring_map,
638                                     (void *)&sc->bnx_ldata.bnx_rx_jumbo_ring,
639                                     &sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
640         if (error) {
641                 if_printf(ifp, "could not create jumbo RX ring\n");
642                 return error;
643         }
644
645         /*
646          * Create DMA stuffs for jumbo buffer block.
647          */
648         error = bnx_dma_block_alloc(sc, BNX_JMEM,
649                                     &sc->bnx_cdata.bnx_jumbo_tag,
650                                     &sc->bnx_cdata.bnx_jumbo_map,
651                                     (void **)&sc->bnx_ldata.bnx_jumbo_buf,
652                                     &paddr);
653         if (error) {
654                 if_printf(ifp, "could not create jumbo buffer\n");
655                 return error;
656         }
657
658         SLIST_INIT(&sc->bnx_jfree_listhead);
659
660         /*
661          * Now divide it up into 9K pieces and save the addresses
662          * in an array. Note that we play an evil trick here by using
663          * the first few bytes in the buffer to hold the the address
664          * of the softc structure for this interface. This is because
665          * bnx_jfree() needs it, but it is called by the mbuf management
666          * code which will not pass it to us explicitly.
667          */
668         for (i = 0, ptr = sc->bnx_ldata.bnx_jumbo_buf; i < BNX_JSLOTS; i++) {
669                 entry = &sc->bnx_cdata.bnx_jslots[i];
670                 entry->bnx_sc = sc;
671                 entry->bnx_buf = ptr;
672                 entry->bnx_paddr = paddr;
673                 entry->bnx_inuse = 0;
674                 entry->bnx_slot = i;
675                 SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, entry, jslot_link);
676
677                 ptr += BNX_JLEN;
678                 paddr += BNX_JLEN;
679         }
680         return 0;
681 }
682
683 static void
684 bnx_free_jumbo_mem(struct bnx_softc *sc)
685 {
686         /* Destroy jumbo RX ring. */
687         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_jumbo_ring_tag,
688                            sc->bnx_cdata.bnx_rx_jumbo_ring_map,
689                            sc->bnx_ldata.bnx_rx_jumbo_ring);
690
691         /* Destroy jumbo buffer block. */
692         bnx_dma_block_free(sc->bnx_cdata.bnx_jumbo_tag,
693                            sc->bnx_cdata.bnx_jumbo_map,
694                            sc->bnx_ldata.bnx_jumbo_buf);
695 }
696
697 /*
698  * Allocate a jumbo buffer.
699  */
700 static struct bnx_jslot *
701 bnx_jalloc(struct bnx_softc *sc)
702 {
703         struct bnx_jslot *entry;
704
705         lwkt_serialize_enter(&sc->bnx_jslot_serializer);
706         entry = SLIST_FIRST(&sc->bnx_jfree_listhead);
707         if (entry) {
708                 SLIST_REMOVE_HEAD(&sc->bnx_jfree_listhead, jslot_link);
709                 entry->bnx_inuse = 1;
710         } else {
711                 if_printf(&sc->arpcom.ac_if, "no free jumbo buffers\n");
712         }
713         lwkt_serialize_exit(&sc->bnx_jslot_serializer);
714         return(entry);
715 }
716
717 /*
718  * Adjust usage count on a jumbo buffer.
719  */
720 static void
721 bnx_jref(void *arg)
722 {
723         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
724         struct bnx_softc *sc = entry->bnx_sc;
725
726         if (sc == NULL)
727                 panic("bnx_jref: can't find softc pointer!");
728
729         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
730                 panic("bnx_jref: asked to reference buffer "
731                     "that we don't manage!");
732         } else if (entry->bnx_inuse == 0) {
733                 panic("bnx_jref: buffer already free!");
734         } else {
735                 atomic_add_int(&entry->bnx_inuse, 1);
736         }
737 }
738
739 /*
740  * Release a jumbo buffer.
741  */
742 static void
743 bnx_jfree(void *arg)
744 {
745         struct bnx_jslot *entry = (struct bnx_jslot *)arg;
746         struct bnx_softc *sc = entry->bnx_sc;
747
748         if (sc == NULL)
749                 panic("bnx_jfree: can't find softc pointer!");
750
751         if (&sc->bnx_cdata.bnx_jslots[entry->bnx_slot] != entry) {
752                 panic("bnx_jfree: asked to free buffer that we don't manage!");
753         } else if (entry->bnx_inuse == 0) {
754                 panic("bnx_jfree: buffer already free!");
755         } else {
756                 /*
757                  * Possible MP race to 0, use the serializer.  The atomic insn
758                  * is still needed for races against bnx_jref().
759                  */
760                 lwkt_serialize_enter(&sc->bnx_jslot_serializer);
761                 atomic_subtract_int(&entry->bnx_inuse, 1);
762                 if (entry->bnx_inuse == 0) {
763                         SLIST_INSERT_HEAD(&sc->bnx_jfree_listhead, 
764                                           entry, jslot_link);
765                 }
766                 lwkt_serialize_exit(&sc->bnx_jslot_serializer);
767         }
768 }
769
770
771 /*
772  * Intialize a standard receive ring descriptor.
773  */
774 static int
775 bnx_newbuf_std(struct bnx_softc *sc, int i, int init)
776 {
777         struct mbuf *m_new = NULL;
778         bus_dma_segment_t seg;
779         bus_dmamap_t map;
780         int error, nsegs;
781
782         m_new = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
783         if (m_new == NULL)
784                 return ENOBUFS;
785         m_new->m_len = m_new->m_pkthdr.len = MCLBYTES;
786         m_adj(m_new, ETHER_ALIGN);
787
788         error = bus_dmamap_load_mbuf_segment(sc->bnx_cdata.bnx_rx_mtag,
789                         sc->bnx_cdata.bnx_rx_tmpmap, m_new,
790                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
791         if (error) {
792                 m_freem(m_new);
793                 return error;
794         }
795
796         if (!init) {
797                 bus_dmamap_sync(sc->bnx_cdata.bnx_rx_mtag,
798                                 sc->bnx_cdata.bnx_rx_std_dmamap[i],
799                                 BUS_DMASYNC_POSTREAD);
800                 bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
801                         sc->bnx_cdata.bnx_rx_std_dmamap[i]);
802         }
803
804         map = sc->bnx_cdata.bnx_rx_tmpmap;
805         sc->bnx_cdata.bnx_rx_tmpmap = sc->bnx_cdata.bnx_rx_std_dmamap[i];
806         sc->bnx_cdata.bnx_rx_std_dmamap[i] = map;
807
808         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_mbuf = m_new;
809         sc->bnx_cdata.bnx_rx_std_chain[i].bnx_paddr = seg.ds_addr;
810
811         bnx_setup_rxdesc_std(sc, i);
812         return 0;
813 }
814
815 static void
816 bnx_setup_rxdesc_std(struct bnx_softc *sc, int i)
817 {
818         struct bnx_rxchain *rc;
819         struct bge_rx_bd *r;
820
821         rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
822         r = &sc->bnx_ldata.bnx_rx_std_ring[i];
823
824         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
825         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
826         r->bge_len = rc->bnx_mbuf->m_len;
827         r->bge_idx = i;
828         r->bge_flags = BGE_RXBDFLAG_END;
829 }
830
831 /*
832  * Initialize a jumbo receive ring descriptor. This allocates
833  * a jumbo buffer from the pool managed internally by the driver.
834  */
835 static int
836 bnx_newbuf_jumbo(struct bnx_softc *sc, int i, int init)
837 {
838         struct mbuf *m_new = NULL;
839         struct bnx_jslot *buf;
840         bus_addr_t paddr;
841
842         /* Allocate the mbuf. */
843         MGETHDR(m_new, init ? MB_WAIT : MB_DONTWAIT, MT_DATA);
844         if (m_new == NULL)
845                 return ENOBUFS;
846
847         /* Allocate the jumbo buffer */
848         buf = bnx_jalloc(sc);
849         if (buf == NULL) {
850                 m_freem(m_new);
851                 return ENOBUFS;
852         }
853
854         /* Attach the buffer to the mbuf. */
855         m_new->m_ext.ext_arg = buf;
856         m_new->m_ext.ext_buf = buf->bnx_buf;
857         m_new->m_ext.ext_free = bnx_jfree;
858         m_new->m_ext.ext_ref = bnx_jref;
859         m_new->m_ext.ext_size = BNX_JUMBO_FRAMELEN;
860
861         m_new->m_flags |= M_EXT;
862
863         m_new->m_data = m_new->m_ext.ext_buf;
864         m_new->m_len = m_new->m_pkthdr.len = m_new->m_ext.ext_size;
865
866         paddr = buf->bnx_paddr;
867         m_adj(m_new, ETHER_ALIGN);
868         paddr += ETHER_ALIGN;
869
870         /* Save necessary information */
871         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_mbuf = m_new;
872         sc->bnx_cdata.bnx_rx_jumbo_chain[i].bnx_paddr = paddr;
873
874         /* Set up the descriptor. */
875         bnx_setup_rxdesc_jumbo(sc, i);
876         return 0;
877 }
878
879 static void
880 bnx_setup_rxdesc_jumbo(struct bnx_softc *sc, int i)
881 {
882         struct bge_rx_bd *r;
883         struct bnx_rxchain *rc;
884
885         r = &sc->bnx_ldata.bnx_rx_jumbo_ring[i];
886         rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
887
888         r->bge_addr.bge_addr_lo = BGE_ADDR_LO(rc->bnx_paddr);
889         r->bge_addr.bge_addr_hi = BGE_ADDR_HI(rc->bnx_paddr);
890         r->bge_len = rc->bnx_mbuf->m_len;
891         r->bge_idx = i;
892         r->bge_flags = BGE_RXBDFLAG_END|BGE_RXBDFLAG_JUMBO_RING;
893 }
894
895 static int
896 bnx_init_rx_ring_std(struct bnx_softc *sc)
897 {
898         int i, error;
899
900         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
901                 error = bnx_newbuf_std(sc, i, 1);
902                 if (error)
903                         return error;
904         };
905
906         sc->bnx_std = BGE_STD_RX_RING_CNT - 1;
907         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
908
909         return(0);
910 }
911
912 static void
913 bnx_free_rx_ring_std(struct bnx_softc *sc)
914 {
915         int i;
916
917         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
918                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_std_chain[i];
919
920                 if (rc->bnx_mbuf != NULL) {
921                         bus_dmamap_unload(sc->bnx_cdata.bnx_rx_mtag,
922                                           sc->bnx_cdata.bnx_rx_std_dmamap[i]);
923                         m_freem(rc->bnx_mbuf);
924                         rc->bnx_mbuf = NULL;
925                 }
926                 bzero(&sc->bnx_ldata.bnx_rx_std_ring[i],
927                     sizeof(struct bge_rx_bd));
928         }
929 }
930
931 static int
932 bnx_init_rx_ring_jumbo(struct bnx_softc *sc)
933 {
934         struct bge_rcb *rcb;
935         int i, error;
936
937         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
938                 error = bnx_newbuf_jumbo(sc, i, 1);
939                 if (error)
940                         return error;
941         };
942
943         sc->bnx_jumbo = BGE_JUMBO_RX_RING_CNT - 1;
944
945         rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
946         rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(0, 0);
947         CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
948
949         bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
950
951         return(0);
952 }
953
954 static void
955 bnx_free_rx_ring_jumbo(struct bnx_softc *sc)
956 {
957         int i;
958
959         for (i = 0; i < BGE_JUMBO_RX_RING_CNT; i++) {
960                 struct bnx_rxchain *rc = &sc->bnx_cdata.bnx_rx_jumbo_chain[i];
961
962                 if (rc->bnx_mbuf != NULL) {
963                         m_freem(rc->bnx_mbuf);
964                         rc->bnx_mbuf = NULL;
965                 }
966                 bzero(&sc->bnx_ldata.bnx_rx_jumbo_ring[i],
967                     sizeof(struct bge_rx_bd));
968         }
969 }
970
971 static void
972 bnx_free_tx_ring(struct bnx_softc *sc)
973 {
974         int i;
975
976         for (i = 0; i < BGE_TX_RING_CNT; i++) {
977                 if (sc->bnx_cdata.bnx_tx_chain[i] != NULL) {
978                         bus_dmamap_unload(sc->bnx_cdata.bnx_tx_mtag,
979                                           sc->bnx_cdata.bnx_tx_dmamap[i]);
980                         m_freem(sc->bnx_cdata.bnx_tx_chain[i]);
981                         sc->bnx_cdata.bnx_tx_chain[i] = NULL;
982                 }
983                 bzero(&sc->bnx_ldata.bnx_tx_ring[i],
984                     sizeof(struct bge_tx_bd));
985         }
986 }
987
988 static int
989 bnx_init_tx_ring(struct bnx_softc *sc)
990 {
991         sc->bnx_txcnt = 0;
992         sc->bnx_tx_saved_considx = 0;
993         sc->bnx_tx_prodidx = 0;
994
995         /* Initialize transmit producer index for host-memory send ring. */
996         bnx_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, sc->bnx_tx_prodidx);
997         bnx_writembx(sc, BGE_MBX_TX_NIC_PROD0_LO, 0);
998
999         return(0);
1000 }
1001
1002 static void
1003 bnx_setmulti(struct bnx_softc *sc)
1004 {
1005         struct ifnet *ifp;
1006         struct ifmultiaddr *ifma;
1007         uint32_t hashes[4] = { 0, 0, 0, 0 };
1008         int h, i;
1009
1010         ifp = &sc->arpcom.ac_if;
1011
1012         if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) {
1013                 for (i = 0; i < 4; i++)
1014                         CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0xFFFFFFFF);
1015                 return;
1016         }
1017
1018         /* First, zot all the existing filters. */
1019         for (i = 0; i < 4; i++)
1020                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), 0);
1021
1022         /* Now program new ones. */
1023         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1024                 if (ifma->ifma_addr->sa_family != AF_LINK)
1025                         continue;
1026                 h = ether_crc32_le(
1027                     LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1028                     ETHER_ADDR_LEN) & 0x7f;
1029                 hashes[(h & 0x60) >> 5] |= 1 << (h & 0x1F);
1030         }
1031
1032         for (i = 0; i < 4; i++)
1033                 CSR_WRITE_4(sc, BGE_MAR0 + (i * 4), hashes[i]);
1034 }
1035
1036 /*
1037  * Do endian, PCI and DMA initialization. Also check the on-board ROM
1038  * self-test results.
1039  */
1040 static int
1041 bnx_chipinit(struct bnx_softc *sc)
1042 {
1043         uint32_t dma_rw_ctl, mode_ctl;
1044         int i;
1045
1046         /* Set endian type before we access any non-PCI registers. */
1047         pci_write_config(sc->bnx_dev, BGE_PCI_MISC_CTL,
1048             BGE_INIT | BGE_PCIMISCCTL_TAGGED_STATUS, 4);
1049
1050         /* Clear the MAC control register */
1051         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
1052
1053         /*
1054          * Clear the MAC statistics block in the NIC's
1055          * internal memory.
1056          */
1057         for (i = BGE_STATS_BLOCK;
1058             i < BGE_STATS_BLOCK_END + 1; i += sizeof(uint32_t))
1059                 BNX_MEMWIN_WRITE(sc, i, 0);
1060
1061         for (i = BGE_STATUS_BLOCK;
1062             i < BGE_STATUS_BLOCK_END + 1; i += sizeof(uint32_t))
1063                 BNX_MEMWIN_WRITE(sc, i, 0);
1064
1065         if (BNX_IS_57765_FAMILY(sc)) {
1066                 uint32_t val;
1067
1068                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0) {
1069                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1070                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1071
1072                         /* Access the lower 1K of PL PCI-E block registers. */
1073                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1074                             val | BGE_MODECTL_PCIE_PL_SEL);
1075
1076                         val = CSR_READ_4(sc, BGE_PCIE_PL_LO_PHYCTL5);
1077                         val |= BGE_PCIE_PL_LO_PHYCTL5_DIS_L2CLKREQ;
1078                         CSR_WRITE_4(sc, BGE_PCIE_PL_LO_PHYCTL5, val);
1079
1080                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1081                 }
1082                 if (sc->bnx_chiprev != BGE_CHIPREV_57765_AX) {
1083                         mode_ctl = CSR_READ_4(sc, BGE_MODE_CTL);
1084                         val = mode_ctl & ~BGE_MODECTL_PCIE_PORTS;
1085
1086                         /* Access the lower 1K of DL PCI-E block registers. */
1087                         CSR_WRITE_4(sc, BGE_MODE_CTL,
1088                             val | BGE_MODECTL_PCIE_DL_SEL);
1089
1090                         val = CSR_READ_4(sc, BGE_PCIE_DL_LO_FTSMAX);
1091                         val &= ~BGE_PCIE_DL_LO_FTSMAX_MASK;
1092                         val |= BGE_PCIE_DL_LO_FTSMAX_VAL;
1093                         CSR_WRITE_4(sc, BGE_PCIE_DL_LO_FTSMAX, val);
1094
1095                         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1096                 }
1097
1098                 val = CSR_READ_4(sc, BGE_CPMU_LSPD_10MB_CLK);
1099                 val &= ~BGE_CPMU_LSPD_10MB_MACCLK_MASK;
1100                 val |= BGE_CPMU_LSPD_10MB_MACCLK_6_25;
1101                 CSR_WRITE_4(sc, BGE_CPMU_LSPD_10MB_CLK, val);
1102         }
1103
1104         /*
1105          * Set up the PCI DMA control register.
1106          */
1107         dma_rw_ctl = pci_read_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, 4);
1108         /*
1109          * Disable 32bytes cache alignment for DMA write to host memory
1110          *
1111          * NOTE:
1112          * 64bytes cache alignment for DMA write to host memory is still
1113          * enabled.
1114          */
1115         dma_rw_ctl |= BGE_PCIDMARWCTL_DIS_CACHE_ALIGNMENT;
1116         if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
1117                 dma_rw_ctl &= ~BGE_PCIDMARWCTL_CRDRDR_RDMA_MRRS_MSK;
1118         /*
1119          * Enable HW workaround for controllers that misinterpret
1120          * a status tag update and leave interrupts permanently
1121          * disabled.
1122          */
1123         if (sc->bnx_asicrev != BGE_ASICREV_BCM5717 &&
1124             !BNX_IS_57765_FAMILY(sc))
1125                 dma_rw_ctl |= BGE_PCIDMARWCTL_TAGGED_STATUS_WA;
1126         if (bootverbose) {
1127                 if_printf(&sc->arpcom.ac_if, "DMA read/write %#x\n",
1128                     dma_rw_ctl);
1129         }
1130         pci_write_config(sc->bnx_dev, BGE_PCI_DMA_RW_CTL, dma_rw_ctl, 4);
1131
1132         /*
1133          * Set up general mode register.
1134          */
1135         mode_ctl = bnx_dma_swap_options(sc) | BGE_MODECTL_MAC_ATTN_INTR |
1136             BGE_MODECTL_HOST_SEND_BDS | BGE_MODECTL_TX_NO_PHDR_CSUM;
1137         CSR_WRITE_4(sc, BGE_MODE_CTL, mode_ctl);
1138
1139         /*
1140          * Disable memory write invalidate.  Apparently it is not supported
1141          * properly by these devices.  Also ensure that INTx isn't disabled,
1142          * as these chips need it even when using MSI.
1143          */
1144         PCI_CLRBIT(sc->bnx_dev, BGE_PCI_CMD,
1145             (PCIM_CMD_MWRICEN | PCIM_CMD_INTxDIS), 4);
1146
1147         /* Set the timer prescaler (always 66Mhz) */
1148         CSR_WRITE_4(sc, BGE_MISC_CFG, 65 << 1/*BGE_32BITTIME_66MHZ*/);
1149
1150         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
1151                 DELAY(40);      /* XXX */
1152
1153                 /* Put PHY into ready state */
1154                 BNX_CLRBIT(sc, BGE_MISC_CFG, BGE_MISCCFG_EPHY_IDDQ);
1155                 CSR_READ_4(sc, BGE_MISC_CFG); /* Flush */
1156                 DELAY(40);
1157         }
1158
1159         return(0);
1160 }
1161
1162 static int
1163 bnx_blockinit(struct bnx_softc *sc)
1164 {
1165         struct bge_rcb *rcb;
1166         bus_size_t vrcb;
1167         bge_hostaddr taddr;
1168         uint32_t val;
1169         int i, limit;
1170
1171         /*
1172          * Initialize the memory window pointer register so that
1173          * we can access the first 32K of internal NIC RAM. This will
1174          * allow us to set up the TX send ring RCBs and the RX return
1175          * ring RCBs, plus other things which live in NIC memory.
1176          */
1177         CSR_WRITE_4(sc, BGE_PCI_MEMWIN_BASEADDR, 0);
1178
1179         /* Configure mbuf pool watermarks */
1180         if (BNX_IS_57765_PLUS(sc)) {
1181                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1182                 if (sc->arpcom.ac_if.if_mtu > ETHERMTU) {
1183                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x7e);
1184                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xea);
1185                 } else {
1186                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x2a);
1187                         CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0xa0);
1188                 }
1189         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
1190                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1191                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x04);
1192                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x10);
1193         } else {
1194                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_READDMA_LOWAT, 0x0);
1195                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_MACRX_LOWAT, 0x10);
1196                 CSR_WRITE_4(sc, BGE_BMAN_MBUFPOOL_HIWAT, 0x60);
1197         }
1198
1199         /* Configure DMA resource watermarks */
1200         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_LOWAT, 5);
1201         CSR_WRITE_4(sc, BGE_BMAN_DMA_DESCPOOL_HIWAT, 10);
1202
1203         /* Enable buffer manager */
1204         val = BGE_BMANMODE_ENABLE | BGE_BMANMODE_LOMBUF_ATTN;
1205         /*
1206          * Change the arbitration algorithm of TXMBUF read request to
1207          * round-robin instead of priority based for BCM5719.  When
1208          * TXFIFO is almost empty, RDMA will hold its request until
1209          * TXFIFO is not almost empty.
1210          */
1211         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719)
1212                 val |= BGE_BMANMODE_NO_TX_UNDERRUN;
1213         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1214             sc->bnx_chipid == BGE_CHIPID_BCM5719_A0 ||
1215             sc->bnx_chipid == BGE_CHIPID_BCM5720_A0)
1216                 val |= BGE_BMANMODE_LOMBUF_ATTN;
1217         CSR_WRITE_4(sc, BGE_BMAN_MODE, val);
1218
1219         /* Poll for buffer manager start indication */
1220         for (i = 0; i < BNX_TIMEOUT; i++) {
1221                 if (CSR_READ_4(sc, BGE_BMAN_MODE) & BGE_BMANMODE_ENABLE)
1222                         break;
1223                 DELAY(10);
1224         }
1225
1226         if (i == BNX_TIMEOUT) {
1227                 if_printf(&sc->arpcom.ac_if,
1228                           "buffer manager failed to start\n");
1229                 return(ENXIO);
1230         }
1231
1232         /* Enable flow-through queues */
1233         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
1234         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
1235
1236         /* Wait until queue initialization is complete */
1237         for (i = 0; i < BNX_TIMEOUT; i++) {
1238                 if (CSR_READ_4(sc, BGE_FTQ_RESET) == 0)
1239                         break;
1240                 DELAY(10);
1241         }
1242
1243         if (i == BNX_TIMEOUT) {
1244                 if_printf(&sc->arpcom.ac_if,
1245                           "flow-through queue init failed\n");
1246                 return(ENXIO);
1247         }
1248
1249         /*
1250          * Summary of rings supported by the controller:
1251          *
1252          * Standard Receive Producer Ring
1253          * - This ring is used to feed receive buffers for "standard"
1254          *   sized frames (typically 1536 bytes) to the controller.
1255          *
1256          * Jumbo Receive Producer Ring
1257          * - This ring is used to feed receive buffers for jumbo sized
1258          *   frames (i.e. anything bigger than the "standard" frames)
1259          *   to the controller.
1260          *
1261          * Mini Receive Producer Ring
1262          * - This ring is used to feed receive buffers for "mini"
1263          *   sized frames to the controller.
1264          * - This feature required external memory for the controller
1265          *   but was never used in a production system.  Should always
1266          *   be disabled.
1267          *
1268          * Receive Return Ring
1269          * - After the controller has placed an incoming frame into a
1270          *   receive buffer that buffer is moved into a receive return
1271          *   ring.  The driver is then responsible to passing the
1272          *   buffer up to the stack.  Many versions of the controller
1273          *   support multiple RR rings.
1274          *
1275          * Send Ring
1276          * - This ring is used for outgoing frames.  Many versions of
1277          *   the controller support multiple send rings.
1278          */
1279
1280         /* Initialize the standard receive producer ring control block. */
1281         rcb = &sc->bnx_ldata.bnx_info.bnx_std_rx_rcb;
1282         rcb->bge_hostaddr.bge_addr_lo =
1283             BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1284         rcb->bge_hostaddr.bge_addr_hi =
1285             BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_std_ring_paddr);
1286         if (BNX_IS_57765_PLUS(sc)) {
1287                 /*
1288                  * Bits 31-16: Programmable ring size (2048, 1024, 512, .., 32)
1289                  * Bits 15-2 : Maximum RX frame size
1290                  * Bit 1     : 1 = Ring Disabled, 0 = Ring ENabled
1291                  * Bit 0     : Reserved
1292                  */
1293                 rcb->bge_maxlen_flags =
1294                     BGE_RCB_MAXLEN_FLAGS(512, BNX_MAX_FRAMELEN << 2);
1295         } else {
1296                 /*
1297                  * Bits 31-16: Programmable ring size (512, 256, 128, 64, 32)
1298                  * Bits 15-2 : Reserved (should be 0)
1299                  * Bit 1     : 1 = Ring Disabled, 0 = Ring Enabled
1300                  * Bit 0     : Reserved
1301                  */
1302                 rcb->bge_maxlen_flags = BGE_RCB_MAXLEN_FLAGS(512, 0);
1303         }
1304         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1305             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1306             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1307                 rcb->bge_nicaddr = BGE_STD_RX_RINGS_5717;
1308         else
1309                 rcb->bge_nicaddr = BGE_STD_RX_RINGS;
1310         /* Write the standard receive producer ring control block. */
1311         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_HI, rcb->bge_hostaddr.bge_addr_hi);
1312         CSR_WRITE_4(sc, BGE_RX_STD_RCB_HADDR_LO, rcb->bge_hostaddr.bge_addr_lo);
1313         CSR_WRITE_4(sc, BGE_RX_STD_RCB_MAXLEN_FLAGS, rcb->bge_maxlen_flags);
1314         CSR_WRITE_4(sc, BGE_RX_STD_RCB_NICADDR, rcb->bge_nicaddr);
1315         /* Reset the standard receive producer ring producer index. */
1316         bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, 0);
1317
1318         /*
1319          * Initialize the jumbo RX producer ring control
1320          * block.  We set the 'ring disabled' bit in the
1321          * flags field until we're actually ready to start
1322          * using this ring (i.e. once we set the MTU
1323          * high enough to require it).
1324          */
1325         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1326                 rcb = &sc->bnx_ldata.bnx_info.bnx_jumbo_rx_rcb;
1327                 /* Get the jumbo receive producer ring RCB parameters. */
1328                 rcb->bge_hostaddr.bge_addr_lo =
1329                     BGE_ADDR_LO(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1330                 rcb->bge_hostaddr.bge_addr_hi =
1331                     BGE_ADDR_HI(sc->bnx_ldata.bnx_rx_jumbo_ring_paddr);
1332                 rcb->bge_maxlen_flags =
1333                     BGE_RCB_MAXLEN_FLAGS(BNX_MAX_FRAMELEN,
1334                     BGE_RCB_FLAG_RING_DISABLED);
1335                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1336                     sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1337                     sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1338                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS_5717;
1339                 else
1340                         rcb->bge_nicaddr = BGE_JUMBO_RX_RINGS;
1341                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_HI,
1342                     rcb->bge_hostaddr.bge_addr_hi);
1343                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_HADDR_LO,
1344                     rcb->bge_hostaddr.bge_addr_lo);
1345                 /* Program the jumbo receive producer ring RCB parameters. */
1346                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_MAXLEN_FLAGS,
1347                     rcb->bge_maxlen_flags);
1348                 CSR_WRITE_4(sc, BGE_RX_JUMBO_RCB_NICADDR, rcb->bge_nicaddr);
1349                 /* Reset the jumbo receive producer ring producer index. */
1350                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, 0);
1351         }
1352
1353         /* Choose de-pipeline mode for BCM5906 A0, A1 and A2. */
1354         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906 &&
1355             (sc->bnx_chipid == BGE_CHIPID_BCM5906_A0 ||
1356              sc->bnx_chipid == BGE_CHIPID_BCM5906_A1 ||
1357              sc->bnx_chipid == BGE_CHIPID_BCM5906_A2)) {
1358                 CSR_WRITE_4(sc, BGE_ISO_PKT_TX,
1359                     (CSR_READ_4(sc, BGE_ISO_PKT_TX) & ~3) | 2);
1360         }
1361
1362         /*
1363          * The BD ring replenish thresholds control how often the
1364          * hardware fetches new BD's from the producer rings in host
1365          * memory.  Setting the value too low on a busy system can
1366          * starve the hardware and recue the throughpout.
1367          *
1368          * Set the BD ring replentish thresholds. The recommended
1369          * values are 1/8th the number of descriptors allocated to
1370          * each ring.
1371          */
1372         val = 8;
1373         CSR_WRITE_4(sc, BGE_RBDI_STD_REPL_THRESH, val);
1374         if (BNX_IS_JUMBO_CAPABLE(sc)) {
1375                 CSR_WRITE_4(sc, BGE_RBDI_JUMBO_REPL_THRESH,
1376                     BGE_JUMBO_RX_RING_CNT/8);
1377         }
1378         if (BNX_IS_57765_PLUS(sc)) {
1379                 CSR_WRITE_4(sc, BGE_STD_REPLENISH_LWM, 32);
1380                 CSR_WRITE_4(sc, BGE_JMB_REPLENISH_LWM, 16);
1381         }
1382
1383         /*
1384          * Disable all send rings by setting the 'ring disabled' bit
1385          * in the flags field of all the TX send ring control blocks,
1386          * located in NIC memory.
1387          */
1388         if (BNX_IS_5717_PLUS(sc))
1389                 limit = 4;
1390         else if (BNX_IS_57765_FAMILY(sc))
1391                 limit = 2;
1392         else
1393                 limit = 1;
1394         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1395         for (i = 0; i < limit; i++) {
1396                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1397                     BGE_RCB_MAXLEN_FLAGS(0, BGE_RCB_FLAG_RING_DISABLED));
1398                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1399                 vrcb += sizeof(struct bge_rcb);
1400         }
1401
1402         /* Configure send ring RCB 0 (we use only the first ring) */
1403         vrcb = BGE_MEMWIN_START + BGE_SEND_RING_RCB;
1404         BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_tx_ring_paddr);
1405         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1406         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1407         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1408             sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1409             sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1410                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, BGE_SEND_RING_5717);
1411         } else {
1412                 RCB_WRITE_4(sc, vrcb, bge_nicaddr,
1413                     BGE_NIC_TXRING_ADDR(0, BGE_TX_RING_CNT));
1414         }
1415         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1416             BGE_RCB_MAXLEN_FLAGS(BGE_TX_RING_CNT, 0));
1417
1418         /*
1419          * Disable all receive return rings by setting the
1420          * 'ring disabled' bit in the flags field of all the receive
1421          * return ring control blocks, located in NIC memory.
1422          */
1423         if (BNX_IS_5717_PLUS(sc)) {
1424                 /* Should be 17, use 16 until we get an SRAM map. */
1425                 limit = 16;
1426         } else if (BNX_IS_57765_FAMILY(sc)) {
1427                 limit = 4;
1428         } else {
1429                 limit = 1;
1430         }
1431         /* Disable all receive return rings. */
1432         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1433         for (i = 0; i < limit; i++) {
1434                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, 0);
1435                 RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, 0);
1436                 RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1437                     BGE_RCB_FLAG_RING_DISABLED);
1438                 RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1439                 bnx_writembx(sc, BGE_MBX_RX_CONS0_LO +
1440                     (i * (sizeof(uint64_t))), 0);
1441                 vrcb += sizeof(struct bge_rcb);
1442         }
1443
1444         /*
1445          * Set up receive return ring 0.  Note that the NIC address
1446          * for RX return rings is 0x0.  The return rings live entirely
1447          * within the host, so the nicaddr field in the RCB isn't used.
1448          */
1449         vrcb = BGE_MEMWIN_START + BGE_RX_RETURN_RING_RCB;
1450         BGE_HOSTADDR(taddr, sc->bnx_ldata.bnx_rx_return_ring_paddr);
1451         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_hi, taddr.bge_addr_hi);
1452         RCB_WRITE_4(sc, vrcb, bge_hostaddr.bge_addr_lo, taddr.bge_addr_lo);
1453         RCB_WRITE_4(sc, vrcb, bge_nicaddr, 0);
1454         RCB_WRITE_4(sc, vrcb, bge_maxlen_flags,
1455             BGE_RCB_MAXLEN_FLAGS(sc->bnx_return_ring_cnt, 0));
1456
1457         /* Set random backoff seed for TX */
1458         CSR_WRITE_4(sc, BGE_TX_RANDOM_BACKOFF,
1459             sc->arpcom.ac_enaddr[0] + sc->arpcom.ac_enaddr[1] +
1460             sc->arpcom.ac_enaddr[2] + sc->arpcom.ac_enaddr[3] +
1461             sc->arpcom.ac_enaddr[4] + sc->arpcom.ac_enaddr[5] +
1462             BGE_TX_BACKOFF_SEED_MASK);
1463
1464         /* Set inter-packet gap */
1465         val = 0x2620;
1466         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1467                 val |= CSR_READ_4(sc, BGE_TX_LENGTHS) &
1468                     (BGE_TXLEN_JMB_FRM_LEN_MSK | BGE_TXLEN_CNT_DN_VAL_MSK);
1469         }
1470         CSR_WRITE_4(sc, BGE_TX_LENGTHS, val);
1471
1472         /*
1473          * Specify which ring to use for packets that don't match
1474          * any RX rules.
1475          */
1476         CSR_WRITE_4(sc, BGE_RX_RULES_CFG, 0x08);
1477
1478         /*
1479          * Configure number of RX lists. One interrupt distribution
1480          * list, sixteen active lists, one bad frames class.
1481          */
1482         CSR_WRITE_4(sc, BGE_RXLP_CFG, 0x181);
1483
1484         /* Inialize RX list placement stats mask. */
1485         CSR_WRITE_4(sc, BGE_RXLP_STATS_ENABLE_MASK, 0x007FFFFF);
1486         CSR_WRITE_4(sc, BGE_RXLP_STATS_CTL, 0x1);
1487
1488         /* Disable host coalescing until we get it set up */
1489         CSR_WRITE_4(sc, BGE_HCC_MODE, 0x00000000);
1490
1491         /* Poll to make sure it's shut down. */
1492         for (i = 0; i < BNX_TIMEOUT; i++) {
1493                 if (!(CSR_READ_4(sc, BGE_HCC_MODE) & BGE_HCCMODE_ENABLE))
1494                         break;
1495                 DELAY(10);
1496         }
1497
1498         if (i == BNX_TIMEOUT) {
1499                 if_printf(&sc->arpcom.ac_if,
1500                           "host coalescing engine failed to idle\n");
1501                 return(ENXIO);
1502         }
1503
1504         /* Set up host coalescing defaults */
1505         CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS, sc->bnx_rx_coal_ticks);
1506         CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS, sc->bnx_tx_coal_ticks);
1507         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS, sc->bnx_rx_coal_bds);
1508         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS, sc->bnx_tx_coal_bds);
1509         CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT, sc->bnx_rx_coal_bds_int);
1510         CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT, sc->bnx_tx_coal_bds_int);
1511
1512         /* Set up address of status block */
1513         bzero(sc->bnx_ldata.bnx_status_block, BGE_STATUS_BLK_SZ);
1514         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_HI,
1515             BGE_ADDR_HI(sc->bnx_ldata.bnx_status_block_paddr));
1516         CSR_WRITE_4(sc, BGE_HCC_STATUSBLK_ADDR_LO,
1517             BGE_ADDR_LO(sc->bnx_ldata.bnx_status_block_paddr));
1518
1519         /* Set up status block partail update size. */
1520         val = BGE_STATBLKSZ_32BYTE;
1521 #if 0
1522         /*
1523          * Does not seem to have visible effect in both
1524          * bulk data (1472B UDP datagram) and tiny data
1525          * (18B UDP datagram) TX tests.
1526          */
1527         val |= BGE_HCCMODE_CLRTICK_TX;
1528 #endif
1529         /* Turn on host coalescing state machine */
1530         CSR_WRITE_4(sc, BGE_HCC_MODE, val | BGE_HCCMODE_ENABLE);
1531
1532         /* Turn on RX BD completion state machine and enable attentions */
1533         CSR_WRITE_4(sc, BGE_RBDC_MODE,
1534             BGE_RBDCMODE_ENABLE|BGE_RBDCMODE_ATTN);
1535
1536         /* Turn on RX list placement state machine */
1537         CSR_WRITE_4(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
1538
1539         val = BGE_MACMODE_TXDMA_ENB | BGE_MACMODE_RXDMA_ENB |
1540             BGE_MACMODE_RX_STATS_CLEAR | BGE_MACMODE_TX_STATS_CLEAR |
1541             BGE_MACMODE_RX_STATS_ENB | BGE_MACMODE_TX_STATS_ENB |
1542             BGE_MACMODE_FRMHDR_DMA_ENB;
1543
1544         if (sc->bnx_flags & BNX_FLAG_TBI)
1545                 val |= BGE_PORTMODE_TBI;
1546         else if (sc->bnx_flags & BNX_FLAG_MII_SERDES)
1547                 val |= BGE_PORTMODE_GMII;
1548         else
1549                 val |= BGE_PORTMODE_MII;
1550
1551         /* Turn on DMA, clear stats */
1552         CSR_WRITE_4(sc, BGE_MAC_MODE, val);
1553
1554         /* Set misc. local control, enable interrupts on attentions */
1555         CSR_WRITE_4(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_INTR_ONATTN);
1556
1557 #ifdef notdef
1558         /* Assert GPIO pins for PHY reset */
1559         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUT0|
1560             BGE_MLC_MISCIO_OUT1|BGE_MLC_MISCIO_OUT2);
1561         BNX_SETBIT(sc, BGE_MISC_LOCAL_CTL, BGE_MLC_MISCIO_OUTEN0|
1562             BGE_MLC_MISCIO_OUTEN1|BGE_MLC_MISCIO_OUTEN2);
1563 #endif
1564
1565         /* Turn on write DMA state machine */
1566         val = BGE_WDMAMODE_ENABLE|BGE_WDMAMODE_ALL_ATTNS;
1567         /* Enable host coalescing bug fix. */
1568         val |= BGE_WDMAMODE_STATUS_TAG_FIX;
1569         if (sc->bnx_asicrev == BGE_ASICREV_BCM5785) {
1570                 /* Request larger DMA burst size to get better performance. */
1571                 val |= BGE_WDMAMODE_BURST_ALL_DATA;
1572         }
1573         CSR_WRITE_4(sc, BGE_WDMA_MODE, val);
1574         DELAY(40);
1575
1576         if (BNX_IS_57765_PLUS(sc)) {
1577                 uint32_t dmactl;
1578
1579                 dmactl = CSR_READ_4(sc, BGE_RDMA_RSRVCTRL);
1580                 /*
1581                  * Adjust tx margin to prevent TX data corruption and
1582                  * fix internal FIFO overflow.
1583                  */
1584                 if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1585                     sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1586                         dmactl &= ~(BGE_RDMA_RSRVCTRL_FIFO_LWM_MASK |
1587                             BGE_RDMA_RSRVCTRL_FIFO_HWM_MASK |
1588                             BGE_RDMA_RSRVCTRL_TXMRGN_MASK);
1589                         dmactl |= BGE_RDMA_RSRVCTRL_FIFO_LWM_1_5K |
1590                             BGE_RDMA_RSRVCTRL_FIFO_HWM_1_5K |
1591                             BGE_RDMA_RSRVCTRL_TXMRGN_320B;
1592                 }
1593                 /*
1594                  * Enable fix for read DMA FIFO overruns.
1595                  * The fix is to limit the number of RX BDs
1596                  * the hardware would fetch at a fime.
1597                  */
1598                 CSR_WRITE_4(sc, BGE_RDMA_RSRVCTRL,
1599                     dmactl | BGE_RDMA_RSRVCTRL_FIFO_OFLW_FIX);
1600         }
1601
1602         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719) {
1603                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1604                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1605                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_4K |
1606                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1607         } else if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1608                 /*
1609                  * Allow 4KB burst length reads for non-LSO frames.
1610                  * Enable 512B burst length reads for buffer descriptors.
1611                  */
1612                 CSR_WRITE_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL,
1613                     CSR_READ_4(sc, BGE_RDMA_LSO_CRPTEN_CTRL) |
1614                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_BD_512 |
1615                     BGE_RDMA_LSO_CRPTEN_CTRL_BLEN_LSO_4K);
1616         }
1617
1618         /* Turn on read DMA state machine */
1619         val = BGE_RDMAMODE_ENABLE | BGE_RDMAMODE_ALL_ATTNS;
1620         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717)
1621                 val |= BGE_RDMAMODE_MULT_DMA_RD_DIS;
1622         if (sc->bnx_asicrev == BGE_ASICREV_BCM5784 ||
1623             sc->bnx_asicrev == BGE_ASICREV_BCM5785 ||
1624             sc->bnx_asicrev == BGE_ASICREV_BCM57780) {
1625                 val |= BGE_RDMAMODE_BD_SBD_CRPT_ATTN |
1626                     BGE_RDMAMODE_MBUF_RBD_CRPT_ATTN |
1627                     BGE_RDMAMODE_MBUF_SBD_CRPT_ATTN;
1628         }
1629         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
1630                 val |= CSR_READ_4(sc, BGE_RDMA_MODE) &
1631                     BGE_RDMAMODE_H2BNC_VLAN_DET;
1632                 /*
1633                  * Allow multiple outstanding read requests from
1634                  * non-LSO read DMA engine.
1635                  */
1636                 val &= ~BGE_RDMAMODE_MULT_DMA_RD_DIS;
1637         }
1638         val |= BGE_RDMAMODE_FIFO_LONG_BURST;
1639         CSR_WRITE_4(sc, BGE_RDMA_MODE, val);
1640         DELAY(40);
1641
1642         /* Turn on RX data completion state machine */
1643         CSR_WRITE_4(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
1644
1645         /* Turn on RX BD initiator state machine */
1646         CSR_WRITE_4(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
1647
1648         /* Turn on RX data and RX BD initiator state machine */
1649         CSR_WRITE_4(sc, BGE_RDBDI_MODE, BGE_RDBDIMODE_ENABLE);
1650
1651         /* Turn on send BD completion state machine */
1652         CSR_WRITE_4(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
1653
1654         /* Turn on send data completion state machine */
1655         val = BGE_SDCMODE_ENABLE;
1656         if (sc->bnx_asicrev == BGE_ASICREV_BCM5761)
1657                 val |= BGE_SDCMODE_CDELAY; 
1658         CSR_WRITE_4(sc, BGE_SDC_MODE, val);
1659
1660         /* Turn on send data initiator state machine */
1661         CSR_WRITE_4(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
1662
1663         /* Turn on send BD initiator state machine */
1664         CSR_WRITE_4(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
1665
1666         /* Turn on send BD selector state machine */
1667         CSR_WRITE_4(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
1668
1669         CSR_WRITE_4(sc, BGE_SDI_STATS_ENABLE_MASK, 0x007FFFFF);
1670         CSR_WRITE_4(sc, BGE_SDI_STATS_CTL,
1671             BGE_SDISTATSCTL_ENABLE|BGE_SDISTATSCTL_FASTER);
1672
1673         /* ack/clear link change events */
1674         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1675             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1676             BGE_MACSTAT_LINK_CHANGED);
1677         CSR_WRITE_4(sc, BGE_MI_STS, 0);
1678
1679         /*
1680          * Enable attention when the link has changed state for
1681          * devices that use auto polling.
1682          */
1683         if (sc->bnx_flags & BNX_FLAG_TBI) {
1684                 CSR_WRITE_4(sc, BGE_MI_STS, BGE_MISTS_LINK);
1685         } else {
1686                 if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
1687                         CSR_WRITE_4(sc, BGE_MI_MODE, sc->bnx_mi_mode);
1688                         DELAY(80);
1689                 }
1690         }
1691
1692         /*
1693          * Clear any pending link state attention.
1694          * Otherwise some link state change events may be lost until attention
1695          * is cleared by bnx_intr() -> bnx_softc.bnx_link_upd() sequence.
1696          * It's not necessary on newer BCM chips - perhaps enabling link
1697          * state change attentions implies clearing pending attention.
1698          */
1699         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED|
1700             BGE_MACSTAT_CFG_CHANGED|BGE_MACSTAT_MI_COMPLETE|
1701             BGE_MACSTAT_LINK_CHANGED);
1702
1703         /* Enable link state change attentions. */
1704         BNX_SETBIT(sc, BGE_MAC_EVT_ENB, BGE_EVTENB_LINK_CHANGED);
1705
1706         return(0);
1707 }
1708
1709 /*
1710  * Probe for a Broadcom chip. Check the PCI vendor and device IDs
1711  * against our list and return its name if we find a match. Note
1712  * that since the Broadcom controller contains VPD support, we
1713  * can get the device name string from the controller itself instead
1714  * of the compiled-in string. This is a little slow, but it guarantees
1715  * we'll always announce the right product name.
1716  */
1717 static int
1718 bnx_probe(device_t dev)
1719 {
1720         const struct bnx_type *t;
1721         uint16_t product, vendor;
1722
1723         if (!pci_is_pcie(dev))
1724                 return ENXIO;
1725
1726         product = pci_get_device(dev);
1727         vendor = pci_get_vendor(dev);
1728
1729         for (t = bnx_devs; t->bnx_name != NULL; t++) {
1730                 if (vendor == t->bnx_vid && product == t->bnx_did)
1731                         break;
1732         }
1733         if (t->bnx_name == NULL)
1734                 return ENXIO;
1735
1736         device_set_desc(dev, t->bnx_name);
1737         return 0;
1738 }
1739
1740 static int
1741 bnx_attach(device_t dev)
1742 {
1743         struct ifnet *ifp;
1744         struct bnx_softc *sc;
1745         uint32_t hwcfg = 0, misccfg;
1746         int error = 0, rid, capmask;
1747         uint8_t ether_addr[ETHER_ADDR_LEN];
1748         uint16_t product, vendor;
1749         driver_intr_t *intr_func;
1750         uintptr_t mii_priv = 0;
1751         u_int intr_flags;
1752
1753         sc = device_get_softc(dev);
1754         sc->bnx_dev = dev;
1755         callout_init_mp(&sc->bnx_stat_timer);
1756         callout_init_mp(&sc->bnx_intr_timer);
1757         lwkt_serialize_init(&sc->bnx_jslot_serializer);
1758
1759         product = pci_get_device(dev);
1760         vendor = pci_get_vendor(dev);
1761
1762 #ifndef BURN_BRIDGES
1763         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
1764                 uint32_t irq, mem;
1765
1766                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
1767                 mem = pci_read_config(dev, BGE_PCI_BAR0, 4);
1768
1769                 device_printf(dev, "chip is in D%d power mode "
1770                     "-- setting to D0\n", pci_get_powerstate(dev));
1771
1772                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
1773
1774                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
1775                 pci_write_config(dev, BGE_PCI_BAR0, mem, 4);
1776         }
1777 #endif  /* !BURN_BRIDGE */
1778
1779         /*
1780          * Map control/status registers.
1781          */
1782         pci_enable_busmaster(dev);
1783
1784         rid = BGE_PCI_BAR0;
1785         sc->bnx_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1786             RF_ACTIVE);
1787
1788         if (sc->bnx_res == NULL) {
1789                 device_printf(dev, "couldn't map memory\n");
1790                 return ENXIO;
1791         }
1792
1793         sc->bnx_btag = rman_get_bustag(sc->bnx_res);
1794         sc->bnx_bhandle = rman_get_bushandle(sc->bnx_res);
1795
1796         /* Save various chip information */
1797         sc->bnx_chipid =
1798             pci_read_config(dev, BGE_PCI_MISC_CTL, 4) >>
1799             BGE_PCIMISCCTL_ASICREV_SHIFT;
1800         if (BGE_ASICREV(sc->bnx_chipid) == BGE_ASICREV_USE_PRODID_REG) {
1801                 /* All chips having dedicated ASICREV register have CPMU */
1802                 sc->bnx_flags |= BNX_FLAG_CPMU;
1803
1804                 switch (product) {
1805                 case PCI_PRODUCT_BROADCOM_BCM5717:
1806                 case PCI_PRODUCT_BROADCOM_BCM5718:
1807                 case PCI_PRODUCT_BROADCOM_BCM5719:
1808                 case PCI_PRODUCT_BROADCOM_BCM5720_ALT:
1809                         sc->bnx_chipid = pci_read_config(dev,
1810                             BGE_PCI_GEN2_PRODID_ASICREV, 4);
1811                         break;
1812
1813                 case PCI_PRODUCT_BROADCOM_BCM57761:
1814                 case PCI_PRODUCT_BROADCOM_BCM57762:
1815                 case PCI_PRODUCT_BROADCOM_BCM57765:
1816                 case PCI_PRODUCT_BROADCOM_BCM57766:
1817                 case PCI_PRODUCT_BROADCOM_BCM57781:
1818                 case PCI_PRODUCT_BROADCOM_BCM57782:
1819                 case PCI_PRODUCT_BROADCOM_BCM57785:
1820                 case PCI_PRODUCT_BROADCOM_BCM57786:
1821                 case PCI_PRODUCT_BROADCOM_BCM57791:
1822                 case PCI_PRODUCT_BROADCOM_BCM57795:
1823                         sc->bnx_chipid = pci_read_config(dev,
1824                             BGE_PCI_GEN15_PRODID_ASICREV, 4);
1825                         break;
1826
1827                 default:
1828                         sc->bnx_chipid = pci_read_config(dev,
1829                             BGE_PCI_PRODID_ASICREV, 4);
1830                         break;
1831                 }
1832         }
1833         sc->bnx_asicrev = BGE_ASICREV(sc->bnx_chipid);
1834         sc->bnx_chiprev = BGE_CHIPREV(sc->bnx_chipid);
1835
1836         switch (sc->bnx_asicrev) {
1837         case BGE_ASICREV_BCM5717:
1838         case BGE_ASICREV_BCM5719:
1839         case BGE_ASICREV_BCM5720:
1840                 sc->bnx_flags |= BNX_FLAG_5717_PLUS | BNX_FLAG_57765_PLUS;
1841                 break;
1842
1843         case BGE_ASICREV_BCM57765:
1844         case BGE_ASICREV_BCM57766:
1845                 sc->bnx_flags |= BNX_FLAG_57765_FAMILY | BNX_FLAG_57765_PLUS;
1846                 break;
1847         }
1848         sc->bnx_flags |= BNX_FLAG_SHORTDMA;
1849
1850         if (sc->bnx_asicrev == BGE_ASICREV_BCM5717 ||
1851             BNX_IS_57765_FAMILY(sc)) {
1852                 /*
1853                  * All BCM57785 and BCM5718 families chips have a bug that
1854                  * under certain situation interrupt will not be enabled
1855                  * even if status tag is written to BGE_MBX_IRQ0_LO mailbox.
1856                  *
1857                  * While BCM5719 and BCM5720 have a hardware workaround
1858                  * which could fix the above bug.
1859                  * See the comment near BGE_PCIDMARWCTL_TAGGED_STATUS_WA in
1860                  * bnx_chipinit().
1861                  *
1862                  * For the rest of the chips in these two families, we will
1863                  * have to poll the status block at high rate (10ms currently)
1864                  * to check whether the interrupt is hosed or not.
1865                  * See bnx_intr_check() for details.
1866                  */
1867                 sc->bnx_flags |= BNX_FLAG_STATUSTAG_BUG;
1868         }
1869
1870         misccfg = CSR_READ_4(sc, BGE_MISC_CFG) & BGE_MISCCFG_BOARD_ID_MASK;
1871
1872         sc->bnx_pciecap = pci_get_pciecap_ptr(sc->bnx_dev);
1873         if (sc->bnx_asicrev == BGE_ASICREV_BCM5719 ||
1874             sc->bnx_asicrev == BGE_ASICREV_BCM5720)
1875                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_2048);
1876         else
1877                 pcie_set_max_readrq(dev, PCIEM_DEVCTL_MAX_READRQ_4096);
1878         device_printf(dev, "CHIP ID 0x%08x; "
1879                       "ASIC REV 0x%02x; CHIP REV 0x%02x\n",
1880                       sc->bnx_chipid, sc->bnx_asicrev, sc->bnx_chiprev);
1881
1882         /*
1883          * Set various PHY quirk flags.
1884          */
1885
1886         capmask = MII_CAPMASK_DEFAULT;
1887         if (product == PCI_PRODUCT_BROADCOM_BCM57791 ||
1888             product == PCI_PRODUCT_BROADCOM_BCM57795) {
1889                 /* 10/100 only */
1890                 capmask &= ~BMSR_EXTSTAT;
1891         }
1892
1893         mii_priv |= BRGPHY_FLAG_WIRESPEED;
1894
1895         /*
1896          * Allocate interrupt
1897          */
1898         sc->bnx_irq_type = pci_alloc_1intr(dev, bnx_msi_enable, &sc->bnx_irq_rid,
1899             &intr_flags);
1900
1901         sc->bnx_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->bnx_irq_rid,
1902             intr_flags);
1903         if (sc->bnx_irq == NULL) {
1904                 device_printf(dev, "couldn't map interrupt\n");
1905                 error = ENXIO;
1906                 goto fail;
1907         }
1908
1909         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
1910                 sc->bnx_flags |= BNX_FLAG_ONESHOT_MSI;
1911                 bnx_enable_msi(sc);
1912         }
1913
1914         /* Initialize if_name earlier, so if_printf could be used */
1915         ifp = &sc->arpcom.ac_if;
1916         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1917
1918         /* Try to reset the chip. */
1919         bnx_reset(sc);
1920
1921         if (bnx_chipinit(sc)) {
1922                 device_printf(dev, "chip initialization failed\n");
1923                 error = ENXIO;
1924                 goto fail;
1925         }
1926
1927         /*
1928          * Get station address
1929          */
1930         error = bnx_get_eaddr(sc, ether_addr);
1931         if (error) {
1932                 device_printf(dev, "failed to read station address\n");
1933                 goto fail;
1934         }
1935
1936         if (BNX_IS_57765_PLUS(sc)) {
1937                 sc->bnx_return_ring_cnt = BGE_RETURN_RING_CNT;
1938         } else {
1939                 /* 5705/5750 limits RX return ring to 512 entries. */
1940                 sc->bnx_return_ring_cnt = BGE_RETURN_RING_CNT_5705;
1941         }
1942
1943         error = bnx_dma_alloc(sc);
1944         if (error)
1945                 goto fail;
1946
1947         /* Set default tuneable values. */
1948         sc->bnx_rx_coal_ticks = BNX_RX_COAL_TICKS_DEF;
1949         sc->bnx_tx_coal_ticks = BNX_TX_COAL_TICKS_DEF;
1950         sc->bnx_rx_coal_bds = BNX_RX_COAL_BDS_DEF;
1951         sc->bnx_tx_coal_bds = BNX_TX_COAL_BDS_DEF;
1952         sc->bnx_rx_coal_bds_int = BNX_RX_COAL_BDS_DEF;
1953         sc->bnx_tx_coal_bds_int = BNX_TX_COAL_BDS_DEF;
1954
1955         /* Set up ifnet structure */
1956         ifp->if_softc = sc;
1957         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1958         ifp->if_ioctl = bnx_ioctl;
1959         ifp->if_start = bnx_start;
1960 #ifdef DEVICE_POLLING
1961         ifp->if_poll = bnx_poll;
1962 #endif
1963         ifp->if_watchdog = bnx_watchdog;
1964         ifp->if_init = bnx_init;
1965         ifp->if_mtu = ETHERMTU;
1966         ifp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
1967         ifq_set_maxlen(&ifp->if_snd, BGE_TX_RING_CNT - 1);
1968         ifq_set_ready(&ifp->if_snd);
1969
1970         ifp->if_capabilities |= IFCAP_HWCSUM;
1971         ifp->if_hwassist = BNX_CSUM_FEATURES;
1972         ifp->if_capenable = ifp->if_capabilities;
1973
1974         /*
1975          * Figure out what sort of media we have by checking the
1976          * hardware config word in the first 32k of NIC internal memory,
1977          * or fall back to examining the EEPROM if necessary.
1978          * Note: on some BCM5700 cards, this value appears to be unset.
1979          * If that's the case, we have to rely on identifying the NIC
1980          * by its PCI subsystem ID, as we do below for the SysKonnect
1981          * SK-9D41.
1982          */
1983         if (bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_SIG) == BGE_MAGIC_NUMBER) {
1984                 hwcfg = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM_NICCFG);
1985         } else {
1986                 if (bnx_read_eeprom(sc, (caddr_t)&hwcfg, BGE_EE_HWCFG_OFFSET,
1987                                     sizeof(hwcfg))) {
1988                         device_printf(dev, "failed to read EEPROM\n");
1989                         error = ENXIO;
1990                         goto fail;
1991                 }
1992                 hwcfg = ntohl(hwcfg);
1993         }
1994
1995         /* The SysKonnect SK-9D41 is a 1000baseSX card. */
1996         if (pci_get_subvendor(dev) == PCI_PRODUCT_SCHNEIDERKOCH_SK_9D41 ||
1997             (hwcfg & BGE_HWCFG_MEDIA) == BGE_MEDIA_FIBER)
1998                 sc->bnx_flags |= BNX_FLAG_TBI;
1999
2000         /* Setup MI MODE */
2001         if (sc->bnx_flags & BNX_FLAG_CPMU)
2002                 sc->bnx_mi_mode = BGE_MIMODE_500KHZ_CONST;
2003         else
2004                 sc->bnx_mi_mode = BGE_MIMODE_BASE;
2005
2006         /* Setup link status update stuffs */
2007         if (sc->bnx_flags & BNX_FLAG_TBI) {
2008                 sc->bnx_link_upd = bnx_tbi_link_upd;
2009                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2010         } else if (sc->bnx_mi_mode & BGE_MIMODE_AUTOPOLL) {
2011                 sc->bnx_link_upd = bnx_autopoll_link_upd;
2012                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2013         } else {
2014                 sc->bnx_link_upd = bnx_copper_link_upd;
2015                 sc->bnx_link_chg = BGE_MACSTAT_LINK_CHANGED;
2016         }
2017
2018         /* Set default PHY address */
2019         sc->bnx_phyno = 1;
2020
2021         /*
2022          * PHY address mapping for various devices.
2023          *
2024          *          | F0 Cu | F0 Sr | F1 Cu | F1 Sr |
2025          * ---------+-------+-------+-------+-------+
2026          * BCM57XX  |   1   |   X   |   X   |   X   |
2027          * BCM5704  |   1   |   X   |   1   |   X   |
2028          * BCM5717  |   1   |   8   |   2   |   9   |
2029          * BCM5719  |   1   |   8   |   2   |   9   |
2030          * BCM5720  |   1   |   8   |   2   |   9   |
2031          *
2032          * Other addresses may respond but they are not
2033          * IEEE compliant PHYs and should be ignored.
2034          */
2035         if (BNX_IS_5717_PLUS(sc)) {
2036                 int f;
2037
2038                 f = pci_get_function(dev);
2039                 if (sc->bnx_chipid == BGE_CHIPID_BCM5717_A0) {
2040                         if (CSR_READ_4(sc, BGE_SGDIG_STS) &
2041                             BGE_SGDIGSTS_IS_SERDES)
2042                                 sc->bnx_phyno = f + 8;
2043                         else
2044                                 sc->bnx_phyno = f + 1;
2045                 } else {
2046                         if (CSR_READ_4(sc, BGE_CPMU_PHY_STRAP) &
2047                             BGE_CPMU_PHY_STRAP_IS_SERDES)
2048                                 sc->bnx_phyno = f + 8;
2049                         else
2050                                 sc->bnx_phyno = f + 1;
2051                 }
2052         }
2053
2054         if (sc->bnx_flags & BNX_FLAG_TBI) {
2055                 ifmedia_init(&sc->bnx_ifmedia, IFM_IMASK,
2056                     bnx_ifmedia_upd, bnx_ifmedia_sts);
2057                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_1000_SX, 0, NULL);
2058                 ifmedia_add(&sc->bnx_ifmedia,
2059                     IFM_ETHER|IFM_1000_SX|IFM_FDX, 0, NULL);
2060                 ifmedia_add(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO, 0, NULL);
2061                 ifmedia_set(&sc->bnx_ifmedia, IFM_ETHER|IFM_AUTO);
2062                 sc->bnx_ifmedia.ifm_media = sc->bnx_ifmedia.ifm_cur->ifm_media;
2063         } else {
2064                 struct mii_probe_args mii_args;
2065
2066                 mii_probe_args_init(&mii_args, bnx_ifmedia_upd, bnx_ifmedia_sts);
2067                 mii_args.mii_probemask = 1 << sc->bnx_phyno;
2068                 mii_args.mii_capmask = capmask;
2069                 mii_args.mii_privtag = MII_PRIVTAG_BRGPHY;
2070                 mii_args.mii_priv = mii_priv;
2071
2072                 error = mii_probe(dev, &sc->bnx_miibus, &mii_args);
2073                 if (error) {
2074                         device_printf(dev, "MII without any PHY!\n");
2075                         goto fail;
2076                 }
2077         }
2078
2079         /*
2080          * Create sysctl nodes.
2081          */
2082         sysctl_ctx_init(&sc->bnx_sysctl_ctx);
2083         sc->bnx_sysctl_tree = SYSCTL_ADD_NODE(&sc->bnx_sysctl_ctx,
2084                                               SYSCTL_STATIC_CHILDREN(_hw),
2085                                               OID_AUTO,
2086                                               device_get_nameunit(dev),
2087                                               CTLFLAG_RD, 0, "");
2088         if (sc->bnx_sysctl_tree == NULL) {
2089                 device_printf(dev, "can't add sysctl node\n");
2090                 error = ENXIO;
2091                 goto fail;
2092         }
2093
2094         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2095                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2096                         OID_AUTO, "rx_coal_ticks",
2097                         CTLTYPE_INT | CTLFLAG_RW,
2098                         sc, 0, bnx_sysctl_rx_coal_ticks, "I",
2099                         "Receive coalescing ticks (usec).");
2100         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2101                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2102                         OID_AUTO, "tx_coal_ticks",
2103                         CTLTYPE_INT | CTLFLAG_RW,
2104                         sc, 0, bnx_sysctl_tx_coal_ticks, "I",
2105                         "Transmit coalescing ticks (usec).");
2106         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2107                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2108                         OID_AUTO, "rx_coal_bds",
2109                         CTLTYPE_INT | CTLFLAG_RW,
2110                         sc, 0, bnx_sysctl_rx_coal_bds, "I",
2111                         "Receive max coalesced BD count.");
2112         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2113                         SYSCTL_CHILDREN(sc->bnx_sysctl_tree),
2114                         OID_AUTO, "tx_coal_bds",
2115                         CTLTYPE_INT | CTLFLAG_RW,
2116                         sc, 0, bnx_sysctl_tx_coal_bds, "I",
2117                         "Transmit max coalesced BD count.");
2118         /*
2119          * A common design characteristic for many Broadcom
2120          * client controllers is that they only support a
2121          * single outstanding DMA read operation on the PCIe
2122          * bus. This means that it will take twice as long to
2123          * fetch a TX frame that is split into header and
2124          * payload buffers as it does to fetch a single,
2125          * contiguous TX frame (2 reads vs. 1 read). For these
2126          * controllers, coalescing buffers to reduce the number
2127          * of memory reads is effective way to get maximum
2128          * performance(about 940Mbps).  Without collapsing TX
2129          * buffers the maximum TCP bulk transfer performance
2130          * is about 850Mbps. However forcing coalescing mbufs
2131          * consumes a lot of CPU cycles, so leave it off by
2132          * default.
2133          */
2134         SYSCTL_ADD_INT(&sc->bnx_sysctl_ctx,
2135             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2136             "force_defrag", CTLFLAG_RW, &sc->bnx_force_defrag, 0,
2137             "Force defragment on TX path");
2138
2139         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2140             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2141             "rx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2142             sc, 0, bnx_sysctl_rx_coal_bds_int, "I",
2143             "Receive max coalesced BD count during interrupt.");
2144         SYSCTL_ADD_PROC(&sc->bnx_sysctl_ctx,
2145             SYSCTL_CHILDREN(sc->bnx_sysctl_tree), OID_AUTO,
2146             "tx_coal_bds_int", CTLTYPE_INT | CTLFLAG_RW,
2147             sc, 0, bnx_sysctl_tx_coal_bds_int, "I",
2148             "Transmit max coalesced BD count during interrupt.");
2149
2150         /*
2151          * Call MI attach routine.
2152          */
2153         ether_ifattach(ifp, ether_addr, NULL);
2154
2155         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
2156                 if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
2157                         intr_func = bnx_msi_oneshot;
2158                         if (bootverbose)
2159                                 device_printf(dev, "oneshot MSI\n");
2160                 } else {
2161                         intr_func = bnx_msi;
2162                 }
2163         } else {
2164                 intr_func = bnx_intr_legacy;
2165         }
2166         error = bus_setup_intr(dev, sc->bnx_irq, INTR_MPSAFE, intr_func, sc,
2167             &sc->bnx_intrhand, ifp->if_serializer);
2168         if (error) {
2169                 ether_ifdetach(ifp);
2170                 device_printf(dev, "couldn't set up irq\n");
2171                 goto fail;
2172         }
2173
2174         ifp->if_cpuid = rman_get_cpuid(sc->bnx_irq);
2175         KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus);
2176
2177         sc->bnx_stat_cpuid = ifp->if_cpuid;
2178         sc->bnx_intr_cpuid = ifp->if_cpuid;
2179
2180         return(0);
2181 fail:
2182         bnx_detach(dev);
2183         return(error);
2184 }
2185
2186 static int
2187 bnx_detach(device_t dev)
2188 {
2189         struct bnx_softc *sc = device_get_softc(dev);
2190
2191         if (device_is_attached(dev)) {
2192                 struct ifnet *ifp = &sc->arpcom.ac_if;
2193
2194                 lwkt_serialize_enter(ifp->if_serializer);
2195                 bnx_stop(sc);
2196                 bnx_reset(sc);
2197                 bus_teardown_intr(dev, sc->bnx_irq, sc->bnx_intrhand);
2198                 lwkt_serialize_exit(ifp->if_serializer);
2199
2200                 ether_ifdetach(ifp);
2201         }
2202
2203         if (sc->bnx_flags & BNX_FLAG_TBI)
2204                 ifmedia_removeall(&sc->bnx_ifmedia);
2205         if (sc->bnx_miibus)
2206                 device_delete_child(dev, sc->bnx_miibus);
2207         bus_generic_detach(dev);
2208
2209         if (sc->bnx_irq != NULL) {
2210                 bus_release_resource(dev, SYS_RES_IRQ, sc->bnx_irq_rid,
2211                     sc->bnx_irq);
2212         }
2213         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI)
2214                 pci_release_msi(dev);
2215
2216         if (sc->bnx_res != NULL) {
2217                 bus_release_resource(dev, SYS_RES_MEMORY,
2218                     BGE_PCI_BAR0, sc->bnx_res);
2219         }
2220
2221         if (sc->bnx_sysctl_tree != NULL)
2222                 sysctl_ctx_free(&sc->bnx_sysctl_ctx);
2223
2224         bnx_dma_free(sc);
2225
2226         return 0;
2227 }
2228
2229 static void
2230 bnx_reset(struct bnx_softc *sc)
2231 {
2232         device_t dev;
2233         uint32_t cachesize, command, pcistate, reset;
2234         void (*write_op)(struct bnx_softc *, uint32_t, uint32_t);
2235         int i, val = 0;
2236         uint16_t devctl;
2237
2238         dev = sc->bnx_dev;
2239
2240         if (sc->bnx_asicrev != BGE_ASICREV_BCM5906)
2241                 write_op = bnx_writemem_direct;
2242         else
2243                 write_op = bnx_writereg_ind;
2244
2245         /* Save some important PCI state. */
2246         cachesize = pci_read_config(dev, BGE_PCI_CACHESZ, 4);
2247         command = pci_read_config(dev, BGE_PCI_CMD, 4);
2248         pcistate = pci_read_config(dev, BGE_PCI_PCISTATE, 4);
2249
2250         pci_write_config(dev, BGE_PCI_MISC_CTL,
2251             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2252             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2253             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2254
2255         /* Disable fastboot on controllers that support it. */
2256         if (bootverbose)
2257                 if_printf(&sc->arpcom.ac_if, "Disabling fastboot\n");
2258         CSR_WRITE_4(sc, BGE_FASTBOOT_PC, 0x0);
2259
2260         /*
2261          * Write the magic number to SRAM at offset 0xB50.
2262          * When firmware finishes its initialization it will
2263          * write ~BGE_MAGIC_NUMBER to the same location.
2264          */
2265         bnx_writemem_ind(sc, BGE_SOFTWARE_GENCOMM, BGE_MAGIC_NUMBER);
2266
2267         reset = BGE_MISCCFG_RESET_CORE_CLOCKS|(65<<1);
2268
2269         /* XXX: Broadcom Linux driver. */
2270         /* Force PCI-E 1.0a mode */
2271         if (!BNX_IS_57765_PLUS(sc) &&
2272             CSR_READ_4(sc, BGE_PCIE_PHY_TSTCTL) ==
2273             (BGE_PCIE_PHY_TSTCTL_PSCRAM |
2274              BGE_PCIE_PHY_TSTCTL_PCIE10)) {
2275                 CSR_WRITE_4(sc, BGE_PCIE_PHY_TSTCTL,
2276                     BGE_PCIE_PHY_TSTCTL_PSCRAM);
2277         }
2278         if (sc->bnx_chipid != BGE_CHIPID_BCM5750_A0) {
2279                 /* Prevent PCIE link training during global reset */
2280                 CSR_WRITE_4(sc, BGE_MISC_CFG, (1<<29));
2281                 reset |= (1<<29);
2282         }
2283
2284         /* 
2285          * Set GPHY Power Down Override to leave GPHY
2286          * powered up in D0 uninitialized.
2287          */
2288         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0)
2289                 reset |= BGE_MISCCFG_GPHY_PD_OVERRIDE;
2290
2291         /* Issue global reset */
2292         write_op(sc, BGE_MISC_CFG, reset);
2293
2294         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
2295                 uint32_t status, ctrl;
2296
2297                 status = CSR_READ_4(sc, BGE_VCPU_STATUS);
2298                 CSR_WRITE_4(sc, BGE_VCPU_STATUS,
2299                     status | BGE_VCPU_STATUS_DRV_RESET);
2300                 ctrl = CSR_READ_4(sc, BGE_VCPU_EXT_CTRL);
2301                 CSR_WRITE_4(sc, BGE_VCPU_EXT_CTRL,
2302                     ctrl & ~BGE_VCPU_EXT_CTRL_HALT_CPU);
2303         }
2304
2305         DELAY(1000);
2306
2307         /* XXX: Broadcom Linux driver. */
2308         if (sc->bnx_chipid == BGE_CHIPID_BCM5750_A0) {
2309                 uint32_t v;
2310
2311                 DELAY(500000); /* wait for link training to complete */
2312                 v = pci_read_config(dev, 0xc4, 4);
2313                 pci_write_config(dev, 0xc4, v | (1<<15), 4);
2314         }
2315
2316         devctl = pci_read_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL, 2);
2317
2318         /* Disable no snoop and disable relaxed ordering. */
2319         devctl &= ~(PCIEM_DEVCTL_RELAX_ORDER | PCIEM_DEVCTL_NOSNOOP);
2320
2321         /* Old PCI-E chips only support 128 bytes Max PayLoad Size. */
2322         if ((sc->bnx_flags & BNX_FLAG_CPMU) == 0) {
2323                 devctl &= ~PCIEM_DEVCTL_MAX_PAYLOAD_MASK;
2324                 devctl |= PCIEM_DEVCTL_MAX_PAYLOAD_128;
2325         }
2326
2327         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVCTRL,
2328             devctl, 2);
2329
2330         /* Clear error status. */
2331         pci_write_config(dev, sc->bnx_pciecap + PCIER_DEVSTS,
2332             PCIEM_DEVSTS_CORR_ERR |
2333             PCIEM_DEVSTS_NFATAL_ERR |
2334             PCIEM_DEVSTS_FATAL_ERR |
2335             PCIEM_DEVSTS_UNSUPP_REQ, 2);
2336
2337         /* Reset some of the PCI state that got zapped by reset */
2338         pci_write_config(dev, BGE_PCI_MISC_CTL,
2339             BGE_PCIMISCCTL_INDIRECT_ACCESS|BGE_PCIMISCCTL_MASK_PCI_INTR|
2340             BGE_HIF_SWAP_OPTIONS|BGE_PCIMISCCTL_PCISTATE_RW|
2341             BGE_PCIMISCCTL_TAGGED_STATUS, 4);
2342         pci_write_config(dev, BGE_PCI_CACHESZ, cachesize, 4);
2343         pci_write_config(dev, BGE_PCI_CMD, command, 4);
2344         write_op(sc, BGE_MISC_CFG, (65 << 1));
2345
2346         /* Enable memory arbiter */
2347         CSR_WRITE_4(sc, BGE_MARB_MODE, BGE_MARBMODE_ENABLE);
2348
2349         if (sc->bnx_asicrev == BGE_ASICREV_BCM5906) {
2350                 for (i = 0; i < BNX_TIMEOUT; i++) {
2351                         val = CSR_READ_4(sc, BGE_VCPU_STATUS);
2352                         if (val & BGE_VCPU_STATUS_INIT_DONE)
2353                                 break;
2354                         DELAY(100);
2355                 }
2356                 if (i == BNX_TIMEOUT) {
2357                         if_printf(&sc->arpcom.ac_if, "reset timed out\n");
2358                         return;
2359                 }
2360         } else {
2361                 /*
2362                  * Poll until we see the 1's complement of the magic number.
2363                  * This indicates that the firmware initialization
2364                  * is complete.
2365                  */
2366                 for (i = 0; i < BNX_FIRMWARE_TIMEOUT; i++) {
2367                         val = bnx_readmem_ind(sc, BGE_SOFTWARE_GENCOMM);
2368                         if (val == ~BGE_MAGIC_NUMBER)
2369                                 break;
2370                         DELAY(10);
2371                 }
2372                 if (i == BNX_FIRMWARE_TIMEOUT) {
2373                         if_printf(&sc->arpcom.ac_if, "firmware handshake "
2374                                   "timed out, found 0x%08x\n", val);
2375                 }
2376
2377                 /* BCM57765 A0 needs additional time before accessing. */
2378                 if (sc->bnx_chipid == BGE_CHIPID_BCM57765_A0)
2379                         DELAY(10 * 1000);
2380         }
2381
2382         /*
2383          * XXX Wait for the value of the PCISTATE register to
2384          * return to its original pre-reset state. This is a
2385          * fairly good indicator of reset completion. If we don't
2386          * wait for the reset to fully complete, trying to read
2387          * from the device's non-PCI registers may yield garbage
2388          * results.
2389          */
2390         for (i = 0; i < BNX_TIMEOUT; i++) {
2391                 if (pci_read_config(dev, BGE_PCI_PCISTATE, 4) == pcistate)
2392                         break;
2393                 DELAY(10);
2394         }
2395
2396         /* Fix up byte swapping */
2397         CSR_WRITE_4(sc, BGE_MODE_CTL, bnx_dma_swap_options(sc));
2398
2399         CSR_WRITE_4(sc, BGE_MAC_MODE, 0);
2400
2401         /*
2402          * The 5704 in TBI mode apparently needs some special
2403          * adjustment to insure the SERDES drive level is set
2404          * to 1.2V.
2405          */
2406         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704 &&
2407             (sc->bnx_flags & BNX_FLAG_TBI)) {
2408                 uint32_t serdescfg;
2409
2410                 serdescfg = CSR_READ_4(sc, BGE_SERDES_CFG);
2411                 serdescfg = (serdescfg & ~0xFFF) | 0x880;
2412                 CSR_WRITE_4(sc, BGE_SERDES_CFG, serdescfg);
2413         }
2414
2415         /* XXX: Broadcom Linux driver. */
2416         if (!BNX_IS_57765_PLUS(sc)) {
2417                 uint32_t v;
2418
2419                 /* Enable Data FIFO protection. */
2420                 v = CSR_READ_4(sc, BGE_PCIE_TLDLPL_PORT);
2421                 CSR_WRITE_4(sc, BGE_PCIE_TLDLPL_PORT, v | (1 << 25));
2422         }
2423
2424         DELAY(10000);
2425
2426         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
2427                 BNX_CLRBIT(sc, BGE_CPMU_CLCK_ORIDE,
2428                     CPMU_CLCK_ORIDE_MAC_ORIDE_EN);
2429         }
2430 }
2431
2432 /*
2433  * Frame reception handling. This is called if there's a frame
2434  * on the receive return list.
2435  *
2436  * Note: we have to be able to handle two possibilities here:
2437  * 1) the frame is from the jumbo recieve ring
2438  * 2) the frame is from the standard receive ring
2439  */
2440
2441 static void
2442 bnx_rxeof(struct bnx_softc *sc, uint16_t rx_prod)
2443 {
2444         struct ifnet *ifp;
2445         int stdcnt = 0, jumbocnt = 0;
2446
2447         ifp = &sc->arpcom.ac_if;
2448
2449         while (sc->bnx_rx_saved_considx != rx_prod) {
2450                 struct bge_rx_bd        *cur_rx;
2451                 uint32_t                rxidx;
2452                 struct mbuf             *m = NULL;
2453                 uint16_t                vlan_tag = 0;
2454                 int                     have_tag = 0;
2455
2456                 cur_rx =
2457             &sc->bnx_ldata.bnx_rx_return_ring[sc->bnx_rx_saved_considx];
2458
2459                 rxidx = cur_rx->bge_idx;
2460                 BNX_INC(sc->bnx_rx_saved_considx, sc->bnx_return_ring_cnt);
2461
2462                 if (cur_rx->bge_flags & BGE_RXBDFLAG_VLAN_TAG) {
2463                         have_tag = 1;
2464                         vlan_tag = cur_rx->bge_vlan_tag;
2465                 }
2466
2467                 if (cur_rx->bge_flags & BGE_RXBDFLAG_JUMBO_RING) {
2468                         BNX_INC(sc->bnx_jumbo, BGE_JUMBO_RX_RING_CNT);
2469                         jumbocnt++;
2470
2471                         if (rxidx != sc->bnx_jumbo) {
2472                                 ifp->if_ierrors++;
2473                                 if_printf(ifp, "sw jumbo index(%d) "
2474                                     "and hw jumbo index(%d) mismatch, drop!\n",
2475                                     sc->bnx_jumbo, rxidx);
2476                                 bnx_setup_rxdesc_jumbo(sc, rxidx);
2477                                 continue;
2478                         }
2479
2480                         m = sc->bnx_cdata.bnx_rx_jumbo_chain[rxidx].bnx_mbuf;
2481                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2482                                 ifp->if_ierrors++;
2483                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2484                                 continue;
2485                         }
2486                         if (bnx_newbuf_jumbo(sc, sc->bnx_jumbo, 0)) {
2487                                 ifp->if_ierrors++;
2488                                 bnx_setup_rxdesc_jumbo(sc, sc->bnx_jumbo);
2489                                 continue;
2490                         }
2491                 } else {
2492                         BNX_INC(sc->bnx_std, BGE_STD_RX_RING_CNT);
2493                         stdcnt++;
2494
2495                         if (rxidx != sc->bnx_std) {
2496                                 ifp->if_ierrors++;
2497                                 if_printf(ifp, "sw std index(%d) "
2498                                     "and hw std index(%d) mismatch, drop!\n",
2499                                     sc->bnx_std, rxidx);
2500                                 bnx_setup_rxdesc_std(sc, rxidx);
2501                                 continue;
2502                         }
2503
2504                         m = sc->bnx_cdata.bnx_rx_std_chain[rxidx].bnx_mbuf;
2505                         if (cur_rx->bge_flags & BGE_RXBDFLAG_ERROR) {
2506                                 ifp->if_ierrors++;
2507                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2508                                 continue;
2509                         }
2510                         if (bnx_newbuf_std(sc, sc->bnx_std, 0)) {
2511                                 ifp->if_ierrors++;
2512                                 bnx_setup_rxdesc_std(sc, sc->bnx_std);
2513                                 continue;
2514                         }
2515                 }
2516
2517                 ifp->if_ipackets++;
2518                 m->m_pkthdr.len = m->m_len = cur_rx->bge_len - ETHER_CRC_LEN;
2519                 m->m_pkthdr.rcvif = ifp;
2520
2521                 if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2522                     (cur_rx->bge_flags & BGE_RXBDFLAG_IPV6) == 0) {
2523                         if (cur_rx->bge_flags & BGE_RXBDFLAG_IP_CSUM) {
2524                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2525                                 if ((cur_rx->bge_error_flag &
2526                                     BGE_RXERRFLAG_IP_CSUM_NOK) == 0)
2527                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2528                         }
2529                         if (cur_rx->bge_flags & BGE_RXBDFLAG_TCP_UDP_CSUM) {
2530                                 m->m_pkthdr.csum_data =
2531                                     cur_rx->bge_tcp_udp_csum;
2532                                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID |
2533                                     CSUM_PSEUDO_HDR;
2534                         }
2535                 }
2536
2537                 /*
2538                  * If we received a packet with a vlan tag, pass it
2539                  * to vlan_input() instead of ether_input().
2540                  */
2541                 if (have_tag) {
2542                         m->m_flags |= M_VLANTAG;
2543                         m->m_pkthdr.ether_vlantag = vlan_tag;
2544                         have_tag = vlan_tag = 0;
2545                 }
2546                 ifp->if_input(ifp, m);
2547         }
2548
2549         bnx_writembx(sc, BGE_MBX_RX_CONS0_LO, sc->bnx_rx_saved_considx);
2550         if (stdcnt)
2551                 bnx_writembx(sc, BGE_MBX_RX_STD_PROD_LO, sc->bnx_std);
2552         if (jumbocnt)
2553                 bnx_writembx(sc, BGE_MBX_RX_JUMBO_PROD_LO, sc->bnx_jumbo);
2554 }
2555
2556 static void
2557 bnx_txeof(struct bnx_softc *sc, uint16_t tx_cons)
2558 {
2559         struct bge_tx_bd *cur_tx = NULL;
2560         struct ifnet *ifp;
2561
2562         ifp = &sc->arpcom.ac_if;
2563
2564         /*
2565          * Go through our tx ring and free mbufs for those
2566          * frames that have been sent.
2567          */
2568         while (sc->bnx_tx_saved_considx != tx_cons) {
2569                 uint32_t idx = 0;
2570
2571                 idx = sc->bnx_tx_saved_considx;
2572                 cur_tx = &sc->bnx_ldata.bnx_tx_ring[idx];
2573                 if (cur_tx->bge_flags & BGE_TXBDFLAG_END)
2574                         ifp->if_opackets++;
2575                 if (sc->bnx_cdata.bnx_tx_chain[idx] != NULL) {
2576                         bus_dmamap_unload(sc->bnx_cdata.bnx_tx_mtag,
2577                             sc->bnx_cdata.bnx_tx_dmamap[idx]);
2578                         m_freem(sc->bnx_cdata.bnx_tx_chain[idx]);
2579                         sc->bnx_cdata.bnx_tx_chain[idx] = NULL;
2580                 }
2581                 sc->bnx_txcnt--;
2582                 BNX_INC(sc->bnx_tx_saved_considx, BGE_TX_RING_CNT);
2583         }
2584
2585         if (cur_tx != NULL &&
2586             (BGE_TX_RING_CNT - sc->bnx_txcnt) >=
2587             (BNX_NSEG_RSVD + BNX_NSEG_SPARE))
2588                 ifp->if_flags &= ~IFF_OACTIVE;
2589
2590         if (sc->bnx_txcnt == 0)
2591                 ifp->if_timer = 0;
2592
2593         if (!ifq_is_empty(&ifp->if_snd))
2594                 if_devstart(ifp);
2595 }
2596
2597 #ifdef DEVICE_POLLING
2598
2599 static void
2600 bnx_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
2601 {
2602         struct bnx_softc *sc = ifp->if_softc;
2603         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2604         uint16_t rx_prod, tx_cons;
2605
2606         switch(cmd) {
2607         case POLL_REGISTER:
2608                 bnx_disable_intr(sc);
2609                 break;
2610         case POLL_DEREGISTER:
2611                 bnx_enable_intr(sc);
2612                 break;
2613         case POLL_AND_CHECK_STATUS:
2614                 /*
2615                  * Process link state changes.
2616                  */
2617                 bnx_link_poll(sc);
2618                 /* Fall through */
2619         case POLL_ONLY:
2620                 sc->bnx_status_tag = sblk->bge_status_tag;
2621                 /*
2622                  * Use a load fence to ensure that status_tag
2623                  * is saved  before rx_prod and tx_cons.
2624                  */
2625                 cpu_lfence();
2626
2627                 rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2628                 tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2629                 if (ifp->if_flags & IFF_RUNNING) {
2630                         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2631                         if (sc->bnx_rx_saved_considx != rx_prod)
2632                                 bnx_rxeof(sc, rx_prod);
2633
2634                         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2635                         if (sc->bnx_tx_saved_considx != tx_cons)
2636                                 bnx_txeof(sc, tx_cons);
2637                 }
2638                 break;
2639         }
2640 }
2641
2642 #endif
2643
2644 static void
2645 bnx_intr_legacy(void *xsc)
2646 {
2647         struct bnx_softc *sc = xsc;
2648         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2649
2650         if (sc->bnx_status_tag == sblk->bge_status_tag) {
2651                 uint32_t val;
2652
2653                 val = pci_read_config(sc->bnx_dev, BGE_PCI_PCISTATE, 4);
2654                 if (val & BGE_PCISTAT_INTR_NOTACT)
2655                         return;
2656         }
2657
2658         /*
2659          * NOTE:
2660          * Interrupt will have to be disabled if tagged status
2661          * is used, else interrupt will always be asserted on
2662          * certain chips (at least on BCM5750 AX/BX).
2663          */
2664         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2665
2666         bnx_intr(sc);
2667 }
2668
2669 static void
2670 bnx_msi(void *xsc)
2671 {
2672         struct bnx_softc *sc = xsc;
2673
2674         /* Disable interrupt first */
2675         bnx_writembx(sc, BGE_MBX_IRQ0_LO, 1);
2676         bnx_intr(sc);
2677 }
2678
2679 static void
2680 bnx_msi_oneshot(void *xsc)
2681 {
2682         bnx_intr(xsc);
2683 }
2684
2685 static void
2686 bnx_intr(struct bnx_softc *sc)
2687 {
2688         struct ifnet *ifp = &sc->arpcom.ac_if;
2689         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
2690         uint16_t rx_prod, tx_cons;
2691         uint32_t status;
2692
2693         sc->bnx_status_tag = sblk->bge_status_tag;
2694         /*
2695          * Use a load fence to ensure that status_tag is saved 
2696          * before rx_prod, tx_cons and status.
2697          */
2698         cpu_lfence();
2699
2700         rx_prod = sblk->bge_idx[0].bge_rx_prod_idx;
2701         tx_cons = sblk->bge_idx[0].bge_tx_cons_idx;
2702         status = sblk->bge_status;
2703
2704         if ((status & BGE_STATFLAG_LINKSTATE_CHANGED) || sc->bnx_link_evt)
2705                 bnx_link_poll(sc);
2706
2707         if (ifp->if_flags & IFF_RUNNING) {
2708                 if (sc->bnx_rx_saved_considx != rx_prod)
2709                         bnx_rxeof(sc, rx_prod);
2710
2711                 if (sc->bnx_tx_saved_considx != tx_cons)
2712                         bnx_txeof(sc, tx_cons);
2713         }
2714
2715         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
2716
2717         if (sc->bnx_coal_chg)
2718                 bnx_coal_change(sc);
2719 }
2720
2721 static void
2722 bnx_tick(void *xsc)
2723 {
2724         struct bnx_softc *sc = xsc;
2725         struct ifnet *ifp = &sc->arpcom.ac_if;
2726
2727         lwkt_serialize_enter(ifp->if_serializer);
2728
2729         KKASSERT(mycpuid == sc->bnx_stat_cpuid);
2730
2731         bnx_stats_update_regs(sc);
2732
2733         if (sc->bnx_flags & BNX_FLAG_TBI) {
2734                 /*
2735                  * Since in TBI mode auto-polling can't be used we should poll
2736                  * link status manually. Here we register pending link event
2737                  * and trigger interrupt.
2738                  */
2739                 sc->bnx_link_evt++;
2740                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
2741         } else if (!sc->bnx_link) {
2742                 mii_tick(device_get_softc(sc->bnx_miibus));
2743         }
2744
2745         callout_reset(&sc->bnx_stat_timer, hz, bnx_tick, sc);
2746
2747         lwkt_serialize_exit(ifp->if_serializer);
2748 }
2749
2750 static void
2751 bnx_stats_update_regs(struct bnx_softc *sc)
2752 {
2753         struct ifnet *ifp = &sc->arpcom.ac_if;
2754         struct bge_mac_stats_regs stats;
2755         uint32_t *s;
2756         int i;
2757
2758         s = (uint32_t *)&stats;
2759         for (i = 0; i < sizeof(struct bge_mac_stats_regs); i += 4) {
2760                 *s = CSR_READ_4(sc, BGE_RX_STATS + i);
2761                 s++;
2762         }
2763
2764         ifp->if_collisions +=
2765            (stats.dot3StatsSingleCollisionFrames +
2766            stats.dot3StatsMultipleCollisionFrames +
2767            stats.dot3StatsExcessiveCollisions +
2768            stats.dot3StatsLateCollisions) -
2769            ifp->if_collisions;
2770 }
2771
2772 /*
2773  * Encapsulate an mbuf chain in the tx ring  by coupling the mbuf data
2774  * pointers to descriptors.
2775  */
2776 static int
2777 bnx_encap(struct bnx_softc *sc, struct mbuf **m_head0, uint32_t *txidx)
2778 {
2779         struct bge_tx_bd *d = NULL;
2780         uint16_t csum_flags = 0;
2781         bus_dma_segment_t segs[BNX_NSEG_NEW];
2782         bus_dmamap_t map;
2783         int error, maxsegs, nsegs, idx, i;
2784         struct mbuf *m_head = *m_head0, *m_new;
2785
2786         if (m_head->m_pkthdr.csum_flags) {
2787                 if (m_head->m_pkthdr.csum_flags & CSUM_IP)
2788                         csum_flags |= BGE_TXBDFLAG_IP_CSUM;
2789                 if (m_head->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP))
2790                         csum_flags |= BGE_TXBDFLAG_TCP_UDP_CSUM;
2791                 if (m_head->m_flags & M_LASTFRAG)
2792                         csum_flags |= BGE_TXBDFLAG_IP_FRAG_END;
2793                 else if (m_head->m_flags & M_FRAG)
2794                         csum_flags |= BGE_TXBDFLAG_IP_FRAG;
2795         }
2796
2797         idx = *txidx;
2798         map = sc->bnx_cdata.bnx_tx_dmamap[idx];
2799
2800         maxsegs = (BGE_TX_RING_CNT - sc->bnx_txcnt) - BNX_NSEG_RSVD;
2801         KASSERT(maxsegs >= BNX_NSEG_SPARE,
2802                 ("not enough segments %d", maxsegs));
2803
2804         if (maxsegs > BNX_NSEG_NEW)
2805                 maxsegs = BNX_NSEG_NEW;
2806
2807         /*
2808          * Pad outbound frame to BGE_MIN_FRAMELEN for an unusual reason.
2809          * The bge hardware will pad out Tx runts to BGE_MIN_FRAMELEN,
2810          * but when such padded frames employ the bge IP/TCP checksum
2811          * offload, the hardware checksum assist gives incorrect results
2812          * (possibly from incorporating its own padding into the UDP/TCP
2813          * checksum; who knows).  If we pad such runts with zeros, the
2814          * onboard checksum comes out correct.
2815          */
2816         if ((csum_flags & BGE_TXBDFLAG_TCP_UDP_CSUM) &&
2817             m_head->m_pkthdr.len < BNX_MIN_FRAMELEN) {
2818                 error = m_devpad(m_head, BNX_MIN_FRAMELEN);
2819                 if (error)
2820                         goto back;
2821         }
2822
2823         if ((sc->bnx_flags & BNX_FLAG_SHORTDMA) && m_head->m_next != NULL) {
2824                 m_new = bnx_defrag_shortdma(m_head);
2825                 if (m_new == NULL) {
2826                         error = ENOBUFS;
2827                         goto back;
2828                 }
2829                 *m_head0 = m_head = m_new;
2830         }
2831         if (sc->bnx_force_defrag && m_head->m_next != NULL) {
2832                 /*
2833                  * Forcefully defragment mbuf chain to overcome hardware
2834                  * limitation which only support a single outstanding
2835                  * DMA read operation.  If it fails, keep moving on using
2836                  * the original mbuf chain.
2837                  */
2838                 m_new = m_defrag(m_head, MB_DONTWAIT);
2839                 if (m_new != NULL)
2840                         *m_head0 = m_head = m_new;
2841         }
2842
2843         error = bus_dmamap_load_mbuf_defrag(sc->bnx_cdata.bnx_tx_mtag, map,
2844                         m_head0, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
2845         if (error)
2846                 goto back;
2847
2848         m_head = *m_head0;
2849         bus_dmamap_sync(sc->bnx_cdata.bnx_tx_mtag, map, BUS_DMASYNC_PREWRITE);
2850
2851         for (i = 0; ; i++) {
2852                 d = &sc->bnx_ldata.bnx_tx_ring[idx];
2853
2854                 d->bge_addr.bge_addr_lo = BGE_ADDR_LO(segs[i].ds_addr);
2855                 d->bge_addr.bge_addr_hi = BGE_ADDR_HI(segs[i].ds_addr);
2856                 d->bge_len = segs[i].ds_len;
2857                 d->bge_flags = csum_flags;
2858
2859                 if (i == nsegs - 1)
2860                         break;
2861                 BNX_INC(idx, BGE_TX_RING_CNT);
2862         }
2863         /* Mark the last segment as end of packet... */
2864         d->bge_flags |= BGE_TXBDFLAG_END;
2865
2866         /* Set vlan tag to the first segment of the packet. */
2867         d = &sc->bnx_ldata.bnx_tx_ring[*txidx];
2868         if (m_head->m_flags & M_VLANTAG) {
2869                 d->bge_flags |= BGE_TXBDFLAG_VLAN_TAG;
2870                 d->bge_vlan_tag = m_head->m_pkthdr.ether_vlantag;
2871         } else {
2872                 d->bge_vlan_tag = 0;
2873         }
2874
2875         /*
2876          * Insure that the map for this transmission is placed at
2877          * the array index of the last descriptor in this chain.
2878          */
2879         sc->bnx_cdata.bnx_tx_dmamap[*txidx] = sc->bnx_cdata.bnx_tx_dmamap[idx];
2880         sc->bnx_cdata.bnx_tx_dmamap[idx] = map;
2881         sc->bnx_cdata.bnx_tx_chain[idx] = m_head;
2882         sc->bnx_txcnt += nsegs;
2883
2884         BNX_INC(idx, BGE_TX_RING_CNT);
2885         *txidx = idx;
2886 back:
2887         if (error) {
2888                 m_freem(*m_head0);
2889                 *m_head0 = NULL;
2890         }
2891         return error;
2892 }
2893
2894 /*
2895  * Main transmit routine. To avoid having to do mbuf copies, we put pointers
2896  * to the mbuf data regions directly in the transmit descriptors.
2897  */
2898 static void
2899 bnx_start(struct ifnet *ifp)
2900 {
2901         struct bnx_softc *sc = ifp->if_softc;
2902         struct mbuf *m_head = NULL;
2903         uint32_t prodidx;
2904         int need_trans;
2905
2906         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
2907                 return;
2908
2909         prodidx = sc->bnx_tx_prodidx;
2910
2911         need_trans = 0;
2912         while (sc->bnx_cdata.bnx_tx_chain[prodidx] == NULL) {
2913                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
2914                 if (m_head == NULL)
2915                         break;
2916
2917                 /*
2918                  * XXX
2919                  * The code inside the if() block is never reached since we
2920                  * must mark CSUM_IP_FRAGS in our if_hwassist to start getting
2921                  * requests to checksum TCP/UDP in a fragmented packet.
2922                  * 
2923                  * XXX
2924                  * safety overkill.  If this is a fragmented packet chain
2925                  * with delayed TCP/UDP checksums, then only encapsulate
2926                  * it if we have enough descriptors to handle the entire
2927                  * chain at once.
2928                  * (paranoia -- may not actually be needed)
2929                  */
2930                 if ((m_head->m_flags & M_FIRSTFRAG) &&
2931                     (m_head->m_pkthdr.csum_flags & CSUM_DELAY_DATA)) {
2932                         if ((BGE_TX_RING_CNT - sc->bnx_txcnt) <
2933                             m_head->m_pkthdr.csum_data + BNX_NSEG_RSVD) {
2934                                 ifp->if_flags |= IFF_OACTIVE;
2935                                 ifq_prepend(&ifp->if_snd, m_head);
2936                                 break;
2937                         }
2938                 }
2939
2940                 /*
2941                  * Sanity check: avoid coming within BGE_NSEG_RSVD
2942                  * descriptors of the end of the ring.  Also make
2943                  * sure there are BGE_NSEG_SPARE descriptors for
2944                  * jumbo buffers' defragmentation.
2945                  */
2946                 if ((BGE_TX_RING_CNT - sc->bnx_txcnt) <
2947                     (BNX_NSEG_RSVD + BNX_NSEG_SPARE)) {
2948                         ifp->if_flags |= IFF_OACTIVE;
2949                         ifq_prepend(&ifp->if_snd, m_head);
2950                         break;
2951                 }
2952
2953                 /*
2954                  * Pack the data into the transmit ring. If we
2955                  * don't have room, set the OACTIVE flag and wait
2956                  * for the NIC to drain the ring.
2957                  */
2958                 if (bnx_encap(sc, &m_head, &prodidx)) {
2959                         ifp->if_flags |= IFF_OACTIVE;
2960                         ifp->if_oerrors++;
2961                         break;
2962                 }
2963                 need_trans = 1;
2964
2965                 ETHER_BPF_MTAP(ifp, m_head);
2966         }
2967
2968         if (!need_trans)
2969                 return;
2970
2971         /* Transmit */
2972         bnx_writembx(sc, BGE_MBX_TX_HOST_PROD0_LO, prodidx);
2973
2974         sc->bnx_tx_prodidx = prodidx;
2975
2976         /*
2977          * Set a timeout in case the chip goes out to lunch.
2978          */
2979         ifp->if_timer = 5;
2980 }
2981
2982 static void
2983 bnx_init(void *xsc)
2984 {
2985         struct bnx_softc *sc = xsc;
2986         struct ifnet *ifp = &sc->arpcom.ac_if;
2987         uint16_t *m;
2988         uint32_t mode;
2989
2990         ASSERT_SERIALIZED(ifp->if_serializer);
2991
2992         /* Cancel pending I/O and flush buffers. */
2993         bnx_stop(sc);
2994         bnx_reset(sc);
2995         bnx_chipinit(sc);
2996
2997         /*
2998          * Init the various state machines, ring
2999          * control blocks and firmware.
3000          */
3001         if (bnx_blockinit(sc)) {
3002                 if_printf(ifp, "initialization failure\n");
3003                 bnx_stop(sc);
3004                 return;
3005         }
3006
3007         /* Specify MTU. */
3008         CSR_WRITE_4(sc, BGE_RX_MTU, ifp->if_mtu +
3009             ETHER_HDR_LEN + ETHER_CRC_LEN + EVL_ENCAPLEN);
3010
3011         /* Load our MAC address. */
3012         m = (uint16_t *)&sc->arpcom.ac_enaddr[0];
3013         CSR_WRITE_4(sc, BGE_MAC_ADDR1_LO, htons(m[0]));
3014         CSR_WRITE_4(sc, BGE_MAC_ADDR1_HI, (htons(m[1]) << 16) | htons(m[2]));
3015
3016         /* Enable or disable promiscuous mode as needed. */
3017         bnx_setpromisc(sc);
3018
3019         /* Program multicast filter. */
3020         bnx_setmulti(sc);
3021
3022         /* Init RX ring. */
3023         if (bnx_init_rx_ring_std(sc)) {
3024                 if_printf(ifp, "RX ring initialization failed\n");
3025                 bnx_stop(sc);
3026                 return;
3027         }
3028
3029         /* Init jumbo RX ring. */
3030         if (ifp->if_mtu > (ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN)) {
3031                 if (bnx_init_rx_ring_jumbo(sc)) {
3032                         if_printf(ifp, "Jumbo RX ring initialization failed\n");
3033                         bnx_stop(sc);
3034                         return;
3035                 }
3036         }
3037
3038         /* Init our RX return ring index */
3039         sc->bnx_rx_saved_considx = 0;
3040
3041         /* Init TX ring. */
3042         bnx_init_tx_ring(sc);
3043
3044         /* Enable TX MAC state machine lockup fix. */
3045         mode = CSR_READ_4(sc, BGE_TX_MODE);
3046         mode |= BGE_TXMODE_MBUF_LOCKUP_FIX;
3047         if (sc->bnx_asicrev == BGE_ASICREV_BCM5720) {
3048                 mode &= ~(BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3049                 mode |= CSR_READ_4(sc, BGE_TX_MODE) &
3050                     (BGE_TXMODE_JMB_FRM_LEN | BGE_TXMODE_CNT_DN_MODE);
3051         }
3052         /* Turn on transmitter */
3053         CSR_WRITE_4(sc, BGE_TX_MODE, mode | BGE_TXMODE_ENABLE);
3054
3055         /* Turn on receiver */
3056         BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3057
3058         /*
3059          * Set the number of good frames to receive after RX MBUF
3060          * Low Watermark has been reached.  After the RX MAC receives
3061          * this number of frames, it will drop subsequent incoming
3062          * frames until the MBUF High Watermark is reached.
3063          */
3064         if (BNX_IS_57765_FAMILY(sc))
3065                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 1);
3066         else
3067                 CSR_WRITE_4(sc, BGE_MAX_RX_FRAME_LOWAT, 2);
3068
3069         if (sc->bnx_irq_type == PCI_INTR_TYPE_MSI) {
3070                 if (bootverbose) {
3071                         if_printf(ifp, "MSI_MODE: %#x\n",
3072                             CSR_READ_4(sc, BGE_MSI_MODE));
3073                 }
3074         }
3075
3076         /* Tell firmware we're alive. */
3077         BNX_SETBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3078
3079         /* Enable host interrupts if polling(4) is not enabled. */
3080         PCI_SETBIT(sc->bnx_dev, BGE_PCI_MISC_CTL, BGE_PCIMISCCTL_CLEAR_INTA, 4);
3081 #ifdef DEVICE_POLLING
3082         if (ifp->if_flags & IFF_POLLING)
3083                 bnx_disable_intr(sc);
3084         else
3085 #endif
3086         bnx_enable_intr(sc);
3087
3088         bnx_ifmedia_upd(ifp);
3089
3090         ifp->if_flags |= IFF_RUNNING;
3091         ifp->if_flags &= ~IFF_OACTIVE;
3092
3093         callout_reset_bycpu(&sc->bnx_stat_timer, hz, bnx_tick, sc,
3094             sc->bnx_stat_cpuid);
3095 }
3096
3097 /*
3098  * Set media options.
3099  */
3100 static int
3101 bnx_ifmedia_upd(struct ifnet *ifp)
3102 {
3103         struct bnx_softc *sc = ifp->if_softc;
3104
3105         /* If this is a 1000baseX NIC, enable the TBI port. */
3106         if (sc->bnx_flags & BNX_FLAG_TBI) {
3107                 struct ifmedia *ifm = &sc->bnx_ifmedia;
3108
3109                 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
3110                         return(EINVAL);
3111
3112                 switch(IFM_SUBTYPE(ifm->ifm_media)) {
3113                 case IFM_AUTO:
3114                         break;
3115
3116                 case IFM_1000_SX:
3117                         if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) {
3118                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3119                                     BGE_MACMODE_HALF_DUPLEX);
3120                         } else {
3121                                 BNX_SETBIT(sc, BGE_MAC_MODE,
3122                                     BGE_MACMODE_HALF_DUPLEX);
3123                         }
3124                         break;
3125                 default:
3126                         return(EINVAL);
3127                 }
3128         } else {
3129                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3130
3131                 sc->bnx_link_evt++;
3132                 sc->bnx_link = 0;
3133                 if (mii->mii_instance) {
3134                         struct mii_softc *miisc;
3135
3136                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
3137                                 mii_phy_reset(miisc);
3138                 }
3139                 mii_mediachg(mii);
3140
3141                 /*
3142                  * Force an interrupt so that we will call bnx_link_upd
3143                  * if needed and clear any pending link state attention.
3144                  * Without this we are not getting any further interrupts
3145                  * for link state changes and thus will not UP the link and
3146                  * not be able to send in bnx_start.  The only way to get
3147                  * things working was to receive a packet and get an RX
3148                  * intr.
3149                  *
3150                  * bnx_tick should help for fiber cards and we might not
3151                  * need to do this here if BNX_FLAG_TBI is set but as
3152                  * we poll for fiber anyway it should not harm.
3153                  */
3154                 BNX_SETBIT(sc, BGE_HCC_MODE, BGE_HCCMODE_COAL_NOW);
3155         }
3156         return(0);
3157 }
3158
3159 /*
3160  * Report current media status.
3161  */
3162 static void
3163 bnx_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
3164 {
3165         struct bnx_softc *sc = ifp->if_softc;
3166
3167         if (sc->bnx_flags & BNX_FLAG_TBI) {
3168                 ifmr->ifm_status = IFM_AVALID;
3169                 ifmr->ifm_active = IFM_ETHER;
3170                 if (CSR_READ_4(sc, BGE_MAC_STS) &
3171                     BGE_MACSTAT_TBI_PCS_SYNCHED) {
3172                         ifmr->ifm_status |= IFM_ACTIVE;
3173                 } else {
3174                         ifmr->ifm_active |= IFM_NONE;
3175                         return;
3176                 }
3177
3178                 ifmr->ifm_active |= IFM_1000_SX;
3179                 if (CSR_READ_4(sc, BGE_MAC_MODE) & BGE_MACMODE_HALF_DUPLEX)
3180                         ifmr->ifm_active |= IFM_HDX;    
3181                 else
3182                         ifmr->ifm_active |= IFM_FDX;
3183         } else {
3184                 struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3185
3186                 mii_pollstat(mii);
3187                 ifmr->ifm_active = mii->mii_media_active;
3188                 ifmr->ifm_status = mii->mii_media_status;
3189         }
3190 }
3191
3192 static int
3193 bnx_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
3194 {
3195         struct bnx_softc *sc = ifp->if_softc;
3196         struct ifreq *ifr = (struct ifreq *)data;
3197         int mask, error = 0;
3198
3199         ASSERT_SERIALIZED(ifp->if_serializer);
3200
3201         switch (command) {
3202         case SIOCSIFMTU:
3203                 if ((!BNX_IS_JUMBO_CAPABLE(sc) && ifr->ifr_mtu > ETHERMTU) ||
3204                     (BNX_IS_JUMBO_CAPABLE(sc) &&
3205                      ifr->ifr_mtu > BNX_JUMBO_MTU)) {
3206                         error = EINVAL;
3207                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
3208                         ifp->if_mtu = ifr->ifr_mtu;
3209                         if (ifp->if_flags & IFF_RUNNING)
3210                                 bnx_init(sc);
3211                 }
3212                 break;
3213         case SIOCSIFFLAGS:
3214                 if (ifp->if_flags & IFF_UP) {
3215                         if (ifp->if_flags & IFF_RUNNING) {
3216                                 mask = ifp->if_flags ^ sc->bnx_if_flags;
3217
3218                                 /*
3219                                  * If only the state of the PROMISC flag
3220                                  * changed, then just use the 'set promisc
3221                                  * mode' command instead of reinitializing
3222                                  * the entire NIC. Doing a full re-init
3223                                  * means reloading the firmware and waiting
3224                                  * for it to start up, which may take a
3225                                  * second or two.  Similarly for ALLMULTI.
3226                                  */
3227                                 if (mask & IFF_PROMISC)
3228                                         bnx_setpromisc(sc);
3229                                 if (mask & IFF_ALLMULTI)
3230                                         bnx_setmulti(sc);
3231                         } else {
3232                                 bnx_init(sc);
3233                         }
3234                 } else if (ifp->if_flags & IFF_RUNNING) {
3235                         bnx_stop(sc);
3236                 }
3237                 sc->bnx_if_flags = ifp->if_flags;
3238                 break;
3239         case SIOCADDMULTI:
3240         case SIOCDELMULTI:
3241                 if (ifp->if_flags & IFF_RUNNING)
3242                         bnx_setmulti(sc);
3243                 break;
3244         case SIOCSIFMEDIA:
3245         case SIOCGIFMEDIA:
3246                 if (sc->bnx_flags & BNX_FLAG_TBI) {
3247                         error = ifmedia_ioctl(ifp, ifr,
3248                             &sc->bnx_ifmedia, command);
3249                 } else {
3250                         struct mii_data *mii;
3251
3252                         mii = device_get_softc(sc->bnx_miibus);
3253                         error = ifmedia_ioctl(ifp, ifr,
3254                                               &mii->mii_media, command);
3255                 }
3256                 break;
3257         case SIOCSIFCAP:
3258                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3259                 if (mask & IFCAP_HWCSUM) {
3260                         ifp->if_capenable ^= (mask & IFCAP_HWCSUM);
3261                         if (IFCAP_HWCSUM & ifp->if_capenable)
3262                                 ifp->if_hwassist = BNX_CSUM_FEATURES;
3263                         else
3264                                 ifp->if_hwassist = 0;
3265                 }
3266                 break;
3267         default:
3268                 error = ether_ioctl(ifp, command, data);
3269                 break;
3270         }
3271         return error;
3272 }
3273
3274 static void
3275 bnx_watchdog(struct ifnet *ifp)
3276 {
3277         struct bnx_softc *sc = ifp->if_softc;
3278
3279         if_printf(ifp, "watchdog timeout -- resetting\n");
3280
3281         bnx_init(sc);
3282
3283         ifp->if_oerrors++;
3284
3285         if (!ifq_is_empty(&ifp->if_snd))
3286                 if_devstart(ifp);
3287 }
3288
3289 /*
3290  * Stop the adapter and free any mbufs allocated to the
3291  * RX and TX lists.
3292  */
3293 static void
3294 bnx_stop(struct bnx_softc *sc)
3295 {
3296         struct ifnet *ifp = &sc->arpcom.ac_if;
3297
3298         ASSERT_SERIALIZED(ifp->if_serializer);
3299
3300         callout_stop(&sc->bnx_stat_timer);
3301
3302         /*
3303          * Disable all of the receiver blocks
3304          */
3305         bnx_stop_block(sc, BGE_RX_MODE, BGE_RXMODE_ENABLE);
3306         bnx_stop_block(sc, BGE_RBDI_MODE, BGE_RBDIMODE_ENABLE);
3307         bnx_stop_block(sc, BGE_RXLP_MODE, BGE_RXLPMODE_ENABLE);
3308         bnx_stop_block(sc, BGE_RDBDI_MODE, BGE_RBDIMODE_ENABLE);
3309         bnx_stop_block(sc, BGE_RDC_MODE, BGE_RDCMODE_ENABLE);
3310         bnx_stop_block(sc, BGE_RBDC_MODE, BGE_RBDCMODE_ENABLE);
3311
3312         /*
3313          * Disable all of the transmit blocks
3314          */
3315         bnx_stop_block(sc, BGE_SRS_MODE, BGE_SRSMODE_ENABLE);
3316         bnx_stop_block(sc, BGE_SBDI_MODE, BGE_SBDIMODE_ENABLE);
3317         bnx_stop_block(sc, BGE_SDI_MODE, BGE_SDIMODE_ENABLE);
3318         bnx_stop_block(sc, BGE_RDMA_MODE, BGE_RDMAMODE_ENABLE);
3319         bnx_stop_block(sc, BGE_SDC_MODE, BGE_SDCMODE_ENABLE);
3320         bnx_stop_block(sc, BGE_SBDC_MODE, BGE_SBDCMODE_ENABLE);
3321
3322         /*
3323          * Shut down all of the memory managers and related
3324          * state machines.
3325          */
3326         bnx_stop_block(sc, BGE_HCC_MODE, BGE_HCCMODE_ENABLE);
3327         bnx_stop_block(sc, BGE_WDMA_MODE, BGE_WDMAMODE_ENABLE);
3328         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0xFFFFFFFF);
3329         CSR_WRITE_4(sc, BGE_FTQ_RESET, 0);
3330
3331         /* Disable host interrupts. */
3332         bnx_disable_intr(sc);
3333
3334         /*
3335          * Tell firmware we're shutting down.
3336          */
3337         BNX_CLRBIT(sc, BGE_MODE_CTL, BGE_MODECTL_STACKUP);
3338
3339         /* Free the RX lists. */
3340         bnx_free_rx_ring_std(sc);
3341
3342         /* Free jumbo RX list. */
3343         if (BNX_IS_JUMBO_CAPABLE(sc))
3344                 bnx_free_rx_ring_jumbo(sc);
3345
3346         /* Free TX buffers. */
3347         bnx_free_tx_ring(sc);
3348
3349         sc->bnx_status_tag = 0;
3350         sc->bnx_link = 0;
3351         sc->bnx_coal_chg = 0;
3352
3353         sc->bnx_tx_saved_considx = BNX_TXCONS_UNSET;
3354
3355         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
3356         ifp->if_timer = 0;
3357 }
3358
3359 /*
3360  * Stop all chip I/O so that the kernel's probe routines don't
3361  * get confused by errant DMAs when rebooting.
3362  */
3363 static void
3364 bnx_shutdown(device_t dev)
3365 {
3366         struct bnx_softc *sc = device_get_softc(dev);
3367         struct ifnet *ifp = &sc->arpcom.ac_if;
3368
3369         lwkt_serialize_enter(ifp->if_serializer);
3370         bnx_stop(sc);
3371         bnx_reset(sc);
3372         lwkt_serialize_exit(ifp->if_serializer);
3373 }
3374
3375 static int
3376 bnx_suspend(device_t dev)
3377 {
3378         struct bnx_softc *sc = device_get_softc(dev);
3379         struct ifnet *ifp = &sc->arpcom.ac_if;
3380
3381         lwkt_serialize_enter(ifp->if_serializer);
3382         bnx_stop(sc);
3383         lwkt_serialize_exit(ifp->if_serializer);
3384
3385         return 0;
3386 }
3387
3388 static int
3389 bnx_resume(device_t dev)
3390 {
3391         struct bnx_softc *sc = device_get_softc(dev);
3392         struct ifnet *ifp = &sc->arpcom.ac_if;
3393
3394         lwkt_serialize_enter(ifp->if_serializer);
3395
3396         if (ifp->if_flags & IFF_UP) {
3397                 bnx_init(sc);
3398
3399                 if (!ifq_is_empty(&ifp->if_snd))
3400                         if_devstart(ifp);
3401         }
3402
3403         lwkt_serialize_exit(ifp->if_serializer);
3404
3405         return 0;
3406 }
3407
3408 static void
3409 bnx_setpromisc(struct bnx_softc *sc)
3410 {
3411         struct ifnet *ifp = &sc->arpcom.ac_if;
3412
3413         if (ifp->if_flags & IFF_PROMISC)
3414                 BNX_SETBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3415         else
3416                 BNX_CLRBIT(sc, BGE_RX_MODE, BGE_RXMODE_RX_PROMISC);
3417 }
3418
3419 static void
3420 bnx_dma_free(struct bnx_softc *sc)
3421 {
3422         int i;
3423
3424         /* Destroy RX mbuf DMA stuffs. */
3425         if (sc->bnx_cdata.bnx_rx_mtag != NULL) {
3426                 for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3427                         bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3428                             sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3429                 }
3430                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3431                                    sc->bnx_cdata.bnx_rx_tmpmap);
3432                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3433         }
3434
3435         /* Destroy TX mbuf DMA stuffs. */
3436         if (sc->bnx_cdata.bnx_tx_mtag != NULL) {
3437                 for (i = 0; i < BGE_TX_RING_CNT; i++) {
3438                         bus_dmamap_destroy(sc->bnx_cdata.bnx_tx_mtag,
3439                             sc->bnx_cdata.bnx_tx_dmamap[i]);
3440                 }
3441                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_tx_mtag);
3442         }
3443
3444         /* Destroy standard RX ring */
3445         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_std_ring_tag,
3446                            sc->bnx_cdata.bnx_rx_std_ring_map,
3447                            sc->bnx_ldata.bnx_rx_std_ring);
3448
3449         if (BNX_IS_JUMBO_CAPABLE(sc))
3450                 bnx_free_jumbo_mem(sc);
3451
3452         /* Destroy RX return ring */
3453         bnx_dma_block_free(sc->bnx_cdata.bnx_rx_return_ring_tag,
3454                            sc->bnx_cdata.bnx_rx_return_ring_map,
3455                            sc->bnx_ldata.bnx_rx_return_ring);
3456
3457         /* Destroy TX ring */
3458         bnx_dma_block_free(sc->bnx_cdata.bnx_tx_ring_tag,
3459                            sc->bnx_cdata.bnx_tx_ring_map,
3460                            sc->bnx_ldata.bnx_tx_ring);
3461
3462         /* Destroy status block */
3463         bnx_dma_block_free(sc->bnx_cdata.bnx_status_tag,
3464                            sc->bnx_cdata.bnx_status_map,
3465                            sc->bnx_ldata.bnx_status_block);
3466
3467         /* Destroy the parent tag */
3468         if (sc->bnx_cdata.bnx_parent_tag != NULL)
3469                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_parent_tag);
3470 }
3471
3472 static int
3473 bnx_dma_alloc(struct bnx_softc *sc)
3474 {
3475         struct ifnet *ifp = &sc->arpcom.ac_if;
3476         int i, error;
3477
3478         /*
3479          * Allocate the parent bus DMA tag appropriate for PCI.
3480          *
3481          * All of the NetExtreme/NetLink controllers have 4GB boundary
3482          * DMA bug.
3483          * Whenever an address crosses a multiple of the 4GB boundary
3484          * (including 4GB, 8Gb, 12Gb, etc.) and makes the transition
3485          * from 0xX_FFFF_FFFF to 0x(X+1)_0000_0000 an internal DMA
3486          * state machine will lockup and cause the device to hang.
3487          */
3488         error = bus_dma_tag_create(NULL, 1, BGE_DMA_BOUNDARY_4G,
3489                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3490                                    NULL, NULL,
3491                                    BUS_SPACE_MAXSIZE_32BIT, 0,
3492                                    BUS_SPACE_MAXSIZE_32BIT,
3493                                    0, &sc->bnx_cdata.bnx_parent_tag);
3494         if (error) {
3495                 if_printf(ifp, "could not allocate parent dma tag\n");
3496                 return error;
3497         }
3498
3499         /*
3500          * Create DMA tag and maps for RX mbufs.
3501          */
3502         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3503                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3504                                    NULL, NULL, MCLBYTES, 1, MCLBYTES,
3505                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK,
3506                                    &sc->bnx_cdata.bnx_rx_mtag);
3507         if (error) {
3508                 if_printf(ifp, "could not allocate RX mbuf dma tag\n");
3509                 return error;
3510         }
3511
3512         error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3513                                   BUS_DMA_WAITOK, &sc->bnx_cdata.bnx_rx_tmpmap);
3514         if (error) {
3515                 bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3516                 sc->bnx_cdata.bnx_rx_mtag = NULL;
3517                 return error;
3518         }
3519
3520         for (i = 0; i < BGE_STD_RX_RING_CNT; i++) {
3521                 error = bus_dmamap_create(sc->bnx_cdata.bnx_rx_mtag,
3522                                           BUS_DMA_WAITOK,
3523                                           &sc->bnx_cdata.bnx_rx_std_dmamap[i]);
3524                 if (error) {
3525                         int j;
3526
3527                         for (j = 0; j < i; ++j) {
3528                                 bus_dmamap_destroy(sc->bnx_cdata.bnx_rx_mtag,
3529                                         sc->bnx_cdata.bnx_rx_std_dmamap[j]);
3530                         }
3531                         bus_dma_tag_destroy(sc->bnx_cdata.bnx_rx_mtag);
3532                         sc->bnx_cdata.bnx_rx_mtag = NULL;
3533
3534                         if_printf(ifp, "could not create DMA map for RX\n");
3535                         return error;
3536                 }
3537         }
3538
3539         /*
3540          * Create DMA tag and maps for TX mbufs.
3541          */
3542         error = bus_dma_tag_create(sc->bnx_cdata.bnx_parent_tag, 1, 0,
3543                                    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3544                                    NULL, NULL,
3545                                    BNX_JUMBO_FRAMELEN, BNX_NSEG_NEW, MCLBYTES,
3546                                    BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK |
3547                                    BUS_DMA_ONEBPAGE,
3548                                    &sc->bnx_cdata.bnx_tx_mtag);
3549         if (error) {
3550                 if_printf(ifp, "could not allocate TX mbuf dma tag\n");
3551                 return error;
3552         }
3553
3554         for (i = 0; i < BGE_TX_RING_CNT; i++) {
3555                 error = bus_dmamap_create(sc->bnx_cdata.bnx_tx_mtag,
3556                                           BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
3557                                           &sc->bnx_cdata.bnx_tx_dmamap[i]);
3558                 if (error) {
3559                         int j;
3560
3561                         for (j = 0; j < i; ++j) {
3562                                 bus_dmamap_destroy(sc->bnx_cdata.bnx_tx_mtag,
3563                                         sc->bnx_cdata.bnx_tx_dmamap[j]);
3564                         }
3565                         bus_dma_tag_destroy(sc->bnx_cdata.bnx_tx_mtag);
3566                         sc->bnx_cdata.bnx_tx_mtag = NULL;
3567
3568                         if_printf(ifp, "could not create DMA map for TX\n");
3569                         return error;
3570                 }
3571         }
3572
3573         /*
3574          * Create DMA stuffs for standard RX ring.
3575          */
3576         error = bnx_dma_block_alloc(sc, BGE_STD_RX_RING_SZ,
3577                                     &sc->bnx_cdata.bnx_rx_std_ring_tag,
3578                                     &sc->bnx_cdata.bnx_rx_std_ring_map,
3579                                     (void *)&sc->bnx_ldata.bnx_rx_std_ring,
3580                                     &sc->bnx_ldata.bnx_rx_std_ring_paddr);
3581         if (error) {
3582                 if_printf(ifp, "could not create std RX ring\n");
3583                 return error;
3584         }
3585
3586         /*
3587          * Create jumbo buffer pool.
3588          */
3589         if (BNX_IS_JUMBO_CAPABLE(sc)) {
3590                 error = bnx_alloc_jumbo_mem(sc);
3591                 if (error) {
3592                         if_printf(ifp, "could not create jumbo buffer pool\n");
3593                         return error;
3594                 }
3595         }
3596
3597         /*
3598          * Create DMA stuffs for RX return ring.
3599          */
3600         error = bnx_dma_block_alloc(sc,
3601             BGE_RX_RTN_RING_SZ(sc->bnx_return_ring_cnt),
3602             &sc->bnx_cdata.bnx_rx_return_ring_tag,
3603             &sc->bnx_cdata.bnx_rx_return_ring_map,
3604             (void *)&sc->bnx_ldata.bnx_rx_return_ring,
3605             &sc->bnx_ldata.bnx_rx_return_ring_paddr);
3606         if (error) {
3607                 if_printf(ifp, "could not create RX ret ring\n");
3608                 return error;
3609         }
3610
3611         /*
3612          * Create DMA stuffs for TX ring.
3613          */
3614         error = bnx_dma_block_alloc(sc, BGE_TX_RING_SZ,
3615                                     &sc->bnx_cdata.bnx_tx_ring_tag,
3616                                     &sc->bnx_cdata.bnx_tx_ring_map,
3617                                     (void *)&sc->bnx_ldata.bnx_tx_ring,
3618                                     &sc->bnx_ldata.bnx_tx_ring_paddr);
3619         if (error) {
3620                 if_printf(ifp, "could not create TX ring\n");
3621                 return error;
3622         }
3623
3624         /*
3625          * Create DMA stuffs for status block.
3626          */
3627         error = bnx_dma_block_alloc(sc, BGE_STATUS_BLK_SZ,
3628                                     &sc->bnx_cdata.bnx_status_tag,
3629                                     &sc->bnx_cdata.bnx_status_map,
3630                                     (void *)&sc->bnx_ldata.bnx_status_block,
3631                                     &sc->bnx_ldata.bnx_status_block_paddr);
3632         if (error) {
3633                 if_printf(ifp, "could not create status block\n");
3634                 return error;
3635         }
3636
3637         return 0;
3638 }
3639
3640 static int
3641 bnx_dma_block_alloc(struct bnx_softc *sc, bus_size_t size, bus_dma_tag_t *tag,
3642                     bus_dmamap_t *map, void **addr, bus_addr_t *paddr)
3643 {
3644         bus_dmamem_t dmem;
3645         int error;
3646
3647         error = bus_dmamem_coherent(sc->bnx_cdata.bnx_parent_tag, PAGE_SIZE, 0,
3648                                     BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3649                                     size, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
3650         if (error)
3651                 return error;
3652
3653         *tag = dmem.dmem_tag;
3654         *map = dmem.dmem_map;
3655         *addr = dmem.dmem_addr;
3656         *paddr = dmem.dmem_busaddr;
3657
3658         return 0;
3659 }
3660
3661 static void
3662 bnx_dma_block_free(bus_dma_tag_t tag, bus_dmamap_t map, void *addr)
3663 {
3664         if (tag != NULL) {
3665                 bus_dmamap_unload(tag, map);
3666                 bus_dmamem_free(tag, addr, map);
3667                 bus_dma_tag_destroy(tag);
3668         }
3669 }
3670
3671 static void
3672 bnx_tbi_link_upd(struct bnx_softc *sc, uint32_t status)
3673 {
3674         struct ifnet *ifp = &sc->arpcom.ac_if;
3675
3676 #define PCS_ENCODE_ERR  (BGE_MACSTAT_PORT_DECODE_ERROR|BGE_MACSTAT_MI_COMPLETE)
3677
3678         /*
3679          * Sometimes PCS encoding errors are detected in
3680          * TBI mode (on fiber NICs), and for some reason
3681          * the chip will signal them as link changes.
3682          * If we get a link change event, but the 'PCS
3683          * encoding error' bit in the MAC status register
3684          * is set, don't bother doing a link check.
3685          * This avoids spurious "gigabit link up" messages
3686          * that sometimes appear on fiber NICs during
3687          * periods of heavy traffic.
3688          */
3689         if (status & BGE_MACSTAT_TBI_PCS_SYNCHED) {
3690                 if (!sc->bnx_link) {
3691                         sc->bnx_link++;
3692                         if (sc->bnx_asicrev == BGE_ASICREV_BCM5704) {
3693                                 BNX_CLRBIT(sc, BGE_MAC_MODE,
3694                                     BGE_MACMODE_TBI_SEND_CFGS);
3695                         }
3696                         CSR_WRITE_4(sc, BGE_MAC_STS, 0xFFFFFFFF);
3697
3698                         if (bootverbose)
3699                                 if_printf(ifp, "link UP\n");
3700
3701                         ifp->if_link_state = LINK_STATE_UP;
3702                         if_link_state_change(ifp);
3703                 }
3704         } else if ((status & PCS_ENCODE_ERR) != PCS_ENCODE_ERR) {
3705                 if (sc->bnx_link) {
3706                         sc->bnx_link = 0;
3707
3708                         if (bootverbose)
3709                                 if_printf(ifp, "link DOWN\n");
3710
3711                         ifp->if_link_state = LINK_STATE_DOWN;
3712                         if_link_state_change(ifp);
3713                 }
3714         }
3715
3716 #undef PCS_ENCODE_ERR
3717
3718         /* Clear the attention. */
3719         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3720             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3721             BGE_MACSTAT_LINK_CHANGED);
3722 }
3723
3724 static void
3725 bnx_copper_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3726 {
3727         struct ifnet *ifp = &sc->arpcom.ac_if;
3728         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3729
3730         mii_pollstat(mii);
3731         bnx_miibus_statchg(sc->bnx_dev);
3732
3733         if (bootverbose) {
3734                 if (sc->bnx_link)
3735                         if_printf(ifp, "link UP\n");
3736                 else
3737                         if_printf(ifp, "link DOWN\n");
3738         }
3739
3740         /* Clear the attention. */
3741         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3742             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3743             BGE_MACSTAT_LINK_CHANGED);
3744 }
3745
3746 static void
3747 bnx_autopoll_link_upd(struct bnx_softc *sc, uint32_t status __unused)
3748 {
3749         struct ifnet *ifp = &sc->arpcom.ac_if;
3750         struct mii_data *mii = device_get_softc(sc->bnx_miibus);
3751
3752         mii_pollstat(mii);
3753
3754         if (!sc->bnx_link &&
3755             (mii->mii_media_status & IFM_ACTIVE) &&
3756             IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
3757                 sc->bnx_link++;
3758                 if (bootverbose)
3759                         if_printf(ifp, "link UP\n");
3760         } else if (sc->bnx_link &&
3761             (!(mii->mii_media_status & IFM_ACTIVE) ||
3762             IFM_SUBTYPE(mii->mii_media_active) == IFM_NONE)) {
3763                 sc->bnx_link = 0;
3764                 if (bootverbose)
3765                         if_printf(ifp, "link DOWN\n");
3766         }
3767
3768         /* Clear the attention. */
3769         CSR_WRITE_4(sc, BGE_MAC_STS, BGE_MACSTAT_SYNC_CHANGED |
3770             BGE_MACSTAT_CFG_CHANGED | BGE_MACSTAT_MI_COMPLETE |
3771             BGE_MACSTAT_LINK_CHANGED);
3772 }
3773
3774 static int
3775 bnx_sysctl_rx_coal_ticks(SYSCTL_HANDLER_ARGS)
3776 {
3777         struct bnx_softc *sc = arg1;
3778
3779         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3780             &sc->bnx_rx_coal_ticks,
3781             BNX_RX_COAL_TICKS_MIN, BNX_RX_COAL_TICKS_MAX,
3782             BNX_RX_COAL_TICKS_CHG);
3783 }
3784
3785 static int
3786 bnx_sysctl_tx_coal_ticks(SYSCTL_HANDLER_ARGS)
3787 {
3788         struct bnx_softc *sc = arg1;
3789
3790         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3791             &sc->bnx_tx_coal_ticks,
3792             BNX_TX_COAL_TICKS_MIN, BNX_TX_COAL_TICKS_MAX,
3793             BNX_TX_COAL_TICKS_CHG);
3794 }
3795
3796 static int
3797 bnx_sysctl_rx_coal_bds(SYSCTL_HANDLER_ARGS)
3798 {
3799         struct bnx_softc *sc = arg1;
3800
3801         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3802             &sc->bnx_rx_coal_bds,
3803             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3804             BNX_RX_COAL_BDS_CHG);
3805 }
3806
3807 static int
3808 bnx_sysctl_tx_coal_bds(SYSCTL_HANDLER_ARGS)
3809 {
3810         struct bnx_softc *sc = arg1;
3811
3812         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3813             &sc->bnx_tx_coal_bds,
3814             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3815             BNX_TX_COAL_BDS_CHG);
3816 }
3817
3818 static int
3819 bnx_sysctl_rx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3820 {
3821         struct bnx_softc *sc = arg1;
3822
3823         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3824             &sc->bnx_rx_coal_bds_int,
3825             BNX_RX_COAL_BDS_MIN, BNX_RX_COAL_BDS_MAX,
3826             BNX_RX_COAL_BDS_INT_CHG);
3827 }
3828
3829 static int
3830 bnx_sysctl_tx_coal_bds_int(SYSCTL_HANDLER_ARGS)
3831 {
3832         struct bnx_softc *sc = arg1;
3833
3834         return bnx_sysctl_coal_chg(oidp, arg1, arg2, req,
3835             &sc->bnx_tx_coal_bds_int,
3836             BNX_TX_COAL_BDS_MIN, BNX_TX_COAL_BDS_MAX,
3837             BNX_TX_COAL_BDS_INT_CHG);
3838 }
3839
3840 static int
3841 bnx_sysctl_coal_chg(SYSCTL_HANDLER_ARGS, uint32_t *coal,
3842     int coal_min, int coal_max, uint32_t coal_chg_mask)
3843 {
3844         struct bnx_softc *sc = arg1;
3845         struct ifnet *ifp = &sc->arpcom.ac_if;
3846         int error = 0, v;
3847
3848         lwkt_serialize_enter(ifp->if_serializer);
3849
3850         v = *coal;
3851         error = sysctl_handle_int(oidp, &v, 0, req);
3852         if (!error && req->newptr != NULL) {
3853                 if (v < coal_min || v > coal_max) {
3854                         error = EINVAL;
3855                 } else {
3856                         *coal = v;
3857                         sc->bnx_coal_chg |= coal_chg_mask;
3858                 }
3859         }
3860
3861         lwkt_serialize_exit(ifp->if_serializer);
3862         return error;
3863 }
3864
3865 static void
3866 bnx_coal_change(struct bnx_softc *sc)
3867 {
3868         struct ifnet *ifp = &sc->arpcom.ac_if;
3869         uint32_t val;
3870
3871         ASSERT_SERIALIZED(ifp->if_serializer);
3872
3873         if (sc->bnx_coal_chg & BNX_RX_COAL_TICKS_CHG) {
3874                 CSR_WRITE_4(sc, BGE_HCC_RX_COAL_TICKS,
3875                             sc->bnx_rx_coal_ticks);
3876                 DELAY(10);
3877                 val = CSR_READ_4(sc, BGE_HCC_RX_COAL_TICKS);
3878
3879                 if (bootverbose) {
3880                         if_printf(ifp, "rx_coal_ticks -> %u\n",
3881                                   sc->bnx_rx_coal_ticks);
3882                 }
3883         }
3884
3885         if (sc->bnx_coal_chg & BNX_TX_COAL_TICKS_CHG) {
3886                 CSR_WRITE_4(sc, BGE_HCC_TX_COAL_TICKS,
3887                             sc->bnx_tx_coal_ticks);
3888                 DELAY(10);
3889                 val = CSR_READ_4(sc, BGE_HCC_TX_COAL_TICKS);
3890
3891                 if (bootverbose) {
3892                         if_printf(ifp, "tx_coal_ticks -> %u\n",
3893                                   sc->bnx_tx_coal_ticks);
3894                 }
3895         }
3896
3897         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_CHG) {
3898                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS,
3899                             sc->bnx_rx_coal_bds);
3900                 DELAY(10);
3901                 val = CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS);
3902
3903                 if (bootverbose) {
3904                         if_printf(ifp, "rx_coal_bds -> %u\n",
3905                                   sc->bnx_rx_coal_bds);
3906                 }
3907         }
3908
3909         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_CHG) {
3910                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS,
3911                             sc->bnx_tx_coal_bds);
3912                 DELAY(10);
3913                 val = CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS);
3914
3915                 if (bootverbose) {
3916                         if_printf(ifp, "tx_max_coal_bds -> %u\n",
3917                                   sc->bnx_tx_coal_bds);
3918                 }
3919         }
3920
3921         if (sc->bnx_coal_chg & BNX_RX_COAL_BDS_INT_CHG) {
3922                 CSR_WRITE_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT,
3923                     sc->bnx_rx_coal_bds_int);
3924                 DELAY(10);
3925                 val = CSR_READ_4(sc, BGE_HCC_RX_MAX_COAL_BDS_INT);
3926
3927                 if (bootverbose) {
3928                         if_printf(ifp, "rx_coal_bds_int -> %u\n",
3929                             sc->bnx_rx_coal_bds_int);
3930                 }
3931         }
3932
3933         if (sc->bnx_coal_chg & BNX_TX_COAL_BDS_INT_CHG) {
3934                 CSR_WRITE_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT,
3935                     sc->bnx_tx_coal_bds_int);
3936                 DELAY(10);
3937                 val = CSR_READ_4(sc, BGE_HCC_TX_MAX_COAL_BDS_INT);
3938
3939                 if (bootverbose) {
3940                         if_printf(ifp, "tx_coal_bds_int -> %u\n",
3941                             sc->bnx_tx_coal_bds_int);
3942                 }
3943         }
3944
3945         sc->bnx_coal_chg = 0;
3946 }
3947
3948 static void
3949 bnx_intr_check(void *xsc)
3950 {
3951         struct bnx_softc *sc = xsc;
3952         struct ifnet *ifp = &sc->arpcom.ac_if;
3953         struct bge_status_block *sblk = sc->bnx_ldata.bnx_status_block;
3954
3955         lwkt_serialize_enter(ifp->if_serializer);
3956
3957         KKASSERT(mycpuid == sc->bnx_intr_cpuid);
3958
3959         if ((ifp->if_flags & (IFF_RUNNING | IFF_POLLING)) != IFF_RUNNING) {
3960                 lwkt_serialize_exit(ifp->if_serializer);
3961                 return;
3962         }
3963
3964         if (sblk->bge_idx[0].bge_rx_prod_idx != sc->bnx_rx_saved_considx ||
3965             sblk->bge_idx[0].bge_tx_cons_idx != sc->bnx_tx_saved_considx) {
3966                 if (sc->bnx_rx_check_considx == sc->bnx_rx_saved_considx &&
3967                     sc->bnx_tx_check_considx == sc->bnx_tx_saved_considx) {
3968                         if (!sc->bnx_intr_maylose) {
3969                                 sc->bnx_intr_maylose = TRUE;
3970                                 goto done;
3971                         }
3972                         if (bootverbose)
3973                                 if_printf(ifp, "lost interrupt\n");
3974                         bnx_msi(sc);
3975                 }
3976         }
3977         sc->bnx_intr_maylose = FALSE;
3978         sc->bnx_rx_check_considx = sc->bnx_rx_saved_considx;
3979         sc->bnx_tx_check_considx = sc->bnx_tx_saved_considx;
3980
3981 done:
3982         callout_reset(&sc->bnx_intr_timer, BNX_INTR_CKINTVL,
3983             bnx_intr_check, sc);
3984         lwkt_serialize_exit(ifp->if_serializer);
3985 }
3986
3987 static void
3988 bnx_enable_intr(struct bnx_softc *sc)
3989 {
3990         struct ifnet *ifp = &sc->arpcom.ac_if;
3991
3992         lwkt_serialize_handler_enable(ifp->if_serializer);
3993
3994         /*
3995          * Enable interrupt.
3996          */
3997         bnx_writembx(sc, BGE_MBX_IRQ0_LO, sc->bnx_status_tag << 24);
3998         if (sc->bnx_flags & BNX_FLAG_ONESHOT_MSI) {
3999                 /* XXX Linux driver */
4000        &n