sbin/newfs_hammer2: Remove "m:" from getopt(3)
[dragonfly.git] / sys / dev / netif / re / if_re.c
1 /*
2  * Copyright (c) 2004
3  *      Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
4  *
5  * Copyright (c) 1997, 1998-2003
6  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. All advertising materials mentioning features or use of this software
17  *    must display the following acknowledgement:
18  *      This product includes software developed by Bill Paul.
19  * 4. Neither the name of the author nor the names of any co-contributors
20  *    may be used to endorse or promote products derived from this software
21  *    without specific prior written permission.
22  *
23  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
27  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33  * THE POSSIBILITY OF SUCH DAMAGE.
34  *
35  * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $
36  */
37
38 /*
39  * RealTek 8169S/8110S/8168/8111/8101E PCI NIC driver
40  *
41  * Written by Bill Paul <wpaul@windriver.com>
42  * Senior Networking Software Engineer
43  * Wind River Systems
44  */
45
46 /*
47  * This driver is designed to support RealTek's next generation of
48  * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently
49  * seven devices in this family: the the RTL8169, the RTL8169S, RTL8110S,
50  * the RTL8168, the RTL8111 and the RTL8101E.
51  *
52  * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC:
53  *
54  *      o Descriptor based DMA mechanism.  Each descriptor represents
55  *        a single packet fragment. Data buffers may be aligned on
56  *        any byte boundary.
57  *
58  *      o 64-bit DMA.
59  *
60  *      o TCP/IP checksum offload for both RX and TX.
61  *
62  *      o High and normal priority transmit DMA rings.
63  *
64  *      o VLAN tag insertion and extraction.
65  *
66  *      o TCP large send (segmentation offload).
67  *
68  *      o 1000Mbps mode.
69  *
70  *      o Jumbo frames.
71  *
72  *      o GMII and TBI ports/registers for interfacing with copper
73  *        or fiber PHYs.
74  *
75  *      o RX and TX DMA rings can have up to 1024 descriptors.
76  *
77  * The 8169 does not have a built-in PHY.  Most reference boards use a
78  * Marvell 88E1000 'Alaska' copper gigE PHY.  8169/8110 is _no longer_
79  * supported.
80  *
81  * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs
82  * (the 'S' stands for 'single-chip').  These devices have the same
83  * programming API as the older 8169, but also have some vendor-specific
84  * registers for the on-board PHY.  The 8110S is a LAN-on-motherboard
85  * part designed to be pin-compatible with the RealTek 8100 10/100 chip.
86  * 
87  * This driver takes advantage of the RX and TX checksum offload and
88  * VLAN tag insertion/extraction features.  It also implements
89  * interrupt moderation using the timer interrupt registers, which
90  * significantly reduces interrupt load.
91  */
92
93 #define _IP_VHL
94
95 #include "opt_ifpoll.h"
96
97 #include <sys/param.h>
98 #include <sys/bus.h>
99 #include <sys/endian.h>
100 #include <sys/kernel.h>
101 #include <sys/in_cksum.h>
102 #include <sys/interrupt.h>
103 #include <sys/malloc.h>
104 #include <sys/mbuf.h>
105 #include <sys/rman.h>
106 #include <sys/serialize.h>
107 #include <sys/socket.h>
108 #include <sys/sockio.h>
109 #include <sys/sysctl.h>
110
111 #include <net/bpf.h>
112 #include <net/ethernet.h>
113 #include <net/if.h>
114 #include <net/ifq_var.h>
115 #include <net/if_arp.h>
116 #include <net/if_dl.h>
117 #include <net/if_media.h>
118 #include <net/if_poll.h>
119 #include <net/if_types.h>
120 #include <net/vlan/if_vlan_var.h>
121 #include <net/vlan/if_vlan_ether.h>
122
123 #include <netinet/ip.h>
124
125 #include "pcidevs.h"
126 #include <bus/pci/pcireg.h>
127 #include <bus/pci/pcivar.h>
128
129 #include <dev/netif/re/if_rereg.h>
130 #include <dev/netif/re/if_revar.h>
131 #include <dev/netif/re/re.h>
132 #include <dev/netif/re/re_dragonfly.h>
133
134 /*
135  * Various supported device vendors/types and their names.
136  */
137 static const struct re_type {
138         uint16_t        re_vid;
139         uint16_t        re_did;
140         const char      *re_name;
141 } re_devs[] = {
142         { PCI_VENDOR_DLINK, PCI_PRODUCT_DLINK_DGE528T,
143           "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
144
145         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8101E,
146           "RealTek 810x PCIe 10/100baseTX" },
147
148         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168,
149           "RealTek 8111/8168 PCIe Gigabit Ethernet" },
150
151         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168_1,
152           "RealTek 8168 PCIe Gigabit Ethernet" },
153
154 #ifdef notyet
155         /*
156          * This driver now only supports built-in PHYs.
157          */
158         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169,
159           "RealTek 8110/8169 Gigabit Ethernet" },
160 #endif
161
162         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169SC,
163           "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" },
164
165         { PCI_VENDOR_COREGA, PCI_PRODUCT_COREGA_CG_LAPCIGT,
166           "Corega CG-LAPCIGT Gigabit Ethernet" },
167
168         { PCI_VENDOR_LINKSYS, PCI_PRODUCT_LINKSYS_EG1032,
169           "Linksys EG1032 Gigabit Ethernet" },
170
171         { PCI_VENDOR_USR2, PCI_PRODUCT_USR2_997902,
172           "US Robotics 997902 Gigabit Ethernet" },
173
174         { PCI_VENDOR_TTTECH, PCI_PRODUCT_TTTECH_MC322,
175           "TTTech MC322 Gigabit Ethernet" },
176
177         { 0, 0, NULL }
178 };
179
180 static int      re_probe(device_t);
181 static int      re_attach(device_t);
182 static int      re_detach(device_t);
183 static int      re_suspend(device_t);
184 static int      re_resume(device_t);
185 static void     re_shutdown(device_t);
186
187 static int      re_allocmem(device_t);
188 static void     re_freemem(device_t);
189 static void     re_freebufmem(struct re_softc *, int, int);
190 static int      re_encap(struct re_softc *, struct mbuf **, int *);
191 static int      re_newbuf_std(struct re_softc *, int, int);
192 #ifdef RE_JUMBO
193 static int      re_newbuf_jumbo(struct re_softc *, int, int);
194 #endif
195 static void     re_setup_rxdesc(struct re_softc *, int);
196 static int      re_rx_list_init(struct re_softc *);
197 static int      re_tx_list_init(struct re_softc *);
198 static int      re_rxeof(struct re_softc *);
199 static int      re_txeof(struct re_softc *);
200 static int      re_tx_collect(struct re_softc *);
201 static void     re_intr(void *);
202 static void     re_tick(void *);
203 static void     re_tick_serialized(void *);
204 static void     re_disable_aspm(device_t);
205 static void     re_link_up(struct re_softc *);
206 static void     re_link_down(struct re_softc *);
207
208 static void     re_start(struct ifnet *, struct ifaltq_subque *);
209 static int      re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
210 static void     re_init(void *);
211 static void     re_stop(struct re_softc *, boolean_t);
212 static void     re_watchdog(struct ifnet *);
213
214 static void     re_setup_hw_im(struct re_softc *);
215 static void     re_setup_sim_im(struct re_softc *);
216 static void     re_disable_hw_im(struct re_softc *);
217 static void     re_disable_sim_im(struct re_softc *);
218 static void     re_config_imtype(struct re_softc *, int);
219 static void     re_setup_intr(struct re_softc *, int, int);
220
221 static int      re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *);
222 static int      re_sysctl_rxtime(SYSCTL_HANDLER_ARGS);
223 static int      re_sysctl_txtime(SYSCTL_HANDLER_ARGS);
224 static int      re_sysctl_simtime(SYSCTL_HANDLER_ARGS);
225 static int      re_sysctl_imtype(SYSCTL_HANDLER_ARGS);
226
227 static int      re_jpool_alloc(struct re_softc *);
228 static void     re_jpool_free(struct re_softc *);
229 #ifdef RE_JUMBO
230 static struct re_jbuf *re_jbuf_alloc(struct re_softc *);
231 static void     re_jbuf_free(void *);
232 static void     re_jbuf_ref(void *);
233 #endif
234
235 #ifdef IFPOLL_ENABLE
236 static void     re_npoll(struct ifnet *, struct ifpoll_info *);
237 static void     re_npoll_compat(struct ifnet *, void *, int);
238 #endif
239
240 static device_method_t re_methods[] = {
241         /* Device interface */
242         DEVMETHOD(device_probe,         re_probe),
243         DEVMETHOD(device_attach,        re_attach),
244         DEVMETHOD(device_detach,        re_detach),
245         DEVMETHOD(device_suspend,       re_suspend),
246         DEVMETHOD(device_resume,        re_resume),
247         DEVMETHOD(device_shutdown,      re_shutdown),
248         DEVMETHOD_END
249 };
250
251 static driver_t re_driver = {
252         "re",
253         re_methods,
254         sizeof(struct re_softc)
255 };
256
257 static devclass_t re_devclass;
258
259 DECLARE_DUMMY_MODULE(if_re);
260 DRIVER_MODULE(if_re, pci, re_driver, re_devclass, NULL, NULL);
261 DRIVER_MODULE(if_re, cardbus, re_driver, re_devclass, NULL, NULL);
262
263 static int      re_rx_desc_count = RE_RX_DESC_CNT_DEF;
264 static int      re_tx_desc_count = RE_TX_DESC_CNT_DEF;
265 static int      re_msi_enable = 1;
266
267 TUNABLE_INT("hw.re.rx_desc_count", &re_rx_desc_count);
268 TUNABLE_INT("hw.re.tx_desc_count", &re_tx_desc_count);
269 TUNABLE_INT("hw.re.msi.enable", &re_msi_enable);
270
271 static __inline void
272 re_free_rxchain(struct re_softc *sc)
273 {
274         if (sc->re_head != NULL) {
275                 m_freem(sc->re_head);
276                 sc->re_head = sc->re_tail = NULL;
277         }
278 }
279
280 static int
281 re_probe(device_t dev)
282 {
283         const struct re_type *t;
284         uint16_t vendor, product;
285
286         vendor = pci_get_vendor(dev);
287         product = pci_get_device(dev);
288
289         /*
290          * Only attach to rev.3 of the Linksys EG1032 adapter.
291          * Rev.2 is supported by sk(4).
292          */
293         if (vendor == PCI_VENDOR_LINKSYS &&
294             product == PCI_PRODUCT_LINKSYS_EG1032 &&
295             pci_get_subdevice(dev) != PCI_SUBDEVICE_LINKSYS_EG1032_REV3)
296                 return ENXIO;
297
298         for (t = re_devs; t->re_name != NULL; t++) {
299                 if (product == t->re_did && vendor == t->re_vid)
300                         break;
301         }
302         if (t->re_name == NULL)
303                 return ENXIO;
304
305         device_set_desc(dev, t->re_name);
306         return 0;
307 }
308
309 static int
310 re_allocmem(device_t dev)
311 {
312         struct re_softc *sc = device_get_softc(dev);
313         bus_dmamem_t dmem;
314         int error, i;
315
316         /*
317          * Allocate list data
318          */
319         sc->re_ldata.re_tx_mbuf =
320         kmalloc(sc->re_tx_desc_cnt * sizeof(struct mbuf *),
321                 M_DEVBUF, M_ZERO | M_WAITOK);
322
323         sc->re_ldata.re_rx_mbuf =
324         kmalloc(sc->re_rx_desc_cnt * sizeof(struct mbuf *),
325                 M_DEVBUF, M_ZERO | M_WAITOK);
326
327         sc->re_ldata.re_rx_paddr =
328         kmalloc(sc->re_rx_desc_cnt * sizeof(bus_addr_t),
329                 M_DEVBUF, M_ZERO | M_WAITOK);
330
331         sc->re_ldata.re_tx_dmamap =
332         kmalloc(sc->re_tx_desc_cnt * sizeof(bus_dmamap_t),
333                 M_DEVBUF, M_ZERO | M_WAITOK);
334
335         sc->re_ldata.re_rx_dmamap =
336         kmalloc(sc->re_rx_desc_cnt * sizeof(bus_dmamap_t),
337                 M_DEVBUF, M_ZERO | M_WAITOK);
338
339         /*
340          * Allocate the parent bus DMA tag appropriate for PCI.
341          */
342         error = bus_dma_tag_create(NULL,        /* parent */
343                         1, 0,                   /* alignment, boundary */
344                         BUS_SPACE_MAXADDR,      /* lowaddr */
345                         BUS_SPACE_MAXADDR,      /* highaddr */
346                         NULL, NULL,             /* filter, filterarg */
347                         BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
348                         0,                      /* nsegments */
349                         BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
350                         0,                      /* flags */
351                         &sc->re_parent_tag);
352         if (error) {
353                 device_printf(dev, "could not allocate parent dma tag\n");
354                 return error;
355         }
356
357         /* Allocate TX descriptor list. */
358         error = bus_dmamem_coherent(sc->re_parent_tag,
359                         RE_RING_ALIGN, 0,
360                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
361                         RE_TX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO,
362                         &dmem);
363         if (error) {
364                 device_printf(dev, "could not allocate TX ring\n");
365                 return error;
366         }
367         sc->re_ldata.re_tx_list_tag = dmem.dmem_tag;
368         sc->re_ldata.re_tx_list_map = dmem.dmem_map;
369         sc->re_ldata.re_tx_list = dmem.dmem_addr;
370         sc->re_ldata.re_tx_list_addr = dmem.dmem_busaddr;
371
372         /* Allocate RX descriptor list. */
373         error = bus_dmamem_coherent(sc->re_parent_tag,
374                         RE_RING_ALIGN, 0,
375                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
376                         RE_RX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO,
377                         &dmem);
378         if (error) {
379                 device_printf(dev, "could not allocate RX ring\n");
380                 return error;
381         }
382         sc->re_ldata.re_rx_list_tag = dmem.dmem_tag;
383         sc->re_ldata.re_rx_list_map = dmem.dmem_map;
384         sc->re_ldata.re_rx_list = dmem.dmem_addr;
385         sc->re_ldata.re_rx_list_addr = dmem.dmem_busaddr;
386
387         /* Allocate maps for TX mbufs. */
388         error = bus_dma_tag_create(sc->re_parent_tag,
389                         1, 0,
390                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
391                         NULL, NULL,
392                         RE_FRAMELEN_MAX, RE_MAXSEGS, MCLBYTES,
393                         BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
394                         &sc->re_ldata.re_tx_mtag);
395         if (error) {
396                 device_printf(dev, "could not allocate TX buf dma tag\n");
397                 return(error);
398         }
399
400         /* Create DMA maps for TX buffers */
401         for (i = 0; i < sc->re_tx_desc_cnt; i++) {
402                 error = bus_dmamap_create(sc->re_ldata.re_tx_mtag,
403                                 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
404                                 &sc->re_ldata.re_tx_dmamap[i]);
405                 if (error) {
406                         device_printf(dev, "can't create DMA map for TX buf\n");
407                         re_freebufmem(sc, i, 0);
408                         return(error);
409                 }
410         }
411
412         /* Allocate maps for RX mbufs. */
413         error = bus_dma_tag_create(sc->re_parent_tag,
414                         RE_RXBUF_ALIGN, 0,
415                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
416                         NULL, NULL,
417                         MCLBYTES, 1, MCLBYTES,
418                         BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED,
419                         &sc->re_ldata.re_rx_mtag);
420         if (error) {
421                 device_printf(dev, "could not allocate RX buf dma tag\n");
422                 return(error);
423         }
424
425         /* Create spare DMA map for RX */
426         error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, BUS_DMA_WAITOK,
427                         &sc->re_ldata.re_rx_spare);
428         if (error) {
429                 device_printf(dev, "can't create spare DMA map for RX\n");
430                 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag);
431                 sc->re_ldata.re_rx_mtag = NULL;
432                 return error;
433         }
434
435         /* Create DMA maps for RX buffers */
436         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
437                 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag,
438                                 BUS_DMA_WAITOK, &sc->re_ldata.re_rx_dmamap[i]);
439                 if (error) {
440                         device_printf(dev, "can't create DMA map for RX buf\n");
441                         re_freebufmem(sc, sc->re_tx_desc_cnt, i);
442                         return(error);
443                 }
444         }
445
446         /* Create jumbo buffer pool for RX if required */
447         if (sc->re_caps & RE_C_CONTIGRX) {
448                 error = re_jpool_alloc(sc);
449                 if (error) {
450                         re_jpool_free(sc);
451 #ifdef RE_JUMBO
452                         /* Disable jumbo frame support */
453                         sc->re_maxmtu = ETHERMTU;
454 #endif
455                 }
456         }
457         return(0);
458 }
459
460 static void
461 re_freebufmem(struct re_softc *sc, int tx_cnt, int rx_cnt)
462 {
463         int i;
464
465         /* Destroy all the RX and TX buffer maps */
466         if (sc->re_ldata.re_tx_mtag) {
467                 for (i = 0; i < tx_cnt; i++) {
468                         bus_dmamap_destroy(sc->re_ldata.re_tx_mtag,
469                                            sc->re_ldata.re_tx_dmamap[i]);
470                 }
471                 bus_dma_tag_destroy(sc->re_ldata.re_tx_mtag);
472                 sc->re_ldata.re_tx_mtag = NULL;
473         }
474
475         if (sc->re_ldata.re_rx_mtag) {
476                 for (i = 0; i < rx_cnt; i++) {
477                         bus_dmamap_destroy(sc->re_ldata.re_rx_mtag,
478                                            sc->re_ldata.re_rx_dmamap[i]);
479                 }
480                 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag,
481                                    sc->re_ldata.re_rx_spare);
482                 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag);
483                 sc->re_ldata.re_rx_mtag = NULL;
484         }
485 }
486
487 static void
488 re_freemem(device_t dev)
489 {
490         struct re_softc *sc = device_get_softc(dev);
491
492         /* Unload and free the RX DMA ring memory and map */
493         if (sc->re_ldata.re_rx_list_tag) {
494                 bus_dmamap_unload(sc->re_ldata.re_rx_list_tag,
495                                   sc->re_ldata.re_rx_list_map);
496                 bus_dmamem_free(sc->re_ldata.re_rx_list_tag,
497                                 sc->re_ldata.re_rx_list,
498                                 sc->re_ldata.re_rx_list_map);
499                 bus_dma_tag_destroy(sc->re_ldata.re_rx_list_tag);
500         }
501
502         /* Unload and free the TX DMA ring memory and map */
503         if (sc->re_ldata.re_tx_list_tag) {
504                 bus_dmamap_unload(sc->re_ldata.re_tx_list_tag,
505                                   sc->re_ldata.re_tx_list_map);
506                 bus_dmamem_free(sc->re_ldata.re_tx_list_tag,
507                                 sc->re_ldata.re_tx_list,
508                                 sc->re_ldata.re_tx_list_map);
509                 bus_dma_tag_destroy(sc->re_ldata.re_tx_list_tag);
510         }
511
512         /* Free RX/TX buf DMA stuffs */
513         re_freebufmem(sc, sc->re_tx_desc_cnt, sc->re_rx_desc_cnt);
514
515         /* Unload and free the stats buffer and map */
516         if (sc->re_ldata.re_stag) {
517                 bus_dmamap_unload(sc->re_ldata.re_stag, sc->re_ldata.re_smap);
518                 bus_dmamem_free(sc->re_ldata.re_stag,
519                                 sc->re_ldata.re_stats,
520                                 sc->re_ldata.re_smap);
521                 bus_dma_tag_destroy(sc->re_ldata.re_stag);
522         }
523
524         if (sc->re_caps & RE_C_CONTIGRX)
525                 re_jpool_free(sc);
526
527         if (sc->re_parent_tag)
528                 bus_dma_tag_destroy(sc->re_parent_tag);
529
530         if (sc->re_ldata.re_tx_mbuf != NULL)
531                 kfree(sc->re_ldata.re_tx_mbuf, M_DEVBUF);
532         if (sc->re_ldata.re_rx_mbuf != NULL)
533                 kfree(sc->re_ldata.re_rx_mbuf, M_DEVBUF);
534         if (sc->re_ldata.re_rx_paddr != NULL)
535                 kfree(sc->re_ldata.re_rx_paddr, M_DEVBUF);
536         if (sc->re_ldata.re_tx_dmamap != NULL)
537                 kfree(sc->re_ldata.re_tx_dmamap, M_DEVBUF);
538         if (sc->re_ldata.re_rx_dmamap != NULL)
539                 kfree(sc->re_ldata.re_rx_dmamap, M_DEVBUF);
540 }
541
542 static boolean_t
543 re_is_faste(struct re_softc *sc)
544 {
545         if (pci_get_vendor(sc->dev) == PCI_VENDOR_REALTEK) {
546                 switch (sc->re_device_id) {
547                 case PCI_PRODUCT_REALTEK_RT8169:
548                 case PCI_PRODUCT_REALTEK_RT8169SC:
549                 case PCI_PRODUCT_REALTEK_RT8168:
550                 case PCI_PRODUCT_REALTEK_RT8168_1:
551                         return FALSE;
552                 default:
553                         return TRUE;
554                 }
555         } else {
556                 return FALSE;
557         }
558 }
559
560 /*
561  * Attach the interface. Allocate softc structures, do ifmedia
562  * setup and ethernet/BPF attach.
563  */
564 static int
565 re_attach(device_t dev)
566 {
567         struct re_softc *sc = device_get_softc(dev);
568         struct ifnet *ifp;
569         struct sysctl_ctx_list *ctx;
570         struct sysctl_oid *tree;
571         uint8_t eaddr[ETHER_ADDR_LEN];
572         int error = 0, qlen, msi_enable;
573         u_int irq_flags;
574
575         callout_init_mp(&sc->re_timer);
576         sc->dev = dev;
577         sc->re_device_id = pci_get_device(dev);
578         sc->re_unit = device_get_unit(dev);
579         ifmedia_init(&sc->media, IFM_IMASK, rtl_ifmedia_upd, rtl_ifmedia_sts);
580
581         sc->re_caps = RE_C_HWIM;
582
583         sc->re_rx_desc_cnt = re_rx_desc_count;
584         if (sc->re_rx_desc_cnt > RE_RX_DESC_CNT_MAX)
585                 sc->re_rx_desc_cnt = RE_RX_DESC_CNT_MAX;
586
587         sc->re_tx_desc_cnt = re_tx_desc_count;
588         if (sc->re_tx_desc_cnt > RE_TX_DESC_CNT_MAX)
589                 sc->re_tx_desc_cnt = RE_TX_DESC_CNT_MAX;
590
591         qlen = RE_IFQ_MAXLEN;
592         if (sc->re_tx_desc_cnt > qlen)
593                 qlen = sc->re_tx_desc_cnt;
594
595         sc->re_rxbuf_size = MCLBYTES;
596         sc->re_newbuf = re_newbuf_std;
597
598         /*
599          * Hardware interrupt moderation settings.
600          * XXX does not seem correct, undocumented.
601          */
602         sc->re_tx_time = 5;             /* 125us */
603         sc->re_rx_time = 2;             /* 50us */
604
605         /* Simulated interrupt moderation setting. */
606         sc->re_sim_time = 150;          /* 150us */
607
608         /* Use simulated interrupt moderation by default. */
609         sc->re_imtype = RE_IMTYPE_SIM;
610         re_config_imtype(sc, sc->re_imtype);
611
612         ctx = device_get_sysctl_ctx(dev);
613         tree = device_get_sysctl_tree(dev);
614         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
615                        "rx_desc_count", CTLFLAG_RD, &sc->re_rx_desc_cnt,
616                        0, "RX desc count");
617         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
618                        "tx_desc_count", CTLFLAG_RD, &sc->re_tx_desc_cnt,
619                        0, "TX desc count");
620         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "sim_time",
621                         CTLTYPE_INT | CTLFLAG_RW,
622                         sc, 0, re_sysctl_simtime, "I",
623                         "Simulated interrupt moderation time (usec).");
624         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "imtype",
625                         CTLTYPE_INT | CTLFLAG_RW,
626                         sc, 0, re_sysctl_imtype, "I",
627                         "Interrupt moderation type -- "
628                         "0:disable, 1:simulated, "
629                         "2:hardware(if supported)");
630         if (sc->re_caps & RE_C_HWIM) {
631                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
632                                 OID_AUTO, "hw_rxtime",
633                                 CTLTYPE_INT | CTLFLAG_RW,
634                                 sc, 0, re_sysctl_rxtime, "I",
635                                 "Hardware interrupt moderation time "
636                                 "(unit: 25usec).");
637                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
638                                 OID_AUTO, "hw_txtime",
639                                 CTLTYPE_INT | CTLFLAG_RW,
640                                 sc, 0, re_sysctl_txtime, "I",
641                                 "Hardware interrupt moderation time "
642                                 "(unit: 25usec).");
643         }
644
645 #ifndef BURN_BRIDGES
646         /*
647          * Handle power management nonsense.
648          */
649
650         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
651                 uint32_t membase, irq;
652
653                 /* Save important PCI config data. */
654                 membase = pci_read_config(dev, RE_PCI_LOMEM, 4);
655                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
656
657                 /* Reset the power state. */
658                 device_printf(dev, "chip is in D%d power mode "
659                     "-- setting to D0\n", pci_get_powerstate(dev));
660
661                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
662
663                 /* Restore PCI config data. */
664                 pci_write_config(dev, RE_PCI_LOMEM, membase, 4);
665                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
666         }
667 #endif
668         /*
669          * Map control/status registers.
670          */
671         pci_enable_busmaster(dev);
672
673         if (pci_is_pcie(dev)) {
674                 sc->re_res_rid = PCIR_BAR(2);
675                 sc->re_res_type = SYS_RES_MEMORY;
676         } else {
677                 sc->re_res_rid = PCIR_BAR(0);
678                 sc->re_res_type = SYS_RES_IOPORT;
679         }
680         sc->re_res = bus_alloc_resource_any(dev, sc->re_res_type,
681             &sc->re_res_rid, RF_ACTIVE);
682         if (sc->re_res == NULL) {
683                 device_printf(dev, "couldn't map IO\n");
684                 error = ENXIO;
685                 goto fail;
686         }
687
688         sc->re_btag = rman_get_bustag(sc->re_res);
689         sc->re_bhandle = rman_get_bushandle(sc->re_res);
690
691         error = rtl_check_mac_version(sc);
692         if (error) {
693                 device_printf(dev, "check mac version failed\n");
694                 goto fail;
695         }
696
697         rtl_init_software_variable(sc);
698         if (pci_is_pcie(dev))
699                 sc->re_if_flags |= RL_FLAG_PCIE;
700         else
701                 sc->re_if_flags &= ~RL_FLAG_PCIE;
702         device_printf(dev, "MAC version 0x%08x, MACFG %u%s%s%s\n",
703             (CSR_READ_4(sc, RE_TXCFG) & 0xFCF00000), sc->re_type,
704             sc->re_coalesce_tx_pkt ? ", software TX defrag" : "",
705             sc->re_pad_runt ? ", pad runt" : "",
706             sc->re_hw_enable_msi_msix ? ", support MSI" : "");
707
708         /*
709          * Allocate interrupt
710          */
711         if (pci_is_pcie(dev) && sc->re_hw_enable_msi_msix)
712                 msi_enable = re_msi_enable;
713         else
714                 msi_enable = 0;
715         sc->re_irq_type = pci_alloc_1intr(dev, msi_enable,
716             &sc->re_irq_rid, &irq_flags);
717
718         sc->re_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->re_irq_rid,
719                                             irq_flags);
720         if (sc->re_irq == NULL) {
721                 device_printf(dev, "couldn't map interrupt\n");
722                 error = ENXIO;
723                 goto fail;
724         }
725
726         /* Disable ASPM */
727         re_disable_aspm(dev);
728
729         rtl_exit_oob(sc);
730         rtl_hw_init(sc);
731
732         /* Reset the adapter. */
733         rtl_reset(sc);
734
735         rtl_get_hw_mac_address(sc, eaddr);
736         if (sc->re_type == MACFG_3)     /* Change PCI Latency time*/
737                 pci_write_config(dev, PCIR_LATTIMER, 0x40, 1);
738
739         /* Allocate DMA stuffs */
740         error = re_allocmem(dev);
741         if (error)
742                 goto fail;
743
744         if (pci_is_pcie(dev)) {
745                 sc->re_bus_speed = 125;
746         } else {
747                 uint8_t cfg2;
748
749                 cfg2 = CSR_READ_1(sc, RE_CFG2);
750                 switch (cfg2 & RE_CFG2_PCICLK_MASK) {
751                 case RE_CFG2_PCICLK_33MHZ:
752                         sc->re_bus_speed = 33;
753                         break;
754                 case RE_CFG2_PCICLK_66MHZ:
755                         sc->re_bus_speed = 66;
756                         break;
757                 default:
758                         device_printf(dev, "unknown bus speed, assume 33MHz\n");
759                         sc->re_bus_speed = 33;
760                         break;
761                 }
762         }
763         device_printf(dev, "bus speed %dMHz\n", sc->re_bus_speed);
764
765         rtl_phy_power_up(sc);
766         rtl_hw_phy_config(sc);
767         rtl_clrwol(sc);
768
769         /* TODO: jumbo frame */
770         CSR_WRITE_2(sc, RE_RxMaxSize, sc->re_rxbuf_size);
771
772         /* Enable hardware checksum if available. */
773         sc->re_tx_cstag = 1;
774         sc->re_rx_cstag = 1;
775
776         ifp = &sc->arpcom.ac_if;
777         ifp->if_softc = sc;
778         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
779         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
780         ifp->if_ioctl = re_ioctl;
781         ifp->if_start = re_start;
782 #ifdef IFPOLL_ENABLE
783         ifp->if_npoll = re_npoll;
784 #endif
785         ifp->if_watchdog = re_watchdog;
786         ifp->if_init = re_init;
787         if (!re_is_faste(sc))
788                 ifp->if_baudrate = 1000000000;
789         else
790                 ifp->if_baudrate = 100000000;
791         ifp->if_nmbclusters = sc->re_rx_desc_cnt;
792         ifq_set_maxlen(&ifp->if_snd, qlen);
793         ifq_set_ready(&ifp->if_snd);
794
795         ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
796             IFCAP_RXCSUM | IFCAP_TXCSUM;
797         ifp->if_capenable = ifp->if_capabilities;
798         /* NOTE: if_hwassist will be setup after the interface is up. */
799
800         /*
801          * Call MI attach routine.
802          */
803         ether_ifattach(ifp, eaddr, NULL);
804
805         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq));
806
807 #ifdef IFPOLL_ENABLE
808         ifpoll_compat_setup(&sc->re_npoll, ctx, (struct sysctl_oid *)tree,
809             device_get_unit(dev), ifp->if_serializer);
810 #endif
811
812         /* Hook interrupt last to avoid having to lock softc */
813         error = bus_setup_intr(dev, sc->re_irq, INTR_MPSAFE | INTR_HIFREQ,
814             re_intr, sc, &sc->re_intrhand, ifp->if_serializer);
815         if (error) {
816                 device_printf(dev, "couldn't set up irq\n");
817                 ether_ifdetach(ifp);
818                 goto fail;
819         }
820
821         ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
822         ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
823         ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);
824         ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
825         if (!re_is_faste(sc)) {
826                 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
827                     0, NULL);
828         }
829         ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
830         ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
831         rtl_ifmedia_upd(ifp);
832
833 fail:
834         if (error)
835                 re_detach(dev);
836
837         return (error);
838 }
839
840 /*
841  * Shutdown hardware and free up resources. This can be called any
842  * time after the mutex has been initialized. It is called in both
843  * the error case in attach and the normal detach case so it needs
844  * to be careful about only freeing resources that have actually been
845  * allocated.
846  */
847 static int
848 re_detach(device_t dev)
849 {
850         struct re_softc *sc = device_get_softc(dev);
851         struct ifnet *ifp = &sc->arpcom.ac_if;
852
853         /* These should only be active if attach succeeded */
854         if (device_is_attached(dev)) {
855                 lwkt_serialize_enter(ifp->if_serializer);
856                 re_stop(sc, TRUE);
857                 bus_teardown_intr(dev, sc->re_irq, sc->re_intrhand);
858                 lwkt_serialize_exit(ifp->if_serializer);
859
860                 ether_ifdetach(ifp);
861         }
862         ifmedia_removeall(&sc->media);
863
864         if (sc->re_irq)
865                 bus_release_resource(dev, SYS_RES_IRQ, sc->re_irq_rid,
866                                      sc->re_irq);
867
868         if (sc->re_irq_type == PCI_INTR_TYPE_MSI)
869                 pci_release_msi(dev);
870
871         if (sc->re_res) {
872                 bus_release_resource(dev, sc->re_res_type, sc->re_res_rid,
873                     sc->re_res);
874         }
875         rtl_cmac_unmap(sc);
876
877         /* Free DMA stuffs */
878         re_freemem(dev);
879
880         return(0);
881 }
882
883 static void
884 re_setup_rxdesc(struct re_softc *sc, int idx)
885 {
886         bus_addr_t paddr;
887         uint32_t cmdstat;
888         struct re_desc *d;
889
890         paddr = sc->re_ldata.re_rx_paddr[idx];
891         d = &sc->re_ldata.re_rx_list[idx];
892
893         d->re_bufaddr_lo = htole32(RE_ADDR_LO(paddr));
894         d->re_bufaddr_hi = htole32(RE_ADDR_HI(paddr));
895
896         cmdstat = sc->re_rxbuf_size | RE_RDESC_CMD_OWN;
897         if (idx == (sc->re_rx_desc_cnt - 1))
898                 cmdstat |= RE_RDESC_CMD_EOR;
899         d->re_cmdstat = htole32(cmdstat);
900 }
901
902 static int
903 re_newbuf_std(struct re_softc *sc, int idx, int init)
904 {
905         bus_dma_segment_t seg;
906         bus_dmamap_t map;
907         struct mbuf *m;
908         int error, nsegs;
909
910         m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR);
911         if (m == NULL) {
912                 error = ENOBUFS;
913
914                 if (init) {
915                         if_printf(&sc->arpcom.ac_if, "m_getcl failed\n");
916                         return error;
917                 } else {
918                         goto back;
919                 }
920         }
921         m->m_len = m->m_pkthdr.len = MCLBYTES;
922
923         /*
924          * NOTE:
925          * re(4) chips need address of the receive buffer to be 8-byte
926          * aligned, so don't call m_adj(m, ETHER_ALIGN) here.
927          */
928
929         error = bus_dmamap_load_mbuf_segment(sc->re_ldata.re_rx_mtag,
930                         sc->re_ldata.re_rx_spare, m,
931                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
932         if (error) {
933                 m_freem(m);
934                 if (init) {
935                         if_printf(&sc->arpcom.ac_if, "can't load RX mbuf\n");
936                         return error;
937                 } else {
938                         goto back;
939                 }
940         }
941
942         if (!init) {
943                 bus_dmamap_sync(sc->re_ldata.re_rx_mtag,
944                                 sc->re_ldata.re_rx_dmamap[idx],
945                                 BUS_DMASYNC_POSTREAD);
946                 bus_dmamap_unload(sc->re_ldata.re_rx_mtag,
947                                   sc->re_ldata.re_rx_dmamap[idx]);
948         }
949         sc->re_ldata.re_rx_mbuf[idx] = m;
950         sc->re_ldata.re_rx_paddr[idx] = seg.ds_addr;
951
952         map = sc->re_ldata.re_rx_dmamap[idx];
953         sc->re_ldata.re_rx_dmamap[idx] = sc->re_ldata.re_rx_spare;
954         sc->re_ldata.re_rx_spare = map;
955 back:
956         re_setup_rxdesc(sc, idx);
957         return error;
958 }
959
960 #ifdef RE_JUMBO
961 static int
962 re_newbuf_jumbo(struct re_softc *sc, int idx, int init)
963 {
964         struct mbuf *m;
965         struct re_jbuf *jbuf;
966         int error = 0;
967
968         MGETHDR(m, init ? M_WAITOK : M_NOWAIT, MT_DATA);
969         if (m == NULL) {
970                 error = ENOBUFS;
971                 if (init) {
972                         if_printf(&sc->arpcom.ac_if, "MGETHDR failed\n");
973                         return error;
974                 } else {
975                         goto back;
976                 }
977         }
978
979         jbuf = re_jbuf_alloc(sc);
980         if (jbuf == NULL) {
981                 m_freem(m);
982
983                 error = ENOBUFS;
984                 if (init) {
985                         if_printf(&sc->arpcom.ac_if, "jpool is empty\n");
986                         return error;
987                 } else {
988                         goto back;
989                 }
990         }
991
992         m->m_ext.ext_arg = jbuf;
993         m->m_ext.ext_buf = jbuf->re_buf;
994         m->m_ext.ext_free = re_jbuf_free;
995         m->m_ext.ext_ref = re_jbuf_ref;
996         m->m_ext.ext_size = sc->re_rxbuf_size;
997
998         m->m_data = m->m_ext.ext_buf;
999         m->m_flags |= M_EXT;
1000         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1001
1002         /*
1003          * NOTE:
1004          * Some re(4) chips(e.g. RTL8101E) need address of the receive buffer
1005          * to be 8-byte aligned, so don't call m_adj(m, ETHER_ALIGN) here.
1006          */
1007
1008         sc->re_ldata.re_rx_mbuf[idx] = m;
1009         sc->re_ldata.re_rx_paddr[idx] = jbuf->re_paddr;
1010 back:
1011         re_setup_rxdesc(sc, idx);
1012         return error;
1013 }
1014 #endif  /* RE_JUMBO */
1015
1016 static int
1017 re_tx_list_init(struct re_softc *sc)
1018 {
1019         bzero(sc->re_ldata.re_tx_list, RE_TX_LIST_SZ(sc));
1020
1021         sc->re_ldata.re_tx_prodidx = 0;
1022         sc->re_ldata.re_tx_considx = 0;
1023         sc->re_ldata.re_tx_free = sc->re_tx_desc_cnt;
1024
1025         return(0);
1026 }
1027
1028 static int
1029 re_rx_list_init(struct re_softc *sc)
1030 {
1031         int i, error;
1032
1033         bzero(sc->re_ldata.re_rx_list, RE_RX_LIST_SZ(sc));
1034
1035         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
1036                 error = sc->re_newbuf(sc, i, 1);
1037                 if (error)
1038                         return(error);
1039         }
1040
1041         sc->re_ldata.re_rx_prodidx = 0;
1042         sc->re_head = sc->re_tail = NULL;
1043
1044         return(0);
1045 }
1046
1047 #define RE_IP4_PACKET   0x1
1048 #define RE_TCP_PACKET   0x2
1049 #define RE_UDP_PACKET   0x4
1050
1051 static __inline uint8_t
1052 re_packet_type(struct re_softc *sc, uint32_t rxstat, uint32_t rxctrl)
1053 {
1054         uint8_t packet_type = 0;
1055
1056         if (sc->re_if_flags & RL_FLAG_DESCV2) {
1057                 if (rxctrl & RE_RDESC_CTL_PROTOIP4)
1058                         packet_type |= RE_IP4_PACKET;
1059         } else {
1060                 if (rxstat & RE_RDESC_STAT_PROTOID)
1061                         packet_type |= RE_IP4_PACKET;
1062         }
1063         if (RE_TCPPKT(rxstat))
1064                 packet_type |= RE_TCP_PACKET;
1065         else if (RE_UDPPKT(rxstat))
1066                 packet_type |= RE_UDP_PACKET;
1067         return packet_type;
1068 }
1069
1070 /*
1071  * RX handler for C+ and 8169. For the gigE chips, we support
1072  * the reception of jumbo frames that have been fragmented
1073  * across multiple 2K mbuf cluster buffers.
1074  */
1075 static int
1076 re_rxeof(struct re_softc *sc)
1077 {
1078         struct ifnet *ifp = &sc->arpcom.ac_if;
1079         struct mbuf *m;
1080         struct re_desc  *cur_rx;
1081         uint32_t rxstat, rxctrl;
1082         int i, total_len, rx = 0;
1083
1084         for (i = sc->re_ldata.re_rx_prodidx;
1085              RE_OWN(&sc->re_ldata.re_rx_list[i]) == 0; RE_RXDESC_INC(sc, i)) {
1086                 cur_rx = &sc->re_ldata.re_rx_list[i];
1087                 m = sc->re_ldata.re_rx_mbuf[i];
1088                 total_len = RE_RXBYTES(cur_rx);
1089                 rxstat = le32toh(cur_rx->re_cmdstat);
1090                 rxctrl = le32toh(cur_rx->re_control);
1091
1092                 rx = 1;
1093
1094 #ifdef INVARIANTS
1095                 if (sc->re_flags & RE_F_USE_JPOOL)
1096                         KKASSERT(rxstat & RE_RDESC_STAT_EOF);
1097 #endif
1098
1099                 if ((rxstat & RE_RDESC_STAT_EOF) == 0) {
1100                         if (sc->re_flags & RE_F_DROP_RXFRAG) {
1101                                 re_setup_rxdesc(sc, i);
1102                                 continue;
1103                         }
1104
1105                         if (sc->re_newbuf(sc, i, 0)) {
1106                                 /* Drop upcoming fragments */
1107                                 sc->re_flags |= RE_F_DROP_RXFRAG;
1108                                 continue;
1109                         }
1110
1111                         m->m_len = MCLBYTES;
1112                         if (sc->re_head == NULL) {
1113                                 sc->re_head = sc->re_tail = m;
1114                         } else {
1115                                 sc->re_tail->m_next = m;
1116                                 sc->re_tail = m;
1117                         }
1118                         continue;
1119                 } else if (sc->re_flags & RE_F_DROP_RXFRAG) {
1120                         /*
1121                          * Last fragment of a multi-fragment packet.
1122                          *
1123                          * Since error already happened, this fragment
1124                          * must be dropped as well as the fragment chain.
1125                          */
1126                         re_setup_rxdesc(sc, i);
1127                         re_free_rxchain(sc);
1128                         sc->re_flags &= ~RE_F_DROP_RXFRAG;
1129                         continue;
1130                 }
1131
1132                 rxstat >>= 1;
1133                 if (rxstat & RE_RDESC_STAT_RXERRSUM) {
1134                         IFNET_STAT_INC(ifp, ierrors, 1);
1135                         /*
1136                          * If this is part of a multi-fragment packet,
1137                          * discard all the pieces.
1138                          */
1139                         re_free_rxchain(sc);
1140                         re_setup_rxdesc(sc, i);
1141                         continue;
1142                 }
1143
1144                 /*
1145                  * If allocating a replacement mbuf fails,
1146                  * reload the current one.
1147                  */
1148
1149                 if (sc->re_newbuf(sc, i, 0)) {
1150                         IFNET_STAT_INC(ifp, ierrors, 1);
1151                         continue;
1152                 }
1153
1154                 if (sc->re_head != NULL) {
1155                         m->m_len = total_len % MCLBYTES;
1156                         /* 
1157                          * Special case: if there's 4 bytes or less
1158                          * in this buffer, the mbuf can be discarded:
1159                          * the last 4 bytes is the CRC, which we don't
1160                          * care about anyway.
1161                          */
1162                         if (m->m_len <= ETHER_CRC_LEN) {
1163                                 sc->re_tail->m_len -=
1164                                     (ETHER_CRC_LEN - m->m_len);
1165                                 m_freem(m);
1166                         } else {
1167                                 m->m_len -= ETHER_CRC_LEN;
1168                                 sc->re_tail->m_next = m;
1169                         }
1170                         m = sc->re_head;
1171                         sc->re_head = sc->re_tail = NULL;
1172                         m->m_pkthdr.len = total_len - ETHER_CRC_LEN;
1173                 } else {
1174                         m->m_pkthdr.len = m->m_len =
1175                             (total_len - ETHER_CRC_LEN);
1176                 }
1177
1178                 IFNET_STAT_INC(ifp, ipackets, 1);
1179                 m->m_pkthdr.rcvif = ifp;
1180
1181                 /* Do RX checksumming if enabled */
1182
1183                 if (ifp->if_capenable & IFCAP_RXCSUM) {
1184                         uint8_t packet_type;
1185
1186                         packet_type = re_packet_type(sc, rxstat, rxctrl);
1187
1188                         /* Check IP header checksum */
1189                         if (packet_type & RE_IP4_PACKET) {
1190                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
1191                                 if ((rxstat & RE_RDESC_STAT_IPSUMBAD) == 0)
1192                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1193                         }
1194
1195                         /* Check TCP/UDP checksum */
1196                         if (((packet_type & RE_TCP_PACKET) &&
1197                              (rxstat & RE_RDESC_STAT_TCPSUMBAD) == 0) ||
1198                             ((packet_type & RE_UDP_PACKET) &&
1199                              (rxstat & RE_RDESC_STAT_UDPSUMBAD) == 0)) {
1200                                 m->m_pkthdr.csum_flags |=
1201                                     CSUM_DATA_VALID|CSUM_PSEUDO_HDR|
1202                                     CSUM_FRAG_NOT_CHECKED;
1203                                 m->m_pkthdr.csum_data = 0xffff;
1204                         }
1205                 }
1206
1207                 if (rxctrl & RE_RDESC_CTL_HASTAG) {
1208                         m->m_flags |= M_VLANTAG;
1209                         m->m_pkthdr.ether_vlantag =
1210                                 be16toh((rxctrl & RE_RDESC_CTL_TAGDATA));
1211                 }
1212                 ifp->if_input(ifp, m, NULL, -1);
1213         }
1214
1215         sc->re_ldata.re_rx_prodidx = i;
1216
1217         return rx;
1218 }
1219
1220 #undef RE_IP4_PACKET
1221 #undef RE_TCP_PACKET
1222 #undef RE_UDP_PACKET
1223
1224 static int
1225 re_tx_collect(struct re_softc *sc)
1226 {
1227         struct ifnet *ifp = &sc->arpcom.ac_if;
1228         uint32_t txstat;
1229         int idx, tx = 0;
1230
1231         for (idx = sc->re_ldata.re_tx_considx;
1232              sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt;
1233              RE_TXDESC_INC(sc, idx)) {
1234                 txstat = le32toh(sc->re_ldata.re_tx_list[idx].re_cmdstat);
1235                 if (txstat & RE_TDESC_CMD_OWN)
1236                         break;
1237
1238                 tx = 1;
1239
1240                 sc->re_ldata.re_tx_list[idx].re_bufaddr_lo = 0;
1241
1242                 /*
1243                  * We only stash mbufs in the last descriptor
1244                  * in a fragment chain, which also happens to
1245                  * be the only place where the TX status bits
1246                  * are valid.
1247                  */
1248                 if (txstat & RE_TDESC_CMD_EOF) {
1249                         bus_dmamap_unload(sc->re_ldata.re_tx_mtag,
1250                             sc->re_ldata.re_tx_dmamap[idx]);
1251                         m_freem(sc->re_ldata.re_tx_mbuf[idx]);
1252                         sc->re_ldata.re_tx_mbuf[idx] = NULL;
1253                         if (txstat & (RE_TDESC_STAT_EXCESSCOL|
1254                             RE_TDESC_STAT_COLCNT))
1255                                 IFNET_STAT_INC(ifp, collisions, 1);
1256                         if (txstat & RE_TDESC_STAT_TXERRSUM)
1257                                 IFNET_STAT_INC(ifp, oerrors, 1);
1258                         else
1259                                 IFNET_STAT_INC(ifp, opackets, 1);
1260                 }
1261                 sc->re_ldata.re_tx_free++;
1262         }
1263         sc->re_ldata.re_tx_considx = idx;
1264
1265         return tx;
1266 }
1267
1268 static int
1269 re_txeof(struct re_softc *sc)
1270 {
1271         struct ifnet *ifp = &sc->arpcom.ac_if;
1272         int tx;
1273
1274         tx = re_tx_collect(sc);
1275
1276         /* There is enough free TX descs */
1277         if (sc->re_ldata.re_tx_free > RE_TXDESC_SPARE)
1278                 ifq_clr_oactive(&ifp->if_snd);
1279
1280         /*
1281          * Some chips will ignore a second TX request issued while an
1282          * existing transmission is in progress. If the transmitter goes
1283          * idle but there are still packets waiting to be sent, we need
1284          * to restart the channel here to flush them out. This only seems
1285          * to be required with the PCIe devices.
1286          */
1287         if (sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt)
1288                 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ);
1289         else
1290                 ifp->if_timer = 0;
1291
1292         return tx;
1293 }
1294
1295 static void
1296 re_tick(void *xsc)
1297 {
1298         struct re_softc *sc = xsc;
1299
1300         lwkt_serialize_enter(sc->arpcom.ac_if.if_serializer);
1301         re_tick_serialized(xsc);
1302         lwkt_serialize_exit(sc->arpcom.ac_if.if_serializer);
1303 }
1304
1305 static void
1306 re_tick_serialized(void *xsc)
1307 {
1308         struct re_softc *sc = xsc;
1309         struct ifnet *ifp = &sc->arpcom.ac_if;
1310
1311         ASSERT_SERIALIZED(ifp->if_serializer);
1312
1313         if ((ifp->if_flags & IFF_RUNNING) == 0)
1314                 return;
1315
1316         if (rtl_link_ok(sc)) {
1317                 if ((sc->re_flags & RE_F_LINKED) == 0)
1318                         re_link_up(sc);
1319         } else if (sc->re_flags & RE_F_LINKED) {
1320                 re_link_down(sc);
1321         }
1322         callout_reset(&sc->re_timer, hz, re_tick, sc);
1323 }
1324
1325 #ifdef IFPOLL_ENABLE
1326
1327 static void
1328 re_npoll_compat(struct ifnet *ifp, void *arg __unused, int count)
1329 {
1330         struct re_softc *sc = ifp->if_softc;
1331
1332         ASSERT_SERIALIZED(ifp->if_serializer);
1333
1334         if (sc->re_npoll.ifpc_stcount-- == 0) {
1335                 uint16_t status;
1336
1337                 sc->re_npoll.ifpc_stcount = sc->re_npoll.ifpc_stfrac;
1338
1339                 status = CSR_READ_2(sc, RE_ISR);
1340                 if (status)
1341                         CSR_WRITE_2(sc, RE_ISR, status);
1342
1343                 /*
1344                  * XXX check behaviour on receiver stalls.
1345                  */
1346
1347                 if (status & RE_ISR_SYSTEM_ERR) {
1348                         rtl_reset(sc);
1349                         re_init(sc);
1350                         /* Done! */
1351                         return;
1352                 }
1353         }
1354
1355         sc->rxcycles = count;
1356         re_rxeof(sc);
1357         re_txeof(sc);
1358
1359         if (!ifq_is_empty(&ifp->if_snd))
1360                 if_devstart(ifp);
1361 }
1362
1363 static void
1364 re_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1365 {
1366         struct re_softc *sc = ifp->if_softc;
1367
1368         ASSERT_SERIALIZED(ifp->if_serializer);
1369
1370         if (info != NULL) {
1371                 int cpuid = sc->re_npoll.ifpc_cpuid;
1372
1373                 info->ifpi_rx[cpuid].poll_func = re_npoll_compat;
1374                 info->ifpi_rx[cpuid].arg = NULL;
1375                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
1376
1377                 if (ifp->if_flags & IFF_RUNNING)
1378                         re_setup_intr(sc, 0, RE_IMTYPE_NONE);
1379                 ifq_set_cpuid(&ifp->if_snd, cpuid);
1380         } else {
1381                 if (ifp->if_flags & IFF_RUNNING)
1382                         re_setup_intr(sc, 1, sc->re_imtype);
1383                 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq));
1384         }
1385 }
1386 #endif /* IFPOLL_ENABLE */
1387
1388 static void
1389 re_intr(void *arg)
1390 {
1391         struct re_softc *sc = arg;
1392         struct ifnet *ifp = &sc->arpcom.ac_if;
1393         uint16_t status;
1394         int proc;
1395
1396         ASSERT_SERIALIZED(ifp->if_serializer);
1397
1398         if ((sc->re_flags & RE_F_SUSPENDED) ||
1399             (ifp->if_flags & IFF_RUNNING) == 0)
1400                 return;
1401
1402         /* Disable interrupts. */
1403         CSR_WRITE_2(sc, RE_IMR, 0);
1404
1405         status = CSR_READ_2(sc, RE_ISR);
1406 again:
1407         proc = 0;
1408         if (status)
1409                 CSR_WRITE_2(sc, RE_ISR, status);
1410         if (status & sc->re_intrs) {
1411                 if (status & RE_ISR_SYSTEM_ERR) {
1412                         rtl_reset(sc);
1413                         re_init(sc);
1414                         /* Done! */
1415                         return;
1416                 }
1417                 proc |= re_rxeof(sc);
1418                 proc |= re_txeof(sc);
1419         }
1420
1421         if (sc->re_imtype == RE_IMTYPE_SIM) {
1422                 if ((sc->re_flags & RE_F_TIMER_INTR)) {
1423                         if (!proc) {
1424                                 /*
1425                                  * Nothing needs to be processed, fallback
1426                                  * to use TX/RX interrupts.
1427                                  *
1428                                  * NOTE: This will re-enable interrupts.
1429                                  */
1430                                 re_setup_intr(sc, 1, RE_IMTYPE_NONE);
1431
1432                                 /*
1433                                  * Recollect, mainly to avoid the possible
1434                                  * race introduced by changing interrupt
1435                                  * masks.
1436                                  */
1437                                 re_rxeof(sc);
1438                                 re_txeof(sc);
1439                         } else {
1440                                 /* Re-enable interrupts. */
1441                                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1442                                 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */
1443                         }
1444                 } else if (proc) {
1445                         /*
1446                          * Assume that using simulated interrupt moderation
1447                          * (hardware timer based) could reduce the interript
1448                          * rate.
1449                          *
1450                          * NOTE: This will re-enable interrupts.
1451                          */
1452                         re_setup_intr(sc, 1, RE_IMTYPE_SIM);
1453                 } else {
1454                         /* Re-enable interrupts. */
1455                         CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1456                 }
1457         } else {
1458                 status = CSR_READ_2(sc, RE_ISR);
1459                 if (status & sc->re_intrs) {
1460                         if (!ifq_is_empty(&ifp->if_snd))
1461                                 if_devstart(ifp);
1462                         /* NOTE: Interrupts are still disabled. */
1463                         goto again;
1464                 }
1465                 /* Re-enable interrupts. */
1466                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1467         }
1468
1469         if (!ifq_is_empty(&ifp->if_snd))
1470                 if_devstart(ifp);
1471 }
1472
1473 static int
1474 re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx0)
1475 {
1476         struct mbuf *m = *m_head;
1477         bus_dma_segment_t segs[RE_MAXSEGS];
1478         bus_dmamap_t map;
1479         int error, maxsegs, idx, i, nsegs;
1480         struct re_desc *d, *tx_ring;
1481         uint32_t cmd_csum, ctl_csum, vlantag;
1482
1483         KASSERT(sc->re_ldata.re_tx_free > RE_TXDESC_SPARE,
1484                 ("not enough free TX desc"));
1485
1486         if (sc->re_coalesce_tx_pkt && m->m_pkthdr.len != m->m_len) {
1487                 struct mbuf *m_new;
1488
1489                 m_new = m_defrag(m, M_NOWAIT);
1490                 if (m_new == NULL) {
1491                         error = ENOBUFS;
1492                         goto back;
1493                 } else {
1494                         *m_head = m = m_new;
1495                         if (m->m_pkthdr.len != m->m_len) {
1496                                 /* Still not configuous; give up. */
1497                                 error = ENOBUFS;
1498                                 goto back;
1499                         }
1500                 }
1501         }
1502
1503         map = sc->re_ldata.re_tx_dmamap[*idx0];
1504
1505         /*
1506          * Set up checksum offload. Note: checksum offload bits must
1507          * appear in all descriptors of a multi-descriptor transmit
1508          * attempt. (This is according to testing done with an 8169
1509          * chip. I'm not sure if this is a requirement or a bug.)
1510          */
1511         cmd_csum = ctl_csum = 0;
1512         if (m->m_pkthdr.csum_flags & CSUM_IP) {
1513                 cmd_csum |= RE_TDESC_CMD_IPCSUM;
1514                 ctl_csum |= RE_TDESC_CTL_IPCSUM;
1515         }
1516         if (m->m_pkthdr.csum_flags & CSUM_TCP) {
1517                 cmd_csum |= RE_TDESC_CMD_TCPCSUM;
1518                 ctl_csum |= RE_TDESC_CTL_TCPCSUM;
1519         }
1520         if (m->m_pkthdr.csum_flags & CSUM_UDP) {
1521                 cmd_csum |= RE_TDESC_CMD_UDPCSUM;
1522                 ctl_csum |= RE_TDESC_CTL_UDPCSUM;
1523         }
1524
1525         /* For version2 descriptor, csum flags are set on re_control */
1526         if (sc->re_if_flags & RL_FLAG_DESCV2)
1527                 cmd_csum = 0;
1528         else
1529                 ctl_csum = 0;
1530
1531         if (sc->re_pad_runt) {
1532                 /*
1533                  * With some of the RealTek chips, using the checksum offload
1534                  * support in conjunction with the autopadding feature results
1535                  * in the transmission of corrupt frames. For example, if we
1536                  * need to send a really small IP fragment that's less than 60
1537                  * bytes in size, and IP header checksumming is enabled, the
1538                  * resulting ethernet frame that appears on the wire will
1539                  * have garbled payload. To work around this, if TX checksum
1540                  * offload is enabled, we always manually pad short frames out
1541                  * to the minimum ethernet frame size.
1542                  *
1543                  * Note: this appears unnecessary for TCP, and doing it for TCP
1544                  * with PCIe adapters seems to result in bad checksums.
1545                  */
1546                 if ((m->m_pkthdr.csum_flags &
1547                      (CSUM_DELAY_IP | CSUM_DELAY_DATA)) &&
1548                     (m->m_pkthdr.csum_flags & CSUM_TCP) == 0 &&
1549                     m->m_pkthdr.len < RE_MIN_FRAMELEN) {
1550                         error = m_devpad(m, RE_MIN_FRAMELEN);
1551                         if (error)
1552                                 goto back;
1553                 }
1554         }
1555
1556         vlantag = 0;
1557         if (m->m_flags & M_VLANTAG) {
1558                 vlantag = htobe16(m->m_pkthdr.ether_vlantag) |
1559                           RE_TDESC_CTL_INSTAG;
1560         }
1561
1562         maxsegs = sc->re_ldata.re_tx_free;
1563         if (maxsegs > RE_MAXSEGS)
1564                 maxsegs = RE_MAXSEGS;
1565
1566         error = bus_dmamap_load_mbuf_defrag(sc->re_ldata.re_tx_mtag, map,
1567                         m_head, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1568         if (error)
1569                 goto back;
1570
1571         m = *m_head;
1572         bus_dmamap_sync(sc->re_ldata.re_tx_mtag, map, BUS_DMASYNC_PREWRITE);
1573
1574         /*
1575          * Map the segment array into descriptors.  We also keep track
1576          * of the end of the ring and set the end-of-ring bits as needed,
1577          * and we set the ownership bits in all except the very first
1578          * descriptor, whose ownership bits will be turned on later.
1579          */
1580         tx_ring = sc->re_ldata.re_tx_list;
1581         idx = *idx0;
1582         i = 0;
1583         for (;;) {
1584                 uint32_t cmdstat;
1585
1586                 d = &tx_ring[idx];
1587
1588                 cmdstat = segs[i].ds_len;
1589                 d->re_bufaddr_lo = htole32(RE_ADDR_LO(segs[i].ds_addr));
1590                 d->re_bufaddr_hi = htole32(RE_ADDR_HI(segs[i].ds_addr));
1591                 if (i == 0)
1592                         cmdstat |= RE_TDESC_CMD_SOF;
1593                 else
1594                         cmdstat |= RE_TDESC_CMD_OWN;
1595                 if (idx == (sc->re_tx_desc_cnt - 1))
1596                         cmdstat |= RE_TDESC_CMD_EOR;
1597                 d->re_cmdstat = htole32(cmdstat | cmd_csum);
1598                 d->re_control = htole32(ctl_csum | vlantag);
1599
1600                 i++;
1601                 if (i == nsegs)
1602                         break;
1603                 RE_TXDESC_INC(sc, idx);
1604         }
1605         d->re_cmdstat |= htole32(RE_TDESC_CMD_EOF);
1606
1607         /* Transfer ownership of packet to the chip. */
1608         d->re_cmdstat |= htole32(RE_TDESC_CMD_OWN);
1609         if (*idx0 != idx)
1610                 tx_ring[*idx0].re_cmdstat |= htole32(RE_TDESC_CMD_OWN);
1611
1612         /*
1613          * Insure that the map for this transmission
1614          * is placed at the array index of the last descriptor
1615          * in this chain.
1616          */
1617         sc->re_ldata.re_tx_dmamap[*idx0] = sc->re_ldata.re_tx_dmamap[idx];
1618         sc->re_ldata.re_tx_dmamap[idx] = map;
1619
1620         sc->re_ldata.re_tx_mbuf[idx] = m;
1621         sc->re_ldata.re_tx_free -= nsegs;
1622
1623         RE_TXDESC_INC(sc, idx);
1624         *idx0 = idx;
1625 back:
1626         if (error) {
1627                 m_freem(*m_head);
1628                 *m_head = NULL;
1629         }
1630         return error;
1631 }
1632
1633 /*
1634  * Main transmit routine for C+ and gigE NICs.
1635  */
1636
1637 static void
1638 re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1639 {
1640         struct re_softc *sc = ifp->if_softc;
1641         struct mbuf *m_head;
1642         int idx, need_trans, oactive, error;
1643
1644         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1645         ASSERT_SERIALIZED(ifp->if_serializer);
1646
1647         if ((sc->re_flags & RE_F_LINKED) == 0) {
1648                 ifq_purge(&ifp->if_snd);
1649                 return;
1650         }
1651
1652         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
1653                 return;
1654
1655         idx = sc->re_ldata.re_tx_prodidx;
1656
1657         need_trans = 0;
1658         oactive = 0;
1659         while (sc->re_ldata.re_tx_mbuf[idx] == NULL) {
1660                 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) {
1661                         if (!oactive) {
1662                                 if (re_tx_collect(sc)) {
1663                                         oactive = 1;
1664                                         continue;
1665                                 }
1666                         }
1667                         ifq_set_oactive(&ifp->if_snd);
1668                         break;
1669                 }
1670
1671                 m_head = ifq_dequeue(&ifp->if_snd);
1672                 if (m_head == NULL)
1673                         break;
1674
1675                 error = re_encap(sc, &m_head, &idx);
1676                 if (error) {
1677                         /* m_head is freed by re_encap(), if we reach here */
1678                         IFNET_STAT_INC(ifp, oerrors, 1);
1679
1680                         if (error == EFBIG && !oactive) {
1681                                 if (re_tx_collect(sc)) {
1682                                         oactive = 1;
1683                                         continue;
1684                                 }
1685                         }
1686                         ifq_set_oactive(&ifp->if_snd);
1687                         break;
1688                 }
1689
1690                 oactive = 0;
1691                 need_trans = 1;
1692
1693                 /*
1694                  * If there's a BPF listener, bounce a copy of this frame
1695                  * to him.
1696                  */
1697                 ETHER_BPF_MTAP(ifp, m_head);
1698         }
1699
1700         /*
1701          * If sc->re_ldata.re_tx_mbuf[idx] is not NULL it is possible
1702          * for OACTIVE to not be properly set when we also do not
1703          * have sufficient free tx descriptors, leaving packet in
1704          * ifp->if_snd.  This can cause if_start_dispatch() to loop
1705          * infinitely so make sure OACTIVE is set properly.
1706          */
1707         if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) {
1708                 if (!ifq_is_oactive(&ifp->if_snd)) {
1709 #if 0
1710                         if_printf(ifp, "Debug: OACTIVE was not set when "
1711                             "re_tx_free was below minimum!\n");
1712 #endif
1713                         ifq_set_oactive(&ifp->if_snd);
1714                 }
1715         }
1716         if (!need_trans)
1717                 return;
1718
1719         sc->re_ldata.re_tx_prodidx = idx;
1720
1721         /*
1722          * RealTek put the TX poll request register in a different
1723          * location on the 8169 gigE chip. I don't know why.
1724          */
1725         CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ);
1726
1727         /*
1728          * Set a timeout in case the chip goes out to lunch.
1729          */
1730         ifp->if_timer = 5;
1731 }
1732
1733 static void
1734 re_link_up(struct re_softc *sc)
1735 {
1736         struct ifnet *ifp = &sc->arpcom.ac_if;
1737         int error;
1738
1739         ASSERT_SERIALIZED(ifp->if_serializer);
1740
1741         rtl_link_on_patch(sc);
1742         re_stop(sc, FALSE);
1743         rtl_set_eaddr(sc);
1744
1745         error = re_rx_list_init(sc);
1746         if (error) {
1747                 re_stop(sc, TRUE);
1748                 return;
1749         }
1750         error = re_tx_list_init(sc);
1751         if (error) {
1752                 re_stop(sc, TRUE);
1753                 return;
1754         }
1755
1756         /*
1757          * Load the addresses of the RX and TX lists into the chip.
1758          */
1759         CSR_WRITE_4(sc, RE_RXLIST_ADDR_HI,
1760             RE_ADDR_HI(sc->re_ldata.re_rx_list_addr));
1761         CSR_WRITE_4(sc, RE_RXLIST_ADDR_LO,
1762             RE_ADDR_LO(sc->re_ldata.re_rx_list_addr));
1763
1764         CSR_WRITE_4(sc, RE_TXLIST_ADDR_HI,
1765             RE_ADDR_HI(sc->re_ldata.re_tx_list_addr));
1766         CSR_WRITE_4(sc, RE_TXLIST_ADDR_LO,
1767             RE_ADDR_LO(sc->re_ldata.re_tx_list_addr));
1768
1769         rtl_hw_start(sc);
1770
1771 #ifdef IFPOLL_ENABLE
1772         /*
1773          * Disable interrupts if we are polling.
1774          */
1775         if (ifp->if_flags & IFF_NPOLLING)
1776                 re_setup_intr(sc, 0, RE_IMTYPE_NONE);
1777         else    /* otherwise ... */
1778 #endif /* IFPOLL_ENABLE */
1779         /*
1780          * Enable interrupts.
1781          */
1782         re_setup_intr(sc, 1, sc->re_imtype);
1783         CSR_WRITE_2(sc, RE_ISR, sc->re_intrs);
1784
1785         sc->re_flags |= RE_F_LINKED;
1786         ifp->if_link_state = LINK_STATE_UP;
1787         if_link_state_change(ifp);
1788
1789         if (bootverbose)
1790                 if_printf(ifp, "link UP\n");
1791
1792         if (!ifq_is_empty(&ifp->if_snd))
1793                 if_devstart(ifp);
1794 }
1795
1796 static void
1797 re_link_down(struct re_softc *sc)
1798 {
1799         struct ifnet *ifp = &sc->arpcom.ac_if;
1800
1801         /* NOTE: re_stop() will reset RE_F_LINKED. */
1802         ifp->if_link_state = LINK_STATE_DOWN;
1803         if_link_state_change(ifp);
1804
1805         re_stop(sc, FALSE);
1806         rtl_ifmedia_upd(ifp);
1807
1808         if (bootverbose)
1809                 if_printf(ifp, "link DOWN\n");
1810 }
1811
1812 static void
1813 re_init(void *xsc)
1814 {
1815         struct re_softc *sc = xsc;
1816         struct ifnet *ifp = &sc->arpcom.ac_if;
1817
1818         ASSERT_SERIALIZED(ifp->if_serializer);
1819
1820         re_stop(sc, TRUE);
1821         if (rtl_link_ok(sc)) {
1822                 if (bootverbose)
1823                         if_printf(ifp, "link is UP in if_init\n");
1824                 re_link_up(sc);
1825         }
1826
1827         ifp->if_flags |= IFF_RUNNING;
1828         ifq_clr_oactive(&ifp->if_snd);
1829
1830         callout_reset(&sc->re_timer, hz, re_tick, sc);
1831 }
1832
1833 static int
1834 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
1835 {
1836         struct re_softc *sc = ifp->if_softc;
1837         struct ifreq *ifr = (struct ifreq *)data;
1838         int error = 0, mask;
1839
1840         ASSERT_SERIALIZED(ifp->if_serializer);
1841
1842         switch(command) {
1843         case SIOCSIFMTU:
1844 #ifdef RE_JUMBO
1845                 if (ifr->ifr_mtu > sc->re_maxmtu) {
1846                         error = EINVAL;
1847                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
1848                         ifp->if_mtu = ifr->ifr_mtu;
1849                         if (ifp->if_flags & IFF_RUNNING)
1850                                 ifp->if_init(sc);
1851                 }
1852 #else
1853                 error = EOPNOTSUPP;
1854 #endif
1855                 break;
1856
1857         case SIOCSIFFLAGS:
1858                 if (ifp->if_flags & IFF_UP) {
1859                         if (ifp->if_flags & IFF_RUNNING) {
1860                                 if ((ifp->if_flags ^ sc->re_saved_ifflags) &
1861                                     (IFF_PROMISC | IFF_ALLMULTI))
1862                                         rtl_set_rx_packet_filter(sc);
1863                         } else {
1864                                 re_init(sc);
1865                         }
1866                 } else if (ifp->if_flags & IFF_RUNNING) {
1867                         re_stop(sc, TRUE);
1868                 }
1869                 sc->re_saved_ifflags = ifp->if_flags;
1870                 break;
1871
1872         case SIOCADDMULTI:
1873         case SIOCDELMULTI:
1874                 rtl_set_rx_packet_filter(sc);
1875                 break;
1876
1877         case SIOCGIFMEDIA:
1878         case SIOCSIFMEDIA:
1879                 error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
1880                 break;
1881
1882         case SIOCSIFCAP:
1883                 mask = (ifr->ifr_reqcap ^ ifp->if_capenable) &
1884                        ifp->if_capabilities;
1885                 ifp->if_capenable ^= mask;
1886
1887                 /* NOTE: re_init will setup if_hwassist. */
1888                 ifp->if_hwassist = 0;
1889
1890                 /* Setup flags for the backend. */
1891                 if (ifp->if_capenable & IFCAP_RXCSUM)
1892                         sc->re_rx_cstag = 1;
1893                 else
1894                         sc->re_rx_cstag = 0;
1895                 if (ifp->if_capenable & IFCAP_TXCSUM)
1896                         sc->re_tx_cstag = 1;
1897                 else
1898                         sc->re_tx_cstag = 0;
1899
1900                 if (mask && (ifp->if_flags & IFF_RUNNING))
1901                         re_init(sc);
1902                 break;
1903
1904         default:
1905                 error = ether_ioctl(ifp, command, data);
1906                 break;
1907         }
1908         return(error);
1909 }
1910
1911 static void
1912 re_watchdog(struct ifnet *ifp)
1913 {
1914         struct re_softc *sc = ifp->if_softc;
1915
1916         ASSERT_SERIALIZED(ifp->if_serializer);
1917
1918         IFNET_STAT_INC(ifp, oerrors, 1);
1919
1920         re_txeof(sc);
1921         re_rxeof(sc);
1922
1923         if (sc->re_ldata.re_tx_free != sc->re_tx_desc_cnt) {
1924                 if_printf(ifp, "watchdog timeout, txd free %d\n",
1925                     sc->re_ldata.re_tx_free);
1926                 rtl_reset(sc);
1927                 re_init(sc);
1928         }
1929 }
1930
1931 /*
1932  * Stop the adapter and free any mbufs allocated to the
1933  * RX and TX lists.
1934  */
1935 static void
1936 re_stop(struct re_softc *sc, boolean_t full_stop)
1937 {
1938         struct ifnet *ifp = &sc->arpcom.ac_if;
1939         int i;
1940
1941         ASSERT_SERIALIZED(ifp->if_serializer);
1942
1943         /* Stop the adapter. */
1944         rtl_stop(sc);
1945
1946         ifp->if_timer = 0;
1947         if (full_stop) {
1948                 callout_stop(&sc->re_timer);
1949                 ifp->if_flags &= ~IFF_RUNNING;
1950         }
1951         ifq_clr_oactive(&ifp->if_snd);
1952         sc->re_flags &= ~(RE_F_TIMER_INTR | RE_F_DROP_RXFRAG | RE_F_LINKED);
1953
1954         re_free_rxchain(sc);
1955
1956         /* Free the TX list buffers. */
1957         for (i = 0; i < sc->re_tx_desc_cnt; i++) {
1958                 if (sc->re_ldata.re_tx_mbuf[i] != NULL) {
1959                         bus_dmamap_unload(sc->re_ldata.re_tx_mtag,
1960                                           sc->re_ldata.re_tx_dmamap[i]);
1961                         m_freem(sc->re_ldata.re_tx_mbuf[i]);
1962                         sc->re_ldata.re_tx_mbuf[i] = NULL;
1963                 }
1964         }
1965
1966         /* Free the RX list buffers. */
1967         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
1968                 if (sc->re_ldata.re_rx_mbuf[i] != NULL) {
1969                         if ((sc->re_flags & RE_F_USE_JPOOL) == 0) {
1970                                 bus_dmamap_unload(sc->re_ldata.re_rx_mtag,
1971                                                   sc->re_ldata.re_rx_dmamap[i]);
1972                         }
1973                         m_freem(sc->re_ldata.re_rx_mbuf[i]);
1974                         sc->re_ldata.re_rx_mbuf[i] = NULL;
1975                 }
1976         }
1977 }
1978
1979 /*
1980  * Device suspend routine.  Stop the interface and save some PCI
1981  * settings in case the BIOS doesn't restore them properly on
1982  * resume.
1983  */
1984 static int
1985 re_suspend(device_t dev)
1986 {
1987 #ifndef BURN_BRIDGES
1988         int i;
1989 #endif
1990         struct re_softc *sc = device_get_softc(dev);
1991         struct ifnet *ifp = &sc->arpcom.ac_if;
1992
1993         lwkt_serialize_enter(ifp->if_serializer);
1994
1995         re_stop(sc, TRUE);
1996
1997 #ifndef BURN_BRIDGES
1998         for (i = 0; i < 5; i++)
1999                 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4);
2000         sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4);
2001         sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1);
2002         sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
2003         sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
2004 #endif
2005
2006         sc->re_flags |= RE_F_SUSPENDED;
2007
2008         lwkt_serialize_exit(ifp->if_serializer);
2009
2010         return (0);
2011 }
2012
2013 /*
2014  * Device resume routine.  Restore some PCI settings in case the BIOS
2015  * doesn't, re-enable busmastering, and restart the interface if
2016  * appropriate.
2017  */
2018 static int
2019 re_resume(device_t dev)
2020 {
2021         struct re_softc *sc = device_get_softc(dev);
2022         struct ifnet *ifp = &sc->arpcom.ac_if;
2023 #ifndef BURN_BRIDGES
2024         int i;
2025 #endif
2026
2027         lwkt_serialize_enter(ifp->if_serializer);
2028
2029 #ifndef BURN_BRIDGES
2030         /* better way to do this? */
2031         for (i = 0; i < 5; i++)
2032                 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4);
2033         pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4);
2034         pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1);
2035         pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1);
2036         pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1);
2037
2038         /* reenable busmastering */
2039         pci_enable_busmaster(dev);
2040         pci_enable_io(dev, SYS_RES_IOPORT);
2041 #endif
2042
2043         /* reinitialize interface if necessary */
2044         if (ifp->if_flags & IFF_UP)
2045                 re_init(sc);
2046
2047         sc->re_flags &= ~RE_F_SUSPENDED;
2048
2049         lwkt_serialize_exit(ifp->if_serializer);
2050
2051         return (0);
2052 }
2053
2054 /*
2055  * Stop all chip I/O so that the kernel's probe routines don't
2056  * get confused by errant DMAs when rebooting.
2057  */
2058 static void
2059 re_shutdown(device_t dev)
2060 {
2061         struct re_softc *sc = device_get_softc(dev);
2062         struct ifnet *ifp = &sc->arpcom.ac_if;
2063
2064         lwkt_serialize_enter(ifp->if_serializer);
2065         re_stop(sc, TRUE);
2066         rtl_hw_d3_para(sc);
2067         rtl_phy_power_down(sc);
2068         lwkt_serialize_exit(ifp->if_serializer);
2069 }
2070
2071 static int
2072 re_sysctl_rxtime(SYSCTL_HANDLER_ARGS)
2073 {
2074         struct re_softc *sc = arg1;
2075
2076         return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_rx_time);
2077 }
2078
2079 static int
2080 re_sysctl_txtime(SYSCTL_HANDLER_ARGS)
2081 {
2082         struct re_softc *sc = arg1;
2083
2084         return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_tx_time);
2085 }
2086
2087 static int
2088 re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *hwtime)
2089 {
2090         struct re_softc *sc = arg1;
2091         struct ifnet *ifp = &sc->arpcom.ac_if;
2092         int error, v;
2093
2094         lwkt_serialize_enter(ifp->if_serializer);
2095
2096         v = *hwtime;
2097         error = sysctl_handle_int(oidp, &v, 0, req);
2098         if (error || req->newptr == NULL)
2099                 goto back;
2100
2101         if (v <= 0) {
2102                 error = EINVAL;
2103                 goto back;
2104         }
2105
2106         if (v != *hwtime) {
2107                 *hwtime = v;
2108
2109                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2110                     IFF_RUNNING && sc->re_imtype == RE_IMTYPE_HW)
2111                         re_setup_hw_im(sc);
2112         }
2113 back:
2114         lwkt_serialize_exit(ifp->if_serializer);
2115         return error;
2116 }
2117
2118 static int
2119 re_sysctl_simtime(SYSCTL_HANDLER_ARGS)
2120 {
2121         struct re_softc *sc = arg1;
2122         struct ifnet *ifp = &sc->arpcom.ac_if;
2123         int error, v;
2124
2125         lwkt_serialize_enter(ifp->if_serializer);
2126
2127         v = sc->re_sim_time;
2128         error = sysctl_handle_int(oidp, &v, 0, req);
2129         if (error || req->newptr == NULL)
2130                 goto back;
2131
2132         if (v <= 0) {
2133                 error = EINVAL;
2134                 goto back;
2135         }
2136
2137         if (v != sc->re_sim_time) {
2138                 sc->re_sim_time = v;
2139
2140                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2141                     IFF_RUNNING && sc->re_imtype == RE_IMTYPE_SIM) {
2142 #ifdef foo
2143                         /*
2144                          * Following code causes various strange
2145                          * performance problems.  Hmm ...
2146                          */
2147                         CSR_WRITE_2(sc, RE_IMR, 0);
2148                         CSR_WRITE_4(sc, RE_TIMERINT, 0);
2149                         CSR_READ_4(sc, RE_TIMERINT); /* flush */
2150
2151                         CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
2152                         re_setup_sim_im(sc);
2153 #else
2154                         re_setup_intr(sc, 0, RE_IMTYPE_NONE);
2155                         DELAY(10);
2156                         re_setup_intr(sc, 1, RE_IMTYPE_SIM);
2157 #endif
2158                 }
2159         }
2160 back:
2161         lwkt_serialize_exit(ifp->if_serializer);
2162         return error;
2163 }
2164
2165 static int
2166 re_sysctl_imtype(SYSCTL_HANDLER_ARGS)
2167 {
2168         struct re_softc *sc = arg1;
2169         struct ifnet *ifp = &sc->arpcom.ac_if;
2170         int error, v;
2171
2172         lwkt_serialize_enter(ifp->if_serializer);
2173
2174         v = sc->re_imtype;
2175         error = sysctl_handle_int(oidp, &v, 0, req);
2176         if (error || req->newptr == NULL)
2177                 goto back;
2178
2179         if (v != RE_IMTYPE_HW && v != RE_IMTYPE_SIM && v != RE_IMTYPE_NONE) {
2180                 error = EINVAL;
2181                 goto back;
2182         }
2183         if (v == RE_IMTYPE_HW && (sc->re_caps & RE_C_HWIM) == 0) {
2184                 /* Can't do hardware interrupt moderation */
2185                 error = EOPNOTSUPP;
2186                 goto back;
2187         }
2188
2189         if (v != sc->re_imtype) {
2190                 sc->re_imtype = v;
2191                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2192                     IFF_RUNNING)
2193                         re_setup_intr(sc, 1, sc->re_imtype);
2194         }
2195 back:
2196         lwkt_serialize_exit(ifp->if_serializer);
2197         return error;
2198 }
2199
2200 static void
2201 re_setup_hw_im(struct re_softc *sc)
2202 {
2203         KKASSERT(sc->re_caps & RE_C_HWIM);
2204
2205         /*
2206          * Interrupt moderation
2207          *
2208          * 0xABCD
2209          * A - unknown (maybe TX related)
2210          * B - TX timer (unit: 25us)
2211          * C - unknown (maybe RX related)
2212          * D - RX timer (unit: 25us)
2213          *
2214          *
2215          * re(4)'s interrupt moderation is actually controlled by
2216          * two variables, like most other NICs (bge, bce etc.)
2217          * o  timer
2218          * o  number of packets [P]
2219          *
2220          * The logic relationship between these two variables is
2221          * similar to other NICs too:
2222          * if (timer expire || packets > [P])
2223          *     Interrupt is delivered
2224          *
2225          * Currently we only know how to set 'timer', but not
2226          * 'number of packets', which should be ~30, as far as I
2227          * tested (sink ~900Kpps, interrupt rate is 30KHz)
2228          */
2229         CSR_WRITE_2(sc, RE_IM,
2230                     RE_IM_RXTIME(sc->re_rx_time) |
2231                     RE_IM_TXTIME(sc->re_tx_time) |
2232                     RE_IM_MAGIC);
2233 }
2234
2235 static void
2236 re_disable_hw_im(struct re_softc *sc)
2237 {
2238         if (sc->re_caps & RE_C_HWIM)
2239                 CSR_WRITE_2(sc, RE_IM, 0);
2240 }
2241
2242 static void
2243 re_setup_sim_im(struct re_softc *sc)
2244 {
2245         uint32_t ticks;
2246
2247         if (sc->re_if_flags & RL_FLAG_PCIE) {
2248                 ticks = sc->re_sim_time * sc->re_bus_speed;
2249         } else {
2250                 /*
2251                  * Datasheet says tick decreases at bus speed,
2252                  * but it seems the clock runs a little bit
2253                  * faster, so we do some compensation here.
2254                  */
2255                 ticks = (sc->re_sim_time * sc->re_bus_speed * 8) / 5;
2256         }
2257         CSR_WRITE_4(sc, RE_TIMERINT, ticks);
2258
2259         CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */
2260         sc->re_flags |= RE_F_TIMER_INTR;
2261 }
2262
2263 static void
2264 re_disable_sim_im(struct re_softc *sc)
2265 {
2266         CSR_WRITE_4(sc, RE_TIMERINT, 0);
2267         sc->re_flags &= ~RE_F_TIMER_INTR;
2268 }
2269
2270 static void
2271 re_config_imtype(struct re_softc *sc, int imtype)
2272 {
2273         switch (imtype) {
2274         case RE_IMTYPE_HW:
2275                 KKASSERT(sc->re_caps & RE_C_HWIM);
2276                 /* FALL THROUGH */
2277         case RE_IMTYPE_NONE:
2278                 sc->re_intrs = RE_INTRS;
2279                 sc->re_rx_ack = RE_ISR_RX_OK | RE_ISR_FIFO_OFLOW |
2280                                 RE_ISR_RX_OVERRUN;
2281                 sc->re_tx_ack = RE_ISR_TX_OK;
2282                 break;
2283
2284         case RE_IMTYPE_SIM:
2285                 sc->re_intrs = RE_INTRS_TIMER;
2286                 sc->re_rx_ack = RE_ISR_PCS_TIMEOUT;
2287                 sc->re_tx_ack = RE_ISR_PCS_TIMEOUT;
2288                 break;
2289
2290         default:
2291                 panic("%s: unknown imtype %d",
2292                       sc->arpcom.ac_if.if_xname, imtype);
2293         }
2294 }
2295
2296 static void
2297 re_setup_intr(struct re_softc *sc, int enable_intrs, int imtype)
2298 {
2299         re_config_imtype(sc, imtype);
2300
2301         if (enable_intrs)
2302                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
2303         else
2304                 CSR_WRITE_2(sc, RE_IMR, 0); 
2305
2306         sc->re_npoll.ifpc_stcount = 0;
2307
2308         switch (imtype) {
2309         case RE_IMTYPE_NONE:
2310                 re_disable_sim_im(sc);
2311                 re_disable_hw_im(sc);
2312                 break;
2313
2314         case RE_IMTYPE_HW:
2315                 KKASSERT(sc->re_caps & RE_C_HWIM);
2316                 re_disable_sim_im(sc);
2317                 re_setup_hw_im(sc);
2318                 break;
2319
2320         case RE_IMTYPE_SIM:
2321                 re_disable_hw_im(sc);
2322                 re_setup_sim_im(sc);
2323                 break;
2324
2325         default:
2326                 panic("%s: unknown imtype %d",
2327                       sc->arpcom.ac_if.if_xname, imtype);
2328         }
2329 }
2330
2331 static int
2332 re_jpool_alloc(struct re_softc *sc)
2333 {
2334         struct re_list_data *ldata = &sc->re_ldata;
2335         struct re_jbuf *jbuf;
2336         bus_addr_t paddr;
2337         bus_size_t jpool_size;
2338         bus_dmamem_t dmem;
2339         caddr_t buf;
2340         int i, error;
2341
2342         lwkt_serialize_init(&ldata->re_jbuf_serializer);
2343
2344         ldata->re_jbuf = kmalloc(sizeof(struct re_jbuf) * RE_JBUF_COUNT(sc),
2345                                  M_DEVBUF, M_WAITOK | M_ZERO);
2346
2347         jpool_size = RE_JBUF_COUNT(sc) * RE_JBUF_SIZE;
2348
2349         error = bus_dmamem_coherent(sc->re_parent_tag,
2350                         RE_RXBUF_ALIGN, 0,
2351                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
2352                         jpool_size, BUS_DMA_WAITOK, &dmem);
2353         if (error) {
2354                 device_printf(sc->dev, "could not allocate jumbo memory\n");
2355                 return error;
2356         }
2357         ldata->re_jpool_tag = dmem.dmem_tag;
2358         ldata->re_jpool_map = dmem.dmem_map;
2359         ldata->re_jpool = dmem.dmem_addr;
2360         paddr = dmem.dmem_busaddr;
2361
2362         /* ..and split it into 9KB chunks */
2363         SLIST_INIT(&ldata->re_jbuf_free);
2364
2365         buf = ldata->re_jpool;
2366         for (i = 0; i < RE_JBUF_COUNT(sc); i++) {
2367                 jbuf = &ldata->re_jbuf[i];
2368
2369                 jbuf->re_sc = sc;
2370                 jbuf->re_inuse = 0;
2371                 jbuf->re_slot = i;
2372                 jbuf->re_buf = buf;
2373                 jbuf->re_paddr = paddr;
2374
2375                 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link);
2376
2377                 buf += RE_JBUF_SIZE;
2378                 paddr += RE_JBUF_SIZE;
2379         }
2380         return 0;
2381 }
2382
2383 static void
2384 re_jpool_free(struct re_softc *sc)
2385 {
2386         struct re_list_data *ldata = &sc->re_ldata;
2387
2388         if (ldata->re_jpool_tag != NULL) {
2389                 bus_dmamap_unload(ldata->re_jpool_tag, ldata->re_jpool_map);
2390                 bus_dmamem_free(ldata->re_jpool_tag, ldata->re_jpool,
2391                                 ldata->re_jpool_map);
2392                 bus_dma_tag_destroy(ldata->re_jpool_tag);
2393                 ldata->re_jpool_tag = NULL;
2394         }
2395
2396         if (ldata->re_jbuf != NULL) {
2397                 kfree(ldata->re_jbuf, M_DEVBUF);
2398                 ldata->re_jbuf = NULL;
2399         }
2400 }
2401
2402 #ifdef RE_JUMBO
2403 static struct re_jbuf *
2404 re_jbuf_alloc(struct re_softc *sc)
2405 {
2406         struct re_list_data *ldata = &sc->re_ldata;
2407         struct re_jbuf *jbuf;
2408
2409         lwkt_serialize_enter(&ldata->re_jbuf_serializer);
2410
2411         jbuf = SLIST_FIRST(&ldata->re_jbuf_free);
2412         if (jbuf != NULL) {
2413                 SLIST_REMOVE_HEAD(&ldata->re_jbuf_free, re_link);
2414                 jbuf->re_inuse = 1;
2415         }
2416
2417         lwkt_serialize_exit(&ldata->re_jbuf_serializer);
2418
2419         return jbuf;
2420 }
2421
2422 static void
2423 re_jbuf_free(void *arg)
2424 {
2425         struct re_jbuf *jbuf = arg;
2426         struct re_softc *sc = jbuf->re_sc;
2427         struct re_list_data *ldata = &sc->re_ldata;
2428
2429         if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) {
2430                 panic("%s: free wrong jumbo buffer",
2431                       sc->arpcom.ac_if.if_xname);
2432         } else if (jbuf->re_inuse == 0) {
2433                 panic("%s: jumbo buffer already freed",
2434                       sc->arpcom.ac_if.if_xname);
2435         }
2436
2437         lwkt_serialize_enter(&ldata->re_jbuf_serializer);
2438         atomic_subtract_int(&jbuf->re_inuse, 1);
2439         if (jbuf->re_inuse == 0)
2440                 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link);
2441         lwkt_serialize_exit(&ldata->re_jbuf_serializer);
2442 }
2443
2444 static void
2445 re_jbuf_ref(void *arg)
2446 {
2447         struct re_jbuf *jbuf = arg;
2448         struct re_softc *sc = jbuf->re_sc;
2449         struct re_list_data *ldata = &sc->re_ldata;
2450
2451         if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) {
2452                 panic("%s: ref wrong jumbo buffer",
2453                       sc->arpcom.ac_if.if_xname);
2454         } else if (jbuf->re_inuse == 0) {
2455                 panic("%s: jumbo buffer already freed",
2456                       sc->arpcom.ac_if.if_xname);
2457         }
2458         atomic_add_int(&jbuf->re_inuse, 1);
2459 }
2460 #endif  /* RE_JUMBO */
2461
2462 static void
2463 re_disable_aspm(device_t dev)
2464 {
2465         uint16_t link_cap, link_ctrl;
2466         uint8_t pcie_ptr, reg;
2467
2468         pcie_ptr = pci_get_pciecap_ptr(dev);
2469         if (pcie_ptr == 0)
2470                 return;
2471
2472         link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2);
2473         if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0)
2474                 return;
2475
2476         if (bootverbose)
2477                 device_printf(dev, "disable ASPM\n");
2478
2479         reg = pcie_ptr + PCIER_LINKCTRL;
2480         link_ctrl = pci_read_config(dev, reg, 2);
2481         link_ctrl &= ~(PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1);
2482         pci_write_config(dev, reg, link_ctrl, 2);
2483 }