sys/dev/netif/re/if_re.c

   1 /*
   2  * Copyright (c) 2004
   3  *      Joerg Sonnenberger <joerg@bec.de>.  All rights reserved.
   4  *
   5  * Copyright (c) 1997, 1998-2003
   6  *      Bill Paul <wpaul@windriver.com>.  All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. All advertising materials mentioning features or use of this software
  17  *    must display the following acknowledgement:
  18  *      This product includes software developed by Bill Paul.
  19  * 4. Neither the name of the author nor the names of any co-contributors
  20  *    may be used to endorse or promote products derived from this software
  21  *    without specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  27  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  33  * THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  * $FreeBSD: src/sys/dev/re/if_re.c,v 1.25 2004/06/09 14:34:01 naddy Exp $
  36  */
  37
  38 /*
  39  * RealTek 8169S/8110S/8168/8111/8101E PCI NIC driver
  40  *
  41  * Written by Bill Paul <wpaul@windriver.com>
  42  * Senior Networking Software Engineer
  43  * Wind River Systems
  44  */
  45
  46 /*
  47  * This driver is designed to support RealTek's next generation of
  48  * 10/100 and 10/100/1000 PCI ethernet controllers. There are currently
  49  * seven devices in this family: the the RTL8169, the RTL8169S, RTL8110S,
  50  * the RTL8168, the RTL8111 and the RTL8101E.
  51  *
  52  * The 8169 is a 64-bit 10/100/1000 gigabit ethernet MAC:
  53  *
  54  *      o Descriptor based DMA mechanism.  Each descriptor represents
  55  *        a single packet fragment. Data buffers may be aligned on
  56  *        any byte boundary.
  57  *
  58  *      o 64-bit DMA.
  59  *
  60  *      o TCP/IP checksum offload for both RX and TX.
  61  *
  62  *      o High and normal priority transmit DMA rings.
  63  *
  64  *      o VLAN tag insertion and extraction.
  65  *
  66  *      o TCP large send (segmentation offload).
  67  *
  68  *      o 1000Mbps mode.
  69  *
  70  *      o Jumbo frames.
  71  *
  72  *      o GMII and TBI ports/registers for interfacing with copper
  73  *        or fiber PHYs.
  74  *
  75  *      o RX and TX DMA rings can have up to 1024 descriptors.
  76  *
  77  * The 8169 does not have a built-in PHY.  Most reference boards use a
  78  * Marvell 88E1000 'Alaska' copper gigE PHY.  8169/8110 is _no longer_
  79  * supported.
  80  *
  81  * The 8169S/8110S 10/100/1000 devices have built-in copper gigE PHYs
  82  * (the 'S' stands for 'single-chip').  These devices have the same
  83  * programming API as the older 8169, but also have some vendor-specific
  84  * registers for the on-board PHY.  The 8110S is a LAN-on-motherboard
  85  * part designed to be pin-compatible with the RealTek 8100 10/100 chip.
  86  *
  87  * This driver takes advantage of the RX and TX checksum offload and
  88  * VLAN tag insertion/extraction features.  It also implements
  89  * interrupt moderation using the timer interrupt registers, which
  90  * significantly reduces interrupt load.
  91  */
  92
  93 #define _IP_VHL
  94
  95 #include "opt_ifpoll.h"
  96
  97 #include <sys/param.h>
  98 #include <sys/bus.h>
  99 #include <sys/endian.h>
 100 #include <sys/kernel.h>
 101 #include <sys/in_cksum.h>
 102 #include <sys/interrupt.h>
 103 #include <sys/malloc.h>
 104 #include <sys/mbuf.h>
 105 #include <sys/rman.h>
 106 #include <sys/serialize.h>
 107 #include <sys/socket.h>
 108 #include <sys/sockio.h>
 109 #include <sys/sysctl.h>
 110
 111 #include <net/bpf.h>
 112 #include <net/ethernet.h>
 113 #include <net/if.h>
 114 #include <net/ifq_var.h>
 115 #include <net/if_arp.h>
 116 #include <net/if_dl.h>
 117 #include <net/if_media.h>
 118 #include <net/if_poll.h>
 119 #include <net/if_types.h>
 120 #include <net/vlan/if_vlan_var.h>
 121 #include <net/vlan/if_vlan_ether.h>
 122
 123 #include <netinet/ip.h>
 124
 125 #include "pcidevs.h"
 126 #include <bus/pci/pcireg.h>
 127 #include <bus/pci/pcivar.h>
 128
 129 #include <dev/netif/re/if_rereg.h>
 130 #include <dev/netif/re/if_revar.h>
 131 #include <dev/netif/re/re.h>
 132 #include <dev/netif/re/re_dragonfly.h>
 133
 134 /*
 135  * Various supported device vendors/types and their names.
 136  */
 137 static const struct re_type {
 138         uint16_t        re_vid;
 139         uint16_t        re_did;
 140         const char      *re_name;
 141 } re_devs[] = {
 142         { PCI_VENDOR_DLINK, PCI_PRODUCT_DLINK_DGE528T,
 143           "D-Link DGE-528(T) Gigabit Ethernet Adapter" },
 144
 145         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8101E,
 146           "RealTek 810x PCIe 10/100baseTX" },
 147
 148         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168,
 149           "RealTek 8111/8168 PCIe Gigabit Ethernet" },
 150
 151         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8168_1,
 152           "RealTek 8168 PCIe Gigabit Ethernet" },
 153
 154 #ifdef notyet
 155         /*
 156          * This driver now only supports built-in PHYs.
 157          */
 158         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169,
 159           "RealTek 8110/8169 Gigabit Ethernet" },
 160 #endif
 161
 162         { PCI_VENDOR_REALTEK, PCI_PRODUCT_REALTEK_RT8169SC,
 163           "RealTek 8169SC/8110SC Single-chip Gigabit Ethernet" },
 164
 165         { PCI_VENDOR_COREGA, PCI_PRODUCT_COREGA_CG_LAPCIGT,
 166           "Corega CG-LAPCIGT Gigabit Ethernet" },
 167
 168         { PCI_VENDOR_LINKSYS, PCI_PRODUCT_LINKSYS_EG1032,
 169           "Linksys EG1032 Gigabit Ethernet" },
 170
 171         { PCI_VENDOR_USR2, PCI_PRODUCT_USR2_997902,
 172           "US Robotics 997902 Gigabit Ethernet" },
 173
 174         { PCI_VENDOR_TTTECH, PCI_PRODUCT_TTTECH_MC322,
 175           "TTTech MC322 Gigabit Ethernet" },
 176
 177         { 0, 0, NULL }
 178 };
 179
 180 static int      re_probe(device_t);
 181 static int      re_attach(device_t);
 182 static int      re_detach(device_t);
 183 static int      re_suspend(device_t);
 184 static int      re_resume(device_t);
 185 static void     re_shutdown(device_t);
 186
 187 static int      re_allocmem(device_t);
 188 static void     re_freemem(device_t);
 189 static void     re_freebufmem(struct re_softc *, int, int);
 190 static int      re_encap(struct re_softc *, struct mbuf **, int *);
 191 static int      re_newbuf_std(struct re_softc *, int, int);
 192 #ifdef RE_JUMBO
 193 static int      re_newbuf_jumbo(struct re_softc *, int, int);
 194 #endif
 195 static void     re_setup_rxdesc(struct re_softc *, int);
 196 static int      re_rx_list_init(struct re_softc *);
 197 static int      re_tx_list_init(struct re_softc *);
 198 static int      re_rxeof(struct re_softc *);
 199 static int      re_txeof(struct re_softc *);
 200 static int      re_tx_collect(struct re_softc *);
 201 static void     re_intr(void *);
 202 static void     re_tick(void *);
 203 static void     re_tick_serialized(void *);
 204 static void     re_disable_aspm(device_t);
 205 static void     re_link_up(struct re_softc *);
 206 static void     re_link_down(struct re_softc *);
 207
 208 static void     re_start(struct ifnet *, struct ifaltq_subque *);
 209 static int      re_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 210 static void     re_init(void *);
 211 static void     re_stop(struct re_softc *, boolean_t);
 212 static void     re_watchdog(struct ifnet *);
 213
 214 static void     re_setup_hw_im(struct re_softc *);
 215 static void     re_setup_sim_im(struct re_softc *);
 216 static void     re_disable_hw_im(struct re_softc *);
 217 static void     re_disable_sim_im(struct re_softc *);
 218 static void     re_config_imtype(struct re_softc *, int);
 219 static void     re_setup_intr(struct re_softc *, int, int);
 220
 221 static int      re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *);
 222 static int      re_sysctl_rxtime(SYSCTL_HANDLER_ARGS);
 223 static int      re_sysctl_txtime(SYSCTL_HANDLER_ARGS);
 224 static int      re_sysctl_simtime(SYSCTL_HANDLER_ARGS);
 225 static int      re_sysctl_imtype(SYSCTL_HANDLER_ARGS);
 226
 227 static int      re_jpool_alloc(struct re_softc *);
 228 static void     re_jpool_free(struct re_softc *);
 229 #ifdef RE_JUMBO
 230 static struct re_jbuf *re_jbuf_alloc(struct re_softc *);
 231 static void     re_jbuf_free(void *);
 232 static void     re_jbuf_ref(void *);
 233 #endif
 234
 235 #ifdef IFPOLL_ENABLE
 236 static void     re_npoll(struct ifnet *, struct ifpoll_info *);
 237 static void     re_npoll_compat(struct ifnet *, void *, int);
 238 #endif
 239
 240 static device_method_t re_methods[] = {
 241         /* Device interface */
 242         DEVMETHOD(device_probe,         re_probe),
 243         DEVMETHOD(device_attach,        re_attach),
 244         DEVMETHOD(device_detach,        re_detach),
 245         DEVMETHOD(device_suspend,       re_suspend),
 246         DEVMETHOD(device_resume,        re_resume),
 247         DEVMETHOD(device_shutdown,      re_shutdown),
 248         DEVMETHOD_END
 249 };
 250
 251 static driver_t re_driver = {
 252         "re",
 253         re_methods,
 254         sizeof(struct re_softc)
 255 };
 256
 257 static devclass_t re_devclass;
 258
 259 DECLARE_DUMMY_MODULE(if_re);
 260 DRIVER_MODULE(if_re, pci, re_driver, re_devclass, NULL, NULL);
 261 DRIVER_MODULE(if_re, cardbus, re_driver, re_devclass, NULL, NULL);
 262
 263 static int      re_rx_desc_count = RE_RX_DESC_CNT_DEF;
 264 static int      re_tx_desc_count = RE_TX_DESC_CNT_DEF;
 265 static int      re_msi_enable = 1;
 266
 267 TUNABLE_INT("hw.re.rx_desc_count", &re_rx_desc_count);
 268 TUNABLE_INT("hw.re.tx_desc_count", &re_tx_desc_count);
 269 TUNABLE_INT("hw.re.msi.enable", &re_msi_enable);
 270
 271 static __inline void
 272 re_free_rxchain(struct re_softc *sc)
 273 {
 274         if (sc->re_head != NULL) {
 275                 m_freem(sc->re_head);
 276                 sc->re_head = sc->re_tail = NULL;
 277         }
 278 }
 279
 280 static int
 281 re_probe(device_t dev)
 282 {
 283         const struct re_type *t;
 284         uint16_t vendor, product;
 285
 286         vendor = pci_get_vendor(dev);
 287         product = pci_get_device(dev);
 288
 289         /*
 290          * Only attach to rev.3 of the Linksys EG1032 adapter.
 291          * Rev.2 is supported by sk(4).
 292          */
 293         if (vendor == PCI_VENDOR_LINKSYS &&
 294             product == PCI_PRODUCT_LINKSYS_EG1032 &&
 295             pci_get_subdevice(dev) != PCI_SUBDEVICE_LINKSYS_EG1032_REV3)
 296                 return ENXIO;
 297
 298         for (t = re_devs; t->re_name != NULL; t++) {
 299                 if (product == t->re_did && vendor == t->re_vid)
 300                         break;
 301         }
 302         if (t->re_name == NULL)
 303                 return ENXIO;
 304
 305         device_set_desc(dev, t->re_name);
 306         return 0;
 307 }
 308
 309 static int
 310 re_allocmem(device_t dev)
 311 {
 312         struct re_softc *sc = device_get_softc(dev);
 313         bus_dmamem_t dmem;
 314         int error, i;
 315
 316         /*
 317          * Allocate list data
 318          */
 319         sc->re_ldata.re_tx_mbuf =
 320         kmalloc(sc->re_tx_desc_cnt * sizeof(struct mbuf *),
 321                 M_DEVBUF, M_ZERO | M_WAITOK);
 322
 323         sc->re_ldata.re_rx_mbuf =
 324         kmalloc(sc->re_rx_desc_cnt * sizeof(struct mbuf *),
 325                 M_DEVBUF, M_ZERO | M_WAITOK);
 326
 327         sc->re_ldata.re_rx_paddr =
 328         kmalloc(sc->re_rx_desc_cnt * sizeof(bus_addr_t),
 329                 M_DEVBUF, M_ZERO | M_WAITOK);
 330
 331         sc->re_ldata.re_tx_dmamap =
 332         kmalloc(sc->re_tx_desc_cnt * sizeof(bus_dmamap_t),
 333                 M_DEVBUF, M_ZERO | M_WAITOK);
 334
 335         sc->re_ldata.re_rx_dmamap =
 336         kmalloc(sc->re_rx_desc_cnt * sizeof(bus_dmamap_t),
 337                 M_DEVBUF, M_ZERO | M_WAITOK);
 338
 339         /*
 340          * Allocate the parent bus DMA tag appropriate for PCI.
 341          */
 342         error = bus_dma_tag_create(NULL,        /* parent */
 343                         1, 0,                   /* alignment, boundary */
 344                         BUS_SPACE_MAXADDR,      /* lowaddr */
 345                         BUS_SPACE_MAXADDR,      /* highaddr */
 346                         NULL, NULL,             /* filter, filterarg */
 347                         BUS_SPACE_MAXSIZE_32BIT,/* maxsize */
 348                         0,                      /* nsegments */
 349                         BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */
 350                         0,                      /* flags */
 351                         &sc->re_parent_tag);
 352         if (error) {
 353                 device_printf(dev, "could not allocate parent dma tag\n");
 354                 return error;
 355         }
 356
 357         /* Allocate TX descriptor list. */
 358         error = bus_dmamem_coherent(sc->re_parent_tag,
 359                         RE_RING_ALIGN, 0,
 360                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 361                         RE_TX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO,
 362                         &dmem);
 363         if (error) {
 364                 device_printf(dev, "could not allocate TX ring\n");
 365                 return error;
 366         }
 367         sc->re_ldata.re_tx_list_tag = dmem.dmem_tag;
 368         sc->re_ldata.re_tx_list_map = dmem.dmem_map;
 369         sc->re_ldata.re_tx_list = dmem.dmem_addr;
 370         sc->re_ldata.re_tx_list_addr = dmem.dmem_busaddr;
 371
 372         /* Allocate RX descriptor list. */
 373         error = bus_dmamem_coherent(sc->re_parent_tag,
 374                         RE_RING_ALIGN, 0,
 375                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 376                         RE_RX_LIST_SZ(sc), BUS_DMA_WAITOK | BUS_DMA_ZERO,
 377                         &dmem);
 378         if (error) {
 379                 device_printf(dev, "could not allocate RX ring\n");
 380                 return error;
 381         }
 382         sc->re_ldata.re_rx_list_tag = dmem.dmem_tag;
 383         sc->re_ldata.re_rx_list_map = dmem.dmem_map;
 384         sc->re_ldata.re_rx_list = dmem.dmem_addr;
 385         sc->re_ldata.re_rx_list_addr = dmem.dmem_busaddr;
 386
 387         /* Allocate maps for TX mbufs. */
 388         error = bus_dma_tag_create(sc->re_parent_tag,
 389                         1, 0,
 390                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 391                         NULL, NULL,
 392                         RE_FRAMELEN_MAX, RE_MAXSEGS, MCLBYTES,
 393                         BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
 394                         &sc->re_ldata.re_tx_mtag);
 395         if (error) {
 396                 device_printf(dev, "could not allocate TX buf dma tag\n");
 397                 return(error);
 398         }
 399
 400         /* Create DMA maps for TX buffers */
 401         for (i = 0; i < sc->re_tx_desc_cnt; i++) {
 402                 error = bus_dmamap_create(sc->re_ldata.re_tx_mtag,
 403                                 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
 404                                 &sc->re_ldata.re_tx_dmamap[i]);
 405                 if (error) {
 406                         device_printf(dev, "can't create DMA map for TX buf\n");
 407                         re_freebufmem(sc, i, 0);
 408                         return(error);
 409                 }
 410         }
 411
 412         /* Allocate maps for RX mbufs. */
 413         error = bus_dma_tag_create(sc->re_parent_tag,
 414                         RE_RXBUF_ALIGN, 0,
 415                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
 416                         NULL, NULL,
 417                         MCLBYTES, 1, MCLBYTES,
 418                         BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED,
 419                         &sc->re_ldata.re_rx_mtag);
 420         if (error) {
 421                 device_printf(dev, "could not allocate RX buf dma tag\n");
 422                 return(error);
 423         }
 424
 425         /* Create spare DMA map for RX */
 426         error = bus_dmamap_create(sc->re_ldata.re_rx_mtag, BUS_DMA_WAITOK,
 427                         &sc->re_ldata.re_rx_spare);
 428         if (error) {
 429                 device_printf(dev, "can't create spare DMA map for RX\n");
 430                 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag);
 431                 sc->re_ldata.re_rx_mtag = NULL;
 432                 return error;
 433         }
 434
 435         /* Create DMA maps for RX buffers */
 436         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
 437                 error = bus_dmamap_create(sc->re_ldata.re_rx_mtag,
 438                                 BUS_DMA_WAITOK, &sc->re_ldata.re_rx_dmamap[i]);
 439                 if (error) {
 440                         device_printf(dev, "can't create DMA map for RX buf\n");
 441                         re_freebufmem(sc, sc->re_tx_desc_cnt, i);
 442                         return(error);
 443                 }
 444         }
 445
 446         /* Create jumbo buffer pool for RX if required */
 447         if (sc->re_caps & RE_C_CONTIGRX) {
 448                 error = re_jpool_alloc(sc);
 449                 if (error) {
 450                         re_jpool_free(sc);
 451 #ifdef RE_JUMBO
 452                         /* Disable jumbo frame support */
 453                         sc->re_maxmtu = ETHERMTU;
 454 #endif
 455                 }
 456         }
 457         return(0);
 458 }
 459
 460 static void
 461 re_freebufmem(struct re_softc *sc, int tx_cnt, int rx_cnt)
 462 {
 463         int i;
 464
 465         /* Destroy all the RX and TX buffer maps */
 466         if (sc->re_ldata.re_tx_mtag) {
 467                 for (i = 0; i < tx_cnt; i++) {
 468                         bus_dmamap_destroy(sc->re_ldata.re_tx_mtag,
 469                                            sc->re_ldata.re_tx_dmamap[i]);
 470                 }
 471                 bus_dma_tag_destroy(sc->re_ldata.re_tx_mtag);
 472                 sc->re_ldata.re_tx_mtag = NULL;
 473         }
 474
 475         if (sc->re_ldata.re_rx_mtag) {
 476                 for (i = 0; i < rx_cnt; i++) {
 477                         bus_dmamap_destroy(sc->re_ldata.re_rx_mtag,
 478                                            sc->re_ldata.re_rx_dmamap[i]);
 479                 }
 480                 bus_dmamap_destroy(sc->re_ldata.re_rx_mtag,
 481                                    sc->re_ldata.re_rx_spare);
 482                 bus_dma_tag_destroy(sc->re_ldata.re_rx_mtag);
 483                 sc->re_ldata.re_rx_mtag = NULL;
 484         }
 485 }
 486
 487 static void
 488 re_freemem(device_t dev)
 489 {
 490         struct re_softc *sc = device_get_softc(dev);
 491
 492         /* Unload and free the RX DMA ring memory and map */
 493         if (sc->re_ldata.re_rx_list_tag) {
 494                 bus_dmamap_unload(sc->re_ldata.re_rx_list_tag,
 495                                   sc->re_ldata.re_rx_list_map);
 496                 bus_dmamem_free(sc->re_ldata.re_rx_list_tag,
 497                                 sc->re_ldata.re_rx_list,
 498                                 sc->re_ldata.re_rx_list_map);
 499                 bus_dma_tag_destroy(sc->re_ldata.re_rx_list_tag);
 500         }
 501
 502         /* Unload and free the TX DMA ring memory and map */
 503         if (sc->re_ldata.re_tx_list_tag) {
 504                 bus_dmamap_unload(sc->re_ldata.re_tx_list_tag,
 505                                   sc->re_ldata.re_tx_list_map);
 506                 bus_dmamem_free(sc->re_ldata.re_tx_list_tag,
 507                                 sc->re_ldata.re_tx_list,
 508                                 sc->re_ldata.re_tx_list_map);
 509                 bus_dma_tag_destroy(sc->re_ldata.re_tx_list_tag);
 510         }
 511
 512         /* Free RX/TX buf DMA stuffs */
 513         re_freebufmem(sc, sc->re_tx_desc_cnt, sc->re_rx_desc_cnt);
 514
 515         /* Unload and free the stats buffer and map */
 516         if (sc->re_ldata.re_stag) {
 517                 bus_dmamap_unload(sc->re_ldata.re_stag, sc->re_ldata.re_smap);
 518                 bus_dmamem_free(sc->re_ldata.re_stag,
 519                                 sc->re_ldata.re_stats,
 520                                 sc->re_ldata.re_smap);
 521                 bus_dma_tag_destroy(sc->re_ldata.re_stag);
 522         }
 523
 524         if (sc->re_caps & RE_C_CONTIGRX)
 525                 re_jpool_free(sc);
 526
 527         if (sc->re_parent_tag)
 528                 bus_dma_tag_destroy(sc->re_parent_tag);
 529
 530         if (sc->re_ldata.re_tx_mbuf != NULL)
 531                 kfree(sc->re_ldata.re_tx_mbuf, M_DEVBUF);
 532         if (sc->re_ldata.re_rx_mbuf != NULL)
 533                 kfree(sc->re_ldata.re_rx_mbuf, M_DEVBUF);
 534         if (sc->re_ldata.re_rx_paddr != NULL)
 535                 kfree(sc->re_ldata.re_rx_paddr, M_DEVBUF);
 536         if (sc->re_ldata.re_tx_dmamap != NULL)
 537                 kfree(sc->re_ldata.re_tx_dmamap, M_DEVBUF);
 538         if (sc->re_ldata.re_rx_dmamap != NULL)
 539                 kfree(sc->re_ldata.re_rx_dmamap, M_DEVBUF);
 540 }
 541
 542 static boolean_t
 543 re_is_faste(struct re_softc *sc)
 544 {
 545         if (pci_get_vendor(sc->dev) == PCI_VENDOR_REALTEK) {
 546                 switch (sc->re_device_id) {
 547                 case PCI_PRODUCT_REALTEK_RT8169:
 548                 case PCI_PRODUCT_REALTEK_RT8169SC:
 549                 case PCI_PRODUCT_REALTEK_RT8168:
 550                 case PCI_PRODUCT_REALTEK_RT8168_1:
 551                         return FALSE;
 552                 default:
 553                         return TRUE;
 554                 }
 555         } else {
 556                 return FALSE;
 557         }
 558 }
 559
 560 /*
 561  * Attach the interface. Allocate softc structures, do ifmedia
 562  * setup and ethernet/BPF attach.
 563  */
 564 static int
 565 re_attach(device_t dev)
 566 {
 567         struct re_softc *sc = device_get_softc(dev);
 568         struct ifnet *ifp;
 569         struct sysctl_ctx_list *ctx;
 570         struct sysctl_oid *tree;
 571         uint8_t eaddr[ETHER_ADDR_LEN];
 572         int error = 0, qlen, msi_enable;
 573         u_int irq_flags;
 574
 575         callout_init_mp(&sc->re_timer);
 576         sc->dev = dev;
 577         sc->re_device_id = pci_get_device(dev);
 578         sc->re_unit = device_get_unit(dev);
 579         ifmedia_init(&sc->media, IFM_IMASK, rtl_ifmedia_upd, rtl_ifmedia_sts);
 580
 581         sc->re_caps = RE_C_HWIM;
 582
 583         sc->re_rx_desc_cnt = re_rx_desc_count;
 584         if (sc->re_rx_desc_cnt > RE_RX_DESC_CNT_MAX)
 585                 sc->re_rx_desc_cnt = RE_RX_DESC_CNT_MAX;
 586
 587         sc->re_tx_desc_cnt = re_tx_desc_count;
 588         if (sc->re_tx_desc_cnt > RE_TX_DESC_CNT_MAX)
 589                 sc->re_tx_desc_cnt = RE_TX_DESC_CNT_MAX;
 590
 591         qlen = RE_IFQ_MAXLEN;
 592         if (sc->re_tx_desc_cnt > qlen)
 593                 qlen = sc->re_tx_desc_cnt;
 594
 595         sc->re_rxbuf_size = MCLBYTES;
 596         sc->re_newbuf = re_newbuf_std;
 597
 598         /*
 599          * Hardware interrupt moderation settings.
 600          * XXX does not seem correct, undocumented.
 601          */
 602         sc->re_tx_time = 5;             /* 125us */
 603         sc->re_rx_time = 2;             /* 50us */
 604
 605         /* Simulated interrupt moderation setting. */
 606         sc->re_sim_time = 150;          /* 150us */
 607
 608         /* Use simulated interrupt moderation by default. */
 609         sc->re_imtype = RE_IMTYPE_SIM;
 610         re_config_imtype(sc, sc->re_imtype);
 611
 612         ctx = device_get_sysctl_ctx(dev);
 613         tree = device_get_sysctl_tree(dev);
 614         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 615                        "rx_desc_count", CTLFLAG_RD, &sc->re_rx_desc_cnt,
 616                        0, "RX desc count");
 617         SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 618                        "tx_desc_count", CTLFLAG_RD, &sc->re_tx_desc_cnt,
 619                        0, "TX desc count");
 620         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "sim_time",
 621                         CTLTYPE_INT | CTLFLAG_RW,
 622                         sc, 0, re_sysctl_simtime, "I",
 623                         "Simulated interrupt moderation time (usec).");
 624         SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "imtype",
 625                         CTLTYPE_INT | CTLFLAG_RW,
 626                         sc, 0, re_sysctl_imtype, "I",
 627                         "Interrupt moderation type -- "
 628                         "0:disable, 1:simulated, "
 629                         "2:hardware(if supported)");
 630         if (sc->re_caps & RE_C_HWIM) {
 631                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
 632                                 OID_AUTO, "hw_rxtime",
 633                                 CTLTYPE_INT | CTLFLAG_RW,
 634                                 sc, 0, re_sysctl_rxtime, "I",
 635                                 "Hardware interrupt moderation time "
 636                                 "(unit: 25usec).");
 637                 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree),
 638                                 OID_AUTO, "hw_txtime",
 639                                 CTLTYPE_INT | CTLFLAG_RW,
 640                                 sc, 0, re_sysctl_txtime, "I",
 641                                 "Hardware interrupt moderation time "
 642                                 "(unit: 25usec).");
 643         }
 644
 645 #ifndef BURN_BRIDGES
 646         /*
 647          * Handle power management nonsense.
 648          */
 649
 650         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
 651                 uint32_t membase, irq;
 652
 653                 /* Save important PCI config data. */
 654                 membase = pci_read_config(dev, RE_PCI_LOMEM, 4);
 655                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
 656
 657                 /* Reset the power state. */
 658                 device_printf(dev, "chip is in D%d power mode "
 659                     "-- setting to D0\n", pci_get_powerstate(dev));
 660
 661                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
 662
 663                 /* Restore PCI config data. */
 664                 pci_write_config(dev, RE_PCI_LOMEM, membase, 4);
 665                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
 666         }
 667 #endif
 668         /*
 669          * Map control/status registers.
 670          */
 671         pci_enable_busmaster(dev);
 672
 673         if (pci_is_pcie(dev)) {
 674                 sc->re_res_rid = PCIR_BAR(2);
 675                 sc->re_res_type = SYS_RES_MEMORY;
 676         } else {
 677                 sc->re_res_rid = PCIR_BAR(0);
 678                 sc->re_res_type = SYS_RES_IOPORT;
 679         }
 680         sc->re_res = bus_alloc_resource_any(dev, sc->re_res_type,
 681             &sc->re_res_rid, RF_ACTIVE);
 682         if (sc->re_res == NULL) {
 683                 device_printf(dev, "couldn't map IO\n");
 684                 error = ENXIO;
 685                 goto fail;
 686         }
 687
 688         sc->re_btag = rman_get_bustag(sc->re_res);
 689         sc->re_bhandle = rman_get_bushandle(sc->re_res);
 690
 691         error = rtl_check_mac_version(sc);
 692         if (error) {
 693                 device_printf(dev, "check mac version failed\n");
 694                 goto fail;
 695         }
 696
 697         rtl_init_software_variable(sc);
 698         if (pci_is_pcie(dev))
 699                 sc->re_if_flags |= RL_FLAG_PCIE;
 700         else
 701                 sc->re_if_flags &= ~RL_FLAG_PCIE;
 702         device_printf(dev, "MAC version 0x%08x, MACFG %u%s%s%s\n",
 703             (CSR_READ_4(sc, RE_TXCFG) & 0xFCF00000), sc->re_type,
 704             sc->re_coalesce_tx_pkt ? ", software TX defrag" : "",
 705             sc->re_pad_runt ? ", pad runt" : "",
 706             sc->re_hw_enable_msi_msix ? ", support MSI" : "");
 707
 708         /*
 709          * Allocate interrupt
 710          */
 711         if (pci_is_pcie(dev) && sc->re_hw_enable_msi_msix)
 712                 msi_enable = re_msi_enable;
 713         else
 714                 msi_enable = 0;
 715         sc->re_irq_type = pci_alloc_1intr(dev, msi_enable,
 716             &sc->re_irq_rid, &irq_flags);
 717
 718         sc->re_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->re_irq_rid,
 719                                             irq_flags);
 720         if (sc->re_irq == NULL) {
 721                 device_printf(dev, "couldn't map interrupt\n");
 722                 error = ENXIO;
 723                 goto fail;
 724         }
 725
 726         /* Disable ASPM */
 727         re_disable_aspm(dev);
 728
 729         rtl_exit_oob(sc);
 730         rtl_hw_init(sc);
 731
 732         /* Reset the adapter. */
 733         rtl_reset(sc);
 734
 735         rtl_get_hw_mac_address(sc, eaddr);
 736         if (sc->re_type == MACFG_3)     /* Change PCI Latency time*/
 737                 pci_write_config(dev, PCIR_LATTIMER, 0x40, 1);
 738
 739         /* Allocate DMA stuffs */
 740         error = re_allocmem(dev);
 741         if (error)
 742                 goto fail;
 743
 744         if (pci_is_pcie(dev)) {
 745                 sc->re_bus_speed = 125;
 746         } else {
 747                 uint8_t cfg2;
 748
 749                 cfg2 = CSR_READ_1(sc, RE_CFG2);
 750                 switch (cfg2 & RE_CFG2_PCICLK_MASK) {
 751                 case RE_CFG2_PCICLK_33MHZ:
 752                         sc->re_bus_speed = 33;
 753                         break;
 754                 case RE_CFG2_PCICLK_66MHZ:
 755                         sc->re_bus_speed = 66;
 756                         break;
 757                 default:
 758                         device_printf(dev, "unknown bus speed, assume 33MHz\n");
 759                         sc->re_bus_speed = 33;
 760                         break;
 761                 }
 762         }
 763         device_printf(dev, "bus speed %dMHz\n", sc->re_bus_speed);
 764
 765         rtl_phy_power_up(sc);
 766         rtl_hw_phy_config(sc);
 767         rtl_clrwol(sc);
 768
 769         /* TODO: jumbo frame */
 770         CSR_WRITE_2(sc, RE_RxMaxSize, sc->re_rxbuf_size);
 771
 772         /* Enable hardware checksum if available. */
 773         sc->re_tx_cstag = 1;
 774         sc->re_rx_cstag = 1;
 775
 776         ifp = &sc->arpcom.ac_if;
 777         ifp->if_softc = sc;
 778         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 779         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 780         ifp->if_ioctl = re_ioctl;
 781         ifp->if_start = re_start;
 782 #ifdef IFPOLL_ENABLE
 783         ifp->if_npoll = re_npoll;
 784 #endif
 785         ifp->if_watchdog = re_watchdog;
 786         ifp->if_init = re_init;
 787         if (!re_is_faste(sc))
 788                 ifp->if_baudrate = 1000000000;
 789         else
 790                 ifp->if_baudrate = 100000000;
 791         ifp->if_nmbclusters = sc->re_rx_desc_cnt;
 792         ifq_set_maxlen(&ifp->if_snd, qlen);
 793         ifq_set_ready(&ifp->if_snd);
 794
 795         ifp->if_capabilities = IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING |
 796             IFCAP_RXCSUM | IFCAP_TXCSUM;
 797         ifp->if_capenable = ifp->if_capabilities;
 798         /* NOTE: if_hwassist will be setup after the interface is up. */
 799
 800         /*
 801          * Call MI attach routine.
 802          */
 803         ether_ifattach(ifp, eaddr, NULL);
 804
 805         ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq));
 806
 807 #ifdef IFPOLL_ENABLE
 808         ifpoll_compat_setup(&sc->re_npoll, ctx, (struct sysctl_oid *)tree,
 809             device_get_unit(dev), ifp->if_serializer);
 810 #endif
 811
 812         /* Hook interrupt last to avoid having to lock softc */
 813         error = bus_setup_intr(dev, sc->re_irq, INTR_MPSAFE | INTR_HIFREQ,
 814             re_intr, sc, &sc->re_intrhand, ifp->if_serializer);
 815         if (error) {
 816                 device_printf(dev, "couldn't set up irq\n");
 817                 ether_ifdetach(ifp);
 818                 goto fail;
 819         }
 820
 821         ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T, 0, NULL);
 822         ifmedia_add(&sc->media, IFM_ETHER | IFM_10_T | IFM_FDX, 0, NULL);
 823         ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);
 824         ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 0, NULL);
 825         if (!re_is_faste(sc)) {
 826                 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX,
 827                     0, NULL);
 828         }
 829         ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
 830         ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
 831         rtl_ifmedia_upd(ifp);
 832
 833 fail:
 834         if (error)
 835                 re_detach(dev);
 836
 837         return (error);
 838 }
 839
 840 /*
 841  * Shutdown hardware and free up resources. This can be called any
 842  * time after the mutex has been initialized. It is called in both
 843  * the error case in attach and the normal detach case so it needs
 844  * to be careful about only freeing resources that have actually been
 845  * allocated.
 846  */
 847 static int
 848 re_detach(device_t dev)
 849 {
 850         struct re_softc *sc = device_get_softc(dev);
 851         struct ifnet *ifp = &sc->arpcom.ac_if;
 852
 853         /* These should only be active if attach succeeded */
 854         if (device_is_attached(dev)) {
 855                 lwkt_serialize_enter(ifp->if_serializer);
 856                 re_stop(sc, TRUE);
 857                 bus_teardown_intr(dev, sc->re_irq, sc->re_intrhand);
 858                 lwkt_serialize_exit(ifp->if_serializer);
 859
 860                 ether_ifdetach(ifp);
 861         }
 862         ifmedia_removeall(&sc->media);
 863
 864         if (sc->re_irq)
 865                 bus_release_resource(dev, SYS_RES_IRQ, sc->re_irq_rid,
 866                                      sc->re_irq);
 867
 868         if (sc->re_irq_type == PCI_INTR_TYPE_MSI)
 869                 pci_release_msi(dev);
 870
 871         if (sc->re_res) {
 872                 bus_release_resource(dev, sc->re_res_type, sc->re_res_rid,
 873                     sc->re_res);
 874         }
 875         rtl_cmac_unmap(sc);
 876
 877         /* Free DMA stuffs */
 878         re_freemem(dev);
 879
 880         return(0);
 881 }
 882
 883 static void
 884 re_setup_rxdesc(struct re_softc *sc, int idx)
 885 {
 886         bus_addr_t paddr;
 887         uint32_t cmdstat;
 888         struct re_desc *d;
 889
 890         paddr = sc->re_ldata.re_rx_paddr[idx];
 891         d = &sc->re_ldata.re_rx_list[idx];
 892
 893         d->re_bufaddr_lo = htole32(RE_ADDR_LO(paddr));
 894         d->re_bufaddr_hi = htole32(RE_ADDR_HI(paddr));
 895
 896         cmdstat = sc->re_rxbuf_size | RE_RDESC_CMD_OWN;
 897         if (idx == (sc->re_rx_desc_cnt - 1))
 898                 cmdstat |= RE_RDESC_CMD_EOR;
 899         d->re_cmdstat = htole32(cmdstat);
 900 }
 901
 902 static int
 903 re_newbuf_std(struct re_softc *sc, int idx, int init)
 904 {
 905         bus_dma_segment_t seg;
 906         bus_dmamap_t map;
 907         struct mbuf *m;
 908         int error, nsegs;
 909
 910         m = m_getcl(init ? M_WAITOK : M_NOWAIT, MT_DATA, M_PKTHDR);
 911         if (m == NULL) {
 912                 error = ENOBUFS;
 913
 914                 if (init) {
 915                         if_printf(&sc->arpcom.ac_if, "m_getcl failed\n");
 916                         return error;
 917                 } else {
 918                         goto back;
 919                 }
 920         }
 921         m->m_len = m->m_pkthdr.len = MCLBYTES;
 922
 923         /*
 924          * NOTE:
 925          * re(4) chips need address of the receive buffer to be 8-byte
 926          * aligned, so don't call m_adj(m, ETHER_ALIGN) here.
 927          */
 928
 929         error = bus_dmamap_load_mbuf_segment(sc->re_ldata.re_rx_mtag,
 930                         sc->re_ldata.re_rx_spare, m,
 931                         &seg, 1, &nsegs, BUS_DMA_NOWAIT);
 932         if (error) {
 933                 m_freem(m);
 934                 if (init) {
 935                         if_printf(&sc->arpcom.ac_if, "can't load RX mbuf\n");
 936                         return error;
 937                 } else {
 938                         goto back;
 939                 }
 940         }
 941
 942         if (!init) {
 943                 bus_dmamap_sync(sc->re_ldata.re_rx_mtag,
 944                                 sc->re_ldata.re_rx_dmamap[idx],
 945                                 BUS_DMASYNC_POSTREAD);
 946                 bus_dmamap_unload(sc->re_ldata.re_rx_mtag,
 947                                   sc->re_ldata.re_rx_dmamap[idx]);
 948         }
 949         sc->re_ldata.re_rx_mbuf[idx] = m;
 950         sc->re_ldata.re_rx_paddr[idx] = seg.ds_addr;
 951
 952         map = sc->re_ldata.re_rx_dmamap[idx];
 953         sc->re_ldata.re_rx_dmamap[idx] = sc->re_ldata.re_rx_spare;
 954         sc->re_ldata.re_rx_spare = map;
 955 back:
 956         re_setup_rxdesc(sc, idx);
 957         return error;
 958 }
 959
 960 #ifdef RE_JUMBO
 961 static int
 962 re_newbuf_jumbo(struct re_softc *sc, int idx, int init)
 963 {
 964         struct mbuf *m;
 965         struct re_jbuf *jbuf;
 966         int error = 0;
 967
 968         MGETHDR(m, init ? M_WAITOK : M_NOWAIT, MT_DATA);
 969         if (m == NULL) {
 970                 error = ENOBUFS;
 971                 if (init) {
 972                         if_printf(&sc->arpcom.ac_if, "MGETHDR failed\n");
 973                         return error;
 974                 } else {
 975                         goto back;
 976                 }
 977         }
 978
 979         jbuf = re_jbuf_alloc(sc);
 980         if (jbuf == NULL) {
 981                 m_freem(m);
 982
 983                 error = ENOBUFS;
 984                 if (init) {
 985                         if_printf(&sc->arpcom.ac_if, "jpool is empty\n");
 986                         return error;
 987                 } else {
 988                         goto back;
 989                 }
 990         }
 991
 992         m->m_ext.ext_arg = jbuf;
 993         m->m_ext.ext_buf = jbuf->re_buf;
 994         m->m_ext.ext_free = re_jbuf_free;
 995         m->m_ext.ext_ref = re_jbuf_ref;
 996         m->m_ext.ext_size = sc->re_rxbuf_size;
 997
 998         m->m_data = m->m_ext.ext_buf;
 999         m->m_flags |= M_EXT;
1000         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1001
1002         /*
1003          * NOTE:
1004          * Some re(4) chips(e.g. RTL8101E) need address of the receive buffer
1005          * to be 8-byte aligned, so don't call m_adj(m, ETHER_ALIGN) here.
1006          */
1007
1008         sc->re_ldata.re_rx_mbuf[idx] = m;
1009         sc->re_ldata.re_rx_paddr[idx] = jbuf->re_paddr;
1010 back:
1011         re_setup_rxdesc(sc, idx);
1012         return error;
1013 }
1014 #endif  /* RE_JUMBO */
1015
1016 static int
1017 re_tx_list_init(struct re_softc *sc)
1018 {
1019         bzero(sc->re_ldata.re_tx_list, RE_TX_LIST_SZ(sc));
1020
1021         sc->re_ldata.re_tx_prodidx = 0;
1022         sc->re_ldata.re_tx_considx = 0;
1023         sc->re_ldata.re_tx_free = sc->re_tx_desc_cnt;
1024
1025         return(0);
1026 }
1027
1028 static int
1029 re_rx_list_init(struct re_softc *sc)
1030 {
1031         int i, error;
1032
1033         bzero(sc->re_ldata.re_rx_list, RE_RX_LIST_SZ(sc));
1034
1035         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
1036                 error = sc->re_newbuf(sc, i, 1);
1037                 if (error)
1038                         return(error);
1039         }
1040
1041         sc->re_ldata.re_rx_prodidx = 0;
1042         sc->re_head = sc->re_tail = NULL;
1043
1044         return(0);
1045 }
1046
1047 #define RE_IP4_PACKET   0x1
1048 #define RE_TCP_PACKET   0x2
1049 #define RE_UDP_PACKET   0x4
1050
1051 static __inline uint8_t
1052 re_packet_type(struct re_softc *sc, uint32_t rxstat, uint32_t rxctrl)
1053 {
1054         uint8_t packet_type = 0;
1055
1056         if (sc->re_if_flags & RL_FLAG_DESCV2) {
1057                 if (rxctrl & RE_RDESC_CTL_PROTOIP4)
1058                         packet_type |= RE_IP4_PACKET;
1059         } else {
1060                 if (rxstat & RE_RDESC_STAT_PROTOID)
1061                         packet_type |= RE_IP4_PACKET;
1062         }
1063         if (RE_TCPPKT(rxstat))
1064                 packet_type |= RE_TCP_PACKET;
1065         else if (RE_UDPPKT(rxstat))
1066                 packet_type |= RE_UDP_PACKET;
1067         return packet_type;
1068 }
1069
1070 /*
1071  * RX handler for C+ and 8169. For the gigE chips, we support
1072  * the reception of jumbo frames that have been fragmented
1073  * across multiple 2K mbuf cluster buffers.
1074  */
1075 static int
1076 re_rxeof(struct re_softc *sc)
1077 {
1078         struct ifnet *ifp = &sc->arpcom.ac_if;
1079         struct mbuf *m;
1080         struct re_desc  *cur_rx;
1081         uint32_t rxstat, rxctrl;
1082         int i, total_len, rx = 0;
1083
1084         for (i = sc->re_ldata.re_rx_prodidx;
1085              RE_OWN(&sc->re_ldata.re_rx_list[i]) == 0; RE_RXDESC_INC(sc, i)) {
1086                 cur_rx = &sc->re_ldata.re_rx_list[i];
1087                 m = sc->re_ldata.re_rx_mbuf[i];
1088                 total_len = RE_RXBYTES(cur_rx);
1089                 rxstat = le32toh(cur_rx->re_cmdstat);
1090                 rxctrl = le32toh(cur_rx->re_control);
1091
1092                 rx = 1;
1093
1094 #ifdef INVARIANTS
1095                 if (sc->re_flags & RE_F_USE_JPOOL)
1096                         KKASSERT(rxstat & RE_RDESC_STAT_EOF);
1097 #endif
1098
1099                 if ((rxstat & RE_RDESC_STAT_EOF) == 0) {
1100                         if (sc->re_flags & RE_F_DROP_RXFRAG) {
1101                                 re_setup_rxdesc(sc, i);
1102                                 continue;
1103                         }
1104
1105                         if (sc->re_newbuf(sc, i, 0)) {
1106                                 /* Drop upcoming fragments */
1107                                 sc->re_flags |= RE_F_DROP_RXFRAG;
1108                                 continue;
1109                         }
1110
1111                         m->m_len = MCLBYTES;
1112                         if (sc->re_head == NULL) {
1113                                 sc->re_head = sc->re_tail = m;
1114                         } else {
1115                                 sc->re_tail->m_next = m;
1116                                 sc->re_tail = m;
1117                         }
1118                         continue;
1119                 } else if (sc->re_flags & RE_F_DROP_RXFRAG) {
1120                         /*
1121                          * Last fragment of a multi-fragment packet.
1122                          *
1123                          * Since error already happened, this fragment
1124                          * must be dropped as well as the fragment chain.
1125                          */
1126                         re_setup_rxdesc(sc, i);
1127                         re_free_rxchain(sc);
1128                         sc->re_flags &= ~RE_F_DROP_RXFRAG;
1129                         continue;
1130                 }
1131
1132                 rxstat >>= 1;
1133                 if (rxstat & RE_RDESC_STAT_RXERRSUM) {
1134                         IFNET_STAT_INC(ifp, ierrors, 1);
1135                         /*
1136                          * If this is part of a multi-fragment packet,
1137                          * discard all the pieces.
1138                          */
1139                         re_free_rxchain(sc);
1140                         re_setup_rxdesc(sc, i);
1141                         continue;
1142                 }
1143
1144                 /*
1145                  * If allocating a replacement mbuf fails,
1146                  * reload the current one.
1147                  */
1148
1149                 if (sc->re_newbuf(sc, i, 0)) {
1150                         IFNET_STAT_INC(ifp, ierrors, 1);
1151                         continue;
1152                 }
1153
1154                 if (sc->re_head != NULL) {
1155                         m->m_len = total_len % MCLBYTES;
1156                         /*
1157                          * Special case: if there's 4 bytes or less
1158                          * in this buffer, the mbuf can be discarded:
1159                          * the last 4 bytes is the CRC, which we don't
1160                          * care about anyway.
1161                          */
1162                         if (m->m_len <= ETHER_CRC_LEN) {
1163                                 sc->re_tail->m_len -=
1164                                     (ETHER_CRC_LEN - m->m_len);
1165                                 m_freem(m);
1166                         } else {
1167                                 m->m_len -= ETHER_CRC_LEN;
1168                                 sc->re_tail->m_next = m;
1169                         }
1170                         m = sc->re_head;
1171                         sc->re_head = sc->re_tail = NULL;
1172                         m->m_pkthdr.len = total_len - ETHER_CRC_LEN;
1173                 } else {
1174                         m->m_pkthdr.len = m->m_len =
1175                             (total_len - ETHER_CRC_LEN);
1176                 }
1177
1178                 IFNET_STAT_INC(ifp, ipackets, 1);
1179                 m->m_pkthdr.rcvif = ifp;
1180
1181                 /* Do RX checksumming if enabled */
1182
1183                 if (ifp->if_capenable & IFCAP_RXCSUM) {
1184                         uint8_t packet_type;
1185
1186                         packet_type = re_packet_type(sc, rxstat, rxctrl);
1187
1188                         /* Check IP header checksum */
1189                         if (packet_type & RE_IP4_PACKET) {
1190                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
1191                                 if ((rxstat & RE_RDESC_STAT_IPSUMBAD) == 0)
1192                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1193                         }
1194
1195                         /* Check TCP/UDP checksum */
1196                         if (((packet_type & RE_TCP_PACKET) &&
1197                              (rxstat & RE_RDESC_STAT_TCPSUMBAD) == 0) ||
1198                             ((packet_type & RE_UDP_PACKET) &&
1199                              (rxstat & RE_RDESC_STAT_UDPSUMBAD) == 0)) {
1200                                 m->m_pkthdr.csum_flags |=
1201                                     CSUM_DATA_VALID|CSUM_PSEUDO_HDR|
1202                                     CSUM_FRAG_NOT_CHECKED;
1203                                 m->m_pkthdr.csum_data = 0xffff;
1204                         }
1205                 }
1206
1207                 if (rxctrl & RE_RDESC_CTL_HASTAG) {
1208                         m->m_flags |= M_VLANTAG;
1209                         m->m_pkthdr.ether_vlantag =
1210                                 be16toh((rxctrl & RE_RDESC_CTL_TAGDATA));
1211                 }
1212                 ifp->if_input(ifp, m, NULL, -1);
1213         }
1214
1215         sc->re_ldata.re_rx_prodidx = i;
1216
1217         return rx;
1218 }
1219
1220 #undef RE_IP4_PACKET
1221 #undef RE_TCP_PACKET
1222 #undef RE_UDP_PACKET
1223
1224 static int
1225 re_tx_collect(struct re_softc *sc)
1226 {
1227         struct ifnet *ifp = &sc->arpcom.ac_if;
1228         uint32_t txstat;
1229         int idx, tx = 0;
1230
1231         for (idx = sc->re_ldata.re_tx_considx;
1232              sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt;
1233              RE_TXDESC_INC(sc, idx)) {
1234                 txstat = le32toh(sc->re_ldata.re_tx_list[idx].re_cmdstat);
1235                 if (txstat & RE_TDESC_CMD_OWN)
1236                         break;
1237
1238                 tx = 1;
1239
1240                 sc->re_ldata.re_tx_list[idx].re_bufaddr_lo = 0;
1241
1242                 /*
1243                  * We only stash mbufs in the last descriptor
1244                  * in a fragment chain, which also happens to
1245                  * be the only place where the TX status bits
1246                  * are valid.
1247                  */
1248                 if (txstat & RE_TDESC_CMD_EOF) {
1249                         bus_dmamap_unload(sc->re_ldata.re_tx_mtag,
1250                             sc->re_ldata.re_tx_dmamap[idx]);
1251                         m_freem(sc->re_ldata.re_tx_mbuf[idx]);
1252                         sc->re_ldata.re_tx_mbuf[idx] = NULL;
1253                         if (txstat & (RE_TDESC_STAT_EXCESSCOL|
1254                             RE_TDESC_STAT_COLCNT))
1255                                 IFNET_STAT_INC(ifp, collisions, 1);
1256                         if (txstat & RE_TDESC_STAT_TXERRSUM)
1257                                 IFNET_STAT_INC(ifp, oerrors, 1);
1258                         else
1259                                 IFNET_STAT_INC(ifp, opackets, 1);
1260                 }
1261                 sc->re_ldata.re_tx_free++;
1262         }
1263         sc->re_ldata.re_tx_considx = idx;
1264
1265         return tx;
1266 }
1267
1268 static int
1269 re_txeof(struct re_softc *sc)
1270 {
1271         struct ifnet *ifp = &sc->arpcom.ac_if;
1272         int tx;
1273
1274         tx = re_tx_collect(sc);
1275
1276         /* There is enough free TX descs */
1277         if (sc->re_ldata.re_tx_free > RE_TXDESC_SPARE)
1278                 ifq_clr_oactive(&ifp->if_snd);
1279
1280         /*
1281          * Some chips will ignore a second TX request issued while an
1282          * existing transmission is in progress. If the transmitter goes
1283          * idle but there are still packets waiting to be sent, we need
1284          * to restart the channel here to flush them out. This only seems
1285          * to be required with the PCIe devices.
1286          */
1287         if (sc->re_ldata.re_tx_free < sc->re_tx_desc_cnt)
1288                 CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ);
1289         else
1290                 ifp->if_timer = 0;
1291
1292         return tx;
1293 }
1294
1295 static void
1296 re_tick(void *xsc)
1297 {
1298         struct re_softc *sc = xsc;
1299
1300         lwkt_serialize_enter(sc->arpcom.ac_if.if_serializer);
1301         re_tick_serialized(xsc);
1302         lwkt_serialize_exit(sc->arpcom.ac_if.if_serializer);
1303 }
1304
1305 static void
1306 re_tick_serialized(void *xsc)
1307 {
1308         struct re_softc *sc = xsc;
1309         struct ifnet *ifp = &sc->arpcom.ac_if;
1310
1311         ASSERT_SERIALIZED(ifp->if_serializer);
1312
1313         if ((ifp->if_flags & IFF_RUNNING) == 0)
1314                 return;
1315
1316         if (rtl_link_ok(sc)) {
1317                 if ((sc->re_flags & RE_F_LINKED) == 0)
1318                         re_link_up(sc);
1319         } else if (sc->re_flags & RE_F_LINKED) {
1320                 re_link_down(sc);
1321         }
1322         callout_reset(&sc->re_timer, hz, re_tick, sc);
1323 }
1324
1325 #ifdef IFPOLL_ENABLE
1326
1327 static void
1328 re_npoll_compat(struct ifnet *ifp, void *arg __unused, int count)
1329 {
1330         struct re_softc *sc = ifp->if_softc;
1331
1332         ASSERT_SERIALIZED(ifp->if_serializer);
1333
1334         if (sc->re_npoll.ifpc_stcount-- == 0) {
1335                 uint16_t status;
1336
1337                 sc->re_npoll.ifpc_stcount = sc->re_npoll.ifpc_stfrac;
1338
1339                 status = CSR_READ_2(sc, RE_ISR);
1340                 if (status)
1341                         CSR_WRITE_2(sc, RE_ISR, status);
1342
1343                 /*
1344                  * XXX check behaviour on receiver stalls.
1345                  */
1346
1347                 if (status & RE_ISR_SYSTEM_ERR) {
1348                         rtl_reset(sc);
1349                         re_init(sc);
1350                         /* Done! */
1351                         return;
1352                 }
1353         }
1354
1355         sc->rxcycles = count;
1356         re_rxeof(sc);
1357         re_txeof(sc);
1358
1359         if (!ifq_is_empty(&ifp->if_snd))
1360                 if_devstart(ifp);
1361 }
1362
1363 static void
1364 re_npoll(struct ifnet *ifp, struct ifpoll_info *info)
1365 {
1366         struct re_softc *sc = ifp->if_softc;
1367
1368         ASSERT_SERIALIZED(ifp->if_serializer);
1369
1370         if (info != NULL) {
1371                 int cpuid = sc->re_npoll.ifpc_cpuid;
1372
1373                 info->ifpi_rx[cpuid].poll_func = re_npoll_compat;
1374                 info->ifpi_rx[cpuid].arg = NULL;
1375                 info->ifpi_rx[cpuid].serializer = ifp->if_serializer;
1376
1377                 if (ifp->if_flags & IFF_RUNNING)
1378                         re_setup_intr(sc, 0, RE_IMTYPE_NONE);
1379                 ifq_set_cpuid(&ifp->if_snd, cpuid);
1380         } else {
1381                 if (ifp->if_flags & IFF_RUNNING)
1382                         re_setup_intr(sc, 1, sc->re_imtype);
1383                 ifq_set_cpuid(&ifp->if_snd, rman_get_cpuid(sc->re_irq));
1384         }
1385 }
1386 #endif /* IFPOLL_ENABLE */
1387
1388 static void
1389 re_intr(void *arg)
1390 {
1391         struct re_softc *sc = arg;
1392         struct ifnet *ifp = &sc->arpcom.ac_if;
1393         uint16_t status;
1394         int proc;
1395
1396         ASSERT_SERIALIZED(ifp->if_serializer);
1397
1398         if ((sc->re_flags & RE_F_SUSPENDED) ||
1399             (ifp->if_flags & IFF_RUNNING) == 0)
1400                 return;
1401
1402         /* Disable interrupts. */
1403         CSR_WRITE_2(sc, RE_IMR, 0);
1404
1405         status = CSR_READ_2(sc, RE_ISR);
1406 again:
1407         proc = 0;
1408         if (status)
1409                 CSR_WRITE_2(sc, RE_ISR, status);
1410         if (status & sc->re_intrs) {
1411                 if (status & RE_ISR_SYSTEM_ERR) {
1412                         rtl_reset(sc);
1413                         re_init(sc);
1414                         /* Done! */
1415                         return;
1416                 }
1417                 proc |= re_rxeof(sc);
1418                 proc |= re_txeof(sc);
1419         }
1420
1421         if (sc->re_imtype == RE_IMTYPE_SIM) {
1422                 if ((sc->re_flags & RE_F_TIMER_INTR)) {
1423                         if (!proc) {
1424                                 /*
1425                                  * Nothing needs to be processed, fallback
1426                                  * to use TX/RX interrupts.
1427                                  *
1428                                  * NOTE: This will re-enable interrupts.
1429                                  */
1430                                 re_setup_intr(sc, 1, RE_IMTYPE_NONE);
1431
1432                                 /*
1433                                  * Recollect, mainly to avoid the possible
1434                                  * race introduced by changing interrupt
1435                                  * masks.
1436                                  */
1437                                 re_rxeof(sc);
1438                                 re_txeof(sc);
1439                         } else {
1440                                 /* Re-enable interrupts. */
1441                                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1442                                 CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */
1443                         }
1444                 } else if (proc) {
1445                         /*
1446                          * Assume that using simulated interrupt moderation
1447                          * (hardware timer based) could reduce the interript
1448                          * rate.
1449                          *
1450                          * NOTE: This will re-enable interrupts.
1451                          */
1452                         re_setup_intr(sc, 1, RE_IMTYPE_SIM);
1453                 } else {
1454                         /* Re-enable interrupts. */
1455                         CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1456                 }
1457         } else {
1458                 status = CSR_READ_2(sc, RE_ISR);
1459                 if (status & sc->re_intrs) {
1460                         if (!ifq_is_empty(&ifp->if_snd))
1461                                 if_devstart(ifp);
1462                         /* NOTE: Interrupts are still disabled. */
1463                         goto again;
1464                 }
1465                 /* Re-enable interrupts. */
1466                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
1467         }
1468
1469         if (!ifq_is_empty(&ifp->if_snd))
1470                 if_devstart(ifp);
1471 }
1472
1473 static int
1474 re_encap(struct re_softc *sc, struct mbuf **m_head, int *idx0)
1475 {
1476         struct mbuf *m = *m_head;
1477         bus_dma_segment_t segs[RE_MAXSEGS];
1478         bus_dmamap_t map;
1479         int error, maxsegs, idx, i, nsegs;
1480         struct re_desc *d, *tx_ring;
1481         uint32_t cmd_csum, ctl_csum, vlantag;
1482
1483         KASSERT(sc->re_ldata.re_tx_free > RE_TXDESC_SPARE,
1484                 ("not enough free TX desc"));
1485
1486         if (sc->re_coalesce_tx_pkt && m->m_pkthdr.len != m->m_len) {
1487                 struct mbuf *m_new;
1488
1489                 m_new = m_defrag(m, M_NOWAIT);
1490                 if (m_new == NULL) {
1491                         error = ENOBUFS;
1492                         goto back;
1493                 } else {
1494                         *m_head = m = m_new;
1495                         if (m->m_pkthdr.len != m->m_len) {
1496                                 /* Still not configuous; give up. */
1497                                 error = ENOBUFS;
1498                                 goto back;
1499                         }
1500                 }
1501         }
1502
1503         map = sc->re_ldata.re_tx_dmamap[*idx0];
1504
1505         /*
1506          * Set up checksum offload. Note: checksum offload bits must
1507          * appear in all descriptors of a multi-descriptor transmit
1508          * attempt. (This is according to testing done with an 8169
1509          * chip. I'm not sure if this is a requirement or a bug.)
1510          */
1511         cmd_csum = ctl_csum = 0;
1512         if (m->m_pkthdr.csum_flags & CSUM_IP) {
1513                 cmd_csum |= RE_TDESC_CMD_IPCSUM;
1514                 ctl_csum |= RE_TDESC_CTL_IPCSUM;
1515         }
1516         if (m->m_pkthdr.csum_flags & CSUM_TCP) {
1517                 cmd_csum |= RE_TDESC_CMD_TCPCSUM;
1518                 ctl_csum |= RE_TDESC_CTL_TCPCSUM;
1519         }
1520         if (m->m_pkthdr.csum_flags & CSUM_UDP) {
1521                 cmd_csum |= RE_TDESC_CMD_UDPCSUM;
1522                 ctl_csum |= RE_TDESC_CTL_UDPCSUM;
1523         }
1524
1525         /* For version2 descriptor, csum flags are set on re_control */
1526         if (sc->re_if_flags & RL_FLAG_DESCV2)
1527                 cmd_csum = 0;
1528         else
1529                 ctl_csum = 0;
1530
1531         if (sc->re_pad_runt) {
1532                 /*
1533                  * With some of the RealTek chips, using the checksum offload
1534                  * support in conjunction with the autopadding feature results
1535                  * in the transmission of corrupt frames. For example, if we
1536                  * need to send a really small IP fragment that's less than 60
1537                  * bytes in size, and IP header checksumming is enabled, the
1538                  * resulting ethernet frame that appears on the wire will
1539                  * have garbled payload. To work around this, if TX checksum
1540                  * offload is enabled, we always manually pad short frames out
1541                  * to the minimum ethernet frame size.
1542                  *
1543                  * Note: this appears unnecessary for TCP, and doing it for TCP
1544                  * with PCIe adapters seems to result in bad checksums.
1545                  */
1546                 if ((m->m_pkthdr.csum_flags &
1547                      (CSUM_DELAY_IP | CSUM_DELAY_DATA)) &&
1548                     (m->m_pkthdr.csum_flags & CSUM_TCP) == 0 &&
1549                     m->m_pkthdr.len < RE_MIN_FRAMELEN) {
1550                         error = m_devpad(m, RE_MIN_FRAMELEN);
1551                         if (error)
1552                                 goto back;
1553                 }
1554         }
1555
1556         vlantag = 0;
1557         if (m->m_flags & M_VLANTAG) {
1558                 vlantag = htobe16(m->m_pkthdr.ether_vlantag) |
1559                           RE_TDESC_CTL_INSTAG;
1560         }
1561
1562         maxsegs = sc->re_ldata.re_tx_free;
1563         if (maxsegs > RE_MAXSEGS)
1564                 maxsegs = RE_MAXSEGS;
1565
1566         error = bus_dmamap_load_mbuf_defrag(sc->re_ldata.re_tx_mtag, map,
1567                         m_head, segs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1568         if (error)
1569                 goto back;
1570
1571         m = *m_head;
1572         bus_dmamap_sync(sc->re_ldata.re_tx_mtag, map, BUS_DMASYNC_PREWRITE);
1573
1574         /*
1575          * Map the segment array into descriptors.  We also keep track
1576          * of the end of the ring and set the end-of-ring bits as needed,
1577          * and we set the ownership bits in all except the very first
1578          * descriptor, whose ownership bits will be turned on later.
1579          */
1580         tx_ring = sc->re_ldata.re_tx_list;
1581         idx = *idx0;
1582         i = 0;
1583         for (;;) {
1584                 uint32_t cmdstat;
1585
1586                 d = &tx_ring[idx];
1587
1588                 cmdstat = segs[i].ds_len;
1589                 d->re_bufaddr_lo = htole32(RE_ADDR_LO(segs[i].ds_addr));
1590                 d->re_bufaddr_hi = htole32(RE_ADDR_HI(segs[i].ds_addr));
1591                 if (i == 0)
1592                         cmdstat |= RE_TDESC_CMD_SOF;
1593                 else
1594                         cmdstat |= RE_TDESC_CMD_OWN;
1595                 if (idx == (sc->re_tx_desc_cnt - 1))
1596                         cmdstat |= RE_TDESC_CMD_EOR;
1597                 d->re_cmdstat = htole32(cmdstat | cmd_csum);
1598                 d->re_control = htole32(ctl_csum | vlantag);
1599
1600                 i++;
1601                 if (i == nsegs)
1602                         break;
1603                 RE_TXDESC_INC(sc, idx);
1604         }
1605         d->re_cmdstat |= htole32(RE_TDESC_CMD_EOF);
1606
1607         /* Transfer ownership of packet to the chip. */
1608         d->re_cmdstat |= htole32(RE_TDESC_CMD_OWN);
1609         if (*idx0 != idx)
1610                 tx_ring[*idx0].re_cmdstat |= htole32(RE_TDESC_CMD_OWN);
1611
1612         /*
1613          * Insure that the map for this transmission
1614          * is placed at the array index of the last descriptor
1615          * in this chain.
1616          */
1617         sc->re_ldata.re_tx_dmamap[*idx0] = sc->re_ldata.re_tx_dmamap[idx];
1618         sc->re_ldata.re_tx_dmamap[idx] = map;
1619
1620         sc->re_ldata.re_tx_mbuf[idx] = m;
1621         sc->re_ldata.re_tx_free -= nsegs;
1622
1623         RE_TXDESC_INC(sc, idx);
1624         *idx0 = idx;
1625 back:
1626         if (error) {
1627                 m_freem(*m_head);
1628                 *m_head = NULL;
1629         }
1630         return error;
1631 }
1632
1633 /*
1634  * Main transmit routine for C+ and gigE NICs.
1635  */
1636
1637 static void
1638 re_start(struct ifnet *ifp, struct ifaltq_subque *ifsq)
1639 {
1640         struct re_softc *sc = ifp->if_softc;
1641         struct mbuf *m_head;
1642         int idx, need_trans, oactive, error;
1643
1644         ASSERT_ALTQ_SQ_DEFAULT(ifp, ifsq);
1645         ASSERT_SERIALIZED(ifp->if_serializer);
1646
1647         if ((sc->re_flags & RE_F_LINKED) == 0) {
1648                 ifq_purge(&ifp->if_snd);
1649                 return;
1650         }
1651
1652         if ((ifp->if_flags & IFF_RUNNING) == 0 || ifq_is_oactive(&ifp->if_snd))
1653                 return;
1654
1655         idx = sc->re_ldata.re_tx_prodidx;
1656
1657         need_trans = 0;
1658         oactive = 0;
1659         while (sc->re_ldata.re_tx_mbuf[idx] == NULL) {
1660                 if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) {
1661                         if (!oactive) {
1662                                 if (re_tx_collect(sc)) {
1663                                         oactive = 1;
1664                                         continue;
1665                                 }
1666                         }
1667                         ifq_set_oactive(&ifp->if_snd);
1668                         break;
1669                 }
1670
1671                 m_head = ifq_dequeue(&ifp->if_snd);
1672                 if (m_head == NULL)
1673                         break;
1674
1675                 error = re_encap(sc, &m_head, &idx);
1676                 if (error) {
1677                         /* m_head is freed by re_encap(), if we reach here */
1678                         IFNET_STAT_INC(ifp, oerrors, 1);
1679
1680                         if (error == EFBIG && !oactive) {
1681                                 if (re_tx_collect(sc)) {
1682                                         oactive = 1;
1683                                         continue;
1684                                 }
1685                         }
1686                         ifq_set_oactive(&ifp->if_snd);
1687                         break;
1688                 }
1689
1690                 oactive = 0;
1691                 need_trans = 1;
1692
1693                 /*
1694                  * If there's a BPF listener, bounce a copy of this frame
1695                  * to him.
1696                  */
1697                 ETHER_BPF_MTAP(ifp, m_head);
1698         }
1699
1700         /*
1701          * If sc->re_ldata.re_tx_mbuf[idx] is not NULL it is possible
1702          * for OACTIVE to not be properly set when we also do not
1703          * have sufficient free tx descriptors, leaving packet in
1704          * ifp->if_snd.  This can cause if_start_dispatch() to loop
1705          * infinitely so make sure OACTIVE is set properly.
1706          */
1707         if (sc->re_ldata.re_tx_free <= RE_TXDESC_SPARE) {
1708                 if (!ifq_is_oactive(&ifp->if_snd)) {
1709 #if 0
1710                         if_printf(ifp, "Debug: OACTIVE was not set when "
1711                             "re_tx_free was below minimum!\n");
1712 #endif
1713                         ifq_set_oactive(&ifp->if_snd);
1714                 }
1715         }
1716         if (!need_trans)
1717                 return;
1718
1719         sc->re_ldata.re_tx_prodidx = idx;
1720
1721         /*
1722          * RealTek put the TX poll request register in a different
1723          * location on the 8169 gigE chip. I don't know why.
1724          */
1725         CSR_WRITE_1(sc, RE_TPPOLL, RE_NPQ);
1726
1727         /*
1728          * Set a timeout in case the chip goes out to lunch.
1729          */
1730         ifp->if_timer = 5;
1731 }
1732
1733 static void
1734 re_link_up(struct re_softc *sc)
1735 {
1736         struct ifnet *ifp = &sc->arpcom.ac_if;
1737         int error;
1738
1739         ASSERT_SERIALIZED(ifp->if_serializer);
1740
1741         rtl_link_on_patch(sc);
1742         re_stop(sc, FALSE);
1743         rtl_set_eaddr(sc);
1744
1745         error = re_rx_list_init(sc);
1746         if (error) {
1747                 re_stop(sc, TRUE);
1748                 return;
1749         }
1750         error = re_tx_list_init(sc);
1751         if (error) {
1752                 re_stop(sc, TRUE);
1753                 return;
1754         }
1755
1756         /*
1757          * Load the addresses of the RX and TX lists into the chip.
1758          */
1759         CSR_WRITE_4(sc, RE_RXLIST_ADDR_HI,
1760             RE_ADDR_HI(sc->re_ldata.re_rx_list_addr));
1761         CSR_WRITE_4(sc, RE_RXLIST_ADDR_LO,
1762             RE_ADDR_LO(sc->re_ldata.re_rx_list_addr));
1763
1764         CSR_WRITE_4(sc, RE_TXLIST_ADDR_HI,
1765             RE_ADDR_HI(sc->re_ldata.re_tx_list_addr));
1766         CSR_WRITE_4(sc, RE_TXLIST_ADDR_LO,
1767             RE_ADDR_LO(sc->re_ldata.re_tx_list_addr));
1768
1769         rtl_hw_start(sc);
1770
1771 #ifdef IFPOLL_ENABLE
1772         /*
1773          * Disable interrupts if we are polling.
1774          */
1775         if (ifp->if_flags & IFF_NPOLLING)
1776                 re_setup_intr(sc, 0, RE_IMTYPE_NONE);
1777         else    /* otherwise ... */
1778 #endif /* IFPOLL_ENABLE */
1779         /*
1780          * Enable interrupts.
1781          */
1782         re_setup_intr(sc, 1, sc->re_imtype);
1783         CSR_WRITE_2(sc, RE_ISR, sc->re_intrs);
1784
1785         sc->re_flags |= RE_F_LINKED;
1786         ifp->if_link_state = LINK_STATE_UP;
1787         if_link_state_change(ifp);
1788
1789         if (bootverbose)
1790                 if_printf(ifp, "link UP\n");
1791
1792         if (!ifq_is_empty(&ifp->if_snd))
1793                 if_devstart(ifp);
1794 }
1795
1796 static void
1797 re_link_down(struct re_softc *sc)
1798 {
1799         struct ifnet *ifp = &sc->arpcom.ac_if;
1800
1801         /* NOTE: re_stop() will reset RE_F_LINKED. */
1802         ifp->if_link_state = LINK_STATE_DOWN;
1803         if_link_state_change(ifp);
1804
1805         re_stop(sc, FALSE);
1806         rtl_ifmedia_upd(ifp);
1807
1808         if (bootverbose)
1809                 if_printf(ifp, "link DOWN\n");
1810 }
1811
1812 static void
1813 re_init(void *xsc)
1814 {
1815         struct re_softc *sc = xsc;
1816         struct ifnet *ifp = &sc->arpcom.ac_if;
1817
1818         ASSERT_SERIALIZED(ifp->if_serializer);
1819
1820         re_stop(sc, TRUE);
1821         if (rtl_link_ok(sc)) {
1822                 if (bootverbose)
1823                         if_printf(ifp, "link is UP in if_init\n");
1824                 re_link_up(sc);
1825         }
1826
1827         ifp->if_flags |= IFF_RUNNING;
1828         ifq_clr_oactive(&ifp->if_snd);
1829
1830         callout_reset(&sc->re_timer, hz, re_tick, sc);
1831 }
1832
1833 static int
1834 re_ioctl(struct ifnet *ifp, u_long command, caddr_t data, struct ucred *cr)
1835 {
1836         struct re_softc *sc = ifp->if_softc;
1837         struct ifreq *ifr = (struct ifreq *)data;
1838         int error = 0, mask;
1839
1840         ASSERT_SERIALIZED(ifp->if_serializer);
1841
1842         switch(command) {
1843         case SIOCSIFMTU:
1844 #ifdef RE_JUMBO
1845                 if (ifr->ifr_mtu > sc->re_maxmtu) {
1846                         error = EINVAL;
1847                 } else if (ifp->if_mtu != ifr->ifr_mtu) {
1848                         ifp->if_mtu = ifr->ifr_mtu;
1849                         if (ifp->if_flags & IFF_RUNNING)
1850                                 ifp->if_init(sc);
1851                 }
1852 #else
1853                 error = EOPNOTSUPP;
1854 #endif
1855                 break;
1856
1857         case SIOCSIFFLAGS:
1858                 if (ifp->if_flags & IFF_UP) {
1859                         if (ifp->if_flags & IFF_RUNNING) {
1860                                 if ((ifp->if_flags ^ sc->re_saved_ifflags) &
1861                                     (IFF_PROMISC | IFF_ALLMULTI))
1862                                         rtl_set_rx_packet_filter(sc);
1863                         } else {
1864                                 re_init(sc);
1865                         }
1866                 } else if (ifp->if_flags & IFF_RUNNING) {
1867                         re_stop(sc, TRUE);
1868                 }
1869                 sc->re_saved_ifflags = ifp->if_flags;
1870                 break;
1871
1872         case SIOCADDMULTI:
1873         case SIOCDELMULTI:
1874                 rtl_set_rx_packet_filter(sc);
1875                 break;
1876
1877         case SIOCGIFMEDIA:
1878         case SIOCSIFMEDIA:
1879                 error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
1880                 break;
1881
1882         case SIOCSIFCAP:
1883                 mask = (ifr->ifr_reqcap ^ ifp->if_capenable) &
1884                        ifp->if_capabilities;
1885                 ifp->if_capenable ^= mask;
1886
1887                 /* NOTE: re_init will setup if_hwassist. */
1888                 ifp->if_hwassist = 0;
1889
1890                 /* Setup flags for the backend. */
1891                 if (ifp->if_capenable & IFCAP_RXCSUM)
1892                         sc->re_rx_cstag = 1;
1893                 else
1894                         sc->re_rx_cstag = 0;
1895                 if (ifp->if_capenable & IFCAP_TXCSUM)
1896                         sc->re_tx_cstag = 1;
1897                 else
1898                         sc->re_tx_cstag = 0;
1899
1900                 if (mask && (ifp->if_flags & IFF_RUNNING))
1901                         re_init(sc);
1902                 break;
1903
1904         default:
1905                 error = ether_ioctl(ifp, command, data);
1906                 break;
1907         }
1908         return(error);
1909 }
1910
1911 static void
1912 re_watchdog(struct ifnet *ifp)
1913 {
1914         struct re_softc *sc = ifp->if_softc;
1915
1916         ASSERT_SERIALIZED(ifp->if_serializer);
1917
1918         IFNET_STAT_INC(ifp, oerrors, 1);
1919
1920         re_txeof(sc);
1921         re_rxeof(sc);
1922
1923         if (sc->re_ldata.re_tx_free != sc->re_tx_desc_cnt) {
1924                 if_printf(ifp, "watchdog timeout, txd free %d\n",
1925                     sc->re_ldata.re_tx_free);
1926                 rtl_reset(sc);
1927                 re_init(sc);
1928         }
1929 }
1930
1931 /*
1932  * Stop the adapter and free any mbufs allocated to the
1933  * RX and TX lists.
1934  */
1935 static void
1936 re_stop(struct re_softc *sc, boolean_t full_stop)
1937 {
1938         struct ifnet *ifp = &sc->arpcom.ac_if;
1939         int i;
1940
1941         ASSERT_SERIALIZED(ifp->if_serializer);
1942
1943         /* Stop the adapter. */
1944         rtl_stop(sc);
1945
1946         ifp->if_timer = 0;
1947         if (full_stop) {
1948                 callout_stop(&sc->re_timer);
1949                 ifp->if_flags &= ~IFF_RUNNING;
1950         }
1951         ifq_clr_oactive(&ifp->if_snd);
1952         sc->re_flags &= ~(RE_F_TIMER_INTR | RE_F_DROP_RXFRAG | RE_F_LINKED);
1953
1954         re_free_rxchain(sc);
1955
1956         /* Free the TX list buffers. */
1957         for (i = 0; i < sc->re_tx_desc_cnt; i++) {
1958                 if (sc->re_ldata.re_tx_mbuf[i] != NULL) {
1959                         bus_dmamap_unload(sc->re_ldata.re_tx_mtag,
1960                                           sc->re_ldata.re_tx_dmamap[i]);
1961                         m_freem(sc->re_ldata.re_tx_mbuf[i]);
1962                         sc->re_ldata.re_tx_mbuf[i] = NULL;
1963                 }
1964         }
1965
1966         /* Free the RX list buffers. */
1967         for (i = 0; i < sc->re_rx_desc_cnt; i++) {
1968                 if (sc->re_ldata.re_rx_mbuf[i] != NULL) {
1969                         if ((sc->re_flags & RE_F_USE_JPOOL) == 0) {
1970                                 bus_dmamap_unload(sc->re_ldata.re_rx_mtag,
1971                                                   sc->re_ldata.re_rx_dmamap[i]);
1972                         }
1973                         m_freem(sc->re_ldata.re_rx_mbuf[i]);
1974                         sc->re_ldata.re_rx_mbuf[i] = NULL;
1975                 }
1976         }
1977 }
1978
1979 /*
1980  * Device suspend routine.  Stop the interface and save some PCI
1981  * settings in case the BIOS doesn't restore them properly on
1982  * resume.
1983  */
1984 static int
1985 re_suspend(device_t dev)
1986 {
1987 #ifndef BURN_BRIDGES
1988         int i;
1989 #endif
1990         struct re_softc *sc = device_get_softc(dev);
1991         struct ifnet *ifp = &sc->arpcom.ac_if;
1992
1993         lwkt_serialize_enter(ifp->if_serializer);
1994
1995         re_stop(sc, TRUE);
1996
1997 #ifndef BURN_BRIDGES
1998         for (i = 0; i < 5; i++)
1999                 sc->saved_maps[i] = pci_read_config(dev, PCIR_MAPS + i * 4, 4);
2000         sc->saved_biosaddr = pci_read_config(dev, PCIR_BIOS, 4);
2001         sc->saved_intline = pci_read_config(dev, PCIR_INTLINE, 1);
2002         sc->saved_cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
2003         sc->saved_lattimer = pci_read_config(dev, PCIR_LATTIMER, 1);
2004 #endif
2005
2006         sc->re_flags |= RE_F_SUSPENDED;
2007
2008         lwkt_serialize_exit(ifp->if_serializer);
2009
2010         return (0);
2011 }
2012
2013 /*
2014  * Device resume routine.  Restore some PCI settings in case the BIOS
2015  * doesn't, re-enable busmastering, and restart the interface if
2016  * appropriate.
2017  */
2018 static int
2019 re_resume(device_t dev)
2020 {
2021         struct re_softc *sc = device_get_softc(dev);
2022         struct ifnet *ifp = &sc->arpcom.ac_if;
2023 #ifndef BURN_BRIDGES
2024         int i;
2025 #endif
2026
2027         lwkt_serialize_enter(ifp->if_serializer);
2028
2029 #ifndef BURN_BRIDGES
2030         /* better way to do this? */
2031         for (i = 0; i < 5; i++)
2032                 pci_write_config(dev, PCIR_MAPS + i * 4, sc->saved_maps[i], 4);
2033         pci_write_config(dev, PCIR_BIOS, sc->saved_biosaddr, 4);
2034         pci_write_config(dev, PCIR_INTLINE, sc->saved_intline, 1);
2035         pci_write_config(dev, PCIR_CACHELNSZ, sc->saved_cachelnsz, 1);
2036         pci_write_config(dev, PCIR_LATTIMER, sc->saved_lattimer, 1);
2037
2038         /* reenable busmastering */
2039         pci_enable_busmaster(dev);
2040         pci_enable_io(dev, SYS_RES_IOPORT);
2041 #endif
2042
2043         /* reinitialize interface if necessary */
2044         if (ifp->if_flags & IFF_UP)
2045                 re_init(sc);
2046
2047         sc->re_flags &= ~RE_F_SUSPENDED;
2048
2049         lwkt_serialize_exit(ifp->if_serializer);
2050
2051         return (0);
2052 }
2053
2054 /*
2055  * Stop all chip I/O so that the kernel's probe routines don't
2056  * get confused by errant DMAs when rebooting.
2057  */
2058 static void
2059 re_shutdown(device_t dev)
2060 {
2061         struct re_softc *sc = device_get_softc(dev);
2062         struct ifnet *ifp = &sc->arpcom.ac_if;
2063
2064         lwkt_serialize_enter(ifp->if_serializer);
2065         re_stop(sc, TRUE);
2066         rtl_hw_d3_para(sc);
2067         rtl_phy_power_down(sc);
2068         lwkt_serialize_exit(ifp->if_serializer);
2069 }
2070
2071 static int
2072 re_sysctl_rxtime(SYSCTL_HANDLER_ARGS)
2073 {
2074         struct re_softc *sc = arg1;
2075
2076         return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_rx_time);
2077 }
2078
2079 static int
2080 re_sysctl_txtime(SYSCTL_HANDLER_ARGS)
2081 {
2082         struct re_softc *sc = arg1;
2083
2084         return re_sysctl_hwtime(oidp, arg1, arg2, req, &sc->re_tx_time);
2085 }
2086
2087 static int
2088 re_sysctl_hwtime(SYSCTL_HANDLER_ARGS, int *hwtime)
2089 {
2090         struct re_softc *sc = arg1;
2091         struct ifnet *ifp = &sc->arpcom.ac_if;
2092         int error, v;
2093
2094         lwkt_serialize_enter(ifp->if_serializer);
2095
2096         v = *hwtime;
2097         error = sysctl_handle_int(oidp, &v, 0, req);
2098         if (error || req->newptr == NULL)
2099                 goto back;
2100
2101         if (v <= 0) {
2102                 error = EINVAL;
2103                 goto back;
2104         }
2105
2106         if (v != *hwtime) {
2107                 *hwtime = v;
2108
2109                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2110                     IFF_RUNNING && sc->re_imtype == RE_IMTYPE_HW)
2111                         re_setup_hw_im(sc);
2112         }
2113 back:
2114         lwkt_serialize_exit(ifp->if_serializer);
2115         return error;
2116 }
2117
2118 static int
2119 re_sysctl_simtime(SYSCTL_HANDLER_ARGS)
2120 {
2121         struct re_softc *sc = arg1;
2122         struct ifnet *ifp = &sc->arpcom.ac_if;
2123         int error, v;
2124
2125         lwkt_serialize_enter(ifp->if_serializer);
2126
2127         v = sc->re_sim_time;
2128         error = sysctl_handle_int(oidp, &v, 0, req);
2129         if (error || req->newptr == NULL)
2130                 goto back;
2131
2132         if (v <= 0) {
2133                 error = EINVAL;
2134                 goto back;
2135         }
2136
2137         if (v != sc->re_sim_time) {
2138                 sc->re_sim_time = v;
2139
2140                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2141                     IFF_RUNNING && sc->re_imtype == RE_IMTYPE_SIM) {
2142 #ifdef foo
2143                         /*
2144                          * Following code causes various strange
2145                          * performance problems.  Hmm ...
2146                          */
2147                         CSR_WRITE_2(sc, RE_IMR, 0);
2148                         CSR_WRITE_4(sc, RE_TIMERINT, 0);
2149                         CSR_READ_4(sc, RE_TIMERINT); /* flush */
2150
2151                         CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
2152                         re_setup_sim_im(sc);
2153 #else
2154                         re_setup_intr(sc, 0, RE_IMTYPE_NONE);
2155                         DELAY(10);
2156                         re_setup_intr(sc, 1, RE_IMTYPE_SIM);
2157 #endif
2158                 }
2159         }
2160 back:
2161         lwkt_serialize_exit(ifp->if_serializer);
2162         return error;
2163 }
2164
2165 static int
2166 re_sysctl_imtype(SYSCTL_HANDLER_ARGS)
2167 {
2168         struct re_softc *sc = arg1;
2169         struct ifnet *ifp = &sc->arpcom.ac_if;
2170         int error, v;
2171
2172         lwkt_serialize_enter(ifp->if_serializer);
2173
2174         v = sc->re_imtype;
2175         error = sysctl_handle_int(oidp, &v, 0, req);
2176         if (error || req->newptr == NULL)
2177                 goto back;
2178
2179         if (v != RE_IMTYPE_HW && v != RE_IMTYPE_SIM && v != RE_IMTYPE_NONE) {
2180                 error = EINVAL;
2181                 goto back;
2182         }
2183         if (v == RE_IMTYPE_HW && (sc->re_caps & RE_C_HWIM) == 0) {
2184                 /* Can't do hardware interrupt moderation */
2185                 error = EOPNOTSUPP;
2186                 goto back;
2187         }
2188
2189         if (v != sc->re_imtype) {
2190                 sc->re_imtype = v;
2191                 if ((ifp->if_flags & (IFF_RUNNING | IFF_NPOLLING)) ==
2192                     IFF_RUNNING)
2193                         re_setup_intr(sc, 1, sc->re_imtype);
2194         }
2195 back:
2196         lwkt_serialize_exit(ifp->if_serializer);
2197         return error;
2198 }
2199
2200 static void
2201 re_setup_hw_im(struct re_softc *sc)
2202 {
2203         KKASSERT(sc->re_caps & RE_C_HWIM);
2204
2205         /*
2206          * Interrupt moderation
2207          *
2208          * 0xABCD
2209          * A - unknown (maybe TX related)
2210          * B - TX timer (unit: 25us)
2211          * C - unknown (maybe RX related)
2212          * D - RX timer (unit: 25us)
2213          *
2214          *
2215          * re(4)'s interrupt moderation is actually controlled by
2216          * two variables, like most other NICs (bge, bce etc.)
2217          * o  timer
2218          * o  number of packets [P]
2219          *
2220          * The logic relationship between these two variables is
2221          * similar to other NICs too:
2222          * if (timer expire || packets > [P])
2223          *     Interrupt is delivered
2224          *
2225          * Currently we only know how to set 'timer', but not
2226          * 'number of packets', which should be ~30, as far as I
2227          * tested (sink ~900Kpps, interrupt rate is 30KHz)
2228          */
2229         CSR_WRITE_2(sc, RE_IM,
2230                     RE_IM_RXTIME(sc->re_rx_time) |
2231                     RE_IM_TXTIME(sc->re_tx_time) |
2232                     RE_IM_MAGIC);
2233 }
2234
2235 static void
2236 re_disable_hw_im(struct re_softc *sc)
2237 {
2238         if (sc->re_caps & RE_C_HWIM)
2239                 CSR_WRITE_2(sc, RE_IM, 0);
2240 }
2241
2242 static void
2243 re_setup_sim_im(struct re_softc *sc)
2244 {
2245         uint32_t ticks;
2246
2247         if (sc->re_if_flags & RL_FLAG_PCIE) {
2248                 ticks = sc->re_sim_time * sc->re_bus_speed;
2249         } else {
2250                 /*
2251                  * Datasheet says tick decreases at bus speed,
2252                  * but it seems the clock runs a little bit
2253                  * faster, so we do some compensation here.
2254                  */
2255                 ticks = (sc->re_sim_time * sc->re_bus_speed * 8) / 5;
2256         }
2257         CSR_WRITE_4(sc, RE_TIMERINT, ticks);
2258
2259         CSR_WRITE_4(sc, RE_TIMERCNT, 1); /* reload */
2260         sc->re_flags |= RE_F_TIMER_INTR;
2261 }
2262
2263 static void
2264 re_disable_sim_im(struct re_softc *sc)
2265 {
2266         CSR_WRITE_4(sc, RE_TIMERINT, 0);
2267         sc->re_flags &= ~RE_F_TIMER_INTR;
2268 }
2269
2270 static void
2271 re_config_imtype(struct re_softc *sc, int imtype)
2272 {
2273         switch (imtype) {
2274         case RE_IMTYPE_HW:
2275                 KKASSERT(sc->re_caps & RE_C_HWIM);
2276                 /* FALL THROUGH */
2277         case RE_IMTYPE_NONE:
2278                 sc->re_intrs = RE_INTRS;
2279                 sc->re_rx_ack = RE_ISR_RX_OK | RE_ISR_FIFO_OFLOW |
2280                                 RE_ISR_RX_OVERRUN;
2281                 sc->re_tx_ack = RE_ISR_TX_OK;
2282                 break;
2283
2284         case RE_IMTYPE_SIM:
2285                 sc->re_intrs = RE_INTRS_TIMER;
2286                 sc->re_rx_ack = RE_ISR_PCS_TIMEOUT;
2287                 sc->re_tx_ack = RE_ISR_PCS_TIMEOUT;
2288                 break;
2289
2290         default:
2291                 panic("%s: unknown imtype %d",
2292                       sc->arpcom.ac_if.if_xname, imtype);
2293         }
2294 }
2295
2296 static void
2297 re_setup_intr(struct re_softc *sc, int enable_intrs, int imtype)
2298 {
2299         re_config_imtype(sc, imtype);
2300
2301         if (enable_intrs)
2302                 CSR_WRITE_2(sc, RE_IMR, sc->re_intrs);
2303         else
2304                 CSR_WRITE_2(sc, RE_IMR, 0);
2305
2306         sc->re_npoll.ifpc_stcount = 0;
2307
2308         switch (imtype) {
2309         case RE_IMTYPE_NONE:
2310                 re_disable_sim_im(sc);
2311                 re_disable_hw_im(sc);
2312                 break;
2313
2314         case RE_IMTYPE_HW:
2315                 KKASSERT(sc->re_caps & RE_C_HWIM);
2316                 re_disable_sim_im(sc);
2317                 re_setup_hw_im(sc);
2318                 break;
2319
2320         case RE_IMTYPE_SIM:
2321                 re_disable_hw_im(sc);
2322                 re_setup_sim_im(sc);
2323                 break;
2324
2325         default:
2326                 panic("%s: unknown imtype %d",
2327                       sc->arpcom.ac_if.if_xname, imtype);
2328         }
2329 }
2330
2331 static int
2332 re_jpool_alloc(struct re_softc *sc)
2333 {
2334         struct re_list_data *ldata = &sc->re_ldata;
2335         struct re_jbuf *jbuf;
2336         bus_addr_t paddr;
2337         bus_size_t jpool_size;
2338         bus_dmamem_t dmem;
2339         caddr_t buf;
2340         int i, error;
2341
2342         lwkt_serialize_init(&ldata->re_jbuf_serializer);
2343
2344         ldata->re_jbuf = kmalloc(sizeof(struct re_jbuf) * RE_JBUF_COUNT(sc),
2345                                  M_DEVBUF, M_WAITOK | M_ZERO);
2346
2347         jpool_size = RE_JBUF_COUNT(sc) * RE_JBUF_SIZE;
2348
2349         error = bus_dmamem_coherent(sc->re_parent_tag,
2350                         RE_RXBUF_ALIGN, 0,
2351                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
2352                         jpool_size, BUS_DMA_WAITOK, &dmem);
2353         if (error) {
2354                 device_printf(sc->dev, "could not allocate jumbo memory\n");
2355                 return error;
2356         }
2357         ldata->re_jpool_tag = dmem.dmem_tag;
2358         ldata->re_jpool_map = dmem.dmem_map;
2359         ldata->re_jpool = dmem.dmem_addr;
2360         paddr = dmem.dmem_busaddr;
2361
2362         /* ..and split it into 9KB chunks */
2363         SLIST_INIT(&ldata->re_jbuf_free);
2364
2365         buf = ldata->re_jpool;
2366         for (i = 0; i < RE_JBUF_COUNT(sc); i++) {
2367                 jbuf = &ldata->re_jbuf[i];
2368
2369                 jbuf->re_sc = sc;
2370                 jbuf->re_inuse = 0;
2371                 jbuf->re_slot = i;
2372                 jbuf->re_buf = buf;
2373                 jbuf->re_paddr = paddr;
2374
2375                 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link);
2376
2377                 buf += RE_JBUF_SIZE;
2378                 paddr += RE_JBUF_SIZE;
2379         }
2380         return 0;
2381 }
2382
2383 static void
2384 re_jpool_free(struct re_softc *sc)
2385 {
2386         struct re_list_data *ldata = &sc->re_ldata;
2387
2388         if (ldata->re_jpool_tag != NULL) {
2389                 bus_dmamap_unload(ldata->re_jpool_tag, ldata->re_jpool_map);
2390                 bus_dmamem_free(ldata->re_jpool_tag, ldata->re_jpool,
2391                                 ldata->re_jpool_map);
2392                 bus_dma_tag_destroy(ldata->re_jpool_tag);
2393                 ldata->re_jpool_tag = NULL;
2394         }
2395
2396         if (ldata->re_jbuf != NULL) {
2397                 kfree(ldata->re_jbuf, M_DEVBUF);
2398                 ldata->re_jbuf = NULL;
2399         }
2400 }
2401
2402 #ifdef RE_JUMBO
2403 static struct re_jbuf *
2404 re_jbuf_alloc(struct re_softc *sc)
2405 {
2406         struct re_list_data *ldata = &sc->re_ldata;
2407         struct re_jbuf *jbuf;
2408
2409         lwkt_serialize_enter(&ldata->re_jbuf_serializer);
2410
2411         jbuf = SLIST_FIRST(&ldata->re_jbuf_free);
2412         if (jbuf != NULL) {
2413                 SLIST_REMOVE_HEAD(&ldata->re_jbuf_free, re_link);
2414                 jbuf->re_inuse = 1;
2415         }
2416
2417         lwkt_serialize_exit(&ldata->re_jbuf_serializer);
2418
2419         return jbuf;
2420 }
2421
2422 static void
2423 re_jbuf_free(void *arg)
2424 {
2425         struct re_jbuf *jbuf = arg;
2426         struct re_softc *sc = jbuf->re_sc;
2427         struct re_list_data *ldata = &sc->re_ldata;
2428
2429         if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) {
2430                 panic("%s: free wrong jumbo buffer",
2431                       sc->arpcom.ac_if.if_xname);
2432         } else if (jbuf->re_inuse == 0) {
2433                 panic("%s: jumbo buffer already freed",
2434                       sc->arpcom.ac_if.if_xname);
2435         }
2436
2437         lwkt_serialize_enter(&ldata->re_jbuf_serializer);
2438         atomic_subtract_int(&jbuf->re_inuse, 1);
2439         if (jbuf->re_inuse == 0)
2440                 SLIST_INSERT_HEAD(&ldata->re_jbuf_free, jbuf, re_link);
2441         lwkt_serialize_exit(&ldata->re_jbuf_serializer);
2442 }
2443
2444 static void
2445 re_jbuf_ref(void *arg)
2446 {
2447         struct re_jbuf *jbuf = arg;
2448         struct re_softc *sc = jbuf->re_sc;
2449         struct re_list_data *ldata = &sc->re_ldata;
2450
2451         if (&ldata->re_jbuf[jbuf->re_slot] != jbuf) {
2452                 panic("%s: ref wrong jumbo buffer",
2453                       sc->arpcom.ac_if.if_xname);
2454         } else if (jbuf->re_inuse == 0) {
2455                 panic("%s: jumbo buffer already freed",
2456                       sc->arpcom.ac_if.if_xname);
2457         }
2458         atomic_add_int(&jbuf->re_inuse, 1);
2459 }
2460 #endif  /* RE_JUMBO */
2461
2462 static void
2463 re_disable_aspm(device_t dev)
2464 {
2465         uint16_t link_cap, link_ctrl;
2466         uint8_t pcie_ptr, reg;
2467
2468         pcie_ptr = pci_get_pciecap_ptr(dev);
2469         if (pcie_ptr == 0)
2470                 return;
2471
2472         link_cap = pci_read_config(dev, pcie_ptr + PCIER_LINKCAP, 2);
2473         if ((link_cap & PCIEM_LNKCAP_ASPM_MASK) == 0)
2474                 return;
2475
2476         if (bootverbose)
2477                 device_printf(dev, "disable ASPM\n");
2478
2479         reg = pcie_ptr + PCIER_LINKCTRL;
2480         link_ctrl = pci_read_config(dev, reg, 2);
2481         link_ctrl &= ~(PCIEM_LNKCTL_ASPM_L0S | PCIEM_LNKCTL_ASPM_L1);
2482         pci_write_config(dev, reg, link_ctrl, 2);
2483 }