sys/dev/netif/nfe/if_nfe.c

   1 /*      $OpenBSD: if_nfe.c,v 1.63 2006/06/17 18:00:43 brad Exp $        */
   2 /*      $DragonFly: src/sys/dev/netif/nfe/if_nfe.c,v 1.5 2006/12/06 13:47:29 sephe Exp $        */
   3
   4 /*
   5  * Copyright (c) 2006 The DragonFly Project.  All rights reserved.
   6  *
   7  * This code is derived from software contributed to The DragonFly Project
   8  * by Sepherosa Ziehau <sepherosa@gmail.com> and
   9  * Matthew Dillon <dillon@apollo.backplane.com>
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  *
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in
  19  *    the documentation and/or other materials provided with the
  20  *    distribution.
  21  * 3. Neither the name of The DragonFly Project nor the names of its
  22  *    contributors may be used to endorse or promote products derived
  23  *    from this software without specific, prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  26  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  28  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  29  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  30  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  31  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  32  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  33  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  34  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  35  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  */
  38
  39 /*
  40  * Copyright (c) 2006 Damien Bergamini <damien.bergamini@free.fr>
  41  * Copyright (c) 2005, 2006 Jonathan Gray <jsg@openbsd.org>
  42  *
  43  * Permission to use, copy, modify, and distribute this software for any
  44  * purpose with or without fee is hereby granted, provided that the above
  45  * copyright notice and this permission notice appear in all copies.
  46  *
  47  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  48  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  49  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  50  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  51  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  52  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  53  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  54  */
  55
  56 /* Driver for NVIDIA nForce MCP Fast Ethernet and Gigabit Ethernet */
  57
  58 #include "opt_polling.h"
  59
  60 #include <sys/param.h>
  61 #include <sys/endian.h>
  62 #include <sys/kernel.h>
  63 #include <sys/bus.h>
  64 #include <sys/proc.h>
  65 #include <sys/rman.h>
  66 #include <sys/serialize.h>
  67 #include <sys/socket.h>
  68 #include <sys/sockio.h>
  69 #include <sys/sysctl.h>
  70
  71 #include <net/ethernet.h>
  72 #include <net/if.h>
  73 #include <net/bpf.h>
  74 #include <net/if_arp.h>
  75 #include <net/if_dl.h>
  76 #include <net/if_media.h>
  77 #include <net/ifq_var.h>
  78 #include <net/if_types.h>
  79 #include <net/if_var.h>
  80 #include <net/vlan/if_vlan_var.h>
  81
  82 #include <bus/pci/pcireg.h>
  83 #include <bus/pci/pcivar.h>
  84 #include <bus/pci/pcidevs.h>
  85
  86 #include <dev/netif/mii_layer/mii.h>
  87 #include <dev/netif/mii_layer/miivar.h>
  88
  89 #include "miibus_if.h"
  90
  91 #include "if_nfereg.h"
  92 #include "if_nfevar.h"
  93
  94 static int      nfe_probe(device_t);
  95 static int      nfe_attach(device_t);
  96 static int      nfe_detach(device_t);
  97 static void     nfe_shutdown(device_t);
  98 static int      nfe_resume(device_t);
  99 static int      nfe_suspend(device_t);
 100
 101 static int      nfe_miibus_readreg(device_t, int, int);
 102 static void     nfe_miibus_writereg(device_t, int, int, int);
 103 static void     nfe_miibus_statchg(device_t);
 104
 105 #ifdef DEVICE_POLLING
 106 static void     nfe_poll(struct ifnet *, enum poll_cmd, int);
 107 #endif
 108 static void     nfe_intr(void *);
 109 static int      nfe_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 110 static void     nfe_rxeof(struct nfe_softc *);
 111 static void     nfe_txeof(struct nfe_softc *);
 112 static int      nfe_encap(struct nfe_softc *, struct nfe_tx_ring *,
 113                           struct mbuf *);
 114 static void     nfe_start(struct ifnet *);
 115 static void     nfe_watchdog(struct ifnet *);
 116 static void     nfe_init(void *);
 117 static void     nfe_stop(struct nfe_softc *);
 118 static struct nfe_jbuf *nfe_jalloc(struct nfe_softc *);
 119 static void     nfe_jfree(void *);
 120 static void     nfe_jref(void *);
 121 static int      nfe_jpool_alloc(struct nfe_softc *, struct nfe_rx_ring *);
 122 static void     nfe_jpool_free(struct nfe_softc *, struct nfe_rx_ring *);
 123 static int      nfe_alloc_rx_ring(struct nfe_softc *, struct nfe_rx_ring *);
 124 static void     nfe_reset_rx_ring(struct nfe_softc *, struct nfe_rx_ring *);
 125 static int      nfe_init_rx_ring(struct nfe_softc *, struct nfe_rx_ring *);
 126 static void     nfe_free_rx_ring(struct nfe_softc *, struct nfe_rx_ring *);
 127 static int      nfe_alloc_tx_ring(struct nfe_softc *, struct nfe_tx_ring *);
 128 static void     nfe_reset_tx_ring(struct nfe_softc *, struct nfe_tx_ring *);
 129 static int      nfe_init_tx_ring(struct nfe_softc *, struct nfe_tx_ring *);
 130 static void     nfe_free_tx_ring(struct nfe_softc *, struct nfe_tx_ring *);
 131 static int      nfe_ifmedia_upd(struct ifnet *);
 132 static void     nfe_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 133 static void     nfe_setmulti(struct nfe_softc *);
 134 static void     nfe_get_macaddr(struct nfe_softc *, uint8_t *);
 135 static void     nfe_set_macaddr(struct nfe_softc *, const uint8_t *);
 136 static void     nfe_tick(void *);
 137 static void     nfe_ring_dma_addr(void *, bus_dma_segment_t *, int, int);
 138 static void     nfe_buf_dma_addr(void *, bus_dma_segment_t *, int, bus_size_t,
 139                                  int);
 140 static void     nfe_set_paddr_rxdesc(struct nfe_softc *, struct nfe_rx_ring *,
 141                                      int, bus_addr_t);
 142 static void     nfe_set_ready_rxdesc(struct nfe_softc *, struct nfe_rx_ring *,
 143                                      int);
 144 static int      nfe_newbuf_std(struct nfe_softc *, struct nfe_rx_ring *, int,
 145                                int);
 146 static int      nfe_newbuf_jumbo(struct nfe_softc *, struct nfe_rx_ring *, int,
 147                                  int);
 148
 149 #define NFE_DEBUG
 150 #ifdef NFE_DEBUG
 151
 152 static int      nfe_debug = 0;
 153
 154 SYSCTL_NODE(_hw, OID_AUTO, nfe, CTLFLAG_RD, 0, "nVidia GigE parameters");
 155 SYSCTL_INT(_hw_nfe, OID_AUTO, debug, CTLFLAG_RW, &nfe_debug, 0,
 156            "control debugging printfs");
 157
 158 #define DPRINTF(sc, fmt, ...) do {              \
 159         if (nfe_debug) {                        \
 160                 if_printf(&(sc)->arpcom.ac_if,  \
 161                           fmt, __VA_ARGS__);    \
 162         }                                       \
 163 } while (0)
 164
 165 #define DPRINTFN(sc, lv, fmt, ...) do {         \
 166         if (nfe_debug >= (lv)) {                \
 167                 if_printf(&(sc)->arpcom.ac_if,  \
 168                           fmt, __VA_ARGS__);    \
 169         }                                       \
 170 } while (0)
 171
 172 #else   /* !NFE_DEBUG */
 173
 174 #define DPRINTF(sc, fmt, ...)
 175 #define DPRINTFN(sc, lv, fmt, ...)
 176
 177 #endif  /* NFE_DEBUG */
 178
 179 struct nfe_dma_ctx {
 180         int                     nsegs;
 181         bus_dma_segment_t       *segs;
 182 };
 183
 184 static const struct nfe_dev {
 185         uint16_t        vid;
 186         uint16_t        did;
 187         const char      *desc;
 188 } nfe_devices[] = {
 189         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE_LAN,
 190           "NVIDIA nForce Gigabit Ethernet" },
 191
 192         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE2_LAN,
 193           "NVIDIA nForce2 Gigabit Ethernet" },
 194
 195         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE3_LAN1,
 196           "NVIDIA nForce3 Gigabit Ethernet" },
 197
 198         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE3_LAN2,
 199           "NVIDIA nForce3 Gigabit Ethernet" },
 200
 201         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE3_LAN3,
 202           "NVIDIA nForce3 Gigabit Ethernet" },
 203
 204         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE3_LAN4,
 205           "NVIDIA nForce3 Gigabit Ethernet" },
 206
 207         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_NFORCE3_LAN5,
 208           "NVIDIA nForce3 Gigabit Ethernet" },
 209
 210         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_CK804_LAN1,
 211           "NVIDIA CK804 Gigabit Ethernet" },
 212
 213         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_CK804_LAN2,
 214           "NVIDIA CK804 Gigabit Ethernet" },
 215
 216         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP04_LAN1,
 217           "NVIDIA MCP04 Gigabit Ethernet" },
 218
 219         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP04_LAN2,
 220           "NVIDIA MCP04 Gigabit Ethernet" },
 221
 222         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP51_LAN1,
 223           "NVIDIA MCP51 Gigabit Ethernet" },
 224
 225         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP51_LAN2,
 226           "NVIDIA MCP51 Gigabit Ethernet" },
 227
 228         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP55_LAN1,
 229           "NVIDIA MCP55 Gigabit Ethernet" },
 230
 231         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP55_LAN2,
 232           "NVIDIA MCP55 Gigabit Ethernet" },
 233
 234         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN1,
 235           "NVIDIA MCP61 Gigabit Ethernet" },
 236
 237         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN2,
 238           "NVIDIA MCP61 Gigabit Ethernet" },
 239
 240         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN3,
 241           "NVIDIA MCP61 Gigabit Ethernet" },
 242
 243         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP61_LAN4,
 244           "NVIDIA MCP61 Gigabit Ethernet" },
 245
 246         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN1,
 247           "NVIDIA MCP65 Gigabit Ethernet" },
 248
 249         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN2,
 250           "NVIDIA MCP65 Gigabit Ethernet" },
 251
 252         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN3,
 253           "NVIDIA MCP65 Gigabit Ethernet" },
 254
 255         { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_LAN4,
 256           "NVIDIA MCP65 Gigabit Ethernet" }
 257 };
 258
 259 static device_method_t nfe_methods[] = {
 260         /* Device interface */
 261         DEVMETHOD(device_probe,         nfe_probe),
 262         DEVMETHOD(device_attach,        nfe_attach),
 263         DEVMETHOD(device_detach,        nfe_detach),
 264         DEVMETHOD(device_suspend,       nfe_suspend),
 265         DEVMETHOD(device_resume,        nfe_resume),
 266         DEVMETHOD(device_shutdown,      nfe_shutdown),
 267
 268         /* Bus interface */
 269         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 270         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 271
 272         /* MII interface */
 273         DEVMETHOD(miibus_readreg,       nfe_miibus_readreg),
 274         DEVMETHOD(miibus_writereg,      nfe_miibus_writereg),
 275         DEVMETHOD(miibus_statchg,       nfe_miibus_statchg),
 276
 277         { 0, 0 }
 278 };
 279
 280 static driver_t nfe_driver = {
 281         "nfe",
 282         nfe_methods,
 283         sizeof(struct nfe_softc)
 284 };
 285
 286 static devclass_t       nfe_devclass;
 287
 288 DECLARE_DUMMY_MODULE(if_nfe);
 289 MODULE_DEPEND(if_nfe, miibus, 1, 1, 1);
 290 DRIVER_MODULE(if_nfe, pci, nfe_driver, nfe_devclass, 0, 0);
 291 DRIVER_MODULE(miibus, nfe, miibus_driver, miibus_devclass, 0, 0);
 292
 293 static int
 294 nfe_probe(device_t dev)
 295 {
 296         const struct nfe_dev *n;
 297         uint16_t vid, did;
 298
 299         vid = pci_get_vendor(dev);
 300         did = pci_get_device(dev);
 301         for (n = nfe_devices; n->desc != NULL; ++n) {
 302                 if (vid == n->vid && did == n->did) {
 303                         struct nfe_softc *sc = device_get_softc(dev);
 304
 305                         switch (did) {
 306                         case PCI_PRODUCT_NVIDIA_NFORCE3_LAN2:
 307                         case PCI_PRODUCT_NVIDIA_NFORCE3_LAN3:
 308                         case PCI_PRODUCT_NVIDIA_NFORCE3_LAN4:
 309                         case PCI_PRODUCT_NVIDIA_NFORCE3_LAN5:
 310                                 sc->sc_flags = NFE_JUMBO_SUP |
 311                                                NFE_HW_CSUM;
 312                                 break;
 313                         case PCI_PRODUCT_NVIDIA_MCP51_LAN1:
 314                         case PCI_PRODUCT_NVIDIA_MCP51_LAN2:
 315                         case PCI_PRODUCT_NVIDIA_MCP61_LAN1:
 316                         case PCI_PRODUCT_NVIDIA_MCP61_LAN2:
 317                         case PCI_PRODUCT_NVIDIA_MCP61_LAN3:
 318                         case PCI_PRODUCT_NVIDIA_MCP61_LAN4:
 319                                 sc->sc_flags = NFE_40BIT_ADDR;
 320                                 break;
 321                         case PCI_PRODUCT_NVIDIA_CK804_LAN1:
 322                         case PCI_PRODUCT_NVIDIA_CK804_LAN2:
 323                         case PCI_PRODUCT_NVIDIA_MCP04_LAN1:
 324                         case PCI_PRODUCT_NVIDIA_MCP04_LAN2:
 325                         case PCI_PRODUCT_NVIDIA_MCP65_LAN1:
 326                         case PCI_PRODUCT_NVIDIA_MCP65_LAN2:
 327                         case PCI_PRODUCT_NVIDIA_MCP65_LAN3:
 328                         case PCI_PRODUCT_NVIDIA_MCP65_LAN4:
 329                                 sc->sc_flags = NFE_JUMBO_SUP |
 330                                                NFE_40BIT_ADDR |
 331                                                NFE_HW_CSUM;
 332                                 break;
 333                         case PCI_PRODUCT_NVIDIA_MCP55_LAN1:
 334                         case PCI_PRODUCT_NVIDIA_MCP55_LAN2:
 335                                 sc->sc_flags = NFE_JUMBO_SUP |
 336                                                NFE_40BIT_ADDR |
 337                                                NFE_HW_CSUM |
 338                                                NFE_HW_VLAN;
 339                                 break;
 340                         }
 341
 342                         /* Enable jumbo frames for adapters that support it */
 343                         if (sc->sc_flags & NFE_JUMBO_SUP)
 344                                 sc->sc_flags |= NFE_USE_JUMBO;
 345
 346                         device_set_desc(dev, n->desc);
 347                         return 0;
 348                 }
 349         }
 350         return ENXIO;
 351 }
 352
 353 static int
 354 nfe_attach(device_t dev)
 355 {
 356         struct nfe_softc *sc = device_get_softc(dev);
 357         struct ifnet *ifp = &sc->arpcom.ac_if;
 358         uint8_t eaddr[ETHER_ADDR_LEN];
 359         int error;
 360
 361         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 362         lwkt_serialize_init(&sc->sc_jbuf_serializer);
 363
 364         sc->sc_mem_rid = PCIR_BAR(0);
 365
 366 #ifndef BURN_BRIDGES
 367         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
 368                 uint32_t mem, irq;
 369
 370                 mem = pci_read_config(dev, sc->sc_mem_rid, 4);
 371                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
 372
 373                 device_printf(dev, "chip is in D%d power mode "
 374                     "-- setting to D0\n", pci_get_powerstate(dev));
 375
 376                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
 377
 378                 pci_write_config(dev, sc->sc_mem_rid, mem, 4);
 379                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
 380         }
 381 #endif  /* !BURN_BRIDGE */
 382
 383         /* Enable bus mastering */
 384         pci_enable_busmaster(dev);
 385
 386         /* Allocate IO memory */
 387         sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 388                                                 &sc->sc_mem_rid, RF_ACTIVE);
 389         if (sc->sc_mem_res == NULL) {
 390                 device_printf(dev, "cound not allocate io memory\n");
 391                 return ENXIO;
 392         }
 393         sc->sc_memh = rman_get_bushandle(sc->sc_mem_res);
 394         sc->sc_memt = rman_get_bustag(sc->sc_mem_res);
 395
 396         /* Allocate IRQ */
 397         sc->sc_irq_rid = 0;
 398         sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
 399                                                 &sc->sc_irq_rid,
 400                                                 RF_SHAREABLE | RF_ACTIVE);
 401         if (sc->sc_irq_res == NULL) {
 402                 device_printf(dev, "could not allocate irq\n");
 403                 error = ENXIO;
 404                 goto fail;
 405         }
 406
 407         nfe_get_macaddr(sc, eaddr);
 408
 409         /*
 410          * Allocate Tx and Rx rings.
 411          */
 412         error = nfe_alloc_tx_ring(sc, &sc->txq);
 413         if (error) {
 414                 device_printf(dev, "could not allocate Tx ring\n");
 415                 goto fail;
 416         }
 417
 418         error = nfe_alloc_rx_ring(sc, &sc->rxq);
 419         if (error) {
 420                 device_printf(dev, "could not allocate Rx ring\n");
 421                 goto fail;
 422         }
 423
 424         error = mii_phy_probe(dev, &sc->sc_miibus, nfe_ifmedia_upd,
 425                               nfe_ifmedia_sts);
 426         if (error) {
 427                 device_printf(dev, "MII without any phy\n");
 428                 goto fail;
 429         }
 430
 431         ifp->if_softc = sc;
 432         ifp->if_mtu = ETHERMTU;
 433         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 434         ifp->if_ioctl = nfe_ioctl;
 435         ifp->if_start = nfe_start;
 436 #ifdef DEVICE_POLLING
 437         ifp->if_poll = nfe_poll;
 438 #endif
 439         ifp->if_watchdog = nfe_watchdog;
 440         ifp->if_init = nfe_init;
 441         ifq_set_maxlen(&ifp->if_snd, NFE_IFQ_MAXLEN);
 442         ifq_set_ready(&ifp->if_snd);
 443
 444         ifp->if_capabilities = IFCAP_VLAN_MTU;
 445
 446 #if 0
 447         if (sc->sc_flags & NFE_USE_JUMBO)
 448                 ifp->if_hardmtu = NFE_JUMBO_MTU;
 449 #endif
 450
 451         if (sc->sc_flags & NFE_HW_VLAN)
 452                 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
 453
 454 #ifdef NFE_CSUM
 455         if (sc->sc_flags & NFE_HW_CSUM) {
 456 #if 0
 457                 ifp->if_capabilities |= IFCAP_CSUM_IPv4 | IFCAP_CSUM_TCPv4 |
 458                     IFCAP_CSUM_UDPv4;
 459 #else
 460                 ifp->if_capabilities = IFCAP_HWCSUM;
 461                 ifp->if_hwassist = CSUM_IP | CSUM_TCP | CSUM_UDP;
 462 #endif
 463         }
 464 #endif
 465         ifp->if_capenable = ifp->if_capabilities;
 466
 467         callout_init(&sc->sc_tick_ch);
 468
 469         ether_ifattach(ifp, eaddr, NULL);
 470
 471         error = bus_setup_intr(dev, sc->sc_irq_res, INTR_MPSAFE, nfe_intr, sc,
 472                                &sc->sc_ih, ifp->if_serializer);
 473         if (error) {
 474                 device_printf(dev, "could not setup intr\n");
 475                 ether_ifdetach(ifp);
 476                 goto fail;
 477         }
 478
 479         return 0;
 480 fail:
 481         nfe_detach(dev);
 482         return error;
 483 }
 484
 485 static int
 486 nfe_detach(device_t dev)
 487 {
 488         struct nfe_softc *sc = device_get_softc(dev);
 489
 490         if (device_is_attached(dev)) {
 491                 struct ifnet *ifp = &sc->arpcom.ac_if;
 492
 493                 lwkt_serialize_enter(ifp->if_serializer);
 494                 nfe_stop(sc);
 495                 bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_ih);
 496                 lwkt_serialize_exit(ifp->if_serializer);
 497
 498                 ether_ifdetach(ifp);
 499         }
 500
 501         if (sc->sc_miibus != NULL)
 502                 device_delete_child(dev, sc->sc_miibus);
 503         bus_generic_detach(dev);
 504
 505         if (sc->sc_irq_res != NULL) {
 506                 bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid,
 507                                      sc->sc_irq_res);
 508         }
 509
 510         if (sc->sc_mem_res != NULL) {
 511                 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_mem_rid,
 512                                      sc->sc_mem_res);
 513         }
 514
 515         nfe_free_tx_ring(sc, &sc->txq);
 516         nfe_free_rx_ring(sc, &sc->rxq);
 517
 518         return 0;
 519 }
 520
 521 static void
 522 nfe_shutdown(device_t dev)
 523 {
 524         struct nfe_softc *sc = device_get_softc(dev);
 525         struct ifnet *ifp = &sc->arpcom.ac_if;
 526
 527         lwkt_serialize_enter(ifp->if_serializer);
 528         nfe_stop(sc);
 529         lwkt_serialize_exit(ifp->if_serializer);
 530 }
 531
 532 static int
 533 nfe_suspend(device_t dev)
 534 {
 535         struct nfe_softc *sc = device_get_softc(dev);
 536         struct ifnet *ifp = &sc->arpcom.ac_if;
 537
 538         lwkt_serialize_enter(ifp->if_serializer);
 539         nfe_stop(sc);
 540         lwkt_serialize_exit(ifp->if_serializer);
 541
 542         return 0;
 543 }
 544
 545 static int
 546 nfe_resume(device_t dev)
 547 {
 548         struct nfe_softc *sc = device_get_softc(dev);
 549         struct ifnet *ifp = &sc->arpcom.ac_if;
 550
 551         lwkt_serialize_enter(ifp->if_serializer);
 552         if (ifp->if_flags & IFF_UP) {
 553                 nfe_init(sc);
 554                 if (ifp->if_flags & IFF_RUNNING)
 555                         ifp->if_start(ifp);
 556         }
 557         lwkt_serialize_exit(ifp->if_serializer);
 558
 559         return 0;
 560 }
 561
 562 static void
 563 nfe_miibus_statchg(device_t dev)
 564 {
 565         struct nfe_softc *sc = device_get_softc(dev);
 566         struct mii_data *mii = device_get_softc(sc->sc_miibus);
 567         uint32_t phy, seed, misc = NFE_MISC1_MAGIC, link = NFE_MEDIA_SET;
 568
 569         phy = NFE_READ(sc, NFE_PHY_IFACE);
 570         phy &= ~(NFE_PHY_HDX | NFE_PHY_100TX | NFE_PHY_1000T);
 571
 572         seed = NFE_READ(sc, NFE_RNDSEED);
 573         seed &= ~NFE_SEED_MASK;
 574
 575         if ((mii->mii_media_active & IFM_GMASK) == IFM_HDX) {
 576                 phy  |= NFE_PHY_HDX;    /* half-duplex */
 577                 misc |= NFE_MISC1_HDX;
 578         }
 579
 580         switch (IFM_SUBTYPE(mii->mii_media_active)) {
 581         case IFM_1000_T:        /* full-duplex only */
 582                 link |= NFE_MEDIA_1000T;
 583                 seed |= NFE_SEED_1000T;
 584                 phy  |= NFE_PHY_1000T;
 585                 break;
 586         case IFM_100_TX:
 587                 link |= NFE_MEDIA_100TX;
 588                 seed |= NFE_SEED_100TX;
 589                 phy  |= NFE_PHY_100TX;
 590                 break;
 591         case IFM_10_T:
 592                 link |= NFE_MEDIA_10T;
 593                 seed |= NFE_SEED_10T;
 594                 break;
 595         }
 596
 597         NFE_WRITE(sc, NFE_RNDSEED, seed);       /* XXX: gigabit NICs only? */
 598
 599         NFE_WRITE(sc, NFE_PHY_IFACE, phy);
 600         NFE_WRITE(sc, NFE_MISC1, misc);
 601         NFE_WRITE(sc, NFE_LINKSPEED, link);
 602 }
 603
 604 static int
 605 nfe_miibus_readreg(device_t dev, int phy, int reg)
 606 {
 607         struct nfe_softc *sc = device_get_softc(dev);
 608         uint32_t val;
 609         int ntries;
 610
 611         NFE_WRITE(sc, NFE_PHY_STATUS, 0xf);
 612
 613         if (NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY) {
 614                 NFE_WRITE(sc, NFE_PHY_CTL, NFE_PHY_BUSY);
 615                 DELAY(100);
 616         }
 617
 618         NFE_WRITE(sc, NFE_PHY_CTL, (phy << NFE_PHYADD_SHIFT) | reg);
 619
 620         for (ntries = 0; ntries < 1000; ntries++) {
 621                 DELAY(100);
 622                 if (!(NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY))
 623                         break;
 624         }
 625         if (ntries == 1000) {
 626                 DPRINTFN(sc, 2, "timeout waiting for PHY %s\n", "");
 627                 return 0;
 628         }
 629
 630         if (NFE_READ(sc, NFE_PHY_STATUS) & NFE_PHY_ERROR) {
 631                 DPRINTFN(sc, 2, "could not read PHY %s\n", "");
 632                 return 0;
 633         }
 634
 635         val = NFE_READ(sc, NFE_PHY_DATA);
 636         if (val != 0xffffffff && val != 0)
 637                 sc->mii_phyaddr = phy;
 638
 639         DPRINTFN(sc, 2, "mii read phy %d reg 0x%x ret 0x%x\n", phy, reg, val);
 640
 641         return val;
 642 }
 643
 644 static void
 645 nfe_miibus_writereg(device_t dev, int phy, int reg, int val)
 646 {
 647         struct nfe_softc *sc = device_get_softc(dev);
 648         uint32_t ctl;
 649         int ntries;
 650
 651         NFE_WRITE(sc, NFE_PHY_STATUS, 0xf);
 652
 653         if (NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY) {
 654                 NFE_WRITE(sc, NFE_PHY_CTL, NFE_PHY_BUSY);
 655                 DELAY(100);
 656         }
 657
 658         NFE_WRITE(sc, NFE_PHY_DATA, val);
 659         ctl = NFE_PHY_WRITE | (phy << NFE_PHYADD_SHIFT) | reg;
 660         NFE_WRITE(sc, NFE_PHY_CTL, ctl);
 661
 662         for (ntries = 0; ntries < 1000; ntries++) {
 663                 DELAY(100);
 664                 if (!(NFE_READ(sc, NFE_PHY_CTL) & NFE_PHY_BUSY))
 665                         break;
 666         }
 667
 668 #ifdef NFE_DEBUG
 669         if (ntries == 1000)
 670                 DPRINTFN(sc, 2, "could not write to PHY %s\n", "");
 671 #endif
 672 }
 673
 674 #ifdef DEVICE_POLLING
 675
 676 static void
 677 nfe_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 678 {
 679         struct nfe_softc *sc = ifp->if_softc;
 680
 681         switch(cmd) {
 682         case POLL_REGISTER:
 683                 /* Disable interrupts */
 684                 NFE_WRITE(sc, NFE_IRQ_MASK, 0);
 685                 break;
 686         case POLL_DEREGISTER:
 687                 /* enable interrupts */
 688                 NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED);
 689                 break;
 690         case POLL_AND_CHECK_STATUS:
 691                 /* fall through */
 692         case POLL_ONLY:
 693                 if (ifp->if_flags & IFF_RUNNING) {
 694                         nfe_rxeof(sc);
 695                         nfe_txeof(sc);
 696                 }
 697                 break;
 698         }
 699 }
 700
 701 #endif
 702
 703 static void
 704 nfe_intr(void *arg)
 705 {
 706         struct nfe_softc *sc = arg;
 707         struct ifnet *ifp = &sc->arpcom.ac_if;
 708         uint32_t r;
 709
 710         r = NFE_READ(sc, NFE_IRQ_STATUS);
 711         if (r == 0)
 712                 return; /* not for us */
 713         NFE_WRITE(sc, NFE_IRQ_STATUS, r);
 714
 715         DPRINTFN(sc, 5, "%s: interrupt register %x\n", __func__, r);
 716
 717         if (r & NFE_IRQ_LINK) {
 718                 NFE_READ(sc, NFE_PHY_STATUS);
 719                 NFE_WRITE(sc, NFE_PHY_STATUS, 0xf);
 720                 DPRINTF(sc, "link state changed %s\n", "");
 721         }
 722
 723         if (ifp->if_flags & IFF_RUNNING) {
 724                 /* check Rx ring */
 725                 nfe_rxeof(sc);
 726
 727                 /* check Tx ring */
 728                 nfe_txeof(sc);
 729         }
 730 }
 731
 732 static int
 733 nfe_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr)
 734 {
 735         struct nfe_softc *sc = ifp->if_softc;
 736         struct ifreq *ifr = (struct ifreq *)data;
 737         struct mii_data *mii;
 738         int error = 0, mask;
 739
 740         switch (cmd) {
 741         case SIOCSIFMTU:
 742                 /* XXX NFE_USE_JUMBO should be set here */
 743                 break;
 744         case SIOCSIFFLAGS:
 745                 if (ifp->if_flags & IFF_UP) {
 746                         /*
 747                          * If only the PROMISC or ALLMULTI flag changes, then
 748                          * don't do a full re-init of the chip, just update
 749                          * the Rx filter.
 750                          */
 751                         if ((ifp->if_flags & IFF_RUNNING) &&
 752                             ((ifp->if_flags ^ sc->sc_if_flags) &
 753                              (IFF_ALLMULTI | IFF_PROMISC)) != 0) {
 754                                 nfe_setmulti(sc);
 755                         } else {
 756                                 if (!(ifp->if_flags & IFF_RUNNING))
 757                                         nfe_init(sc);
 758                         }
 759                 } else {
 760                         if (ifp->if_flags & IFF_RUNNING)
 761                                 nfe_stop(sc);
 762                 }
 763                 sc->sc_if_flags = ifp->if_flags;
 764                 break;
 765         case SIOCADDMULTI:
 766         case SIOCDELMULTI:
 767                 if (ifp->if_flags & IFF_RUNNING)
 768                         nfe_setmulti(sc);
 769                 break;
 770         case SIOCSIFMEDIA:
 771         case SIOCGIFMEDIA:
 772                 mii = device_get_softc(sc->sc_miibus);
 773                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
 774                 break;
 775         case SIOCSIFCAP:
 776                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 777                 if (mask & IFCAP_HWCSUM) {
 778                         if (IFCAP_HWCSUM & ifp->if_capenable)
 779                                 ifp->if_capenable &= ~IFCAP_HWCSUM;
 780                         else
 781                                 ifp->if_capenable |= IFCAP_HWCSUM;
 782                 }
 783                 break;
 784         default:
 785                 error = ether_ioctl(ifp, cmd, data);
 786                 break;
 787         }
 788         return error;
 789 }
 790
 791 static void
 792 nfe_rxeof(struct nfe_softc *sc)
 793 {
 794         struct ifnet *ifp = &sc->arpcom.ac_if;
 795         struct nfe_rx_ring *ring = &sc->rxq;
 796         int reap;
 797
 798         reap = 0;
 799         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_POSTREAD);
 800
 801         for (;;) {
 802                 struct nfe_rx_data *data = &ring->data[ring->cur];
 803                 struct mbuf *m;
 804                 uint16_t flags;
 805                 int len, error;
 806
 807                 if (sc->sc_flags & NFE_40BIT_ADDR) {
 808                         struct nfe_desc64 *desc64 = &ring->desc64[ring->cur];
 809
 810                         flags = le16toh(desc64->flags);
 811                         len = le16toh(desc64->length) & 0x3fff;
 812                 } else {
 813                         struct nfe_desc32 *desc32 = &ring->desc32[ring->cur];
 814
 815                         flags = le16toh(desc32->flags);
 816                         len = le16toh(desc32->length) & 0x3fff;
 817                 }
 818
 819                 if (flags & NFE_RX_READY)
 820                         break;
 821
 822                 reap = 1;
 823
 824                 if ((sc->sc_flags & (NFE_JUMBO_SUP | NFE_40BIT_ADDR)) == 0) {
 825                         if (!(flags & NFE_RX_VALID_V1))
 826                                 goto skip;
 827
 828                         if ((flags & NFE_RX_FIXME_V1) == NFE_RX_FIXME_V1) {
 829                                 flags &= ~NFE_RX_ERROR;
 830                                 len--;  /* fix buffer length */
 831                         }
 832                 } else {
 833                         if (!(flags & NFE_RX_VALID_V2))
 834                                 goto skip;
 835
 836                         if ((flags & NFE_RX_FIXME_V2) == NFE_RX_FIXME_V2) {
 837                                 flags &= ~NFE_RX_ERROR;
 838                                 len--;  /* fix buffer length */
 839                         }
 840                 }
 841
 842                 if (flags & NFE_RX_ERROR) {
 843                         ifp->if_ierrors++;
 844                         goto skip;
 845                 }
 846
 847                 m = data->m;
 848
 849                 if (sc->sc_flags & NFE_USE_JUMBO)
 850                         error = nfe_newbuf_jumbo(sc, ring, ring->cur, 0);
 851                 else
 852                         error = nfe_newbuf_std(sc, ring, ring->cur, 0);
 853                 if (error) {
 854                         ifp->if_ierrors++;
 855                         goto skip;
 856                 }
 857
 858                 /* finalize mbuf */
 859                 m->m_pkthdr.len = m->m_len = len;
 860                 m->m_pkthdr.rcvif = ifp;
 861
 862 #ifdef notyet
 863                 if (sc->sc_flags & NFE_HW_CSUM) {
 864                         if (flags & NFE_RX_IP_CSUMOK)
 865                                 m->m_pkthdr.csum_flags |= M_IPV4_CSUM_IN_OK;
 866                         if (flags & NFE_RX_UDP_CSUMOK)
 867                                 m->m_pkthdr.csum_flags |= M_UDP_CSUM_IN_OK;
 868                         if (flags & NFE_RX_TCP_CSUMOK)
 869                                 m->m_pkthdr.csum_flags |= M_TCP_CSUM_IN_OK;
 870                 }
 871 #elif defined(NFE_CSUM)
 872                 if ((sc->sc_flags & NFE_HW_CSUM) && (flags & NFE_RX_CSUMOK))
 873                         m->m_pkthdr.csum_flags = M_IPV4_CSUM_IN_OK;
 874 #endif
 875
 876                 ifp->if_ipackets++;
 877                 ifp->if_input(ifp, m);
 878 skip:
 879                 nfe_set_ready_rxdesc(sc, ring, ring->cur);
 880                 sc->rxq.cur = (sc->rxq.cur + 1) % NFE_RX_RING_COUNT;
 881         }
 882
 883         if (reap)
 884                 bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_PREWRITE);
 885 }
 886
 887 static void
 888 nfe_txeof(struct nfe_softc *sc)
 889 {
 890         struct ifnet *ifp = &sc->arpcom.ac_if;
 891         struct nfe_tx_ring *ring = &sc->txq;
 892         struct nfe_tx_data *data = NULL;
 893
 894         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_POSTREAD);
 895         while (ring->next != ring->cur) {
 896                 uint16_t flags;
 897
 898                 if (sc->sc_flags & NFE_40BIT_ADDR)
 899                         flags = le16toh(ring->desc64[ring->next].flags);
 900                 else
 901                         flags = le16toh(ring->desc32[ring->next].flags);
 902
 903                 if (flags & NFE_TX_VALID)
 904                         break;
 905
 906                 data = &ring->data[ring->next];
 907
 908                 if ((sc->sc_flags & (NFE_JUMBO_SUP | NFE_40BIT_ADDR)) == 0) {
 909                         if (!(flags & NFE_TX_LASTFRAG_V1) && data->m == NULL)
 910                                 goto skip;
 911
 912                         if ((flags & NFE_TX_ERROR_V1) != 0) {
 913                                 if_printf(ifp, "tx v1 error 0x%4b\n", flags,
 914                                           NFE_V1_TXERR);
 915                                 ifp->if_oerrors++;
 916                         } else {
 917                                 ifp->if_opackets++;
 918                         }
 919                 } else {
 920                         if (!(flags & NFE_TX_LASTFRAG_V2) && data->m == NULL)
 921                                 goto skip;
 922
 923                         if ((flags & NFE_TX_ERROR_V2) != 0) {
 924                                 if_printf(ifp, "tx v2 error 0x%4b\n", flags,
 925                                           NFE_V2_TXERR);
 926                                 ifp->if_oerrors++;
 927                         } else {
 928                                 ifp->if_opackets++;
 929                         }
 930                 }
 931
 932                 if (data->m == NULL) {  /* should not get there */
 933                         if_printf(ifp,
 934                                   "last fragment bit w/o associated mbuf!\n");
 935                         goto skip;
 936                 }
 937
 938                 /* last fragment of the mbuf chain transmitted */
 939                 bus_dmamap_sync(ring->data_tag, data->map,
 940                                 BUS_DMASYNC_POSTWRITE);
 941                 bus_dmamap_unload(ring->data_tag, data->map);
 942                 m_freem(data->m);
 943                 data->m = NULL;
 944
 945                 ifp->if_timer = 0;
 946 skip:
 947                 ring->queued--;
 948                 KKASSERT(ring->queued >= 0);
 949                 ring->next = (ring->next + 1) % NFE_TX_RING_COUNT;
 950         }
 951
 952         if (data != NULL) {     /* at least one slot freed */
 953                 ifp->if_flags &= ~IFF_OACTIVE;
 954                 ifp->if_start(ifp);
 955         }
 956 }
 957
 958 static int
 959 nfe_encap(struct nfe_softc *sc, struct nfe_tx_ring *ring, struct mbuf *m0)
 960 {
 961         struct nfe_dma_ctx ctx;
 962         bus_dma_segment_t segs[NFE_MAX_SCATTER];
 963         struct nfe_tx_data *data, *data_map;
 964         bus_dmamap_t map;
 965         struct nfe_desc64 *desc64 = NULL;
 966         struct nfe_desc32 *desc32 = NULL;
 967         uint16_t flags = 0;
 968         uint32_t vtag = 0;
 969         int error, i, j;
 970
 971         data = &ring->data[ring->cur];
 972         map = data->map;
 973         data_map = data;        /* Remember who owns the DMA map */
 974
 975         ctx.nsegs = NFE_MAX_SCATTER;
 976         ctx.segs = segs;
 977         error = bus_dmamap_load_mbuf(ring->data_tag, map, m0,
 978                                      nfe_buf_dma_addr, &ctx, BUS_DMA_NOWAIT);
 979         if (error && error != EFBIG) {
 980                 if_printf(&sc->arpcom.ac_if, "could not map TX mbuf\n");
 981                 goto back;
 982         }
 983
 984         if (error) {    /* error == EFBIG */
 985                 struct mbuf *m_new;
 986
 987                 m_new = m_defrag(m0, MB_DONTWAIT);
 988                 if (m_new == NULL) {
 989                         if_printf(&sc->arpcom.ac_if,
 990                                   "could not defrag TX mbuf\n");
 991                         error = ENOBUFS;
 992                         goto back;
 993                 } else {
 994                         m0 = m_new;
 995                 }
 996
 997                 ctx.nsegs = NFE_MAX_SCATTER;
 998                 ctx.segs = segs;
 999                 error = bus_dmamap_load_mbuf(ring->data_tag, map, m0,
1000                                              nfe_buf_dma_addr, &ctx,
1001                                              BUS_DMA_NOWAIT);
1002                 if (error) {
1003                         if_printf(&sc->arpcom.ac_if,
1004                                   "could not map defraged TX mbuf\n");
1005                         goto back;
1006                 }
1007         }
1008
1009         error = 0;
1010
1011         if (ring->queued + ctx.nsegs >= NFE_TX_RING_COUNT - 1) {
1012                 bus_dmamap_unload(ring->data_tag, map);
1013                 error = ENOBUFS;
1014                 goto back;
1015         }
1016
1017         /* setup h/w VLAN tagging */
1018         if ((m0->m_flags & (M_PROTO1 | M_PKTHDR)) == (M_PROTO1 | M_PKTHDR) &&
1019             m0->m_pkthdr.rcvif != NULL &&
1020             m0->m_pkthdr.rcvif->if_type == IFT_L2VLAN) {
1021                 struct ifvlan *ifv = m0->m_pkthdr.rcvif->if_softc;
1022
1023                 if (ifv != NULL)
1024                         vtag = NFE_TX_VTAG | htons(ifv->ifv_tag);
1025         }
1026
1027 #ifdef NFE_CSUM
1028         if (m0->m_pkthdr.csum_flags & M_IPV4_CSUM_OUT)
1029                 flags |= NFE_TX_IP_CSUM;
1030         if (m0->m_pkthdr.csum_flags & (M_TCPV4_CSUM_OUT | M_UDPV4_CSUM_OUT))
1031                 flags |= NFE_TX_TCP_CSUM;
1032 #endif
1033
1034         /*
1035          * XXX urm. somebody is unaware of how hardware works.  You
1036          * absolutely CANNOT set NFE_TX_VALID on the next descriptor in
1037          * the ring until the entire chain is actually *VALID*.  Otherwise
1038          * the hardware may encounter a partially initialized chain that
1039          * is marked as being ready to go when it in fact is not ready to
1040          * go.
1041          */
1042
1043         for (i = 0; i < ctx.nsegs; i++) {
1044                 j = (ring->cur + i) % NFE_TX_RING_COUNT;
1045                 data = &ring->data[j];
1046
1047                 if (sc->sc_flags & NFE_40BIT_ADDR) {
1048                         desc64 = &ring->desc64[j];
1049 #if defined(__LP64__)
1050                         desc64->physaddr[0] =
1051                             htole32(segs[i].ds_addr >> 32);
1052 #endif
1053                         desc64->physaddr[1] =
1054                             htole32(segs[i].ds_addr & 0xffffffff);
1055                         desc64->length = htole16(segs[i].ds_len - 1);
1056                         desc64->vtag = htole32(vtag);
1057                         desc64->flags = htole16(flags);
1058                 } else {
1059                         desc32 = &ring->desc32[j];
1060                         desc32->physaddr = htole32(segs[i].ds_addr);
1061                         desc32->length = htole16(segs[i].ds_len - 1);
1062                         desc32->flags = htole16(flags);
1063                 }
1064
1065                 /* csum flags and vtag belong to the first fragment only */
1066                 flags &= ~(NFE_TX_IP_CSUM | NFE_TX_TCP_CSUM);
1067                 vtag = 0;
1068
1069                 ring->queued++;
1070                 KKASSERT(ring->queued <= NFE_TX_RING_COUNT);
1071         }
1072
1073         /* the whole mbuf chain has been DMA mapped, fix last descriptor */
1074         if (sc->sc_flags & NFE_40BIT_ADDR) {
1075                 desc64->flags |= htole16(NFE_TX_LASTFRAG_V2);
1076         } else {
1077                 if (sc->sc_flags & NFE_JUMBO_SUP)
1078                         flags = NFE_TX_LASTFRAG_V2;
1079                 else
1080                         flags = NFE_TX_LASTFRAG_V1;
1081                 desc32->flags |= htole16(flags);
1082         }
1083
1084         /*
1085          * Set NFE_TX_VALID backwards so the hardware doesn't see the
1086          * whole mess until the first descriptor in the map is flagged.
1087          */
1088         for (i = ctx.nsegs - 1; i >= 0; --i) {
1089                 j = (ring->cur + i) % NFE_TX_RING_COUNT;
1090                 if (sc->sc_flags & NFE_40BIT_ADDR) {
1091                         desc64 = &ring->desc64[j];
1092                         desc64->flags |= htole16(NFE_TX_VALID);
1093                 } else {
1094                         desc32 = &ring->desc32[j];
1095                         desc32->flags |= htole16(NFE_TX_VALID);
1096                 }
1097         }
1098         ring->cur = (ring->cur + ctx.nsegs) % NFE_TX_RING_COUNT;
1099
1100         /* Exchange DMA map */
1101         data_map->map = data->map;
1102         data->map = map;
1103         data->m = m0;
1104
1105         bus_dmamap_sync(ring->data_tag, map, BUS_DMASYNC_PREWRITE);
1106 back:
1107         if (error)
1108                 m_freem(m0);
1109         return error;
1110 }
1111
1112 static void
1113 nfe_start(struct ifnet *ifp)
1114 {
1115         struct nfe_softc *sc = ifp->if_softc;
1116         struct nfe_tx_ring *ring = &sc->txq;
1117         int count = 0;
1118         struct mbuf *m0;
1119
1120         if (ifp->if_flags & IFF_OACTIVE)
1121                 return;
1122
1123         if (ifq_is_empty(&ifp->if_snd))
1124                 return;
1125
1126         for (;;) {
1127                 m0 = ifq_dequeue(&ifp->if_snd, NULL);
1128                 if (m0 == NULL)
1129                         break;
1130
1131                 BPF_MTAP(ifp, m0);
1132
1133                 if (nfe_encap(sc, ring, m0) != 0) {
1134                         ifp->if_flags |= IFF_OACTIVE;
1135                         break;
1136                 }
1137                 ++count;
1138
1139                 /*
1140                  * NOTE:
1141                  * `m0' may be freed in nfe_encap(), so
1142                  * it should not be touched any more.
1143                  */
1144         }
1145         if (count == 0) /* nothing sent */
1146                 return;
1147
1148         /* Sync TX descriptor ring */
1149         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_PREWRITE);
1150
1151         /* Kick Tx */
1152         NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_KICKTX | sc->rxtxctl);
1153
1154         /*
1155          * Set a timeout in case the chip goes out to lunch.
1156          */
1157         ifp->if_timer = 5;
1158 }
1159
1160 static void
1161 nfe_watchdog(struct ifnet *ifp)
1162 {
1163         struct nfe_softc *sc = ifp->if_softc;
1164
1165         if (ifp->if_flags & IFF_RUNNING) {
1166                 if_printf(ifp, "watchdog timeout - lost interrupt recovered\n");
1167                 nfe_txeof(sc);
1168                 return;
1169         }
1170
1171         if_printf(ifp, "watchdog timeout\n");
1172
1173         nfe_init(ifp->if_softc);
1174
1175         ifp->if_oerrors++;
1176
1177         if (!ifq_is_empty(&ifp->if_snd))
1178                 ifp->if_start(ifp);
1179 }
1180
1181 static void
1182 nfe_init(void *xsc)
1183 {
1184         struct nfe_softc *sc = xsc;
1185         struct ifnet *ifp = &sc->arpcom.ac_if;
1186         uint32_t tmp;
1187         int error;
1188
1189         nfe_stop(sc);
1190
1191         error = nfe_init_tx_ring(sc, &sc->txq);
1192         if (error) {
1193                 nfe_stop(sc);
1194                 return;
1195         }
1196
1197         error = nfe_init_rx_ring(sc, &sc->rxq);
1198         if (error) {
1199                 nfe_stop(sc);
1200                 return;
1201         }
1202
1203         NFE_WRITE(sc, NFE_TX_UNK, 0);
1204         NFE_WRITE(sc, NFE_STATUS, 0);
1205
1206         sc->rxtxctl = NFE_RXTX_BIT2;
1207         if (sc->sc_flags & NFE_40BIT_ADDR)
1208                 sc->rxtxctl |= NFE_RXTX_V3MAGIC;
1209         else if (sc->sc_flags & NFE_JUMBO_SUP)
1210                 sc->rxtxctl |= NFE_RXTX_V2MAGIC;
1211 #ifdef NFE_CSUM
1212         if (sc->sc_flags & NFE_HW_CSUM)
1213                 sc->rxtxctl |= NFE_RXTX_RXCSUM;
1214 #endif
1215
1216         /*
1217          * Although the adapter is capable of stripping VLAN tags from received
1218          * frames (NFE_RXTX_VTAG_STRIP), we do not enable this functionality on
1219          * purpose.  This will be done in software by our network stack.
1220          */
1221         if (sc->sc_flags & NFE_HW_VLAN)
1222                 sc->rxtxctl |= NFE_RXTX_VTAG_INSERT;
1223
1224         NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_RESET | sc->rxtxctl);
1225         DELAY(10);
1226         NFE_WRITE(sc, NFE_RXTX_CTL, sc->rxtxctl);
1227
1228         if (sc->sc_flags & NFE_HW_VLAN)
1229                 NFE_WRITE(sc, NFE_VTAG_CTL, NFE_VTAG_ENABLE);
1230
1231         NFE_WRITE(sc, NFE_SETUP_R6, 0);
1232
1233         /* set MAC address */
1234         nfe_set_macaddr(sc, sc->arpcom.ac_enaddr);
1235
1236         /* tell MAC where rings are in memory */
1237 #ifdef __LP64__
1238         NFE_WRITE(sc, NFE_RX_RING_ADDR_HI, sc->rxq.physaddr >> 32);
1239 #endif
1240         NFE_WRITE(sc, NFE_RX_RING_ADDR_LO, sc->rxq.physaddr & 0xffffffff);
1241 #ifdef __LP64__
1242         NFE_WRITE(sc, NFE_TX_RING_ADDR_HI, sc->txq.physaddr >> 32);
1243 #endif
1244         NFE_WRITE(sc, NFE_TX_RING_ADDR_LO, sc->txq.physaddr & 0xffffffff);
1245
1246         NFE_WRITE(sc, NFE_RING_SIZE,
1247             (NFE_RX_RING_COUNT - 1) << 16 |
1248             (NFE_TX_RING_COUNT - 1));
1249
1250         NFE_WRITE(sc, NFE_RXBUFSZ, sc->rxq.bufsz);
1251
1252         /* force MAC to wakeup */
1253         tmp = NFE_READ(sc, NFE_PWR_STATE);
1254         NFE_WRITE(sc, NFE_PWR_STATE, tmp | NFE_PWR_WAKEUP);
1255         DELAY(10);
1256         tmp = NFE_READ(sc, NFE_PWR_STATE);
1257         NFE_WRITE(sc, NFE_PWR_STATE, tmp | NFE_PWR_VALID);
1258
1259 #if 1
1260         /* configure interrupts coalescing/mitigation */
1261         NFE_WRITE(sc, NFE_IMTIMER, NFE_IM_DEFAULT);
1262 #else
1263         /* no interrupt mitigation: one interrupt per packet */
1264         NFE_WRITE(sc, NFE_IMTIMER, 970);
1265 #endif
1266
1267         NFE_WRITE(sc, NFE_SETUP_R1, NFE_R1_MAGIC);
1268         NFE_WRITE(sc, NFE_SETUP_R2, NFE_R2_MAGIC);
1269         NFE_WRITE(sc, NFE_SETUP_R6, NFE_R6_MAGIC);
1270
1271         /* update MAC knowledge of PHY; generates a NFE_IRQ_LINK interrupt */
1272         NFE_WRITE(sc, NFE_STATUS, sc->mii_phyaddr << 24 | NFE_STATUS_MAGIC);
1273
1274         NFE_WRITE(sc, NFE_SETUP_R4, NFE_R4_MAGIC);
1275         NFE_WRITE(sc, NFE_WOL_CTL, NFE_WOL_MAGIC);
1276
1277         sc->rxtxctl &= ~NFE_RXTX_BIT2;
1278         NFE_WRITE(sc, NFE_RXTX_CTL, sc->rxtxctl);
1279         DELAY(10);
1280         NFE_WRITE(sc, NFE_RXTX_CTL, NFE_RXTX_BIT1 | sc->rxtxctl);
1281
1282         /* set Rx filter */
1283         nfe_setmulti(sc);
1284
1285         nfe_ifmedia_upd(ifp);
1286
1287         /* enable Rx */
1288         NFE_WRITE(sc, NFE_RX_CTL, NFE_RX_START);
1289
1290         /* enable Tx */
1291         NFE_WRITE(sc, NFE_TX_CTL, NFE_TX_START);
1292
1293         NFE_WRITE(sc, NFE_PHY_STATUS, 0xf);
1294
1295 #ifdef DEVICE_POLLING
1296         if ((ifp->if_flags & IFF_POLLING) == 0)
1297 #endif
1298         /* enable interrupts */
1299         NFE_WRITE(sc, NFE_IRQ_MASK, NFE_IRQ_WANTED);
1300
1301         callout_reset(&sc->sc_tick_ch, hz, nfe_tick, sc);
1302
1303         ifp->if_flags |= IFF_RUNNING;
1304         ifp->if_flags &= ~IFF_OACTIVE;
1305 }
1306
1307 static void
1308 nfe_stop(struct nfe_softc *sc)
1309 {
1310         struct ifnet *ifp = &sc->arpcom.ac_if;
1311
1312         callout_stop(&sc->sc_tick_ch);
1313
1314         ifp->if_timer = 0;
1315         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1316
1317         /* Abort Tx */
1318         NFE_WRITE(sc, NFE_TX_CTL, 0);
1319
1320         /* Disable Rx */
1321         NFE_WRITE(sc, NFE_RX_CTL, 0);
1322
1323         /* Disable interrupts */
1324         NFE_WRITE(sc, NFE_IRQ_MASK, 0);
1325
1326         /* Reset Tx and Rx rings */
1327         nfe_reset_tx_ring(sc, &sc->txq);
1328         nfe_reset_rx_ring(sc, &sc->rxq);
1329 }
1330
1331 static int
1332 nfe_alloc_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1333 {
1334         int i, j, error, descsize;
1335         void **desc;
1336
1337         if (sc->sc_flags & NFE_40BIT_ADDR) {
1338                 desc = (void **)&ring->desc64;
1339                 descsize = sizeof(struct nfe_desc64);
1340         } else {
1341                 desc = (void **)&ring->desc32;
1342                 descsize = sizeof(struct nfe_desc32);
1343         }
1344
1345         ring->bufsz = MCLBYTES;
1346         ring->cur = ring->next = 0;
1347
1348         error = bus_dma_tag_create(NULL, PAGE_SIZE, 0,
1349                                    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1350                                    NULL, NULL,
1351                                    NFE_RX_RING_COUNT * descsize, 1,
1352                                    NFE_RX_RING_COUNT * descsize,
1353                                    0, &ring->tag);
1354         if (error) {
1355                 if_printf(&sc->arpcom.ac_if,
1356                           "could not create desc RX DMA tag\n");
1357                 return error;
1358         }
1359
1360         error = bus_dmamem_alloc(ring->tag, desc, BUS_DMA_WAITOK | BUS_DMA_ZERO,
1361                                  &ring->map);
1362         if (error) {
1363                 if_printf(&sc->arpcom.ac_if,
1364                           "could not allocate RX desc DMA memory\n");
1365                 bus_dma_tag_destroy(ring->tag);
1366                 ring->tag = NULL;
1367                 return error;
1368         }
1369
1370         error = bus_dmamap_load(ring->tag, ring->map, *desc,
1371                                 NFE_RX_RING_COUNT * descsize,
1372                                 nfe_ring_dma_addr, &ring->physaddr,
1373                                 BUS_DMA_WAITOK);
1374         if (error) {
1375                 if_printf(&sc->arpcom.ac_if,
1376                           "could not load RX desc DMA map\n");
1377                 bus_dmamem_free(ring->tag, *desc, ring->map);
1378                 bus_dma_tag_destroy(ring->tag);
1379                 ring->tag = NULL;
1380                 return error;
1381         }
1382
1383         if (sc->sc_flags & NFE_USE_JUMBO) {
1384                 ring->bufsz = NFE_JBYTES;
1385
1386                 error = nfe_jpool_alloc(sc, ring);
1387                 if (error) {
1388                         if_printf(&sc->arpcom.ac_if,
1389                                   "could not allocate jumbo frames\n");
1390                         return error;
1391                 }
1392         }
1393
1394         error = bus_dma_tag_create(NULL, 1, 0,
1395                                    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1396                                    NULL, NULL,
1397                                    MCLBYTES, 1, MCLBYTES,
1398                                    0, &ring->data_tag);
1399         if (error) {
1400                 if_printf(&sc->arpcom.ac_if,
1401                           "could not create RX mbuf DMA tag\n");
1402                 return error;
1403         }
1404
1405         /* Create a spare RX mbuf DMA map */
1406         error = bus_dmamap_create(ring->data_tag, 0, &ring->data_tmpmap);
1407         if (error) {
1408                 if_printf(&sc->arpcom.ac_if,
1409                           "could not create spare RX mbuf DMA map\n");
1410                 bus_dma_tag_destroy(ring->data_tag);
1411                 ring->data_tag = NULL;
1412                 return error;
1413         }
1414
1415         for (i = 0; i < NFE_RX_RING_COUNT; i++) {
1416                 error = bus_dmamap_create(ring->data_tag, 0,
1417                                           &ring->data[i].map);
1418                 if (error) {
1419                         if_printf(&sc->arpcom.ac_if,
1420                                   "could not create %dth RX mbuf DMA mapn", i);
1421                         goto fail;
1422                 }
1423         }
1424         return 0;
1425 fail:
1426         for (j = 0; j < i; ++j)
1427                 bus_dmamap_destroy(ring->data_tag, ring->data[i].map);
1428         bus_dmamap_destroy(ring->data_tag, ring->data_tmpmap);
1429         bus_dma_tag_destroy(ring->data_tag);
1430         ring->data_tag = NULL;
1431         return error;
1432 }
1433
1434 static void
1435 nfe_reset_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1436 {
1437         int i;
1438
1439         for (i = 0; i < NFE_RX_RING_COUNT; i++) {
1440                 struct nfe_rx_data *data = &ring->data[i];
1441
1442                 if (data->m != NULL) {
1443                         bus_dmamap_unload(ring->data_tag, data->map);
1444                         m_freem(data->m);
1445                         data->m = NULL;
1446                 }
1447         }
1448         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_PREWRITE);
1449
1450         ring->cur = ring->next = 0;
1451 }
1452
1453 static int
1454 nfe_init_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1455 {
1456         int i;
1457
1458         for (i = 0; i < NFE_RX_RING_COUNT; ++i) {
1459                 int error;
1460
1461                 /* XXX should use a function pointer */
1462                 if (sc->sc_flags & NFE_USE_JUMBO)
1463                         error = nfe_newbuf_jumbo(sc, ring, i, 1);
1464                 else
1465                         error = nfe_newbuf_std(sc, ring, i, 1);
1466                 if (error) {
1467                         if_printf(&sc->arpcom.ac_if,
1468                                   "could not allocate RX buffer\n");
1469                         return error;
1470                 }
1471
1472                 nfe_set_ready_rxdesc(sc, ring, i);
1473         }
1474         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_PREWRITE);
1475
1476         return 0;
1477 }
1478
1479 static void
1480 nfe_free_rx_ring(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1481 {
1482         if (ring->data_tag != NULL) {
1483                 struct nfe_rx_data *data;
1484                 int i;
1485
1486                 for (i = 0; i < NFE_RX_RING_COUNT; i++) {
1487                         data = &ring->data[i];
1488
1489                         if (data->m != NULL) {
1490                                 bus_dmamap_unload(ring->data_tag, data->map);
1491                                 m_freem(data->m);
1492                         }
1493                         bus_dmamap_destroy(ring->data_tag, data->map);
1494                 }
1495                 bus_dmamap_destroy(ring->data_tag, ring->data_tmpmap);
1496                 bus_dma_tag_destroy(ring->data_tag);
1497         }
1498
1499         nfe_jpool_free(sc, ring);
1500
1501         if (ring->tag != NULL) {
1502                 void *desc;
1503
1504                 if (sc->sc_flags & NFE_40BIT_ADDR)
1505                         desc = ring->desc64;
1506                 else
1507                         desc = ring->desc32;
1508
1509                 bus_dmamap_unload(ring->tag, ring->map);
1510                 bus_dmamem_free(ring->tag, desc, ring->map);
1511                 bus_dma_tag_destroy(ring->tag);
1512         }
1513 }
1514
1515 static struct nfe_jbuf *
1516 nfe_jalloc(struct nfe_softc *sc)
1517 {
1518         struct ifnet *ifp = &sc->arpcom.ac_if;
1519         struct nfe_jbuf *jbuf;
1520
1521         lwkt_serialize_enter(&sc->sc_jbuf_serializer);
1522
1523         jbuf = SLIST_FIRST(&sc->rxq.jfreelist);
1524         if (jbuf != NULL) {
1525                 SLIST_REMOVE_HEAD(&sc->rxq.jfreelist, jnext);
1526                 jbuf->inuse = 1;
1527         } else {
1528                 if_printf(ifp, "no free jumbo buffer\n");
1529         }
1530
1531         lwkt_serialize_exit(&sc->sc_jbuf_serializer);
1532
1533         return jbuf;
1534 }
1535
1536 static void
1537 nfe_jfree(void *arg)
1538 {
1539         struct nfe_jbuf *jbuf = arg;
1540         struct nfe_softc *sc = jbuf->sc;
1541         struct nfe_rx_ring *ring = jbuf->ring;
1542
1543         if (&ring->jbuf[jbuf->slot] != jbuf)
1544                 panic("%s: free wrong jumbo buffer\n", __func__);
1545         else if (jbuf->inuse == 0)
1546                 panic("%s: jumbo buffer already freed\n", __func__);
1547
1548         lwkt_serialize_enter(&sc->sc_jbuf_serializer);
1549         atomic_subtract_int(&jbuf->inuse, 1);
1550         if (jbuf->inuse == 0)
1551                 SLIST_INSERT_HEAD(&ring->jfreelist, jbuf, jnext);
1552         lwkt_serialize_exit(&sc->sc_jbuf_serializer);
1553 }
1554
1555 static void
1556 nfe_jref(void *arg)
1557 {
1558         struct nfe_jbuf *jbuf = arg;
1559         struct nfe_rx_ring *ring = jbuf->ring;
1560
1561         if (&ring->jbuf[jbuf->slot] != jbuf)
1562                 panic("%s: ref wrong jumbo buffer\n", __func__);
1563         else if (jbuf->inuse == 0)
1564                 panic("%s: jumbo buffer already freed\n", __func__);
1565
1566         atomic_add_int(&jbuf->inuse, 1);
1567 }
1568
1569 static int
1570 nfe_jpool_alloc(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1571 {
1572         struct nfe_jbuf *jbuf;
1573         bus_addr_t physaddr;
1574         caddr_t buf;
1575         int i, error;
1576
1577         /*
1578          * Allocate a big chunk of DMA'able memory.
1579          */
1580         error = bus_dma_tag_create(NULL, PAGE_SIZE, 0,
1581                                    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1582                                    NULL, NULL,
1583                                    NFE_JPOOL_SIZE, 1, NFE_JPOOL_SIZE,
1584                                    0, &ring->jtag);
1585         if (error) {
1586                 if_printf(&sc->arpcom.ac_if,
1587                           "could not create jumbo DMA tag\n");
1588                 return error;
1589         }
1590
1591         error = bus_dmamem_alloc(ring->jtag, (void **)&ring->jpool,
1592                                  BUS_DMA_WAITOK, &ring->jmap);
1593         if (error) {
1594                 if_printf(&sc->arpcom.ac_if,
1595                           "could not allocate jumbo DMA memory\n");
1596                 bus_dma_tag_destroy(ring->jtag);
1597                 ring->jtag = NULL;
1598                 return error;
1599         }
1600
1601         error = bus_dmamap_load(ring->jtag, ring->jmap, ring->jpool,
1602                                 NFE_JPOOL_SIZE, nfe_ring_dma_addr, &physaddr,
1603                                 BUS_DMA_WAITOK);
1604         if (error) {
1605                 if_printf(&sc->arpcom.ac_if,
1606                           "could not load jumbo DMA map\n");
1607                 bus_dmamem_free(ring->jtag, ring->jpool, ring->jmap);
1608                 bus_dma_tag_destroy(ring->jtag);
1609                 ring->jtag = NULL;
1610                 return error;
1611         }
1612
1613         /* ..and split it into 9KB chunks */
1614         SLIST_INIT(&ring->jfreelist);
1615
1616         buf = ring->jpool;
1617         for (i = 0; i < NFE_JPOOL_COUNT; i++) {
1618                 jbuf = &ring->jbuf[i];
1619
1620                 jbuf->sc = sc;
1621                 jbuf->ring = ring;
1622                 jbuf->inuse = 0;
1623                 jbuf->slot = i;
1624                 jbuf->buf = buf;
1625                 jbuf->physaddr = physaddr;
1626
1627                 SLIST_INSERT_HEAD(&ring->jfreelist, jbuf, jnext);
1628
1629                 buf += NFE_JBYTES;
1630                 physaddr += NFE_JBYTES;
1631         }
1632
1633         return 0;
1634 }
1635
1636 static void
1637 nfe_jpool_free(struct nfe_softc *sc, struct nfe_rx_ring *ring)
1638 {
1639         if (ring->jtag != NULL) {
1640                 bus_dmamap_unload(ring->jtag, ring->jmap);
1641                 bus_dmamem_free(ring->jtag, ring->jpool, ring->jmap);
1642                 bus_dma_tag_destroy(ring->jtag);
1643         }
1644 }
1645
1646 static int
1647 nfe_alloc_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring)
1648 {
1649         int i, j, error, descsize;
1650         void **desc;
1651
1652         if (sc->sc_flags & NFE_40BIT_ADDR) {
1653                 desc = (void **)&ring->desc64;
1654                 descsize = sizeof(struct nfe_desc64);
1655         } else {
1656                 desc = (void **)&ring->desc32;
1657                 descsize = sizeof(struct nfe_desc32);
1658         }
1659
1660         ring->queued = 0;
1661         ring->cur = ring->next = 0;
1662
1663         error = bus_dma_tag_create(NULL, PAGE_SIZE, 0,
1664                                    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1665                                    NULL, NULL,
1666                                    NFE_TX_RING_COUNT * descsize, 1,
1667                                    NFE_TX_RING_COUNT * descsize,
1668                                    0, &ring->tag);
1669         if (error) {
1670                 if_printf(&sc->arpcom.ac_if,
1671                           "could not create TX desc DMA map\n");
1672                 return error;
1673         }
1674
1675         error = bus_dmamem_alloc(ring->tag, desc, BUS_DMA_WAITOK | BUS_DMA_ZERO,
1676                                  &ring->map);
1677         if (error) {
1678                 if_printf(&sc->arpcom.ac_if,
1679                           "could not allocate TX desc DMA memory\n");
1680                 bus_dma_tag_destroy(ring->tag);
1681                 ring->tag = NULL;
1682                 return error;
1683         }
1684
1685         error = bus_dmamap_load(ring->tag, ring->map, *desc,
1686                                 NFE_TX_RING_COUNT * descsize,
1687                                 nfe_ring_dma_addr, &ring->physaddr,
1688                                 BUS_DMA_WAITOK);
1689         if (error) {
1690                 if_printf(&sc->arpcom.ac_if,
1691                           "could not load TX desc DMA map\n");
1692                 bus_dmamem_free(ring->tag, *desc, ring->map);
1693                 bus_dma_tag_destroy(ring->tag);
1694                 ring->tag = NULL;
1695                 return error;
1696         }
1697
1698         error = bus_dma_tag_create(NULL, PAGE_SIZE, 0,
1699                                    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR,
1700                                    NULL, NULL,
1701                                    NFE_JBYTES * NFE_MAX_SCATTER,
1702                                    NFE_MAX_SCATTER, NFE_JBYTES,
1703                                    0, &ring->data_tag);
1704         if (error) {
1705                 if_printf(&sc->arpcom.ac_if,
1706                           "could not create TX buf DMA tag\n");
1707                 return error;
1708         }
1709
1710         for (i = 0; i < NFE_TX_RING_COUNT; i++) {
1711                 error = bus_dmamap_create(ring->data_tag, 0,
1712                                           &ring->data[i].map);
1713                 if (error) {
1714                         if_printf(&sc->arpcom.ac_if,
1715                                   "could not create %dth TX buf DMA map\n", i);
1716                         goto fail;
1717                 }
1718         }
1719
1720         return 0;
1721 fail:
1722         for (j = 0; j < i; ++j)
1723                 bus_dmamap_destroy(ring->data_tag, ring->data[i].map);
1724         bus_dma_tag_destroy(ring->data_tag);
1725         ring->data_tag = NULL;
1726         return error;
1727 }
1728
1729 static void
1730 nfe_reset_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring)
1731 {
1732         int i;
1733
1734         for (i = 0; i < NFE_TX_RING_COUNT; i++) {
1735                 struct nfe_tx_data *data = &ring->data[i];
1736
1737                 if (sc->sc_flags & NFE_40BIT_ADDR)
1738                         ring->desc64[i].flags = 0;
1739                 else
1740                         ring->desc32[i].flags = 0;
1741
1742                 if (data->m != NULL) {
1743                         bus_dmamap_sync(ring->data_tag, data->map,
1744                                         BUS_DMASYNC_POSTWRITE);
1745                         bus_dmamap_unload(ring->data_tag, data->map);
1746                         m_freem(data->m);
1747                         data->m = NULL;
1748                 }
1749         }
1750         bus_dmamap_sync(ring->tag, ring->map, BUS_DMASYNC_PREWRITE);
1751
1752         ring->queued = 0;
1753         ring->cur = ring->next = 0;
1754 }
1755
1756 static int
1757 nfe_init_tx_ring(struct nfe_softc *sc __unused,
1758                  struct nfe_tx_ring *ring __unused)
1759 {
1760         return 0;
1761 }
1762
1763 static void
1764 nfe_free_tx_ring(struct nfe_softc *sc, struct nfe_tx_ring *ring)
1765 {
1766         if (ring->data_tag != NULL) {
1767                 struct nfe_tx_data *data;
1768                 int i;
1769
1770                 for (i = 0; i < NFE_TX_RING_COUNT; ++i) {
1771                         data = &ring->data[i];
1772
1773                         if (data->m != NULL) {
1774                                 bus_dmamap_unload(ring->data_tag, data->map);
1775                                 m_freem(data->m);
1776                         }
1777                         bus_dmamap_destroy(ring->data_tag, data->map);
1778                 }
1779
1780                 bus_dma_tag_destroy(ring->data_tag);
1781         }
1782
1783         if (ring->tag != NULL) {
1784                 void *desc;
1785
1786                 if (sc->sc_flags & NFE_40BIT_ADDR)
1787                         desc = ring->desc64;
1788                 else
1789                         desc = ring->desc32;
1790
1791                 bus_dmamap_unload(ring->tag, ring->map);
1792                 bus_dmamem_free(ring->tag, desc, ring->map);
1793                 bus_dma_tag_destroy(ring->tag);
1794         }
1795 }
1796
1797 static int
1798 nfe_ifmedia_upd(struct ifnet *ifp)
1799 {
1800         struct nfe_softc *sc = ifp->if_softc;
1801         struct mii_data *mii = device_get_softc(sc->sc_miibus);
1802
1803         if (mii->mii_instance != 0) {
1804                 struct mii_softc *miisc;
1805
1806                 LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
1807                         mii_phy_reset(miisc);
1808         }
1809         mii_mediachg(mii);
1810
1811         return 0;
1812 }
1813
1814 static void
1815 nfe_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
1816 {
1817         struct nfe_softc *sc = ifp->if_softc;
1818         struct mii_data *mii = device_get_softc(sc->sc_miibus);
1819
1820         mii_pollstat(mii);
1821         ifmr->ifm_status = mii->mii_media_status;
1822         ifmr->ifm_active = mii->mii_media_active;
1823 }
1824
1825 static void
1826 nfe_setmulti(struct nfe_softc *sc)
1827 {
1828         struct ifnet *ifp = &sc->arpcom.ac_if;
1829         struct ifmultiaddr *ifma;
1830         uint8_t addr[ETHER_ADDR_LEN], mask[ETHER_ADDR_LEN];
1831         uint32_t filter = NFE_RXFILTER_MAGIC;
1832         int i;
1833
1834         if ((ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)) != 0) {
1835                 bzero(addr, ETHER_ADDR_LEN);
1836                 bzero(mask, ETHER_ADDR_LEN);
1837                 goto done;
1838         }
1839
1840         bcopy(etherbroadcastaddr, addr, ETHER_ADDR_LEN);
1841         bcopy(etherbroadcastaddr, mask, ETHER_ADDR_LEN);
1842
1843         LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1844                 caddr_t maddr;
1845
1846                 if (ifma->ifma_addr->sa_family != AF_LINK)
1847                         continue;
1848
1849                 maddr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1850                 for (i = 0; i < ETHER_ADDR_LEN; i++) {
1851                         addr[i] &= maddr[i];
1852                         mask[i] &= ~maddr[i];
1853                 }
1854         }
1855
1856         for (i = 0; i < ETHER_ADDR_LEN; i++)
1857                 mask[i] |= addr[i];
1858
1859 done:
1860         addr[0] |= 0x01;        /* make sure multicast bit is set */
1861
1862         NFE_WRITE(sc, NFE_MULTIADDR_HI,
1863             addr[3] << 24 | addr[2] << 16 | addr[1] << 8 | addr[0]);
1864         NFE_WRITE(sc, NFE_MULTIADDR_LO,
1865             addr[5] <<  8 | addr[4]);
1866         NFE_WRITE(sc, NFE_MULTIMASK_HI,
1867             mask[3] << 24 | mask[2] << 16 | mask[1] << 8 | mask[0]);
1868         NFE_WRITE(sc, NFE_MULTIMASK_LO,
1869             mask[5] <<  8 | mask[4]);
1870
1871         filter |= (ifp->if_flags & IFF_PROMISC) ? NFE_PROMISC : NFE_U2M;
1872         NFE_WRITE(sc, NFE_RXFILTER, filter);
1873 }
1874
1875 static void
1876 nfe_get_macaddr(struct nfe_softc *sc, uint8_t *addr)
1877 {
1878         uint32_t tmp;
1879
1880         tmp = NFE_READ(sc, NFE_MACADDR_LO);
1881         addr[0] = (tmp >> 8) & 0xff;
1882         addr[1] = (tmp & 0xff);
1883
1884         tmp = NFE_READ(sc, NFE_MACADDR_HI);
1885         addr[2] = (tmp >> 24) & 0xff;
1886         addr[3] = (tmp >> 16) & 0xff;
1887         addr[4] = (tmp >>  8) & 0xff;
1888         addr[5] = (tmp & 0xff);
1889 }
1890
1891 static void
1892 nfe_set_macaddr(struct nfe_softc *sc, const uint8_t *addr)
1893 {
1894         NFE_WRITE(sc, NFE_MACADDR_LO,
1895             addr[5] <<  8 | addr[4]);
1896         NFE_WRITE(sc, NFE_MACADDR_HI,
1897             addr[3] << 24 | addr[2] << 16 | addr[1] << 8 | addr[0]);
1898 }
1899
1900 static void
1901 nfe_tick(void *arg)
1902 {
1903         struct nfe_softc *sc = arg;
1904         struct ifnet *ifp = &sc->arpcom.ac_if;
1905         struct mii_data *mii = device_get_softc(sc->sc_miibus);
1906
1907         lwkt_serialize_enter(ifp->if_serializer);
1908
1909         mii_tick(mii);
1910         callout_reset(&sc->sc_tick_ch, hz, nfe_tick, sc);
1911
1912         lwkt_serialize_exit(ifp->if_serializer);
1913 }
1914
1915 static void
1916 nfe_ring_dma_addr(void *arg, bus_dma_segment_t *seg, int nseg, int error)
1917 {
1918         if (error)
1919                 return;
1920
1921         KASSERT(nseg == 1, ("too many segments, should be 1\n"));
1922
1923         *((uint32_t *)arg) = seg->ds_addr;
1924 }
1925
1926 static void
1927 nfe_buf_dma_addr(void *arg, bus_dma_segment_t *segs, int nsegs,
1928                  bus_size_t mapsz __unused, int error)
1929 {
1930         struct nfe_dma_ctx *ctx = arg;
1931         int i;
1932
1933         if (error)
1934                 return;
1935
1936         KASSERT(nsegs <= ctx->nsegs,
1937                 ("too many segments(%d), should be <= %d\n",
1938                  nsegs, ctx->nsegs));
1939
1940         ctx->nsegs = nsegs;
1941         for (i = 0; i < nsegs; ++i)
1942                 ctx->segs[i] = segs[i];
1943 }
1944
1945 static int
1946 nfe_newbuf_std(struct nfe_softc *sc, struct nfe_rx_ring *ring, int idx,
1947                int wait)
1948 {
1949         struct nfe_rx_data *data = &ring->data[idx];
1950         struct nfe_dma_ctx ctx;
1951         bus_dma_segment_t seg;
1952         bus_dmamap_t map;
1953         struct mbuf *m;
1954         int error;
1955
1956         m = m_getcl(wait ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
1957         if (m == NULL)
1958                 return ENOBUFS;
1959         m->m_len = m->m_pkthdr.len = MCLBYTES;
1960
1961         ctx.nsegs = 1;
1962         ctx.segs = &seg;
1963         error = bus_dmamap_load_mbuf(ring->data_tag, ring->data_tmpmap,
1964                                      m, nfe_buf_dma_addr, &ctx,
1965                                      wait ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT);
1966         if (error) {
1967                 m_freem(m);
1968                 if_printf(&sc->arpcom.ac_if, "could map RX mbuf %d\n", error);
1969                 return error;
1970         }
1971
1972         /* Unload originally mapped mbuf */
1973         bus_dmamap_unload(ring->data_tag, data->map);
1974
1975         /* Swap this DMA map with tmp DMA map */
1976         map = data->map;
1977         data->map = ring->data_tmpmap;
1978         ring->data_tmpmap = map;
1979
1980         /* Caller is assumed to have collected the old mbuf */
1981         data->m = m;
1982
1983         nfe_set_paddr_rxdesc(sc, ring, idx, seg.ds_addr);
1984
1985         bus_dmamap_sync(ring->data_tag, data->map, BUS_DMASYNC_PREREAD);
1986         return 0;
1987 }
1988
1989 static int
1990 nfe_newbuf_jumbo(struct nfe_softc *sc, struct nfe_rx_ring *ring, int idx,
1991                  int wait)
1992 {
1993         struct nfe_rx_data *data = &ring->data[idx];
1994         struct nfe_jbuf *jbuf;
1995         struct mbuf *m;
1996
1997         MGETHDR(m, wait ? MB_WAIT : MB_DONTWAIT, MT_DATA);
1998         if (m == NULL)
1999                 return ENOBUFS;
2000
2001         jbuf = nfe_jalloc(sc);
2002         if (jbuf == NULL) {
2003                 m_freem(m);
2004                 if_printf(&sc->arpcom.ac_if, "jumbo allocation failed "
2005                     "-- packet dropped!\n");
2006                 return ENOBUFS;
2007         }
2008
2009         m->m_ext.ext_arg = jbuf;
2010         m->m_ext.ext_buf = jbuf->buf;
2011         m->m_ext.ext_free = nfe_jfree;
2012         m->m_ext.ext_ref = nfe_jref;
2013         m->m_ext.ext_size = NFE_JBYTES;
2014
2015         m->m_data = m->m_ext.ext_buf;
2016         m->m_flags |= M_EXT;
2017         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
2018
2019         /* Caller is assumed to have collected the old mbuf */
2020         data->m = m;
2021
2022         nfe_set_paddr_rxdesc(sc, ring, idx, jbuf->physaddr);
2023
2024         bus_dmamap_sync(ring->jtag, ring->jmap, BUS_DMASYNC_PREREAD);
2025         return 0;
2026 }
2027
2028 static void
2029 nfe_set_paddr_rxdesc(struct nfe_softc *sc, struct nfe_rx_ring *ring, int idx,
2030                      bus_addr_t physaddr)
2031 {
2032         if (sc->sc_flags & NFE_40BIT_ADDR) {
2033                 struct nfe_desc64 *desc64 = &ring->desc64[idx];
2034
2035 #if defined(__LP64__)
2036                 desc64->physaddr[0] = htole32(physaddr >> 32);
2037 #endif
2038                 desc64->physaddr[1] = htole32(physaddr & 0xffffffff);
2039         } else {
2040                 struct nfe_desc32 *desc32 = &ring->desc32[idx];
2041
2042                 desc32->physaddr = htole32(physaddr);
2043         }
2044 }
2045
2046 static void
2047 nfe_set_ready_rxdesc(struct nfe_softc *sc, struct nfe_rx_ring *ring, int idx)
2048 {
2049         if (sc->sc_flags & NFE_40BIT_ADDR) {
2050                 struct nfe_desc64 *desc64 = &ring->desc64[idx];
2051
2052                 desc64->length = htole16(ring->bufsz);
2053                 desc64->flags = htole16(NFE_RX_READY);
2054         } else {
2055                 struct nfe_desc32 *desc32 = &ring->desc32[idx];
2056
2057                 desc32->length = htole16(ring->bufsz);
2058                 desc32->flags = htole16(NFE_RX_READY);
2059         }
2060 }