sys/dev/netif/jme/if_jme.c

   1 /*-
   2  * Copyright (c) 2008, Pyun YongHyeon <yongari@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * $FreeBSD: src/sys/dev/jme/if_jme.c,v 1.2 2008/07/18 04:20:48 yongari Exp $
  28  */
  29
  30 #include "opt_polling.h"
  31 #include "opt_jme.h"
  32
  33 #include <sys/param.h>
  34 #include <sys/endian.h>
  35 #include <sys/kernel.h>
  36 #include <sys/bus.h>
  37 #include <sys/interrupt.h>
  38 #include <sys/malloc.h>
  39 #include <sys/proc.h>
  40 #include <sys/rman.h>
  41 #include <sys/serialize.h>
  42 #include <sys/serialize2.h>
  43 #include <sys/socket.h>
  44 #include <sys/sockio.h>
  45 #include <sys/sysctl.h>
  46
  47 #include <net/ethernet.h>
  48 #include <net/if.h>
  49 #include <net/bpf.h>
  50 #include <net/if_arp.h>
  51 #include <net/if_dl.h>
  52 #include <net/if_media.h>
  53 #include <net/ifq_var.h>
  54 #include <net/toeplitz.h>
  55 #include <net/toeplitz2.h>
  56 #include <net/vlan/if_vlan_var.h>
  57 #include <net/vlan/if_vlan_ether.h>
  58
  59 #include <netinet/in.h>
  60
  61 #include <dev/netif/mii_layer/miivar.h>
  62 #include <dev/netif/mii_layer/jmphyreg.h>
  63
  64 #include <bus/pci/pcireg.h>
  65 #include <bus/pci/pcivar.h>
  66 #include <bus/pci/pcidevs.h>
  67
  68 #include <dev/netif/jme/if_jmereg.h>
  69 #include <dev/netif/jme/if_jmevar.h>
  70
  71 #include "miibus_if.h"
  72
  73 #define JME_TX_SERIALIZE        1
  74 #define JME_RX_SERIALIZE        2
  75
  76 #define JME_CSUM_FEATURES       (CSUM_IP | CSUM_TCP | CSUM_UDP)
  77
  78 #ifdef JME_RSS_DEBUG
  79 #define JME_RSS_DPRINTF(sc, lvl, fmt, ...) \
  80 do { \
  81         if ((sc)->jme_rss_debug >= (lvl)) \
  82                 if_printf(&(sc)->arpcom.ac_if, fmt, __VA_ARGS__); \
  83 } while (0)
  84 #else   /* !JME_RSS_DEBUG */
  85 #define JME_RSS_DPRINTF(sc, lvl, fmt, ...)      ((void)0)
  86 #endif  /* JME_RSS_DEBUG */
  87
  88 static int      jme_probe(device_t);
  89 static int      jme_attach(device_t);
  90 static int      jme_detach(device_t);
  91 static int      jme_shutdown(device_t);
  92 static int      jme_suspend(device_t);
  93 static int      jme_resume(device_t);
  94
  95 static int      jme_miibus_readreg(device_t, int, int);
  96 static int      jme_miibus_writereg(device_t, int, int, int);
  97 static void     jme_miibus_statchg(device_t);
  98
  99 static void     jme_init(void *);
 100 static int      jme_ioctl(struct ifnet *, u_long, caddr_t, struct ucred *);
 101 static void     jme_start(struct ifnet *);
 102 static void     jme_watchdog(struct ifnet *);
 103 static void     jme_mediastatus(struct ifnet *, struct ifmediareq *);
 104 static int      jme_mediachange(struct ifnet *);
 105 #ifdef DEVICE_POLLING
 106 static void     jme_poll(struct ifnet *, enum poll_cmd, int);
 107 #endif
 108 static void     jme_serialize(struct ifnet *, enum ifnet_serialize);
 109 static void     jme_deserialize(struct ifnet *, enum ifnet_serialize);
 110 static int      jme_tryserialize(struct ifnet *, enum ifnet_serialize);
 111 #ifdef INVARIANTS
 112 static void     jme_serialize_assert(struct ifnet *, enum ifnet_serialize,
 113                     boolean_t);
 114 #endif
 115
 116 static void     jme_intr(void *);
 117 static void     jme_msix_tx(void *);
 118 static void     jme_msix_rx(void *);
 119 static void     jme_txeof(struct jme_softc *);
 120 static void     jme_rxeof(struct jme_rxdata *, int);
 121 static void     jme_rx_intr(struct jme_softc *, uint32_t);
 122
 123 static int      jme_msix_setup(device_t);
 124 static void     jme_msix_teardown(device_t, int);
 125 static int      jme_intr_setup(device_t);
 126 static void     jme_intr_teardown(device_t);
 127 static void     jme_msix_try_alloc(device_t);
 128 static void     jme_msix_free(device_t);
 129 static int      jme_intr_alloc(device_t);
 130 static void     jme_intr_free(device_t);
 131 static int      jme_dma_alloc(struct jme_softc *);
 132 static void     jme_dma_free(struct jme_softc *);
 133 static int      jme_init_rx_ring(struct jme_rxdata *);
 134 static void     jme_init_tx_ring(struct jme_softc *);
 135 static void     jme_init_ssb(struct jme_softc *);
 136 static int      jme_newbuf(struct jme_rxdata *, struct jme_rxdesc *, int);
 137 static int      jme_encap(struct jme_softc *, struct mbuf **);
 138 static void     jme_rxpkt(struct jme_rxdata *);
 139 static int      jme_rxring_dma_alloc(struct jme_rxdata *);
 140 static int      jme_rxbuf_dma_alloc(struct jme_rxdata *);
 141 static int      jme_rxbuf_dma_filter(void *, bus_addr_t);
 142
 143 static void     jme_tick(void *);
 144 static void     jme_stop(struct jme_softc *);
 145 static void     jme_reset(struct jme_softc *);
 146 static void     jme_set_msinum(struct jme_softc *);
 147 static void     jme_set_vlan(struct jme_softc *);
 148 static void     jme_set_filter(struct jme_softc *);
 149 static void     jme_stop_tx(struct jme_softc *);
 150 static void     jme_stop_rx(struct jme_softc *);
 151 static void     jme_mac_config(struct jme_softc *);
 152 static void     jme_reg_macaddr(struct jme_softc *, uint8_t[]);
 153 static int      jme_eeprom_macaddr(struct jme_softc *, uint8_t[]);
 154 static int      jme_eeprom_read_byte(struct jme_softc *, uint8_t, uint8_t *);
 155 #ifdef notyet
 156 static void     jme_setwol(struct jme_softc *);
 157 static void     jme_setlinkspeed(struct jme_softc *);
 158 #endif
 159 static void     jme_set_tx_coal(struct jme_softc *);
 160 static void     jme_set_rx_coal(struct jme_softc *);
 161 static void     jme_enable_rss(struct jme_softc *);
 162 static void     jme_disable_rss(struct jme_softc *);
 163 static void     jme_serialize_skipmain(struct jme_softc *);
 164 static void     jme_deserialize_skipmain(struct jme_softc *);
 165
 166 static void     jme_sysctl_node(struct jme_softc *);
 167 static int      jme_sysctl_tx_coal_to(SYSCTL_HANDLER_ARGS);
 168 static int      jme_sysctl_tx_coal_pkt(SYSCTL_HANDLER_ARGS);
 169 static int      jme_sysctl_rx_coal_to(SYSCTL_HANDLER_ARGS);
 170 static int      jme_sysctl_rx_coal_pkt(SYSCTL_HANDLER_ARGS);
 171
 172 /*
 173  * Devices supported by this driver.
 174  */
 175 static const struct jme_dev {
 176         uint16_t        jme_vendorid;
 177         uint16_t        jme_deviceid;
 178         uint32_t        jme_caps;
 179         const char      *jme_name;
 180 } jme_devs[] = {
 181         { PCI_VENDOR_JMICRON, PCI_PRODUCT_JMICRON_JMC250,
 182             JME_CAP_JUMBO,
 183             "JMicron Inc, JMC250 Gigabit Ethernet" },
 184         { PCI_VENDOR_JMICRON, PCI_PRODUCT_JMICRON_JMC260,
 185             JME_CAP_FASTETH,
 186             "JMicron Inc, JMC260 Fast Ethernet" },
 187         { 0, 0, 0, NULL }
 188 };
 189
 190 static device_method_t jme_methods[] = {
 191         /* Device interface. */
 192         DEVMETHOD(device_probe,         jme_probe),
 193         DEVMETHOD(device_attach,        jme_attach),
 194         DEVMETHOD(device_detach,        jme_detach),
 195         DEVMETHOD(device_shutdown,      jme_shutdown),
 196         DEVMETHOD(device_suspend,       jme_suspend),
 197         DEVMETHOD(device_resume,        jme_resume),
 198
 199         /* Bus interface. */
 200         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 201         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 202
 203         /* MII interface. */
 204         DEVMETHOD(miibus_readreg,       jme_miibus_readreg),
 205         DEVMETHOD(miibus_writereg,      jme_miibus_writereg),
 206         DEVMETHOD(miibus_statchg,       jme_miibus_statchg),
 207
 208         { NULL, NULL }
 209 };
 210
 211 static driver_t jme_driver = {
 212         "jme",
 213         jme_methods,
 214         sizeof(struct jme_softc)
 215 };
 216
 217 static devclass_t jme_devclass;
 218
 219 DECLARE_DUMMY_MODULE(if_jme);
 220 MODULE_DEPEND(if_jme, miibus, 1, 1, 1);
 221 DRIVER_MODULE(if_jme, pci, jme_driver, jme_devclass, NULL, NULL);
 222 DRIVER_MODULE(miibus, jme, miibus_driver, miibus_devclass, NULL, NULL);
 223
 224 static const struct {
 225         uint32_t        jme_coal;
 226         uint32_t        jme_comp;
 227         uint32_t        jme_empty;
 228 } jme_rx_status[JME_NRXRING_MAX] = {
 229         { INTR_RXQ0_COAL | INTR_RXQ0_COAL_TO, INTR_RXQ0_COMP,
 230           INTR_RXQ0_DESC_EMPTY },
 231         { INTR_RXQ1_COAL | INTR_RXQ1_COAL_TO, INTR_RXQ1_COMP,
 232           INTR_RXQ1_DESC_EMPTY },
 233         { INTR_RXQ2_COAL | INTR_RXQ2_COAL_TO, INTR_RXQ2_COMP,
 234           INTR_RXQ2_DESC_EMPTY },
 235         { INTR_RXQ3_COAL | INTR_RXQ3_COAL_TO, INTR_RXQ3_COMP,
 236           INTR_RXQ3_DESC_EMPTY }
 237 };
 238
 239 static int      jme_rx_desc_count = JME_RX_DESC_CNT_DEF;
 240 static int      jme_tx_desc_count = JME_TX_DESC_CNT_DEF;
 241 static int      jme_rx_ring_count = 0;
 242 static int      jme_msi_enable = 1;
 243 static int      jme_msix_enable = 1;
 244
 245 TUNABLE_INT("hw.jme.rx_desc_count", &jme_rx_desc_count);
 246 TUNABLE_INT("hw.jme.tx_desc_count", &jme_tx_desc_count);
 247 TUNABLE_INT("hw.jme.rx_ring_count", &jme_rx_ring_count);
 248 TUNABLE_INT("hw.jme.msi.enable", &jme_msi_enable);
 249 TUNABLE_INT("hw.jme.msix.enable", &jme_msix_enable);
 250
 251 static __inline void
 252 jme_setup_rxdesc(struct jme_rxdesc *rxd)
 253 {
 254         struct jme_desc *desc;
 255
 256         desc = rxd->rx_desc;
 257         desc->buflen = htole32(MCLBYTES);
 258         desc->addr_lo = htole32(JME_ADDR_LO(rxd->rx_paddr));
 259         desc->addr_hi = htole32(JME_ADDR_HI(rxd->rx_paddr));
 260         desc->flags = htole32(JME_RD_OWN | JME_RD_INTR | JME_RD_64BIT);
 261 }
 262
 263 /*
 264  *      Read a PHY register on the MII of the JMC250.
 265  */
 266 static int
 267 jme_miibus_readreg(device_t dev, int phy, int reg)
 268 {
 269         struct jme_softc *sc = device_get_softc(dev);
 270         uint32_t val;
 271         int i;
 272
 273         /* For FPGA version, PHY address 0 should be ignored. */
 274         if (sc->jme_caps & JME_CAP_FPGA) {
 275                 if (phy == 0)
 276                         return (0);
 277         } else {
 278                 if (sc->jme_phyaddr != phy)
 279                         return (0);
 280         }
 281
 282         CSR_WRITE_4(sc, JME_SMI, SMI_OP_READ | SMI_OP_EXECUTE |
 283             SMI_PHY_ADDR(phy) | SMI_REG_ADDR(reg));
 284
 285         for (i = JME_PHY_TIMEOUT; i > 0; i--) {
 286                 DELAY(1);
 287                 if (((val = CSR_READ_4(sc, JME_SMI)) & SMI_OP_EXECUTE) == 0)
 288                         break;
 289         }
 290         if (i == 0) {
 291                 device_printf(sc->jme_dev, "phy read timeout: "
 292                               "phy %d, reg %d\n", phy, reg);
 293                 return (0);
 294         }
 295
 296         return ((val & SMI_DATA_MASK) >> SMI_DATA_SHIFT);
 297 }
 298
 299 /*
 300  *      Write a PHY register on the MII of the JMC250.
 301  */
 302 static int
 303 jme_miibus_writereg(device_t dev, int phy, int reg, int val)
 304 {
 305         struct jme_softc *sc = device_get_softc(dev);
 306         int i;
 307
 308         /* For FPGA version, PHY address 0 should be ignored. */
 309         if (sc->jme_caps & JME_CAP_FPGA) {
 310                 if (phy == 0)
 311                         return (0);
 312         } else {
 313                 if (sc->jme_phyaddr != phy)
 314                         return (0);
 315         }
 316
 317         CSR_WRITE_4(sc, JME_SMI, SMI_OP_WRITE | SMI_OP_EXECUTE |
 318             ((val << SMI_DATA_SHIFT) & SMI_DATA_MASK) |
 319             SMI_PHY_ADDR(phy) | SMI_REG_ADDR(reg));
 320
 321         for (i = JME_PHY_TIMEOUT; i > 0; i--) {
 322                 DELAY(1);
 323                 if (((val = CSR_READ_4(sc, JME_SMI)) & SMI_OP_EXECUTE) == 0)
 324                         break;
 325         }
 326         if (i == 0) {
 327                 device_printf(sc->jme_dev, "phy write timeout: "
 328                               "phy %d, reg %d\n", phy, reg);
 329         }
 330
 331         return (0);
 332 }
 333
 334 /*
 335  *      Callback from MII layer when media changes.
 336  */
 337 static void
 338 jme_miibus_statchg(device_t dev)
 339 {
 340         struct jme_softc *sc = device_get_softc(dev);
 341         struct ifnet *ifp = &sc->arpcom.ac_if;
 342         struct mii_data *mii;
 343         struct jme_txdesc *txd;
 344         bus_addr_t paddr;
 345         int i, r;
 346
 347         if (sc->jme_in_tick)
 348                 jme_serialize_skipmain(sc);
 349         ASSERT_IFNET_SERIALIZED_ALL(ifp);
 350
 351         if ((ifp->if_flags & IFF_RUNNING) == 0)
 352                 goto done;
 353
 354         mii = device_get_softc(sc->jme_miibus);
 355
 356         sc->jme_has_link = FALSE;
 357         if ((mii->mii_media_status & IFM_AVALID) != 0) {
 358                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
 359                 case IFM_10_T:
 360                 case IFM_100_TX:
 361                         sc->jme_has_link = TRUE;
 362                         break;
 363                 case IFM_1000_T:
 364                         if (sc->jme_caps & JME_CAP_FASTETH)
 365                                 break;
 366                         sc->jme_has_link = TRUE;
 367                         break;
 368                 default:
 369                         break;
 370                 }
 371         }
 372
 373         /*
 374          * Disabling Rx/Tx MACs have a side-effect of resetting
 375          * JME_TXNDA/JME_RXNDA register to the first address of
 376          * Tx/Rx descriptor address. So driver should reset its
 377          * internal procucer/consumer pointer and reclaim any
 378          * allocated resources.  Note, just saving the value of
 379          * JME_TXNDA and JME_RXNDA registers before stopping MAC
 380          * and restoring JME_TXNDA/JME_RXNDA register is not
 381          * sufficient to make sure correct MAC state because
 382          * stopping MAC operation can take a while and hardware
 383          * might have updated JME_TXNDA/JME_RXNDA registers
 384          * during the stop operation.
 385          */
 386
 387         /* Disable interrupts */
 388         CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS);
 389
 390         /* Stop driver */
 391         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
 392         ifp->if_timer = 0;
 393         callout_stop(&sc->jme_tick_ch);
 394
 395         /* Stop receiver/transmitter. */
 396         jme_stop_rx(sc);
 397         jme_stop_tx(sc);
 398
 399         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
 400                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r];
 401
 402                 jme_rxeof(rdata, -1);
 403                 if (rdata->jme_rxhead != NULL)
 404                         m_freem(rdata->jme_rxhead);
 405                 JME_RXCHAIN_RESET(rdata);
 406
 407                 /*
 408                  * Reuse configured Rx descriptors and reset
 409                  * procuder/consumer index.
 410                  */
 411                 rdata->jme_rx_cons = 0;
 412         }
 413         if (JME_ENABLE_HWRSS(sc))
 414                 jme_enable_rss(sc);
 415         else
 416                 jme_disable_rss(sc);
 417
 418         jme_txeof(sc);
 419         if (sc->jme_cdata.jme_tx_cnt != 0) {
 420                 /* Remove queued packets for transmit. */
 421                 for (i = 0; i < sc->jme_cdata.jme_tx_desc_cnt; i++) {
 422                         txd = &sc->jme_cdata.jme_txdesc[i];
 423                         if (txd->tx_m != NULL) {
 424                                 bus_dmamap_unload(
 425                                     sc->jme_cdata.jme_tx_tag,
 426                                     txd->tx_dmamap);
 427                                 m_freem(txd->tx_m);
 428                                 txd->tx_m = NULL;
 429                                 txd->tx_ndesc = 0;
 430                                 ifp->if_oerrors++;
 431                         }
 432                 }
 433         }
 434         jme_init_tx_ring(sc);
 435
 436         /* Initialize shadow status block. */
 437         jme_init_ssb(sc);
 438
 439         /* Program MAC with resolved speed/duplex/flow-control. */
 440         if (sc->jme_has_link) {
 441                 jme_mac_config(sc);
 442
 443                 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr);
 444
 445                 /* Set Tx ring address to the hardware. */
 446                 paddr = sc->jme_cdata.jme_tx_ring_paddr;
 447                 CSR_WRITE_4(sc, JME_TXDBA_HI, JME_ADDR_HI(paddr));
 448                 CSR_WRITE_4(sc, JME_TXDBA_LO, JME_ADDR_LO(paddr));
 449
 450                 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
 451                         CSR_WRITE_4(sc, JME_RXCSR,
 452                             sc->jme_rxcsr | RXCSR_RXQ_N_SEL(r));
 453
 454                         /* Set Rx ring address to the hardware. */
 455                         paddr = sc->jme_cdata.jme_rx_data[r].jme_rx_ring_paddr;
 456                         CSR_WRITE_4(sc, JME_RXDBA_HI, JME_ADDR_HI(paddr));
 457                         CSR_WRITE_4(sc, JME_RXDBA_LO, JME_ADDR_LO(paddr));
 458                 }
 459
 460                 /* Restart receiver/transmitter. */
 461                 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | RXCSR_RX_ENB |
 462                     RXCSR_RXQ_START);
 463                 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr | TXCSR_TX_ENB);
 464         }
 465
 466         ifp->if_flags |= IFF_RUNNING;
 467         ifp->if_flags &= ~IFF_OACTIVE;
 468         callout_reset(&sc->jme_tick_ch, hz, jme_tick, sc);
 469
 470 #ifdef DEVICE_POLLING
 471         if (!(ifp->if_flags & IFF_POLLING))
 472 #endif
 473         /* Reenable interrupts. */
 474         CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS);
 475
 476 done:
 477         if (sc->jme_in_tick)
 478                 jme_deserialize_skipmain(sc);
 479 }
 480
 481 /*
 482  *      Get the current interface media status.
 483  */
 484 static void
 485 jme_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
 486 {
 487         struct jme_softc *sc = ifp->if_softc;
 488         struct mii_data *mii = device_get_softc(sc->jme_miibus);
 489
 490         ASSERT_IFNET_SERIALIZED_ALL(ifp);
 491
 492         mii_pollstat(mii);
 493         ifmr->ifm_status = mii->mii_media_status;
 494         ifmr->ifm_active = mii->mii_media_active;
 495 }
 496
 497 /*
 498  *      Set hardware to newly-selected media.
 499  */
 500 static int
 501 jme_mediachange(struct ifnet *ifp)
 502 {
 503         struct jme_softc *sc = ifp->if_softc;
 504         struct mii_data *mii = device_get_softc(sc->jme_miibus);
 505         int error;
 506
 507         ASSERT_IFNET_SERIALIZED_ALL(ifp);
 508
 509         if (mii->mii_instance != 0) {
 510                 struct mii_softc *miisc;
 511
 512                 LIST_FOREACH(miisc, &mii->mii_phys, mii_list)
 513                         mii_phy_reset(miisc);
 514         }
 515         error = mii_mediachg(mii);
 516
 517         return (error);
 518 }
 519
 520 static int
 521 jme_probe(device_t dev)
 522 {
 523         const struct jme_dev *sp;
 524         uint16_t vid, did;
 525
 526         vid = pci_get_vendor(dev);
 527         did = pci_get_device(dev);
 528         for (sp = jme_devs; sp->jme_name != NULL; ++sp) {
 529                 if (vid == sp->jme_vendorid && did == sp->jme_deviceid) {
 530                         struct jme_softc *sc = device_get_softc(dev);
 531
 532                         sc->jme_caps = sp->jme_caps;
 533                         device_set_desc(dev, sp->jme_name);
 534                         return (0);
 535                 }
 536         }
 537         return (ENXIO);
 538 }
 539
 540 static int
 541 jme_eeprom_read_byte(struct jme_softc *sc, uint8_t addr, uint8_t *val)
 542 {
 543         uint32_t reg;
 544         int i;
 545
 546         *val = 0;
 547         for (i = JME_TIMEOUT; i > 0; i--) {
 548                 reg = CSR_READ_4(sc, JME_SMBCSR);
 549                 if ((reg & SMBCSR_HW_BUSY_MASK) == SMBCSR_HW_IDLE)
 550                         break;
 551                 DELAY(1);
 552         }
 553
 554         if (i == 0) {
 555                 device_printf(sc->jme_dev, "EEPROM idle timeout!\n");
 556                 return (ETIMEDOUT);
 557         }
 558
 559         reg = ((uint32_t)addr << SMBINTF_ADDR_SHIFT) & SMBINTF_ADDR_MASK;
 560         CSR_WRITE_4(sc, JME_SMBINTF, reg | SMBINTF_RD | SMBINTF_CMD_TRIGGER);
 561         for (i = JME_TIMEOUT; i > 0; i--) {
 562                 DELAY(1);
 563                 reg = CSR_READ_4(sc, JME_SMBINTF);
 564                 if ((reg & SMBINTF_CMD_TRIGGER) == 0)
 565                         break;
 566         }
 567
 568         if (i == 0) {
 569                 device_printf(sc->jme_dev, "EEPROM read timeout!\n");
 570                 return (ETIMEDOUT);
 571         }
 572
 573         reg = CSR_READ_4(sc, JME_SMBINTF);
 574         *val = (reg & SMBINTF_RD_DATA_MASK) >> SMBINTF_RD_DATA_SHIFT;
 575
 576         return (0);
 577 }
 578
 579 static int
 580 jme_eeprom_macaddr(struct jme_softc *sc, uint8_t eaddr[])
 581 {
 582         uint8_t fup, reg, val;
 583         uint32_t offset;
 584         int match;
 585
 586         offset = 0;
 587         if (jme_eeprom_read_byte(sc, offset++, &fup) != 0 ||
 588             fup != JME_EEPROM_SIG0)
 589                 return (ENOENT);
 590         if (jme_eeprom_read_byte(sc, offset++, &fup) != 0 ||
 591             fup != JME_EEPROM_SIG1)
 592                 return (ENOENT);
 593         match = 0;
 594         do {
 595                 if (jme_eeprom_read_byte(sc, offset, &fup) != 0)
 596                         break;
 597                 if (JME_EEPROM_MKDESC(JME_EEPROM_FUNC0, JME_EEPROM_PAGE_BAR1) ==
 598                     (fup & (JME_EEPROM_FUNC_MASK | JME_EEPROM_PAGE_MASK))) {
 599                         if (jme_eeprom_read_byte(sc, offset + 1, &reg) != 0)
 600                                 break;
 601                         if (reg >= JME_PAR0 &&
 602                             reg < JME_PAR0 + ETHER_ADDR_LEN) {
 603                                 if (jme_eeprom_read_byte(sc, offset + 2,
 604                                     &val) != 0)
 605                                         break;
 606                                 eaddr[reg - JME_PAR0] = val;
 607                                 match++;
 608                         }
 609                 }
 610                 /* Check for the end of EEPROM descriptor. */
 611                 if ((fup & JME_EEPROM_DESC_END) == JME_EEPROM_DESC_END)
 612                         break;
 613                 /* Try next eeprom descriptor. */
 614                 offset += JME_EEPROM_DESC_BYTES;
 615         } while (match != ETHER_ADDR_LEN && offset < JME_EEPROM_END);
 616
 617         if (match == ETHER_ADDR_LEN)
 618                 return (0);
 619
 620         return (ENOENT);
 621 }
 622
 623 static void
 624 jme_reg_macaddr(struct jme_softc *sc, uint8_t eaddr[])
 625 {
 626         uint32_t par0, par1;
 627
 628         /* Read station address. */
 629         par0 = CSR_READ_4(sc, JME_PAR0);
 630         par1 = CSR_READ_4(sc, JME_PAR1);
 631         par1 &= 0xFFFF;
 632         if ((par0 == 0 && par1 == 0) || (par0 & 0x1)) {
 633                 device_printf(sc->jme_dev,
 634                     "generating fake ethernet address.\n");
 635                 par0 = karc4random();
 636                 /* Set OUI to JMicron. */
 637                 eaddr[0] = 0x00;
 638                 eaddr[1] = 0x1B;
 639                 eaddr[2] = 0x8C;
 640                 eaddr[3] = (par0 >> 16) & 0xff;
 641                 eaddr[4] = (par0 >> 8) & 0xff;
 642                 eaddr[5] = par0 & 0xff;
 643         } else {
 644                 eaddr[0] = (par0 >> 0) & 0xFF;
 645                 eaddr[1] = (par0 >> 8) & 0xFF;
 646                 eaddr[2] = (par0 >> 16) & 0xFF;
 647                 eaddr[3] = (par0 >> 24) & 0xFF;
 648                 eaddr[4] = (par1 >> 0) & 0xFF;
 649                 eaddr[5] = (par1 >> 8) & 0xFF;
 650         }
 651 }
 652
 653 static int
 654 jme_attach(device_t dev)
 655 {
 656         struct jme_softc *sc = device_get_softc(dev);
 657         struct ifnet *ifp = &sc->arpcom.ac_if;
 658         uint32_t reg;
 659         uint16_t did;
 660         uint8_t pcie_ptr, rev;
 661         int error = 0, i, j, rx_desc_cnt;
 662         uint8_t eaddr[ETHER_ADDR_LEN];
 663
 664         lwkt_serialize_init(&sc->jme_serialize);
 665         lwkt_serialize_init(&sc->jme_cdata.jme_tx_serialize);
 666         for (i = 0; i < JME_NRXRING_MAX; ++i) {
 667                 lwkt_serialize_init(
 668                     &sc->jme_cdata.jme_rx_data[i].jme_rx_serialize);
 669         }
 670
 671         rx_desc_cnt = device_getenv_int(dev, "rx_desc_count",
 672             jme_rx_desc_count);
 673         rx_desc_cnt = roundup(rx_desc_cnt, JME_NDESC_ALIGN);
 674         if (rx_desc_cnt > JME_NDESC_MAX)
 675                 rx_desc_cnt = JME_NDESC_MAX;
 676
 677         sc->jme_cdata.jme_tx_desc_cnt = device_getenv_int(dev, "tx_desc_count",
 678             jme_tx_desc_count);
 679         sc->jme_cdata.jme_tx_desc_cnt = roundup(sc->jme_cdata.jme_tx_desc_cnt,
 680             JME_NDESC_ALIGN);
 681         if (sc->jme_cdata.jme_tx_desc_cnt > JME_NDESC_MAX)
 682                 sc->jme_cdata.jme_tx_desc_cnt = JME_NDESC_MAX;
 683
 684         /*
 685          * Calculate rx rings
 686          */
 687         sc->jme_cdata.jme_rx_ring_cnt = device_getenv_int(dev, "rx_ring_count",
 688             jme_rx_ring_count);
 689         sc->jme_cdata.jme_rx_ring_cnt =
 690             if_ring_count2(sc->jme_cdata.jme_rx_ring_cnt, JME_NRXRING_MAX);
 691
 692         i = 0;
 693         sc->jme_serialize_arr[i++] = &sc->jme_serialize;
 694
 695         KKASSERT(i == JME_TX_SERIALIZE);
 696         sc->jme_serialize_arr[i++] = &sc->jme_cdata.jme_tx_serialize;
 697
 698         KKASSERT(i == JME_RX_SERIALIZE);
 699         for (j = 0; j < sc->jme_cdata.jme_rx_ring_cnt; ++j) {
 700                 sc->jme_serialize_arr[i++] =
 701                     &sc->jme_cdata.jme_rx_data[j].jme_rx_serialize;
 702         }
 703         KKASSERT(i <= JME_NSERIALIZE);
 704         sc->jme_serialize_cnt = i;
 705
 706         sc->jme_cdata.jme_sc = sc;
 707         for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) {
 708                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[i];
 709
 710                 rdata->jme_sc = sc;
 711                 rdata->jme_rx_coal = jme_rx_status[i].jme_coal;
 712                 rdata->jme_rx_comp = jme_rx_status[i].jme_comp;
 713                 rdata->jme_rx_empty = jme_rx_status[i].jme_empty;
 714                 rdata->jme_rx_idx = i;
 715                 rdata->jme_rx_desc_cnt = rx_desc_cnt;
 716         }
 717
 718         sc->jme_dev = dev;
 719         sc->jme_lowaddr = BUS_SPACE_MAXADDR;
 720
 721         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 722
 723         callout_init(&sc->jme_tick_ch);
 724
 725 #ifndef BURN_BRIDGES
 726         if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) {
 727                 uint32_t irq, mem;
 728
 729                 irq = pci_read_config(dev, PCIR_INTLINE, 4);
 730                 mem = pci_read_config(dev, JME_PCIR_BAR, 4);
 731
 732                 device_printf(dev, "chip is in D%d power mode "
 733                     "-- setting to D0\n", pci_get_powerstate(dev));
 734
 735                 pci_set_powerstate(dev, PCI_POWERSTATE_D0);
 736
 737                 pci_write_config(dev, PCIR_INTLINE, irq, 4);
 738                 pci_write_config(dev, JME_PCIR_BAR, mem, 4);
 739         }
 740 #endif  /* !BURN_BRIDGE */
 741
 742         /* Enable bus mastering */
 743         pci_enable_busmaster(dev);
 744
 745         /*
 746          * Allocate IO memory
 747          *
 748          * JMC250 supports both memory mapped and I/O register space
 749          * access.  Because I/O register access should use different
 750          * BARs to access registers it's waste of time to use I/O
 751          * register spce access.  JMC250 uses 16K to map entire memory
 752          * space.
 753          */
 754         sc->jme_mem_rid = JME_PCIR_BAR;
 755         sc->jme_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 756                                                  &sc->jme_mem_rid, RF_ACTIVE);
 757         if (sc->jme_mem_res == NULL) {
 758                 device_printf(dev, "can't allocate IO memory\n");
 759                 return ENXIO;
 760         }
 761         sc->jme_mem_bt = rman_get_bustag(sc->jme_mem_res);
 762         sc->jme_mem_bh = rman_get_bushandle(sc->jme_mem_res);
 763
 764         /*
 765          * Allocate IRQ
 766          */
 767         error = jme_intr_alloc(dev);
 768         if (error)
 769                 goto fail;
 770
 771         /*
 772          * Extract revisions
 773          */
 774         reg = CSR_READ_4(sc, JME_CHIPMODE);
 775         if (((reg & CHIPMODE_FPGA_REV_MASK) >> CHIPMODE_FPGA_REV_SHIFT) !=
 776             CHIPMODE_NOT_FPGA) {
 777                 sc->jme_caps |= JME_CAP_FPGA;
 778                 if (bootverbose) {
 779                         device_printf(dev, "FPGA revision: 0x%04x\n",
 780                                       (reg & CHIPMODE_FPGA_REV_MASK) >>
 781                                       CHIPMODE_FPGA_REV_SHIFT);
 782                 }
 783         }
 784
 785         /* NOTE: FM revision is put in the upper 4 bits */
 786         rev = ((reg & CHIPMODE_REVFM_MASK) >> CHIPMODE_REVFM_SHIFT) << 4;
 787         rev |= (reg & CHIPMODE_REVECO_MASK) >> CHIPMODE_REVECO_SHIFT;
 788         if (bootverbose)
 789                 device_printf(dev, "Revision (FM/ECO): 0x%02x\n", rev);
 790
 791         did = pci_get_device(dev);
 792         switch (did) {
 793         case PCI_PRODUCT_JMICRON_JMC250:
 794                 if (rev == JME_REV1_A2)
 795                         sc->jme_workaround |= JME_WA_EXTFIFO | JME_WA_HDX;
 796                 break;
 797
 798         case PCI_PRODUCT_JMICRON_JMC260:
 799                 if (rev == JME_REV2)
 800                         sc->jme_lowaddr = BUS_SPACE_MAXADDR_32BIT;
 801                 break;
 802
 803         default:
 804                 panic("unknown device id 0x%04x", did);
 805         }
 806         if (rev >= JME_REV2) {
 807                 sc->jme_clksrc = GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC;
 808                 sc->jme_clksrc_1000 = GHC_TXOFL_CLKSRC_1000 |
 809                                       GHC_TXMAC_CLKSRC_1000;
 810         }
 811
 812         /* Reset the ethernet controller. */
 813         jme_reset(sc);
 814
 815         /* Map MSI/MSI-X vectors */
 816         jme_set_msinum(sc);
 817
 818         /* Get station address. */
 819         reg = CSR_READ_4(sc, JME_SMBCSR);
 820         if (reg & SMBCSR_EEPROM_PRESENT)
 821                 error = jme_eeprom_macaddr(sc, eaddr);
 822         if (error != 0 || (reg & SMBCSR_EEPROM_PRESENT) == 0) {
 823                 if (error != 0 && (bootverbose)) {
 824                         device_printf(dev, "ethernet hardware address "
 825                                       "not found in EEPROM.\n");
 826                 }
 827                 jme_reg_macaddr(sc, eaddr);
 828         }
 829
 830         /*
 831          * Save PHY address.
 832          * Integrated JR0211 has fixed PHY address whereas FPGA version
 833          * requires PHY probing to get correct PHY address.
 834          */
 835         if ((sc->jme_caps & JME_CAP_FPGA) == 0) {
 836                 sc->jme_phyaddr = CSR_READ_4(sc, JME_GPREG0) &
 837                     GPREG0_PHY_ADDR_MASK;
 838                 if (bootverbose) {
 839                         device_printf(dev, "PHY is at address %d.\n",
 840                             sc->jme_phyaddr);
 841                 }
 842         } else {
 843                 sc->jme_phyaddr = 0;
 844         }
 845
 846         /* Set max allowable DMA size. */
 847         pcie_ptr = pci_get_pciecap_ptr(dev);
 848         if (pcie_ptr != 0) {
 849                 uint16_t ctrl;
 850
 851                 sc->jme_caps |= JME_CAP_PCIE;
 852                 ctrl = pci_read_config(dev, pcie_ptr + PCIER_DEVCTRL, 2);
 853                 if (bootverbose) {
 854                         device_printf(dev, "Read request size : %d bytes.\n",
 855                             128 << ((ctrl >> 12) & 0x07));
 856                         device_printf(dev, "TLP payload size : %d bytes.\n",
 857                             128 << ((ctrl >> 5) & 0x07));
 858                 }
 859                 switch (ctrl & PCIEM_DEVCTL_MAX_READRQ_MASK) {
 860                 case PCIEM_DEVCTL_MAX_READRQ_128:
 861                         sc->jme_tx_dma_size = TXCSR_DMA_SIZE_128;
 862                         break;
 863                 case PCIEM_DEVCTL_MAX_READRQ_256:
 864                         sc->jme_tx_dma_size = TXCSR_DMA_SIZE_256;
 865                         break;
 866                 default:
 867                         sc->jme_tx_dma_size = TXCSR_DMA_SIZE_512;
 868                         break;
 869                 }
 870                 sc->jme_rx_dma_size = RXCSR_DMA_SIZE_128;
 871         } else {
 872                 sc->jme_tx_dma_size = TXCSR_DMA_SIZE_512;
 873                 sc->jme_rx_dma_size = RXCSR_DMA_SIZE_128;
 874         }
 875
 876 #ifdef notyet
 877         if (pci_find_extcap(dev, PCIY_PMG, &pmc) == 0)
 878                 sc->jme_caps |= JME_CAP_PMCAP;
 879 #endif
 880
 881         /*
 882          * Create sysctl tree
 883          */
 884         jme_sysctl_node(sc);
 885
 886         /* Allocate DMA stuffs */
 887         error = jme_dma_alloc(sc);
 888         if (error)
 889                 goto fail;
 890
 891         ifp->if_softc = sc;
 892         ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 893         ifp->if_init = jme_init;
 894         ifp->if_ioctl = jme_ioctl;
 895         ifp->if_start = jme_start;
 896 #ifdef DEVICE_POLLING
 897         ifp->if_poll = jme_poll;
 898 #endif
 899         ifp->if_watchdog = jme_watchdog;
 900         ifp->if_serialize = jme_serialize;
 901         ifp->if_deserialize = jme_deserialize;
 902         ifp->if_tryserialize = jme_tryserialize;
 903 #ifdef INVARIANTS
 904         ifp->if_serialize_assert = jme_serialize_assert;
 905 #endif
 906         ifq_set_maxlen(&ifp->if_snd,
 907             sc->jme_cdata.jme_tx_desc_cnt - JME_TXD_RSVD);
 908         ifq_set_ready(&ifp->if_snd);
 909
 910         /* JMC250 supports Tx/Rx checksum offload and hardware vlan tagging. */
 911         ifp->if_capabilities = IFCAP_HWCSUM |
 912                                IFCAP_VLAN_MTU |
 913                                IFCAP_VLAN_HWTAGGING;
 914         if (sc->jme_cdata.jme_rx_ring_cnt > JME_NRXRING_MIN)
 915                 ifp->if_capabilities |= IFCAP_RSS;
 916         ifp->if_capenable = ifp->if_capabilities;
 917
 918         /*
 919          * Disable TXCSUM by default to improve bulk data
 920          * transmit performance (+20Mbps improvement).
 921          */
 922         ifp->if_capenable &= ~IFCAP_TXCSUM;
 923
 924         if (ifp->if_capenable & IFCAP_TXCSUM)
 925                 ifp->if_hwassist = JME_CSUM_FEATURES;
 926
 927         /* Set up MII bus. */
 928         error = mii_phy_probe(dev, &sc->jme_miibus,
 929                               jme_mediachange, jme_mediastatus);
 930         if (error) {
 931                 device_printf(dev, "no PHY found!\n");
 932                 goto fail;
 933         }
 934
 935         /*
 936          * Save PHYADDR for FPGA mode PHY.
 937          */
 938         if (sc->jme_caps & JME_CAP_FPGA) {
 939                 struct mii_data *mii = device_get_softc(sc->jme_miibus);
 940
 941                 if (mii->mii_instance != 0) {
 942                         struct mii_softc *miisc;
 943
 944                         LIST_FOREACH(miisc, &mii->mii_phys, mii_list) {
 945                                 if (miisc->mii_phy != 0) {
 946                                         sc->jme_phyaddr = miisc->mii_phy;
 947                                         break;
 948                                 }
 949                         }
 950                         if (sc->jme_phyaddr != 0) {
 951                                 device_printf(sc->jme_dev,
 952                                     "FPGA PHY is at %d\n", sc->jme_phyaddr);
 953                                 /* vendor magic. */
 954                                 jme_miibus_writereg(dev, sc->jme_phyaddr,
 955                                     JMPHY_CONF, JMPHY_CONF_DEFFIFO);
 956
 957                                 /* XXX should we clear JME_WA_EXTFIFO */
 958                         }
 959                 }
 960         }
 961
 962         ether_ifattach(ifp, eaddr, NULL);
 963
 964         /* Tell the upper layer(s) we support long frames. */
 965         ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
 966
 967         error = jme_intr_setup(dev);
 968         if (error) {
 969                 ether_ifdetach(ifp);
 970                 goto fail;
 971         }
 972
 973         return 0;
 974 fail:
 975         jme_detach(dev);
 976         return (error);
 977 }
 978
 979 static int
 980 jme_detach(device_t dev)
 981 {
 982         struct jme_softc *sc = device_get_softc(dev);
 983
 984         if (device_is_attached(dev)) {
 985                 struct ifnet *ifp = &sc->arpcom.ac_if;
 986
 987                 ifnet_serialize_all(ifp);
 988                 jme_stop(sc);
 989                 jme_intr_teardown(dev);
 990                 ifnet_deserialize_all(ifp);
 991
 992                 ether_ifdetach(ifp);
 993         }
 994
 995         if (sc->jme_sysctl_tree != NULL)
 996                 sysctl_ctx_free(&sc->jme_sysctl_ctx);
 997
 998         if (sc->jme_miibus != NULL)
 999                 device_delete_child(dev, sc->jme_miibus);
1000         bus_generic_detach(dev);
1001
1002         jme_intr_free(dev);
1003
1004         if (sc->jme_mem_res != NULL) {
1005                 bus_release_resource(dev, SYS_RES_MEMORY, sc->jme_mem_rid,
1006                                      sc->jme_mem_res);
1007         }
1008
1009         jme_dma_free(sc);
1010
1011         return (0);
1012 }
1013
1014 static void
1015 jme_sysctl_node(struct jme_softc *sc)
1016 {
1017         int coal_max;
1018 #ifdef JME_RSS_DEBUG
1019         int r;
1020 #endif
1021
1022         sysctl_ctx_init(&sc->jme_sysctl_ctx);
1023         sc->jme_sysctl_tree = SYSCTL_ADD_NODE(&sc->jme_sysctl_ctx,
1024                                 SYSCTL_STATIC_CHILDREN(_hw), OID_AUTO,
1025                                 device_get_nameunit(sc->jme_dev),
1026                                 CTLFLAG_RD, 0, "");
1027         if (sc->jme_sysctl_tree == NULL) {
1028                 device_printf(sc->jme_dev, "can't add sysctl node\n");
1029                 return;
1030         }
1031
1032         SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx,
1033             SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1034             "tx_coal_to", CTLTYPE_INT | CTLFLAG_RW,
1035             sc, 0, jme_sysctl_tx_coal_to, "I", "jme tx coalescing timeout");
1036
1037         SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx,
1038             SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1039             "tx_coal_pkt", CTLTYPE_INT | CTLFLAG_RW,
1040             sc, 0, jme_sysctl_tx_coal_pkt, "I", "jme tx coalescing packet");
1041
1042         SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx,
1043             SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1044             "rx_coal_to", CTLTYPE_INT | CTLFLAG_RW,
1045             sc, 0, jme_sysctl_rx_coal_to, "I", "jme rx coalescing timeout");
1046
1047         SYSCTL_ADD_PROC(&sc->jme_sysctl_ctx,
1048             SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1049             "rx_coal_pkt", CTLTYPE_INT | CTLFLAG_RW,
1050             sc, 0, jme_sysctl_rx_coal_pkt, "I", "jme rx coalescing packet");
1051
1052         SYSCTL_ADD_INT(&sc->jme_sysctl_ctx,
1053                        SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1054                        "rx_desc_count", CTLFLAG_RD,
1055                        &sc->jme_cdata.jme_rx_data[0].jme_rx_desc_cnt,
1056                        0, "RX desc count");
1057         SYSCTL_ADD_INT(&sc->jme_sysctl_ctx,
1058                        SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1059                        "tx_desc_count", CTLFLAG_RD,
1060                        &sc->jme_cdata.jme_tx_desc_cnt,
1061                        0, "TX desc count");
1062         SYSCTL_ADD_INT(&sc->jme_sysctl_ctx,
1063                        SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1064                        "rx_ring_count", CTLFLAG_RD,
1065                        &sc->jme_cdata.jme_rx_ring_cnt,
1066                        0, "RX ring count");
1067 #ifdef JME_RSS_DEBUG
1068         SYSCTL_ADD_INT(&sc->jme_sysctl_ctx,
1069                        SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1070                        "rss_debug", CTLFLAG_RW, &sc->jme_rss_debug,
1071                        0, "RSS debug level");
1072         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
1073                 char rx_ring_pkt[32];
1074
1075                 ksnprintf(rx_ring_pkt, sizeof(rx_ring_pkt), "rx_ring%d_pkt", r);
1076                 SYSCTL_ADD_ULONG(&sc->jme_sysctl_ctx,
1077                     SYSCTL_CHILDREN(sc->jme_sysctl_tree), OID_AUTO,
1078                     rx_ring_pkt, CTLFLAG_RW,
1079                     &sc->jme_cdata.jme_rx_data[r].jme_rx_pkt, "RXed packets");
1080         }
1081 #endif
1082
1083         /*
1084          * Set default coalesce valves
1085          */
1086         sc->jme_tx_coal_to = PCCTX_COAL_TO_DEFAULT;
1087         sc->jme_tx_coal_pkt = PCCTX_COAL_PKT_DEFAULT;
1088         sc->jme_rx_coal_to = PCCRX_COAL_TO_DEFAULT;
1089         sc->jme_rx_coal_pkt = PCCRX_COAL_PKT_DEFAULT;
1090
1091         /*
1092          * Adjust coalesce valves, in case that the number of TX/RX
1093          * descs are set to small values by users.
1094          *
1095          * NOTE: coal_max will not be zero, since number of descs
1096          * must aligned by JME_NDESC_ALIGN (16 currently)
1097          */
1098         coal_max = sc->jme_cdata.jme_tx_desc_cnt / 6;
1099         if (coal_max < sc->jme_tx_coal_pkt)
1100                 sc->jme_tx_coal_pkt = coal_max;
1101
1102         coal_max = sc->jme_cdata.jme_rx_data[0].jme_rx_desc_cnt / 4;
1103         if (coal_max < sc->jme_rx_coal_pkt)
1104                 sc->jme_rx_coal_pkt = coal_max;
1105 }
1106
1107 static int
1108 jme_dma_alloc(struct jme_softc *sc)
1109 {
1110         struct jme_txdesc *txd;
1111         bus_dmamem_t dmem;
1112         int error, i, asize;
1113
1114         sc->jme_cdata.jme_txdesc =
1115         kmalloc(sc->jme_cdata.jme_tx_desc_cnt * sizeof(struct jme_txdesc),
1116                 M_DEVBUF, M_WAITOK | M_ZERO);
1117         for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) {
1118                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[i];
1119
1120                 rdata->jme_rxdesc =
1121                 kmalloc(rdata->jme_rx_desc_cnt * sizeof(struct jme_rxdesc),
1122                         M_DEVBUF, M_WAITOK | M_ZERO);
1123         }
1124
1125         /* Create parent ring tag. */
1126         error = bus_dma_tag_create(NULL,/* parent */
1127             1, JME_RING_BOUNDARY,       /* algnmnt, boundary */
1128             sc->jme_lowaddr,            /* lowaddr */
1129             BUS_SPACE_MAXADDR,          /* highaddr */
1130             NULL, NULL,                 /* filter, filterarg */
1131             BUS_SPACE_MAXSIZE_32BIT,    /* maxsize */
1132             0,                          /* nsegments */
1133             BUS_SPACE_MAXSIZE_32BIT,    /* maxsegsize */
1134             0,                          /* flags */
1135             &sc->jme_cdata.jme_ring_tag);
1136         if (error) {
1137                 device_printf(sc->jme_dev,
1138                     "could not create parent ring DMA tag.\n");
1139                 return error;
1140         }
1141
1142         /*
1143          * Create DMA stuffs for TX ring
1144          */
1145         asize = roundup2(JME_TX_RING_SIZE(sc), JME_TX_RING_ALIGN);
1146         error = bus_dmamem_coherent(sc->jme_cdata.jme_ring_tag,
1147                         JME_TX_RING_ALIGN, 0,
1148                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1149                         asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
1150         if (error) {
1151                 device_printf(sc->jme_dev, "could not allocate Tx ring.\n");
1152                 return error;
1153         }
1154         sc->jme_cdata.jme_tx_ring_tag = dmem.dmem_tag;
1155         sc->jme_cdata.jme_tx_ring_map = dmem.dmem_map;
1156         sc->jme_cdata.jme_tx_ring = dmem.dmem_addr;
1157         sc->jme_cdata.jme_tx_ring_paddr = dmem.dmem_busaddr;
1158
1159         /*
1160          * Create DMA stuffs for RX rings
1161          */
1162         for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) {
1163                 error = jme_rxring_dma_alloc(&sc->jme_cdata.jme_rx_data[i]);
1164                 if (error)
1165                         return error;
1166         }
1167
1168         /* Create parent buffer tag. */
1169         error = bus_dma_tag_create(NULL,/* parent */
1170             1, 0,                       /* algnmnt, boundary */
1171             sc->jme_lowaddr,            /* lowaddr */
1172             BUS_SPACE_MAXADDR,          /* highaddr */
1173             NULL, NULL,                 /* filter, filterarg */
1174             BUS_SPACE_MAXSIZE_32BIT,    /* maxsize */
1175             0,                          /* nsegments */
1176             BUS_SPACE_MAXSIZE_32BIT,    /* maxsegsize */
1177             0,                          /* flags */
1178             &sc->jme_cdata.jme_buffer_tag);
1179         if (error) {
1180                 device_printf(sc->jme_dev,
1181                     "could not create parent buffer DMA tag.\n");
1182                 return error;
1183         }
1184
1185         /*
1186          * Create DMA stuffs for shadow status block
1187          */
1188         asize = roundup2(JME_SSB_SIZE, JME_SSB_ALIGN);
1189         error = bus_dmamem_coherent(sc->jme_cdata.jme_buffer_tag,
1190                         JME_SSB_ALIGN, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1191                         asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
1192         if (error) {
1193                 device_printf(sc->jme_dev,
1194                     "could not create shadow status block.\n");
1195                 return error;
1196         }
1197         sc->jme_cdata.jme_ssb_tag = dmem.dmem_tag;
1198         sc->jme_cdata.jme_ssb_map = dmem.dmem_map;
1199         sc->jme_cdata.jme_ssb_block = dmem.dmem_addr;
1200         sc->jme_cdata.jme_ssb_block_paddr = dmem.dmem_busaddr;
1201
1202         /*
1203          * Create DMA stuffs for TX buffers
1204          */
1205
1206         /* Create tag for Tx buffers. */
1207         error = bus_dma_tag_create(sc->jme_cdata.jme_buffer_tag,/* parent */
1208             1, 0,                       /* algnmnt, boundary */
1209             BUS_SPACE_MAXADDR,          /* lowaddr */
1210             BUS_SPACE_MAXADDR,          /* highaddr */
1211             NULL, NULL,                 /* filter, filterarg */
1212             JME_JUMBO_FRAMELEN,         /* maxsize */
1213             JME_MAXTXSEGS,              /* nsegments */
1214             JME_MAXSEGSIZE,             /* maxsegsize */
1215             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,/* flags */
1216             &sc->jme_cdata.jme_tx_tag);
1217         if (error != 0) {
1218                 device_printf(sc->jme_dev, "could not create Tx DMA tag.\n");
1219                 return error;
1220         }
1221
1222         /* Create DMA maps for Tx buffers. */
1223         for (i = 0; i < sc->jme_cdata.jme_tx_desc_cnt; i++) {
1224                 txd = &sc->jme_cdata.jme_txdesc[i];
1225                 error = bus_dmamap_create(sc->jme_cdata.jme_tx_tag,
1226                                 BUS_DMA_WAITOK | BUS_DMA_ONEBPAGE,
1227                                 &txd->tx_dmamap);
1228                 if (error) {
1229                         int j;
1230
1231                         device_printf(sc->jme_dev,
1232                             "could not create %dth Tx dmamap.\n", i);
1233
1234                         for (j = 0; j < i; ++j) {
1235                                 txd = &sc->jme_cdata.jme_txdesc[j];
1236                                 bus_dmamap_destroy(sc->jme_cdata.jme_tx_tag,
1237                                                    txd->tx_dmamap);
1238                         }
1239                         bus_dma_tag_destroy(sc->jme_cdata.jme_tx_tag);
1240                         sc->jme_cdata.jme_tx_tag = NULL;
1241                         return error;
1242                 }
1243         }
1244
1245         /*
1246          * Create DMA stuffs for RX buffers
1247          */
1248         for (i = 0; i < sc->jme_cdata.jme_rx_ring_cnt; ++i) {
1249                 error = jme_rxbuf_dma_alloc(&sc->jme_cdata.jme_rx_data[i]);
1250                 if (error)
1251                         return error;
1252         }
1253         return 0;
1254 }
1255
1256 static void
1257 jme_dma_free(struct jme_softc *sc)
1258 {
1259         struct jme_txdesc *txd;
1260         struct jme_rxdesc *rxd;
1261         struct jme_rxdata *rdata;
1262         int i, r;
1263
1264         /* Tx ring */
1265         if (sc->jme_cdata.jme_tx_ring_tag != NULL) {
1266                 bus_dmamap_unload(sc->jme_cdata.jme_tx_ring_tag,
1267                     sc->jme_cdata.jme_tx_ring_map);
1268                 bus_dmamem_free(sc->jme_cdata.jme_tx_ring_tag,
1269                     sc->jme_cdata.jme_tx_ring,
1270                     sc->jme_cdata.jme_tx_ring_map);
1271                 bus_dma_tag_destroy(sc->jme_cdata.jme_tx_ring_tag);
1272                 sc->jme_cdata.jme_tx_ring_tag = NULL;
1273         }
1274
1275         /* Rx ring */
1276         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
1277                 rdata = &sc->jme_cdata.jme_rx_data[r];
1278                 if (rdata->jme_rx_ring_tag != NULL) {
1279                         bus_dmamap_unload(rdata->jme_rx_ring_tag,
1280                                           rdata->jme_rx_ring_map);
1281                         bus_dmamem_free(rdata->jme_rx_ring_tag,
1282                                         rdata->jme_rx_ring,
1283                                         rdata->jme_rx_ring_map);
1284                         bus_dma_tag_destroy(rdata->jme_rx_ring_tag);
1285                         rdata->jme_rx_ring_tag = NULL;
1286                 }
1287         }
1288
1289         /* Tx buffers */
1290         if (sc->jme_cdata.jme_tx_tag != NULL) {
1291                 for (i = 0; i < sc->jme_cdata.jme_tx_desc_cnt; i++) {
1292                         txd = &sc->jme_cdata.jme_txdesc[i];
1293                         bus_dmamap_destroy(sc->jme_cdata.jme_tx_tag,
1294                             txd->tx_dmamap);
1295                 }
1296                 bus_dma_tag_destroy(sc->jme_cdata.jme_tx_tag);
1297                 sc->jme_cdata.jme_tx_tag = NULL;
1298         }
1299
1300         /* Rx buffers */
1301         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
1302                 rdata = &sc->jme_cdata.jme_rx_data[r];
1303                 if (rdata->jme_rx_tag != NULL) {
1304                         for (i = 0; i < rdata->jme_rx_desc_cnt; i++) {
1305                                 rxd = &rdata->jme_rxdesc[i];
1306                                 bus_dmamap_destroy(rdata->jme_rx_tag,
1307                                                    rxd->rx_dmamap);
1308                         }
1309                         bus_dmamap_destroy(rdata->jme_rx_tag,
1310                                            rdata->jme_rx_sparemap);
1311                         bus_dma_tag_destroy(rdata->jme_rx_tag);
1312                         rdata->jme_rx_tag = NULL;
1313                 }
1314         }
1315
1316         /* Shadow status block. */
1317         if (sc->jme_cdata.jme_ssb_tag != NULL) {
1318                 bus_dmamap_unload(sc->jme_cdata.jme_ssb_tag,
1319                     sc->jme_cdata.jme_ssb_map);
1320                 bus_dmamem_free(sc->jme_cdata.jme_ssb_tag,
1321                     sc->jme_cdata.jme_ssb_block,
1322                     sc->jme_cdata.jme_ssb_map);
1323                 bus_dma_tag_destroy(sc->jme_cdata.jme_ssb_tag);
1324                 sc->jme_cdata.jme_ssb_tag = NULL;
1325         }
1326
1327         if (sc->jme_cdata.jme_buffer_tag != NULL) {
1328                 bus_dma_tag_destroy(sc->jme_cdata.jme_buffer_tag);
1329                 sc->jme_cdata.jme_buffer_tag = NULL;
1330         }
1331         if (sc->jme_cdata.jme_ring_tag != NULL) {
1332                 bus_dma_tag_destroy(sc->jme_cdata.jme_ring_tag);
1333                 sc->jme_cdata.jme_ring_tag = NULL;
1334         }
1335
1336         if (sc->jme_cdata.jme_txdesc != NULL) {
1337                 kfree(sc->jme_cdata.jme_txdesc, M_DEVBUF);
1338                 sc->jme_cdata.jme_txdesc = NULL;
1339         }
1340         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
1341                 rdata = &sc->jme_cdata.jme_rx_data[r];
1342                 if (rdata->jme_rxdesc != NULL) {
1343                         kfree(rdata->jme_rxdesc, M_DEVBUF);
1344                         rdata->jme_rxdesc = NULL;
1345                 }
1346         }
1347 }
1348
1349 /*
1350  *      Make sure the interface is stopped at reboot time.
1351  */
1352 static int
1353 jme_shutdown(device_t dev)
1354 {
1355         return jme_suspend(dev);
1356 }
1357
1358 #ifdef notyet
1359 /*
1360  * Unlike other ethernet controllers, JMC250 requires
1361  * explicit resetting link speed to 10/100Mbps as gigabit
1362  * link will cunsume more power than 375mA.
1363  * Note, we reset the link speed to 10/100Mbps with
1364  * auto-negotiation but we don't know whether that operation
1365  * would succeed or not as we have no control after powering
1366  * off. If the renegotiation fail WOL may not work. Running
1367  * at 1Gbps draws more power than 375mA at 3.3V which is
1368  * specified in PCI specification and that would result in
1369  * complete shutdowning power to ethernet controller.
1370  *
1371  * TODO
1372  *  Save current negotiated media speed/duplex/flow-control
1373  *  to softc and restore the same link again after resuming.
1374  *  PHY handling such as power down/resetting to 100Mbps
1375  *  may be better handled in suspend method in phy driver.
1376  */
1377 static void
1378 jme_setlinkspeed(struct jme_softc *sc)
1379 {
1380         struct mii_data *mii;
1381         int aneg, i;
1382
1383         JME_LOCK_ASSERT(sc);
1384
1385         mii = device_get_softc(sc->jme_miibus);
1386         mii_pollstat(mii);
1387         aneg = 0;
1388         if ((mii->mii_media_status & IFM_AVALID) != 0) {
1389                 switch IFM_SUBTYPE(mii->mii_media_active) {
1390                 case IFM_10_T:
1391                 case IFM_100_TX:
1392                         return;
1393                 case IFM_1000_T:
1394                         aneg++;
1395                 default:
1396                         break;
1397                 }
1398         }
1399         jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_100T2CR, 0);
1400         jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_ANAR,
1401             ANAR_TX_FD | ANAR_TX | ANAR_10_FD | ANAR_10 | ANAR_CSMA);
1402         jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr, MII_BMCR,
1403             BMCR_AUTOEN | BMCR_STARTNEG);
1404         DELAY(1000);
1405         if (aneg != 0) {
1406                 /* Poll link state until jme(4) get a 10/100 link. */
1407                 for (i = 0; i < MII_ANEGTICKS_GIGE; i++) {
1408                         mii_pollstat(mii);
1409                         if ((mii->mii_media_status & IFM_AVALID) != 0) {
1410                                 switch (IFM_SUBTYPE(mii->mii_media_active)) {
1411                                 case IFM_10_T:
1412                                 case IFM_100_TX:
1413                                         jme_mac_config(sc);
1414                                         return;
1415                                 default:
1416                                         break;
1417                                 }
1418                         }
1419                         JME_UNLOCK(sc);
1420                         pause("jmelnk", hz);
1421                         JME_LOCK(sc);
1422                 }
1423                 if (i == MII_ANEGTICKS_GIGE)
1424                         device_printf(sc->jme_dev, "establishing link failed, "
1425                             "WOL may not work!");
1426         }
1427         /*
1428          * No link, force MAC to have 100Mbps, full-duplex link.
1429          * This is the last resort and may/may not work.
1430          */
1431         mii->mii_media_status = IFM_AVALID | IFM_ACTIVE;
1432         mii->mii_media_active = IFM_ETHER | IFM_100_TX | IFM_FDX;
1433         jme_mac_config(sc);
1434 }
1435
1436 static void
1437 jme_setwol(struct jme_softc *sc)
1438 {
1439         struct ifnet *ifp = &sc->arpcom.ac_if;
1440         uint32_t gpr, pmcs;
1441         uint16_t pmstat;
1442         int pmc;
1443
1444         if (pci_find_extcap(sc->jme_dev, PCIY_PMG, &pmc) != 0) {
1445                 /* No PME capability, PHY power down. */
1446                 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr,
1447                     MII_BMCR, BMCR_PDOWN);
1448                 return;
1449         }
1450
1451         gpr = CSR_READ_4(sc, JME_GPREG0) & ~GPREG0_PME_ENB;
1452         pmcs = CSR_READ_4(sc, JME_PMCS);
1453         pmcs &= ~PMCS_WOL_ENB_MASK;
1454         if ((ifp->if_capenable & IFCAP_WOL_MAGIC) != 0) {
1455                 pmcs |= PMCS_MAGIC_FRAME | PMCS_MAGIC_FRAME_ENB;
1456                 /* Enable PME message. */
1457                 gpr |= GPREG0_PME_ENB;
1458                 /* For gigabit controllers, reset link speed to 10/100. */
1459                 if ((sc->jme_caps & JME_CAP_FASTETH) == 0)
1460                         jme_setlinkspeed(sc);
1461         }
1462
1463         CSR_WRITE_4(sc, JME_PMCS, pmcs);
1464         CSR_WRITE_4(sc, JME_GPREG0, gpr);
1465
1466         /* Request PME. */
1467         pmstat = pci_read_config(sc->jme_dev, pmc + PCIR_POWER_STATUS, 2);
1468         pmstat &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE);
1469         if ((ifp->if_capenable & IFCAP_WOL) != 0)
1470                 pmstat |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
1471         pci_write_config(sc->jme_dev, pmc + PCIR_POWER_STATUS, pmstat, 2);
1472         if ((ifp->if_capenable & IFCAP_WOL) == 0) {
1473                 /* No WOL, PHY power down. */
1474                 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr,
1475                     MII_BMCR, BMCR_PDOWN);
1476         }
1477 }
1478 #endif
1479
1480 static int
1481 jme_suspend(device_t dev)
1482 {
1483         struct jme_softc *sc = device_get_softc(dev);
1484         struct ifnet *ifp = &sc->arpcom.ac_if;
1485
1486         ifnet_serialize_all(ifp);
1487         jme_stop(sc);
1488 #ifdef notyet
1489         jme_setwol(sc);
1490 #endif
1491         ifnet_deserialize_all(ifp);
1492
1493         return (0);
1494 }
1495
1496 static int
1497 jme_resume(device_t dev)
1498 {
1499         struct jme_softc *sc = device_get_softc(dev);
1500         struct ifnet *ifp = &sc->arpcom.ac_if;
1501 #ifdef notyet
1502         int pmc;
1503 #endif
1504
1505         ifnet_serialize_all(ifp);
1506
1507 #ifdef notyet
1508         if (pci_find_extcap(sc->jme_dev, PCIY_PMG, &pmc) != 0) {
1509                 uint16_t pmstat;
1510
1511                 pmstat = pci_read_config(sc->jme_dev,
1512                     pmc + PCIR_POWER_STATUS, 2);
1513                 /* Disable PME clear PME status. */
1514                 pmstat &= ~PCIM_PSTAT_PMEENABLE;
1515                 pci_write_config(sc->jme_dev,
1516                     pmc + PCIR_POWER_STATUS, pmstat, 2);
1517         }
1518 #endif
1519
1520         if (ifp->if_flags & IFF_UP)
1521                 jme_init(sc);
1522
1523         ifnet_deserialize_all(ifp);
1524
1525         return (0);
1526 }
1527
1528 static int
1529 jme_encap(struct jme_softc *sc, struct mbuf **m_head)
1530 {
1531         struct jme_txdesc *txd;
1532         struct jme_desc *desc;
1533         struct mbuf *m;
1534         bus_dma_segment_t txsegs[JME_MAXTXSEGS];
1535         int maxsegs, nsegs;
1536         int error, i, prod, symbol_desc;
1537         uint32_t cflags, flag64;
1538
1539         M_ASSERTPKTHDR((*m_head));
1540
1541         prod = sc->jme_cdata.jme_tx_prod;
1542         txd = &sc->jme_cdata.jme_txdesc[prod];
1543
1544         if (sc->jme_lowaddr != BUS_SPACE_MAXADDR_32BIT)
1545                 symbol_desc = 1;
1546         else
1547                 symbol_desc = 0;
1548
1549         maxsegs = (sc->jme_cdata.jme_tx_desc_cnt - sc->jme_cdata.jme_tx_cnt) -
1550                   (JME_TXD_RSVD + symbol_desc);
1551         if (maxsegs > JME_MAXTXSEGS)
1552                 maxsegs = JME_MAXTXSEGS;
1553         KASSERT(maxsegs >= (sc->jme_txd_spare - symbol_desc),
1554                 ("not enough segments %d", maxsegs));
1555
1556         error = bus_dmamap_load_mbuf_defrag(sc->jme_cdata.jme_tx_tag,
1557                         txd->tx_dmamap, m_head,
1558                         txsegs, maxsegs, &nsegs, BUS_DMA_NOWAIT);
1559         if (error)
1560                 goto fail;
1561
1562         bus_dmamap_sync(sc->jme_cdata.jme_tx_tag, txd->tx_dmamap,
1563                         BUS_DMASYNC_PREWRITE);
1564
1565         m = *m_head;
1566         cflags = 0;
1567
1568         /* Configure checksum offload. */
1569         if (m->m_pkthdr.csum_flags & CSUM_IP)
1570                 cflags |= JME_TD_IPCSUM;
1571         if (m->m_pkthdr.csum_flags & CSUM_TCP)
1572                 cflags |= JME_TD_TCPCSUM;
1573         if (m->m_pkthdr.csum_flags & CSUM_UDP)
1574                 cflags |= JME_TD_UDPCSUM;
1575
1576         /* Configure VLAN. */
1577         if (m->m_flags & M_VLANTAG) {
1578                 cflags |= (m->m_pkthdr.ether_vlantag & JME_TD_VLAN_MASK);
1579                 cflags |= JME_TD_VLAN_TAG;
1580         }
1581
1582         desc = &sc->jme_cdata.jme_tx_ring[prod];
1583         desc->flags = htole32(cflags);
1584         desc->addr_hi = htole32(m->m_pkthdr.len);
1585         if (sc->jme_lowaddr != BUS_SPACE_MAXADDR_32BIT) {
1586                 /*
1587                  * Use 64bits TX desc chain format.
1588                  *
1589                  * The first TX desc of the chain, which is setup here,
1590                  * is just a symbol TX desc carrying no payload.
1591                  */
1592                 flag64 = JME_TD_64BIT;
1593                 desc->buflen = 0;
1594                 desc->addr_lo = 0;
1595
1596                 /* No effective TX desc is consumed */
1597                 i = 0;
1598         } else {
1599                 /*
1600                  * Use 32bits TX desc chain format.
1601                  *
1602                  * The first TX desc of the chain, which is setup here,
1603                  * is an effective TX desc carrying the first segment of
1604                  * the mbuf chain.
1605                  */
1606                 flag64 = 0;
1607                 desc->buflen = htole32(txsegs[0].ds_len);
1608                 desc->addr_lo = htole32(JME_ADDR_LO(txsegs[0].ds_addr));
1609
1610                 /* One effective TX desc is consumed */
1611                 i = 1;
1612         }
1613         sc->jme_cdata.jme_tx_cnt++;
1614         KKASSERT(sc->jme_cdata.jme_tx_cnt - i <
1615                  sc->jme_cdata.jme_tx_desc_cnt - JME_TXD_RSVD);
1616         JME_DESC_INC(prod, sc->jme_cdata.jme_tx_desc_cnt);
1617
1618         txd->tx_ndesc = 1 - i;
1619         for (; i < nsegs; i++) {
1620                 desc = &sc->jme_cdata.jme_tx_ring[prod];
1621                 desc->buflen = htole32(txsegs[i].ds_len);
1622                 desc->addr_hi = htole32(JME_ADDR_HI(txsegs[i].ds_addr));
1623                 desc->addr_lo = htole32(JME_ADDR_LO(txsegs[i].ds_addr));
1624                 desc->flags = htole32(JME_TD_OWN | flag64);
1625
1626                 sc->jme_cdata.jme_tx_cnt++;
1627                 KKASSERT(sc->jme_cdata.jme_tx_cnt <=
1628                          sc->jme_cdata.jme_tx_desc_cnt - JME_TXD_RSVD);
1629                 JME_DESC_INC(prod, sc->jme_cdata.jme_tx_desc_cnt);
1630         }
1631
1632         /* Update producer index. */
1633         sc->jme_cdata.jme_tx_prod = prod;
1634         /*
1635          * Finally request interrupt and give the first descriptor
1636          * owenership to hardware.
1637          */
1638         desc = txd->tx_desc;
1639         desc->flags |= htole32(JME_TD_OWN | JME_TD_INTR);
1640
1641         txd->tx_m = m;
1642         txd->tx_ndesc += nsegs;
1643
1644         return 0;
1645 fail:
1646         m_freem(*m_head);
1647         *m_head = NULL;
1648         return error;
1649 }
1650
1651 static void
1652 jme_start(struct ifnet *ifp)
1653 {
1654         struct jme_softc *sc = ifp->if_softc;
1655         struct mbuf *m_head;
1656         int enq = 0;
1657
1658         ASSERT_SERIALIZED(&sc->jme_cdata.jme_tx_serialize);
1659
1660         if (!sc->jme_has_link) {
1661                 ifq_purge(&ifp->if_snd);
1662                 return;
1663         }
1664
1665         if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
1666                 return;
1667
1668         if (sc->jme_cdata.jme_tx_cnt >= JME_TX_DESC_HIWAT(sc))
1669                 jme_txeof(sc);
1670
1671         while (!ifq_is_empty(&ifp->if_snd)) {
1672                 /*
1673                  * Check number of available TX descs, always
1674                  * leave JME_TXD_RSVD free TX descs.
1675                  */
1676                 if (sc->jme_cdata.jme_tx_cnt + sc->jme_txd_spare >
1677                     sc->jme_cdata.jme_tx_desc_cnt - JME_TXD_RSVD) {
1678                         ifp->if_flags |= IFF_OACTIVE;
1679                         break;
1680                 }
1681
1682                 m_head = ifq_dequeue(&ifp->if_snd, NULL);
1683                 if (m_head == NULL)
1684                         break;
1685
1686                 /*
1687                  * Pack the data into the transmit ring. If we
1688                  * don't have room, set the OACTIVE flag and wait
1689                  * for the NIC to drain the ring.
1690                  */
1691                 if (jme_encap(sc, &m_head)) {
1692                         KKASSERT(m_head == NULL);
1693                         ifp->if_oerrors++;
1694                         ifp->if_flags |= IFF_OACTIVE;
1695                         break;
1696                 }
1697                 enq++;
1698
1699                 /*
1700                  * If there's a BPF listener, bounce a copy of this frame
1701                  * to him.
1702                  */
1703                 ETHER_BPF_MTAP(ifp, m_head);
1704         }
1705
1706         if (enq > 0) {
1707                 /*
1708                  * Reading TXCSR takes very long time under heavy load
1709                  * so cache TXCSR value and writes the ORed value with
1710                  * the kick command to the TXCSR. This saves one register
1711                  * access cycle.
1712                  */
1713                 CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr | TXCSR_TX_ENB |
1714                     TXCSR_TXQ_N_START(TXCSR_TXQ0));
1715                 /* Set a timeout in case the chip goes out to lunch. */
1716                 ifp->if_timer = JME_TX_TIMEOUT;
1717         }
1718 }
1719
1720 static void
1721 jme_watchdog(struct ifnet *ifp)
1722 {
1723         struct jme_softc *sc = ifp->if_softc;
1724
1725         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1726
1727         if (!sc->jme_has_link) {
1728                 if_printf(ifp, "watchdog timeout (missed link)\n");
1729                 ifp->if_oerrors++;
1730                 jme_init(sc);
1731                 return;
1732         }
1733
1734         jme_txeof(sc);
1735         if (sc->jme_cdata.jme_tx_cnt == 0) {
1736                 if_printf(ifp, "watchdog timeout (missed Tx interrupts) "
1737                           "-- recovering\n");
1738                 if (!ifq_is_empty(&ifp->if_snd))
1739                         if_devstart(ifp);
1740                 return;
1741         }
1742
1743         if_printf(ifp, "watchdog timeout\n");
1744         ifp->if_oerrors++;
1745         jme_init(sc);
1746         if (!ifq_is_empty(&ifp->if_snd))
1747                 if_devstart(ifp);
1748 }
1749
1750 static int
1751 jme_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data, struct ucred *cr)
1752 {
1753         struct jme_softc *sc = ifp->if_softc;
1754         struct mii_data *mii = device_get_softc(sc->jme_miibus);
1755         struct ifreq *ifr = (struct ifreq *)data;
1756         int error = 0, mask;
1757
1758         ASSERT_IFNET_SERIALIZED_ALL(ifp);
1759
1760         switch (cmd) {
1761         case SIOCSIFMTU:
1762                 if (ifr->ifr_mtu < ETHERMIN || ifr->ifr_mtu > JME_JUMBO_MTU ||
1763                     (!(sc->jme_caps & JME_CAP_JUMBO) &&
1764                      ifr->ifr_mtu > JME_MAX_MTU)) {
1765                         error = EINVAL;
1766                         break;
1767                 }
1768
1769                 if (ifp->if_mtu != ifr->ifr_mtu) {
1770                         /*
1771                          * No special configuration is required when interface
1772                          * MTU is changed but availability of Tx checksum
1773                          * offload should be chcked against new MTU size as
1774                          * FIFO size is just 2K.
1775                          */
1776                         if (ifr->ifr_mtu >= JME_TX_FIFO_SIZE) {
1777                                 ifp->if_capenable &= ~IFCAP_TXCSUM;
1778                                 ifp->if_hwassist &= ~JME_CSUM_FEATURES;
1779                         }
1780                         ifp->if_mtu = ifr->ifr_mtu;
1781                         if (ifp->if_flags & IFF_RUNNING)
1782                                 jme_init(sc);
1783                 }
1784                 break;
1785
1786         case SIOCSIFFLAGS:
1787                 if (ifp->if_flags & IFF_UP) {
1788                         if (ifp->if_flags & IFF_RUNNING) {
1789                                 if ((ifp->if_flags ^ sc->jme_if_flags) &
1790                                     (IFF_PROMISC | IFF_ALLMULTI))
1791                                         jme_set_filter(sc);
1792                         } else {
1793                                 jme_init(sc);
1794                         }
1795                 } else {
1796                         if (ifp->if_flags & IFF_RUNNING)
1797                                 jme_stop(sc);
1798                 }
1799                 sc->jme_if_flags = ifp->if_flags;
1800                 break;
1801
1802         case SIOCADDMULTI:
1803         case SIOCDELMULTI:
1804                 if (ifp->if_flags & IFF_RUNNING)
1805                         jme_set_filter(sc);
1806                 break;
1807
1808         case SIOCSIFMEDIA:
1809         case SIOCGIFMEDIA:
1810                 error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd);
1811                 break;
1812
1813         case SIOCSIFCAP:
1814                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1815
1816                 if ((mask & IFCAP_TXCSUM) && ifp->if_mtu < JME_TX_FIFO_SIZE) {
1817                         ifp->if_capenable ^= IFCAP_TXCSUM;
1818                         if (IFCAP_TXCSUM & ifp->if_capenable)
1819                                 ifp->if_hwassist |= JME_CSUM_FEATURES;
1820                         else
1821                                 ifp->if_hwassist &= ~JME_CSUM_FEATURES;
1822                 }
1823                 if (mask & IFCAP_RXCSUM) {
1824                         uint32_t reg;
1825
1826                         ifp->if_capenable ^= IFCAP_RXCSUM;
1827                         reg = CSR_READ_4(sc, JME_RXMAC);
1828                         reg &= ~RXMAC_CSUM_ENB;
1829                         if (ifp->if_capenable & IFCAP_RXCSUM)
1830                                 reg |= RXMAC_CSUM_ENB;
1831                         CSR_WRITE_4(sc, JME_RXMAC, reg);
1832                 }
1833
1834                 if (mask & IFCAP_VLAN_HWTAGGING) {
1835                         ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1836                         jme_set_vlan(sc);
1837                 }
1838
1839                 if (mask & IFCAP_RSS)
1840                         ifp->if_capenable ^= IFCAP_RSS;
1841                 break;
1842
1843         default:
1844                 error = ether_ioctl(ifp, cmd, data);
1845                 break;
1846         }
1847         return (error);
1848 }
1849
1850 static void
1851 jme_mac_config(struct jme_softc *sc)
1852 {
1853         struct mii_data *mii;
1854         uint32_t ghc, rxmac, txmac, txpause, gp1;
1855         int phyconf = JMPHY_CONF_DEFFIFO, hdx = 0;
1856
1857         mii = device_get_softc(sc->jme_miibus);
1858
1859         CSR_WRITE_4(sc, JME_GHC, GHC_RESET);
1860         DELAY(10);
1861         CSR_WRITE_4(sc, JME_GHC, 0);
1862         ghc = 0;
1863         rxmac = CSR_READ_4(sc, JME_RXMAC);
1864         rxmac &= ~RXMAC_FC_ENB;
1865         txmac = CSR_READ_4(sc, JME_TXMAC);
1866         txmac &= ~(TXMAC_CARRIER_EXT | TXMAC_FRAME_BURST);
1867         txpause = CSR_READ_4(sc, JME_TXPFC);
1868         txpause &= ~TXPFC_PAUSE_ENB;
1869         if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) {
1870                 ghc |= GHC_FULL_DUPLEX;
1871                 rxmac &= ~RXMAC_COLL_DET_ENB;
1872                 txmac &= ~(TXMAC_COLL_ENB | TXMAC_CARRIER_SENSE |
1873                     TXMAC_BACKOFF | TXMAC_CARRIER_EXT |
1874                     TXMAC_FRAME_BURST);
1875 #ifdef notyet
1876                 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_TXPAUSE) != 0)
1877                         txpause |= TXPFC_PAUSE_ENB;
1878                 if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_RXPAUSE) != 0)
1879                         rxmac |= RXMAC_FC_ENB;
1880 #endif
1881                 /* Disable retry transmit timer/retry limit. */
1882                 CSR_WRITE_4(sc, JME_TXTRHD, CSR_READ_4(sc, JME_TXTRHD) &
1883                     ~(TXTRHD_RT_PERIOD_ENB | TXTRHD_RT_LIMIT_ENB));
1884         } else {
1885                 rxmac |= RXMAC_COLL_DET_ENB;
1886                 txmac |= TXMAC_COLL_ENB | TXMAC_CARRIER_SENSE | TXMAC_BACKOFF;
1887                 /* Enable retry transmit timer/retry limit. */
1888                 CSR_WRITE_4(sc, JME_TXTRHD, CSR_READ_4(sc, JME_TXTRHD) |
1889                     TXTRHD_RT_PERIOD_ENB | TXTRHD_RT_LIMIT_ENB);
1890         }
1891
1892         /*
1893          * Reprogram Tx/Rx MACs with resolved speed/duplex.
1894          */
1895         gp1 = CSR_READ_4(sc, JME_GPREG1);
1896         gp1 &= ~GPREG1_WA_HDX;
1897
1898         if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) == 0)
1899                 hdx = 1;
1900
1901         switch (IFM_SUBTYPE(mii->mii_media_active)) {
1902         case IFM_10_T:
1903                 ghc |= GHC_SPEED_10 | sc->jme_clksrc;
1904                 if (hdx)
1905                         gp1 |= GPREG1_WA_HDX;
1906                 break;
1907
1908         case IFM_100_TX:
1909                 ghc |= GHC_SPEED_100 | sc->jme_clksrc;
1910                 if (hdx)
1911                         gp1 |= GPREG1_WA_HDX;
1912
1913                 /*
1914                  * Use extended FIFO depth to workaround CRC errors
1915                  * emitted by chips before JMC250B
1916                  */
1917                 phyconf = JMPHY_CONF_EXTFIFO;
1918                 break;
1919
1920         case IFM_1000_T:
1921                 if (sc->jme_caps & JME_CAP_FASTETH)
1922                         break;
1923
1924                 ghc |= GHC_SPEED_1000 | sc->jme_clksrc_1000;
1925                 if (hdx)
1926                         txmac |= TXMAC_CARRIER_EXT | TXMAC_FRAME_BURST;
1927                 break;
1928
1929         default:
1930                 break;
1931         }
1932         CSR_WRITE_4(sc, JME_GHC, ghc);
1933         CSR_WRITE_4(sc, JME_RXMAC, rxmac);
1934         CSR_WRITE_4(sc, JME_TXMAC, txmac);
1935         CSR_WRITE_4(sc, JME_TXPFC, txpause);
1936
1937         if (sc->jme_workaround & JME_WA_EXTFIFO) {
1938                 jme_miibus_writereg(sc->jme_dev, sc->jme_phyaddr,
1939                                     JMPHY_CONF, phyconf);
1940         }
1941         if (sc->jme_workaround & JME_WA_HDX)
1942                 CSR_WRITE_4(sc, JME_GPREG1, gp1);
1943 }
1944
1945 static void
1946 jme_intr(void *xsc)
1947 {
1948         struct jme_softc *sc = xsc;
1949         struct ifnet *ifp = &sc->arpcom.ac_if;
1950         uint32_t status;
1951         int r;
1952
1953         ASSERT_SERIALIZED(&sc->jme_serialize);
1954
1955         status = CSR_READ_4(sc, JME_INTR_REQ_STATUS);
1956         if (status == 0 || status == 0xFFFFFFFF)
1957                 return;
1958
1959         /* Disable interrupts. */
1960         CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS);
1961
1962         status = CSR_READ_4(sc, JME_INTR_STATUS);
1963         if ((status & JME_INTRS) == 0 || status == 0xFFFFFFFF)
1964                 goto back;
1965
1966         /* Reset PCC counter/timer and Ack interrupts. */
1967         status &= ~(INTR_TXQ_COMP | INTR_RXQ_COMP);
1968
1969         if (status & (INTR_TXQ_COAL | INTR_TXQ_COAL_TO))
1970                 status |= INTR_TXQ_COAL | INTR_TXQ_COAL_TO | INTR_TXQ_COMP;
1971
1972         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
1973                 if (status & jme_rx_status[r].jme_coal) {
1974                         status |= jme_rx_status[r].jme_coal |
1975                                   jme_rx_status[r].jme_comp;
1976                 }
1977         }
1978
1979         CSR_WRITE_4(sc, JME_INTR_STATUS, status);
1980
1981         if (ifp->if_flags & IFF_RUNNING) {
1982                 if (status & (INTR_RXQ_COAL | INTR_RXQ_COAL_TO))
1983                         jme_rx_intr(sc, status);
1984
1985                 if (status & INTR_RXQ_DESC_EMPTY) {
1986                         /*
1987                          * Notify hardware availability of new Rx buffers.
1988                          * Reading RXCSR takes very long time under heavy
1989                          * load so cache RXCSR value and writes the ORed
1990                          * value with the kick command to the RXCSR. This
1991                          * saves one register access cycle.
1992                          */
1993                         CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr |
1994                             RXCSR_RX_ENB | RXCSR_RXQ_START);
1995                 }
1996
1997                 if (status & (INTR_TXQ_COAL | INTR_TXQ_COAL_TO)) {
1998                         lwkt_serialize_enter(&sc->jme_cdata.jme_tx_serialize);
1999                         jme_txeof(sc);
2000                         if (!ifq_is_empty(&ifp->if_snd))
2001                                 if_devstart(ifp);
2002                         lwkt_serialize_exit(&sc->jme_cdata.jme_tx_serialize);
2003                 }
2004         }
2005 back:
2006         /* Reenable interrupts. */
2007         CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS);
2008 }
2009
2010 static void
2011 jme_txeof(struct jme_softc *sc)
2012 {
2013         struct ifnet *ifp = &sc->arpcom.ac_if;
2014         int cons;
2015
2016         cons = sc->jme_cdata.jme_tx_cons;
2017         if (cons == sc->jme_cdata.jme_tx_prod)
2018                 return;
2019
2020         /*
2021          * Go through our Tx list and free mbufs for those
2022          * frames which have been transmitted.
2023          */
2024         while (cons != sc->jme_cdata.jme_tx_prod) {
2025                 struct jme_txdesc *txd, *next_txd;
2026                 uint32_t status, next_status;
2027                 int next_cons, nsegs;
2028
2029                 txd = &sc->jme_cdata.jme_txdesc[cons];
2030                 KASSERT(txd->tx_m != NULL,
2031                         ("%s: freeing NULL mbuf!", __func__));
2032
2033                 status = le32toh(txd->tx_desc->flags);
2034                 if ((status & JME_TD_OWN) == JME_TD_OWN)
2035                         break;
2036
2037                 /*
2038                  * NOTE:
2039                  * This chip will always update the TX descriptor's
2040                  * buflen field and this updating always happens
2041                  * after clearing the OWN bit, so even if the OWN
2042                  * bit is cleared by the chip, we still don't sure
2043                  * about whether the buflen field has been updated
2044                  * by the chip or not.  To avoid this race, we wait
2045                  * for the next TX descriptor's OWN bit to be cleared
2046                  * by the chip before reusing this TX descriptor.
2047                  */
2048                 next_cons = cons;
2049                 JME_DESC_ADD(next_cons, txd->tx_ndesc,
2050                     sc->jme_cdata.jme_tx_desc_cnt);
2051                 next_txd = &sc->jme_cdata.jme_txdesc[next_cons];
2052                 if (next_txd->tx_m == NULL)
2053                         break;
2054                 next_status = le32toh(next_txd->tx_desc->flags);
2055                 if ((next_status & JME_TD_OWN) == JME_TD_OWN)
2056                         break;
2057
2058                 if (status & (JME_TD_TMOUT | JME_TD_RETRY_EXP)) {
2059                         ifp->if_oerrors++;
2060                 } else {
2061                         ifp->if_opackets++;
2062                         if (status & JME_TD_COLLISION) {
2063                                 ifp->if_collisions +=
2064                                     le32toh(txd->tx_desc->buflen) &
2065                                     JME_TD_BUF_LEN_MASK;
2066                         }
2067                 }
2068
2069                 /*
2070                  * Only the first descriptor of multi-descriptor
2071                  * transmission is updated so driver have to skip entire
2072                  * chained buffers for the transmiited frame. In other
2073                  * words, JME_TD_OWN bit is valid only at the first
2074                  * descriptor of a multi-descriptor transmission.
2075                  */
2076                 for (nsegs = 0; nsegs < txd->tx_ndesc; nsegs++) {
2077                         sc->jme_cdata.jme_tx_ring[cons].flags = 0;
2078                         JME_DESC_INC(cons, sc->jme_cdata.jme_tx_desc_cnt);
2079                 }
2080
2081                 /* Reclaim transferred mbufs. */
2082                 bus_dmamap_unload(sc->jme_cdata.jme_tx_tag, txd->tx_dmamap);
2083                 m_freem(txd->tx_m);
2084                 txd->tx_m = NULL;
2085                 sc->jme_cdata.jme_tx_cnt -= txd->tx_ndesc;
2086                 KASSERT(sc->jme_cdata.jme_tx_cnt >= 0,
2087                         ("%s: Active Tx desc counter was garbled", __func__));
2088                 txd->tx_ndesc = 0;
2089         }
2090         sc->jme_cdata.jme_tx_cons = cons;
2091
2092         if (sc->jme_cdata.jme_tx_cnt < JME_MAXTXSEGS + 1)
2093                 ifp->if_timer = 0;
2094
2095         if (sc->jme_cdata.jme_tx_cnt + sc->jme_txd_spare <=
2096             sc->jme_cdata.jme_tx_desc_cnt - JME_TXD_RSVD)
2097                 ifp->if_flags &= ~IFF_OACTIVE;
2098 }
2099
2100 static __inline void
2101 jme_discard_rxbufs(struct jme_rxdata *rdata, int cons, int count)
2102 {
2103         int i;
2104
2105         for (i = 0; i < count; ++i) {
2106                 jme_setup_rxdesc(&rdata->jme_rxdesc[cons]);
2107                 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt);
2108         }
2109 }
2110
2111 static __inline struct pktinfo *
2112 jme_pktinfo(struct pktinfo *pi, uint32_t flags)
2113 {
2114         if (flags & JME_RD_IPV4)
2115                 pi->pi_netisr = NETISR_IP;
2116         else if (flags & JME_RD_IPV6)
2117                 pi->pi_netisr = NETISR_IPV6;
2118         else
2119                 return NULL;
2120
2121         pi->pi_flags = 0;
2122         pi->pi_l3proto = IPPROTO_UNKNOWN;
2123
2124         if (flags & JME_RD_MORE_FRAG)
2125                 pi->pi_flags |= PKTINFO_FLAG_FRAG;
2126         else if (flags & JME_RD_TCP)
2127                 pi->pi_l3proto = IPPROTO_TCP;
2128         else if (flags & JME_RD_UDP)
2129                 pi->pi_l3proto = IPPROTO_UDP;
2130         else
2131                 pi = NULL;
2132         return pi;
2133 }
2134
2135 /* Receive a frame. */
2136 static void
2137 jme_rxpkt(struct jme_rxdata *rdata)
2138 {
2139         struct ifnet *ifp = &rdata->jme_sc->arpcom.ac_if;
2140         struct jme_desc *desc;
2141         struct jme_rxdesc *rxd;
2142         struct mbuf *mp, *m;
2143         uint32_t flags, status, hash, hashinfo;
2144         int cons, count, nsegs;
2145
2146         cons = rdata->jme_rx_cons;
2147         desc = &rdata->jme_rx_ring[cons];
2148
2149         flags = le32toh(desc->flags);
2150         status = le32toh(desc->buflen);
2151         hash = le32toh(desc->addr_hi);
2152         hashinfo = le32toh(desc->addr_lo);
2153         nsegs = JME_RX_NSEGS(status);
2154
2155         if (nsegs > 1) {
2156                 /* Skip the first descriptor. */
2157                 JME_DESC_INC(cons, rdata->jme_rx_desc_cnt);
2158
2159                 /*
2160                  * Clear the OWN bit of the following RX descriptors;
2161                  * hardware will not clear the OWN bit except the first
2162                  * RX descriptor.
2163                  *
2164                  * Since the first RX descriptor is setup, i.e. OWN bit
2165                  * on, before its followins RX descriptors, leaving the
2166                  * OWN bit on the following RX descriptors will trick
2167                  * the hardware into thinking that the following RX
2168                  * descriptors are ready to be used too.
2169                  */
2170                 for (count = 1; count < nsegs; count++,
2171                      JME_DESC_INC(cons, rdata->jme_rx_desc_cnt))
2172                         rdata->jme_rx_ring[cons].flags = 0;
2173
2174                 cons = rdata->jme_rx_cons;
2175         }
2176
2177         JME_RSS_DPRINTF(rdata->jme_sc, 15, "ring%d, flags 0x%08x, "
2178                         "hash 0x%08x, hash info 0x%08x\n",
2179                         rdata->jme_rx_idx, flags, hash, hashinfo);
2180
2181         if (status & JME_RX_ERR_STAT) {
2182                 ifp->if_ierrors++;
2183                 jme_discard_rxbufs(rdata, cons, nsegs);
2184 #ifdef JME_SHOW_ERRORS
2185                 if_printf(ifp, "%s : receive error = 0x%b\n",
2186                     __func__, JME_RX_ERR(status), JME_RX_ERR_BITS);
2187 #endif
2188                 rdata->jme_rx_cons += nsegs;
2189                 rdata->jme_rx_cons %= rdata->jme_rx_desc_cnt;
2190                 return;
2191         }
2192
2193         rdata->jme_rxlen = JME_RX_BYTES(status) - JME_RX_PAD_BYTES;
2194         for (count = 0; count < nsegs; count++,
2195              JME_DESC_INC(cons, rdata->jme_rx_desc_cnt)) {
2196                 rxd = &rdata->jme_rxdesc[cons];
2197                 mp = rxd->rx_m;
2198
2199                 /* Add a new receive buffer to the ring. */
2200                 if (jme_newbuf(rdata, rxd, 0) != 0) {
2201                         ifp->if_iqdrops++;
2202                         /* Reuse buffer. */
2203                         jme_discard_rxbufs(rdata, cons, nsegs - count);
2204                         if (rdata->jme_rxhead != NULL) {
2205                                 m_freem(rdata->jme_rxhead);
2206                                 JME_RXCHAIN_RESET(rdata);
2207                         }
2208                         break;
2209                 }
2210
2211                 /*
2212                  * Assume we've received a full sized frame.
2213                  * Actual size is fixed when we encounter the end of
2214                  * multi-segmented frame.
2215                  */
2216                 mp->m_len = MCLBYTES;
2217
2218                 /* Chain received mbufs. */
2219                 if (rdata->jme_rxhead == NULL) {
2220                         rdata->jme_rxhead = mp;
2221                         rdata->jme_rxtail = mp;
2222                 } else {
2223                         /*
2224                          * Receive processor can receive a maximum frame
2225                          * size of 65535 bytes.
2226                          */
2227                         rdata->jme_rxtail->m_next = mp;
2228                         rdata->jme_rxtail = mp;
2229                 }
2230
2231                 if (count == nsegs - 1) {
2232                         struct pktinfo pi0, *pi;
2233
2234                         /* Last desc. for this frame. */
2235                         m = rdata->jme_rxhead;
2236                         m->m_pkthdr.len = rdata->jme_rxlen;
2237                         if (nsegs > 1) {
2238                                 /* Set first mbuf size. */
2239                                 m->m_len = MCLBYTES - JME_RX_PAD_BYTES;
2240                                 /* Set last mbuf size. */
2241                                 mp->m_len = rdata->jme_rxlen -
2242                                     ((MCLBYTES - JME_RX_PAD_BYTES) +
2243                                     (MCLBYTES * (nsegs - 2)));
2244                         } else {
2245                                 m->m_len = rdata->jme_rxlen;
2246                         }
2247                         m->m_pkthdr.rcvif = ifp;
2248
2249                         /*
2250                          * Account for 10bytes auto padding which is used
2251                          * to align IP header on 32bit boundary. Also note,
2252                          * CRC bytes is automatically removed by the
2253                          * hardware.
2254                          */
2255                         m->m_data += JME_RX_PAD_BYTES;
2256
2257                         /* Set checksum information. */
2258                         if ((ifp->if_capenable & IFCAP_RXCSUM) &&
2259                             (flags & JME_RD_IPV4)) {
2260                                 m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
2261                                 if (flags & JME_RD_IPCSUM)
2262                                         m->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2263                                 if ((flags & JME_RD_MORE_FRAG) == 0 &&
2264                                     ((flags & (JME_RD_TCP | JME_RD_TCPCSUM)) ==
2265                                      (JME_RD_TCP | JME_RD_TCPCSUM) ||
2266                                      (flags & (JME_RD_UDP | JME_RD_UDPCSUM)) ==
2267                                      (JME_RD_UDP | JME_RD_UDPCSUM))) {
2268                                         m->m_pkthdr.csum_flags |=
2269                                             CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
2270                                         m->m_pkthdr.csum_data = 0xffff;
2271                                 }
2272                         }
2273
2274                         /* Check for VLAN tagged packets. */
2275                         if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) &&
2276                             (flags & JME_RD_VLAN_TAG)) {
2277                                 m->m_pkthdr.ether_vlantag =
2278                                     flags & JME_RD_VLAN_MASK;
2279                                 m->m_flags |= M_VLANTAG;
2280                         }
2281
2282                         ifp->if_ipackets++;
2283
2284                         if (ifp->if_capenable & IFCAP_RSS)
2285                                 pi = jme_pktinfo(&pi0, flags);
2286                         else
2287                                 pi = NULL;
2288
2289                         if (pi != NULL &&
2290                             (hashinfo & JME_RD_HASH_FN_MASK) != 0) {
2291                                 m->m_flags |= M_HASH;
2292                                 m->m_pkthdr.hash = toeplitz_hash(hash);
2293                         }
2294
2295 #ifdef JME_RSS_DEBUG
2296                         if (pi != NULL) {
2297                                 JME_RSS_DPRINTF(rdata->jme_sc, 10,
2298                                     "isr %d flags %08x, l3 %d %s\n",
2299                                     pi->pi_netisr, pi->pi_flags,
2300                                     pi->pi_l3proto,
2301                                     (m->m_flags & M_HASH) ? "hash" : "");
2302                         }
2303 #endif
2304
2305                         /* Pass it on. */
2306                         ether_input_pkt(ifp, m, pi);
2307
2308                         /* Reset mbuf chains. */
2309                         JME_RXCHAIN_RESET(rdata);
2310 #ifdef JME_RSS_DEBUG
2311                         rdata->jme_rx_pkt++;
2312 #endif
2313                 }
2314         }
2315
2316         rdata->jme_rx_cons += nsegs;
2317         rdata->jme_rx_cons %= rdata->jme_rx_desc_cnt;
2318 }
2319
2320 static void
2321 jme_rxeof(struct jme_rxdata *rdata, int count)
2322 {
2323         struct jme_desc *desc;
2324         int nsegs, pktlen;
2325
2326         for (;;) {
2327 #ifdef DEVICE_POLLING
2328                 if (count >= 0 && count-- == 0)
2329                         break;
2330 #endif
2331                 desc = &rdata->jme_rx_ring[rdata->jme_rx_cons];
2332                 if ((le32toh(desc->flags) & JME_RD_OWN) == JME_RD_OWN)
2333                         break;
2334                 if ((le32toh(desc->buflen) & JME_RD_VALID) == 0)
2335                         break;
2336
2337                 /*
2338                  * Check number of segments against received bytes.
2339                  * Non-matching value would indicate that hardware
2340                  * is still trying to update Rx descriptors. I'm not
2341                  * sure whether this check is needed.
2342                  */
2343                 nsegs = JME_RX_NSEGS(le32toh(desc->buflen));
2344                 pktlen = JME_RX_BYTES(le32toh(desc->buflen));
2345                 if (nsegs != howmany(pktlen, MCLBYTES)) {
2346                         if_printf(&rdata->jme_sc->arpcom.ac_if,
2347                             "RX fragment count(%d) and "
2348                             "packet size(%d) mismach\n", nsegs, pktlen);
2349                         break;
2350                 }
2351
2352                 /*
2353                  * NOTE:
2354                  * RSS hash and hash information may _not_ be set by the
2355                  * hardware even if the OWN bit is cleared and VALID bit
2356                  * is set.
2357                  *
2358                  * If the RSS information is not delivered by the hardware
2359                  * yet, we MUST NOT accept this packet, let alone reusing
2360                  * its RX descriptor.  If this packet was accepted and its
2361                  * RX descriptor was reused before hardware delivering the
2362                  * RSS information, the RX buffer's address would be trashed
2363                  * by the RSS information delivered by the hardware.
2364                  */
2365                 if (JME_ENABLE_HWRSS(rdata->jme_sc)) {
2366                         struct jme_rxdesc *rxd;
2367                         uint32_t hashinfo;
2368
2369                         hashinfo = le32toh(desc->addr_lo);
2370                         rxd = &rdata->jme_rxdesc[rdata->jme_rx_cons];
2371
2372                         /*
2373                          * This test should be enough to detect the pending
2374                          * RSS information delivery, given:
2375                          * - If RSS hash is not calculated, the hashinfo
2376                          *   will be 0.  Howvever, the lower 32bits of RX
2377                          *   buffers' physical address will never be 0.
2378                          *   (see jme_rxbuf_dma_filter)
2379                          * - If RSS hash is calculated, the lowest 4 bits
2380                          *   of hashinfo will be set, while the RX buffers
2381                          *   are at least 2K aligned.
2382                          */
2383                         if (hashinfo == JME_ADDR_LO(rxd->rx_paddr)) {
2384 #ifdef JME_SHOW_RSSWB
2385                                 if_printf(&rdata->jme_sc->arpcom.ac_if,
2386                                     "RSS is not written back yet\n");
2387 #endif
2388                                 break;
2389                         }
2390                 }
2391
2392                 /* Received a frame. */
2393                 jme_rxpkt(rdata);
2394         }
2395 }
2396
2397 static void
2398 jme_tick(void *xsc)
2399 {
2400         struct jme_softc *sc = xsc;
2401         struct mii_data *mii = device_get_softc(sc->jme_miibus);
2402
2403         lwkt_serialize_enter(&sc->jme_serialize);
2404
2405         sc->jme_in_tick = TRUE;
2406         mii_tick(mii);
2407         sc->jme_in_tick = FALSE;
2408
2409         callout_reset(&sc->jme_tick_ch, hz, jme_tick, sc);
2410
2411         lwkt_serialize_exit(&sc->jme_serialize);
2412 }
2413
2414 static void
2415 jme_reset(struct jme_softc *sc)
2416 {
2417         uint32_t val;
2418
2419         /* Make sure that TX and RX are stopped */
2420         jme_stop_tx(sc);
2421         jme_stop_rx(sc);
2422
2423         /* Start reset */
2424         CSR_WRITE_4(sc, JME_GHC, GHC_RESET);
2425         DELAY(20);
2426
2427         /*
2428          * Hold reset bit before stop reset
2429          */
2430
2431         /* Disable TXMAC and TXOFL clock sources */
2432         CSR_WRITE_4(sc, JME_GHC, GHC_RESET);
2433         /* Disable RXMAC clock source */
2434         val = CSR_READ_4(sc, JME_GPREG1);
2435         CSR_WRITE_4(sc, JME_GPREG1, val | GPREG1_DIS_RXMAC_CLKSRC);
2436         /* Flush */
2437         CSR_READ_4(sc, JME_GHC);
2438
2439         /* Stop reset */
2440         CSR_WRITE_4(sc, JME_GHC, 0);
2441         /* Flush */
2442         CSR_READ_4(sc, JME_GHC);
2443
2444         /*
2445          * Clear reset bit after stop reset
2446          */
2447
2448         /* Enable TXMAC and TXOFL clock sources */
2449         CSR_WRITE_4(sc, JME_GHC, GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC);
2450         /* Enable RXMAC clock source */
2451         val = CSR_READ_4(sc, JME_GPREG1);
2452         CSR_WRITE_4(sc, JME_GPREG1, val & ~GPREG1_DIS_RXMAC_CLKSRC);
2453         /* Flush */
2454         CSR_READ_4(sc, JME_GHC);
2455
2456         /* Disable TXMAC and TXOFL clock sources */
2457         CSR_WRITE_4(sc, JME_GHC, 0);
2458         /* Disable RXMAC clock source */
2459         val = CSR_READ_4(sc, JME_GPREG1);
2460         CSR_WRITE_4(sc, JME_GPREG1, val | GPREG1_DIS_RXMAC_CLKSRC);
2461         /* Flush */
2462         CSR_READ_4(sc, JME_GHC);
2463
2464         /* Enable TX and RX */
2465         val = CSR_READ_4(sc, JME_TXCSR);
2466         CSR_WRITE_4(sc, JME_TXCSR, val | TXCSR_TX_ENB);
2467         val = CSR_READ_4(sc, JME_RXCSR);
2468         CSR_WRITE_4(sc, JME_RXCSR, val | RXCSR_RX_ENB);
2469         /* Flush */
2470         CSR_READ_4(sc, JME_TXCSR);
2471         CSR_READ_4(sc, JME_RXCSR);
2472
2473         /* Enable TXMAC and TXOFL clock sources */
2474         CSR_WRITE_4(sc, JME_GHC, GHC_TXOFL_CLKSRC | GHC_TXMAC_CLKSRC);
2475         /* Eisable RXMAC clock source */
2476         val = CSR_READ_4(sc, JME_GPREG1);
2477         CSR_WRITE_4(sc, JME_GPREG1, val & ~GPREG1_DIS_RXMAC_CLKSRC);
2478         /* Flush */
2479         CSR_READ_4(sc, JME_GHC);
2480
2481         /* Stop TX and RX */
2482         jme_stop_tx(sc);
2483         jme_stop_rx(sc);
2484 }
2485
2486 static void
2487 jme_init(void *xsc)
2488 {
2489         struct jme_softc *sc = xsc;
2490         struct ifnet *ifp = &sc->arpcom.ac_if;
2491         struct mii_data *mii;
2492         uint8_t eaddr[ETHER_ADDR_LEN];
2493         bus_addr_t paddr;
2494         uint32_t reg;
2495         int error, r;
2496
2497         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2498
2499         /*
2500          * Cancel any pending I/O.
2501          */
2502         jme_stop(sc);
2503
2504         /*
2505          * Reset the chip to a known state.
2506          */
2507         jme_reset(sc);
2508
2509         /*
2510          * Setup MSI/MSI-X vectors to interrupts mapping
2511          */
2512         jme_set_msinum(sc);
2513
2514         sc->jme_txd_spare =
2515         howmany(ifp->if_mtu + sizeof(struct ether_vlan_header), MCLBYTES);
2516         KKASSERT(sc->jme_txd_spare >= 1);
2517
2518         /*
2519          * If we use 64bit address mode for transmitting, each Tx request
2520          * needs one more symbol descriptor.
2521          */
2522         if (sc->jme_lowaddr != BUS_SPACE_MAXADDR_32BIT)
2523                 sc->jme_txd_spare += 1;
2524
2525         if (JME_ENABLE_HWRSS(sc))
2526                 jme_enable_rss(sc);
2527         else
2528                 jme_disable_rss(sc);
2529
2530         /* Init RX descriptors */
2531         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
2532                 error = jme_init_rx_ring(&sc->jme_cdata.jme_rx_data[r]);
2533                 if (error) {
2534                         if_printf(ifp, "initialization failed: "
2535                                   "no memory for %dth RX ring.\n", r);
2536                         jme_stop(sc);
2537                         return;
2538                 }
2539         }
2540
2541         /* Init TX descriptors */
2542         jme_init_tx_ring(sc);
2543
2544         /* Initialize shadow status block. */
2545         jme_init_ssb(sc);
2546
2547         /* Reprogram the station address. */
2548         bcopy(IF_LLADDR(ifp), eaddr, ETHER_ADDR_LEN);
2549         CSR_WRITE_4(sc, JME_PAR0,
2550             eaddr[3] << 24 | eaddr[2] << 16 | eaddr[1] << 8 | eaddr[0]);
2551         CSR_WRITE_4(sc, JME_PAR1, eaddr[5] << 8 | eaddr[4]);
2552
2553         /*
2554          * Configure Tx queue.
2555          *  Tx priority queue weight value : 0
2556          *  Tx FIFO threshold for processing next packet : 16QW
2557          *  Maximum Tx DMA length : 512
2558          *  Allow Tx DMA burst.
2559          */
2560         sc->jme_txcsr = TXCSR_TXQ_N_SEL(TXCSR_TXQ0);
2561         sc->jme_txcsr |= TXCSR_TXQ_WEIGHT(TXCSR_TXQ_WEIGHT_MIN);
2562         sc->jme_txcsr |= TXCSR_FIFO_THRESH_16QW;
2563         sc->jme_txcsr |= sc->jme_tx_dma_size;
2564         sc->jme_txcsr |= TXCSR_DMA_BURST;
2565         CSR_WRITE_4(sc, JME_TXCSR, sc->jme_txcsr);
2566
2567         /* Set Tx descriptor counter. */
2568         CSR_WRITE_4(sc, JME_TXQDC, sc->jme_cdata.jme_tx_desc_cnt);
2569
2570         /* Set Tx ring address to the hardware. */
2571         paddr = sc->jme_cdata.jme_tx_ring_paddr;
2572         CSR_WRITE_4(sc, JME_TXDBA_HI, JME_ADDR_HI(paddr));
2573         CSR_WRITE_4(sc, JME_TXDBA_LO, JME_ADDR_LO(paddr));
2574
2575         /* Configure TxMAC parameters. */
2576         reg = TXMAC_IFG1_DEFAULT | TXMAC_IFG2_DEFAULT | TXMAC_IFG_ENB;
2577         reg |= TXMAC_THRESH_1_PKT;
2578         reg |= TXMAC_CRC_ENB | TXMAC_PAD_ENB;
2579         CSR_WRITE_4(sc, JME_TXMAC, reg);
2580
2581         /*
2582          * Configure Rx queue.
2583          *  FIFO full threshold for transmitting Tx pause packet : 128T
2584          *  FIFO threshold for processing next packet : 128QW
2585          *  Rx queue 0 select
2586          *  Max Rx DMA length : 128
2587          *  Rx descriptor retry : 32
2588          *  Rx descriptor retry time gap : 256ns
2589          *  Don't receive runt/bad frame.
2590          */
2591         sc->jme_rxcsr = RXCSR_FIFO_FTHRESH_128T;
2592 #if 0
2593         /*
2594          * Since Rx FIFO size is 4K bytes, receiving frames larger
2595          * than 4K bytes will suffer from Rx FIFO overruns. So
2596          * decrease FIFO threshold to reduce the FIFO overruns for
2597          * frames larger than 4000 bytes.
2598          * For best performance of standard MTU sized frames use
2599          * maximum allowable FIFO threshold, 128QW.
2600          */
2601         if ((ifp->if_mtu + ETHER_HDR_LEN + EVL_ENCAPLEN + ETHER_CRC_LEN) >
2602             JME_RX_FIFO_SIZE)
2603                 sc->jme_rxcsr |= RXCSR_FIFO_THRESH_16QW;
2604         else
2605                 sc->jme_rxcsr |= RXCSR_FIFO_THRESH_128QW;
2606 #else
2607         /* Improve PCI Express compatibility */
2608         sc->jme_rxcsr |= RXCSR_FIFO_THRESH_16QW;
2609 #endif
2610         sc->jme_rxcsr |= sc->jme_rx_dma_size;
2611         sc->jme_rxcsr |= RXCSR_DESC_RT_CNT(RXCSR_DESC_RT_CNT_DEFAULT);
2612         sc->jme_rxcsr |= RXCSR_DESC_RT_GAP_256 & RXCSR_DESC_RT_GAP_MASK;
2613         /* XXX TODO DROP_BAD */
2614
2615         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
2616                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r];
2617
2618                 CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr | RXCSR_RXQ_N_SEL(r));
2619
2620                 /* Set Rx descriptor counter. */
2621                 CSR_WRITE_4(sc, JME_RXQDC, rdata->jme_rx_desc_cnt);
2622
2623                 /* Set Rx ring address to the hardware. */
2624                 paddr = rdata->jme_rx_ring_paddr;
2625                 CSR_WRITE_4(sc, JME_RXDBA_HI, JME_ADDR_HI(paddr));
2626                 CSR_WRITE_4(sc, JME_RXDBA_LO, JME_ADDR_LO(paddr));
2627         }
2628
2629         /* Clear receive filter. */
2630         CSR_WRITE_4(sc, JME_RXMAC, 0);
2631
2632         /* Set up the receive filter. */
2633         jme_set_filter(sc);
2634         jme_set_vlan(sc);
2635
2636         /*
2637          * Disable all WOL bits as WOL can interfere normal Rx
2638          * operation. Also clear WOL detection status bits.
2639          */
2640         reg = CSR_READ_4(sc, JME_PMCS);
2641         reg &= ~PMCS_WOL_ENB_MASK;
2642         CSR_WRITE_4(sc, JME_PMCS, reg);
2643
2644         /*
2645          * Pad 10bytes right before received frame. This will greatly
2646          * help Rx performance on strict-alignment architectures as
2647          * it does not need to copy the frame to align the payload.
2648          */
2649         reg = CSR_READ_4(sc, JME_RXMAC);
2650         reg |= RXMAC_PAD_10BYTES;
2651
2652         if (ifp->if_capenable & IFCAP_RXCSUM)
2653                 reg |= RXMAC_CSUM_ENB;
2654         CSR_WRITE_4(sc, JME_RXMAC, reg);
2655
2656         /* Configure general purpose reg0 */
2657         reg = CSR_READ_4(sc, JME_GPREG0);
2658         reg &= ~GPREG0_PCC_UNIT_MASK;
2659         /* Set PCC timer resolution to micro-seconds unit. */
2660         reg |= GPREG0_PCC_UNIT_US;
2661         /*
2662          * Disable all shadow register posting as we have to read
2663          * JME_INTR_STATUS register in jme_intr. Also it seems
2664          * that it's hard to synchronize interrupt status between
2665          * hardware and software with shadow posting due to
2666          * requirements of bus_dmamap_sync(9).
2667          */
2668         reg |= GPREG0_SH_POST_DW7_DIS | GPREG0_SH_POST_DW6_DIS |
2669             GPREG0_SH_POST_DW5_DIS | GPREG0_SH_POST_DW4_DIS |
2670             GPREG0_SH_POST_DW3_DIS | GPREG0_SH_POST_DW2_DIS |
2671             GPREG0_SH_POST_DW1_DIS | GPREG0_SH_POST_DW0_DIS;
2672         /* Disable posting of DW0. */
2673         reg &= ~GPREG0_POST_DW0_ENB;
2674         /* Clear PME message. */
2675         reg &= ~GPREG0_PME_ENB;
2676         /* Set PHY address. */
2677         reg &= ~GPREG0_PHY_ADDR_MASK;
2678         reg |= sc->jme_phyaddr;
2679         CSR_WRITE_4(sc, JME_GPREG0, reg);
2680
2681         /* Configure Tx queue 0 packet completion coalescing. */
2682         jme_set_tx_coal(sc);
2683
2684         /* Configure Rx queues packet completion coalescing. */
2685         jme_set_rx_coal(sc);
2686
2687         /* Configure shadow status block but don't enable posting. */
2688         paddr = sc->jme_cdata.jme_ssb_block_paddr;
2689         CSR_WRITE_4(sc, JME_SHBASE_ADDR_HI, JME_ADDR_HI(paddr));
2690         CSR_WRITE_4(sc, JME_SHBASE_ADDR_LO, JME_ADDR_LO(paddr));
2691
2692         /* Disable Timer 1 and Timer 2. */
2693         CSR_WRITE_4(sc, JME_TIMER1, 0);
2694         CSR_WRITE_4(sc, JME_TIMER2, 0);
2695
2696         /* Configure retry transmit period, retry limit value. */
2697         CSR_WRITE_4(sc, JME_TXTRHD,
2698             ((TXTRHD_RT_PERIOD_DEFAULT << TXTRHD_RT_PERIOD_SHIFT) &
2699             TXTRHD_RT_PERIOD_MASK) |
2700             ((TXTRHD_RT_LIMIT_DEFAULT << TXTRHD_RT_LIMIT_SHIFT) &
2701             TXTRHD_RT_LIMIT_SHIFT));
2702
2703 #ifdef DEVICE_POLLING
2704         if (!(ifp->if_flags & IFF_POLLING))
2705 #endif
2706         /* Initialize the interrupt mask. */
2707         CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS);
2708         CSR_WRITE_4(sc, JME_INTR_STATUS, 0xFFFFFFFF);
2709
2710         /*
2711          * Enabling Tx/Rx DMA engines and Rx queue processing is
2712          * done after detection of valid link in jme_miibus_statchg.
2713          */
2714         sc->jme_has_link = FALSE;
2715
2716         /* Set the current media. */
2717         mii = device_get_softc(sc->jme_miibus);
2718         mii_mediachg(mii);
2719
2720         callout_reset(&sc->jme_tick_ch, hz, jme_tick, sc);
2721
2722         ifp->if_flags |= IFF_RUNNING;
2723         ifp->if_flags &= ~IFF_OACTIVE;
2724 }
2725
2726 static void
2727 jme_stop(struct jme_softc *sc)
2728 {
2729         struct ifnet *ifp = &sc->arpcom.ac_if;
2730         struct jme_txdesc *txd;
2731         struct jme_rxdesc *rxd;
2732         struct jme_rxdata *rdata;
2733         int i, r;
2734
2735         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2736
2737         /*
2738          * Mark the interface down and cancel the watchdog timer.
2739          */
2740         ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
2741         ifp->if_timer = 0;
2742
2743         callout_stop(&sc->jme_tick_ch);
2744         sc->jme_has_link = FALSE;
2745
2746         /*
2747          * Disable interrupts.
2748          */
2749         CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS);
2750         CSR_WRITE_4(sc, JME_INTR_STATUS, 0xFFFFFFFF);
2751
2752         /* Disable updating shadow status block. */
2753         CSR_WRITE_4(sc, JME_SHBASE_ADDR_LO,
2754             CSR_READ_4(sc, JME_SHBASE_ADDR_LO) & ~SHBASE_POST_ENB);
2755
2756         /* Stop receiver, transmitter. */
2757         jme_stop_rx(sc);
2758         jme_stop_tx(sc);
2759
2760         /*
2761          * Free partial finished RX segments
2762          */
2763         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
2764                 rdata = &sc->jme_cdata.jme_rx_data[r];
2765                 if (rdata->jme_rxhead != NULL)
2766                         m_freem(rdata->jme_rxhead);
2767                 JME_RXCHAIN_RESET(rdata);
2768         }
2769
2770         /*
2771          * Free RX and TX mbufs still in the queues.
2772          */
2773         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
2774                 rdata = &sc->jme_cdata.jme_rx_data[r];
2775                 for (i = 0; i < rdata->jme_rx_desc_cnt; i++) {
2776                         rxd = &rdata->jme_rxdesc[i];
2777                         if (rxd->rx_m != NULL) {
2778                                 bus_dmamap_unload(rdata->jme_rx_tag,
2779                                                   rxd->rx_dmamap);
2780                                 m_freem(rxd->rx_m);
2781                                 rxd->rx_m = NULL;
2782                         }
2783                 }
2784         }
2785         for (i = 0; i < sc->jme_cdata.jme_tx_desc_cnt; i++) {
2786                 txd = &sc->jme_cdata.jme_txdesc[i];
2787                 if (txd->tx_m != NULL) {
2788                         bus_dmamap_unload(sc->jme_cdata.jme_tx_tag,
2789                             txd->tx_dmamap);
2790                         m_freem(txd->tx_m);
2791                         txd->tx_m = NULL;
2792                         txd->tx_ndesc = 0;
2793                 }
2794         }
2795 }
2796
2797 static void
2798 jme_stop_tx(struct jme_softc *sc)
2799 {
2800         uint32_t reg;
2801         int i;
2802
2803         reg = CSR_READ_4(sc, JME_TXCSR);
2804         if ((reg & TXCSR_TX_ENB) == 0)
2805                 return;
2806         reg &= ~TXCSR_TX_ENB;
2807         CSR_WRITE_4(sc, JME_TXCSR, reg);
2808         for (i = JME_TIMEOUT; i > 0; i--) {
2809                 DELAY(1);
2810                 if ((CSR_READ_4(sc, JME_TXCSR) & TXCSR_TX_ENB) == 0)
2811                         break;
2812         }
2813         if (i == 0)
2814                 device_printf(sc->jme_dev, "stopping transmitter timeout!\n");
2815 }
2816
2817 static void
2818 jme_stop_rx(struct jme_softc *sc)
2819 {
2820         uint32_t reg;
2821         int i;
2822
2823         reg = CSR_READ_4(sc, JME_RXCSR);
2824         if ((reg & RXCSR_RX_ENB) == 0)
2825                 return;
2826         reg &= ~RXCSR_RX_ENB;
2827         CSR_WRITE_4(sc, JME_RXCSR, reg);
2828         for (i = JME_TIMEOUT; i > 0; i--) {
2829                 DELAY(1);
2830                 if ((CSR_READ_4(sc, JME_RXCSR) & RXCSR_RX_ENB) == 0)
2831                         break;
2832         }
2833         if (i == 0)
2834                 device_printf(sc->jme_dev, "stopping recevier timeout!\n");
2835 }
2836
2837 static void
2838 jme_init_tx_ring(struct jme_softc *sc)
2839 {
2840         struct jme_chain_data *cd;
2841         struct jme_txdesc *txd;
2842         int i;
2843
2844         sc->jme_cdata.jme_tx_prod = 0;
2845         sc->jme_cdata.jme_tx_cons = 0;
2846         sc->jme_cdata.jme_tx_cnt = 0;
2847
2848         cd = &sc->jme_cdata;
2849         bzero(cd->jme_tx_ring, JME_TX_RING_SIZE(sc));
2850         for (i = 0; i < sc->jme_cdata.jme_tx_desc_cnt; i++) {
2851                 txd = &sc->jme_cdata.jme_txdesc[i];
2852                 txd->tx_m = NULL;
2853                 txd->tx_desc = &cd->jme_tx_ring[i];
2854                 txd->tx_ndesc = 0;
2855         }
2856 }
2857
2858 static void
2859 jme_init_ssb(struct jme_softc *sc)
2860 {
2861         struct jme_chain_data *cd;
2862
2863         cd = &sc->jme_cdata;
2864         bzero(cd->jme_ssb_block, JME_SSB_SIZE);
2865 }
2866
2867 static int
2868 jme_init_rx_ring(struct jme_rxdata *rdata)
2869 {
2870         struct jme_rxdesc *rxd;
2871         int i;
2872
2873         KKASSERT(rdata->jme_rxhead == NULL &&
2874                  rdata->jme_rxtail == NULL &&
2875                  rdata->jme_rxlen == 0);
2876         rdata->jme_rx_cons = 0;
2877
2878         bzero(rdata->jme_rx_ring, JME_RX_RING_SIZE(rdata));
2879         for (i = 0; i < rdata->jme_rx_desc_cnt; i++) {
2880                 int error;
2881
2882                 rxd = &rdata->jme_rxdesc[i];
2883                 rxd->rx_m = NULL;
2884                 rxd->rx_desc = &rdata->jme_rx_ring[i];
2885                 error = jme_newbuf(rdata, rxd, 1);
2886                 if (error)
2887                         return error;
2888         }
2889         return 0;
2890 }
2891
2892 static int
2893 jme_newbuf(struct jme_rxdata *rdata, struct jme_rxdesc *rxd, int init)
2894 {
2895         struct mbuf *m;
2896         bus_dma_segment_t segs;
2897         bus_dmamap_t map;
2898         int error, nsegs;
2899
2900         m = m_getcl(init ? MB_WAIT : MB_DONTWAIT, MT_DATA, M_PKTHDR);
2901         if (m == NULL)
2902                 return ENOBUFS;
2903         /*
2904          * JMC250 has 64bit boundary alignment limitation so jme(4)
2905          * takes advantage of 10 bytes padding feature of hardware
2906          * in order not to copy entire frame to align IP header on
2907          * 32bit boundary.
2908          */
2909         m->m_len = m->m_pkthdr.len = MCLBYTES;
2910
2911         error = bus_dmamap_load_mbuf_segment(rdata->jme_rx_tag,
2912                         rdata->jme_rx_sparemap, m, &segs, 1, &nsegs,
2913                         BUS_DMA_NOWAIT);
2914         if (error) {
2915                 m_freem(m);
2916                 if (init) {
2917                         if_printf(&rdata->jme_sc->arpcom.ac_if,
2918                             "can't load RX mbuf\n");
2919                 }
2920                 return error;
2921         }
2922
2923         if (rxd->rx_m != NULL) {
2924                 bus_dmamap_sync(rdata->jme_rx_tag, rxd->rx_dmamap,
2925                                 BUS_DMASYNC_POSTREAD);
2926                 bus_dmamap_unload(rdata->jme_rx_tag, rxd->rx_dmamap);
2927         }
2928         map = rxd->rx_dmamap;
2929         rxd->rx_dmamap = rdata->jme_rx_sparemap;
2930         rdata->jme_rx_sparemap = map;
2931         rxd->rx_m = m;
2932         rxd->rx_paddr = segs.ds_addr;
2933
2934         jme_setup_rxdesc(rxd);
2935         return 0;
2936 }
2937
2938 static void
2939 jme_set_vlan(struct jme_softc *sc)
2940 {
2941         struct ifnet *ifp = &sc->arpcom.ac_if;
2942         uint32_t reg;
2943
2944         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2945
2946         reg = CSR_READ_4(sc, JME_RXMAC);
2947         reg &= ~RXMAC_VLAN_ENB;
2948         if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2949                 reg |= RXMAC_VLAN_ENB;
2950         CSR_WRITE_4(sc, JME_RXMAC, reg);
2951 }
2952
2953 static void
2954 jme_set_filter(struct jme_softc *sc)
2955 {
2956         struct ifnet *ifp = &sc->arpcom.ac_if;
2957         struct ifmultiaddr *ifma;
2958         uint32_t crc;
2959         uint32_t mchash[2];
2960         uint32_t rxcfg;
2961
2962         ASSERT_IFNET_SERIALIZED_ALL(ifp);
2963
2964         rxcfg = CSR_READ_4(sc, JME_RXMAC);
2965         rxcfg &= ~(RXMAC_BROADCAST | RXMAC_PROMISC | RXMAC_MULTICAST |
2966             RXMAC_ALLMULTI);
2967
2968         /*
2969          * Always accept frames destined to our station address.
2970          * Always accept broadcast frames.
2971          */
2972         rxcfg |= RXMAC_UNICAST | RXMAC_BROADCAST;
2973
2974         if (ifp->if_flags & (IFF_PROMISC | IFF_ALLMULTI)) {
2975                 if (ifp->if_flags & IFF_PROMISC)
2976                         rxcfg |= RXMAC_PROMISC;
2977                 if (ifp->if_flags & IFF_ALLMULTI)
2978                         rxcfg |= RXMAC_ALLMULTI;
2979                 CSR_WRITE_4(sc, JME_MAR0, 0xFFFFFFFF);
2980                 CSR_WRITE_4(sc, JME_MAR1, 0xFFFFFFFF);
2981                 CSR_WRITE_4(sc, JME_RXMAC, rxcfg);
2982                 return;
2983         }
2984
2985         /*
2986          * Set up the multicast address filter by passing all multicast
2987          * addresses through a CRC generator, and then using the low-order
2988          * 6 bits as an index into the 64 bit multicast hash table.  The
2989          * high order bits select the register, while the rest of the bits
2990          * select the bit within the register.
2991          */
2992         rxcfg |= RXMAC_MULTICAST;
2993         bzero(mchash, sizeof(mchash));
2994
2995         TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2996                 if (ifma->ifma_addr->sa_family != AF_LINK)
2997                         continue;
2998                 crc = ether_crc32_be(LLADDR((struct sockaddr_dl *)
2999                     ifma->ifma_addr), ETHER_ADDR_LEN);
3000
3001                 /* Just want the 6 least significant bits. */
3002                 crc &= 0x3f;
3003
3004                 /* Set the corresponding bit in the hash table. */
3005                 mchash[crc >> 5] |= 1 << (crc & 0x1f);
3006         }
3007
3008         CSR_WRITE_4(sc, JME_MAR0, mchash[0]);
3009         CSR_WRITE_4(sc, JME_MAR1, mchash[1]);
3010         CSR_WRITE_4(sc, JME_RXMAC, rxcfg);
3011 }
3012
3013 static int
3014 jme_sysctl_tx_coal_to(SYSCTL_HANDLER_ARGS)
3015 {
3016         struct jme_softc *sc = arg1;
3017         struct ifnet *ifp = &sc->arpcom.ac_if;
3018         int error, v;
3019
3020         ifnet_serialize_all(ifp);
3021
3022         v = sc->jme_tx_coal_to;
3023         error = sysctl_handle_int(oidp, &v, 0, req);
3024         if (error || req->newptr == NULL)
3025                 goto back;
3026
3027         if (v < PCCTX_COAL_TO_MIN || v > PCCTX_COAL_TO_MAX) {
3028                 error = EINVAL;
3029                 goto back;
3030         }
3031
3032         if (v != sc->jme_tx_coal_to) {
3033                 sc->jme_tx_coal_to = v;
3034                 if (ifp->if_flags & IFF_RUNNING)
3035                         jme_set_tx_coal(sc);
3036         }
3037 back:
3038         ifnet_deserialize_all(ifp);
3039         return error;
3040 }
3041
3042 static int
3043 jme_sysctl_tx_coal_pkt(SYSCTL_HANDLER_ARGS)
3044 {
3045         struct jme_softc *sc = arg1;
3046         struct ifnet *ifp = &sc->arpcom.ac_if;
3047         int error, v;
3048
3049         ifnet_serialize_all(ifp);
3050
3051         v = sc->jme_tx_coal_pkt;
3052         error = sysctl_handle_int(oidp, &v, 0, req);
3053         if (error || req->newptr == NULL)
3054                 goto back;
3055
3056         if (v < PCCTX_COAL_PKT_MIN || v > PCCTX_COAL_PKT_MAX) {
3057                 error = EINVAL;
3058                 goto back;
3059         }
3060
3061         if (v != sc->jme_tx_coal_pkt) {
3062                 sc->jme_tx_coal_pkt = v;
3063                 if (ifp->if_flags & IFF_RUNNING)
3064                         jme_set_tx_coal(sc);
3065         }
3066 back:
3067         ifnet_deserialize_all(ifp);
3068         return error;
3069 }
3070
3071 static int
3072 jme_sysctl_rx_coal_to(SYSCTL_HANDLER_ARGS)
3073 {
3074         struct jme_softc *sc = arg1;
3075         struct ifnet *ifp = &sc->arpcom.ac_if;
3076         int error, v;
3077
3078         ifnet_serialize_all(ifp);
3079
3080         v = sc->jme_rx_coal_to;
3081         error = sysctl_handle_int(oidp, &v, 0, req);
3082         if (error || req->newptr == NULL)
3083                 goto back;
3084
3085         if (v < PCCRX_COAL_TO_MIN || v > PCCRX_COAL_TO_MAX) {
3086                 error = EINVAL;
3087                 goto back;
3088         }
3089
3090         if (v != sc->jme_rx_coal_to) {
3091                 sc->jme_rx_coal_to = v;
3092                 if (ifp->if_flags & IFF_RUNNING)
3093                         jme_set_rx_coal(sc);
3094         }
3095 back:
3096         ifnet_deserialize_all(ifp);
3097         return error;
3098 }
3099
3100 static int
3101 jme_sysctl_rx_coal_pkt(SYSCTL_HANDLER_ARGS)
3102 {
3103         struct jme_softc *sc = arg1;
3104         struct ifnet *ifp = &sc->arpcom.ac_if;
3105         int error, v;
3106
3107         ifnet_serialize_all(ifp);
3108
3109         v = sc->jme_rx_coal_pkt;
3110         error = sysctl_handle_int(oidp, &v, 0, req);
3111         if (error || req->newptr == NULL)
3112                 goto back;
3113
3114         if (v < PCCRX_COAL_PKT_MIN || v > PCCRX_COAL_PKT_MAX) {
3115                 error = EINVAL;
3116                 goto back;
3117         }
3118
3119         if (v != sc->jme_rx_coal_pkt) {
3120                 sc->jme_rx_coal_pkt = v;
3121                 if (ifp->if_flags & IFF_RUNNING)
3122                         jme_set_rx_coal(sc);
3123         }
3124 back:
3125         ifnet_deserialize_all(ifp);
3126         return error;
3127 }
3128
3129 static void
3130 jme_set_tx_coal(struct jme_softc *sc)
3131 {
3132         uint32_t reg;
3133
3134         reg = (sc->jme_tx_coal_to << PCCTX_COAL_TO_SHIFT) &
3135             PCCTX_COAL_TO_MASK;
3136         reg |= (sc->jme_tx_coal_pkt << PCCTX_COAL_PKT_SHIFT) &
3137             PCCTX_COAL_PKT_MASK;
3138         reg |= PCCTX_COAL_TXQ0;
3139         CSR_WRITE_4(sc, JME_PCCTX, reg);
3140 }
3141
3142 static void
3143 jme_set_rx_coal(struct jme_softc *sc)
3144 {
3145         uint32_t reg;
3146         int r;
3147
3148         reg = (sc->jme_rx_coal_to << PCCRX_COAL_TO_SHIFT) &
3149             PCCRX_COAL_TO_MASK;
3150         reg |= (sc->jme_rx_coal_pkt << PCCRX_COAL_PKT_SHIFT) &
3151             PCCRX_COAL_PKT_MASK;
3152         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r)
3153                 CSR_WRITE_4(sc, JME_PCCRX(r), reg);
3154 }
3155
3156 #ifdef DEVICE_POLLING
3157
3158 static void
3159 jme_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
3160 {
3161         struct jme_softc *sc = ifp->if_softc;
3162         uint32_t status;
3163         int r;
3164
3165         ASSERT_SERIALIZED(&sc->jme_serialize);
3166
3167         switch (cmd) {
3168         case POLL_REGISTER:
3169                 CSR_WRITE_4(sc, JME_INTR_MASK_CLR, JME_INTRS);
3170                 break;
3171
3172         case POLL_DEREGISTER:
3173                 CSR_WRITE_4(sc, JME_INTR_MASK_SET, JME_INTRS);
3174                 break;
3175
3176         case POLL_AND_CHECK_STATUS:
3177         case POLL_ONLY:
3178                 status = CSR_READ_4(sc, JME_INTR_STATUS);
3179
3180                 for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
3181                         struct jme_rxdata *rdata =
3182                             &sc->jme_cdata.jme_rx_data[r];
3183
3184                         lwkt_serialize_enter(&rdata->jme_rx_serialize);
3185                         jme_rxeof(rdata, count);
3186                         lwkt_serialize_exit(&rdata->jme_rx_serialize);
3187                 }
3188
3189                 if (status & INTR_RXQ_DESC_EMPTY) {
3190                         CSR_WRITE_4(sc, JME_INTR_STATUS, status);
3191                         CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr |
3192                             RXCSR_RX_ENB | RXCSR_RXQ_START);
3193                 }
3194
3195                 lwkt_serialize_enter(&sc->jme_cdata.jme_tx_serialize);
3196                 jme_txeof(sc);
3197                 if (!ifq_is_empty(&ifp->if_snd))
3198                         if_devstart(ifp);
3199                 lwkt_serialize_exit(&sc->jme_cdata.jme_tx_serialize);
3200                 break;
3201         }
3202 }
3203
3204 #endif  /* DEVICE_POLLING */
3205
3206 static int
3207 jme_rxring_dma_alloc(struct jme_rxdata *rdata)
3208 {
3209         bus_dmamem_t dmem;
3210         int error, asize;
3211
3212         asize = roundup2(JME_RX_RING_SIZE(rdata), JME_RX_RING_ALIGN);
3213         error = bus_dmamem_coherent(rdata->jme_sc->jme_cdata.jme_ring_tag,
3214                         JME_RX_RING_ALIGN, 0,
3215                         BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
3216                         asize, BUS_DMA_WAITOK | BUS_DMA_ZERO, &dmem);
3217         if (error) {
3218                 device_printf(rdata->jme_sc->jme_dev,
3219                     "could not allocate %dth Rx ring.\n", rdata->jme_rx_idx);
3220                 return error;
3221         }
3222         rdata->jme_rx_ring_tag = dmem.dmem_tag;
3223         rdata->jme_rx_ring_map = dmem.dmem_map;
3224         rdata->jme_rx_ring = dmem.dmem_addr;
3225         rdata->jme_rx_ring_paddr = dmem.dmem_busaddr;
3226
3227         return 0;
3228 }
3229
3230 static int
3231 jme_rxbuf_dma_filter(void *arg __unused, bus_addr_t paddr)
3232 {
3233         if ((paddr & 0xffffffff) == 0) {
3234                 /*
3235                  * Don't allow lower 32bits of the RX buffer's
3236                  * physical address to be 0, else it will break
3237                  * hardware pending RSS information delivery
3238                  * detection on RX path.
3239                  */
3240                 return 1;
3241         }
3242         return 0;
3243 }
3244
3245 static int
3246 jme_rxbuf_dma_alloc(struct jme_rxdata *rdata)
3247 {
3248         bus_addr_t lowaddr;
3249         int i, error;
3250
3251         lowaddr = BUS_SPACE_MAXADDR;
3252         if (JME_ENABLE_HWRSS(rdata->jme_sc)) {
3253                 /* jme_rxbuf_dma_filter will be called */
3254                 lowaddr = BUS_SPACE_MAXADDR_32BIT;
3255         }
3256
3257         /* Create tag for Rx buffers. */
3258         error = bus_dma_tag_create(
3259             rdata->jme_sc->jme_cdata.jme_buffer_tag,/* parent */
3260             JME_RX_BUF_ALIGN, 0,        /* algnmnt, boundary */
3261             lowaddr,                    /* lowaddr */
3262             BUS_SPACE_MAXADDR,          /* highaddr */
3263             jme_rxbuf_dma_filter, NULL, /* filter, filterarg */
3264             MCLBYTES,                   /* maxsize */
3265             1,                          /* nsegments */
3266             MCLBYTES,                   /* maxsegsize */
3267             BUS_DMA_ALLOCNOW | BUS_DMA_WAITOK | BUS_DMA_ALIGNED,/* flags */
3268             &rdata->jme_rx_tag);
3269         if (error) {
3270                 device_printf(rdata->jme_sc->jme_dev,
3271                     "could not create %dth Rx DMA tag.\n", rdata->jme_rx_idx);
3272                 return error;
3273         }
3274
3275         /* Create DMA maps for Rx buffers. */
3276         error = bus_dmamap_create(rdata->jme_rx_tag, BUS_DMA_WAITOK,
3277                                   &rdata->jme_rx_sparemap);
3278         if (error) {
3279                 device_printf(rdata->jme_sc->jme_dev,
3280                     "could not create %dth spare Rx dmamap.\n",
3281                     rdata->jme_rx_idx);
3282                 bus_dma_tag_destroy(rdata->jme_rx_tag);
3283                 rdata->jme_rx_tag = NULL;
3284                 return error;
3285         }
3286         for (i = 0; i < rdata->jme_rx_desc_cnt; i++) {
3287                 struct jme_rxdesc *rxd = &rdata->jme_rxdesc[i];
3288
3289                 error = bus_dmamap_create(rdata->jme_rx_tag, BUS_DMA_WAITOK,
3290                                           &rxd->rx_dmamap);
3291                 if (error) {
3292                         int j;
3293
3294                         device_printf(rdata->jme_sc->jme_dev,
3295                             "could not create %dth Rx dmamap "
3296                             "for %dth RX ring.\n", i, rdata->jme_rx_idx);
3297
3298                         for (j = 0; j < i; ++j) {
3299                                 rxd = &rdata->jme_rxdesc[j];
3300                                 bus_dmamap_destroy(rdata->jme_rx_tag,
3301                                                    rxd->rx_dmamap);
3302                         }
3303                         bus_dmamap_destroy(rdata->jme_rx_tag,
3304                                            rdata->jme_rx_sparemap);
3305                         bus_dma_tag_destroy(rdata->jme_rx_tag);
3306                         rdata->jme_rx_tag = NULL;
3307                         return error;
3308                 }
3309         }
3310         return 0;
3311 }
3312
3313 static void
3314 jme_rx_intr(struct jme_softc *sc, uint32_t status)
3315 {
3316         int r;
3317
3318         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
3319                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r];
3320
3321                 if (status & rdata->jme_rx_coal) {
3322                         lwkt_serialize_enter(&rdata->jme_rx_serialize);
3323                         jme_rxeof(rdata, -1);
3324                         lwkt_serialize_exit(&rdata->jme_rx_serialize);
3325                 }
3326         }
3327 }
3328
3329 static void
3330 jme_enable_rss(struct jme_softc *sc)
3331 {
3332         uint32_t rssc, ind;
3333         uint8_t key[RSSKEY_NREGS * RSSKEY_REGSIZE];
3334         int i;
3335
3336         KASSERT(sc->jme_cdata.jme_rx_ring_cnt == JME_NRXRING_2 ||
3337                 sc->jme_cdata.jme_rx_ring_cnt == JME_NRXRING_4,
3338                 ("%s: invalid # of RX rings (%d)",
3339                  sc->arpcom.ac_if.if_xname, sc->jme_cdata.jme_rx_ring_cnt));
3340
3341         rssc = RSSC_HASH_64_ENTRY;
3342         rssc |= RSSC_HASH_IPV4 | RSSC_HASH_IPV4_TCP;
3343         rssc |= sc->jme_cdata.jme_rx_ring_cnt >> 1;
3344         JME_RSS_DPRINTF(sc, 1, "rssc 0x%08x\n", rssc);
3345         CSR_WRITE_4(sc, JME_RSSC, rssc);
3346
3347         toeplitz_get_key(key, sizeof(key));
3348         for (i = 0; i < RSSKEY_NREGS; ++i) {
3349                 uint32_t keyreg;
3350
3351                 keyreg = RSSKEY_REGVAL(key, i);
3352                 JME_RSS_DPRINTF(sc, 5, "keyreg%d 0x%08x\n", i, keyreg);
3353
3354                 CSR_WRITE_4(sc, RSSKEY_REG(i), keyreg);
3355         }
3356
3357         /*
3358          * Create redirect table in following fashion:
3359          * (hash & ring_cnt_mask) == rdr_table[(hash & rdr_table_mask)]
3360          */
3361         ind = 0;
3362         for (i = 0; i < RSSTBL_REGSIZE; ++i) {
3363                 int q;
3364
3365                 q = i % sc->jme_cdata.jme_rx_ring_cnt;
3366                 ind |= q << (i * 8);
3367         }
3368         JME_RSS_DPRINTF(sc, 1, "ind 0x%08x\n", ind);
3369
3370         for (i = 0; i < RSSTBL_NREGS; ++i)
3371                 CSR_WRITE_4(sc, RSSTBL_REG(i), ind);
3372 }
3373
3374 static void
3375 jme_disable_rss(struct jme_softc *sc)
3376 {
3377         CSR_WRITE_4(sc, JME_RSSC, RSSC_DIS_RSS);
3378 }
3379
3380 static void
3381 jme_serialize(struct ifnet *ifp, enum ifnet_serialize slz)
3382 {
3383         struct jme_softc *sc = ifp->if_softc;
3384
3385         ifnet_serialize_array_enter(sc->jme_serialize_arr,
3386             sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz);
3387 }
3388
3389 static void
3390 jme_deserialize(struct ifnet *ifp, enum ifnet_serialize slz)
3391 {
3392         struct jme_softc *sc = ifp->if_softc;
3393
3394         ifnet_serialize_array_exit(sc->jme_serialize_arr,
3395             sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz);
3396 }
3397
3398 static int
3399 jme_tryserialize(struct ifnet *ifp, enum ifnet_serialize slz)
3400 {
3401         struct jme_softc *sc = ifp->if_softc;
3402
3403         return ifnet_serialize_array_try(sc->jme_serialize_arr,
3404             sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE, slz);
3405 }
3406
3407 #ifdef INVARIANTS
3408
3409 static void
3410 jme_serialize_assert(struct ifnet *ifp, enum ifnet_serialize slz,
3411     boolean_t serialized)
3412 {
3413         struct jme_softc *sc = ifp->if_softc;
3414
3415         ifnet_serialize_array_assert(sc->jme_serialize_arr,
3416             sc->jme_serialize_cnt, JME_TX_SERIALIZE, JME_RX_SERIALIZE,
3417             slz, serialized);
3418 }
3419
3420 #endif  /* INVARIANTS */
3421
3422 static void
3423 jme_msix_try_alloc(device_t dev)
3424 {
3425         struct jme_softc *sc = device_get_softc(dev);
3426         struct jme_msix_data *msix;
3427         int error, i, r, msix_enable, msix_count;
3428
3429         msix_count = 1 + sc->jme_cdata.jme_rx_ring_cnt;
3430         KKASSERT(msix_count <= JME_NMSIX);
3431
3432         msix_enable = device_getenv_int(dev, "msix.enable", jme_msix_enable);
3433
3434         /*
3435          * We leave the 1st MSI-X vector unused, so we
3436          * actually need msix_count + 1 MSI-X vectors.
3437          */
3438         if (!msix_enable || pci_msix_count(dev) < (msix_count + 1))
3439                 return;
3440
3441         for (i = 0; i < msix_count; ++i)
3442                 sc->jme_msix[i].jme_msix_rid = -1;
3443
3444         i = 0;
3445
3446         msix = &sc->jme_msix[i++];
3447         msix->jme_msix_cpuid = 0;               /* XXX Put TX to cpu0 */
3448         msix->jme_msix_arg = &sc->jme_cdata;
3449         msix->jme_msix_func = jme_msix_tx;
3450         msix->jme_msix_intrs = INTR_TXQ_COAL | INTR_TXQ_COAL_TO;
3451         msix->jme_msix_serialize = &sc->jme_cdata.jme_tx_serialize;
3452         ksnprintf(msix->jme_msix_desc, sizeof(msix->jme_msix_desc), "%s tx",
3453             device_get_nameunit(dev));
3454
3455         for (r = 0; r < sc->jme_cdata.jme_rx_ring_cnt; ++r) {
3456                 struct jme_rxdata *rdata = &sc->jme_cdata.jme_rx_data[r];
3457
3458                 msix = &sc->jme_msix[i++];
3459                 msix->jme_msix_cpuid = r;       /* XXX Put RX to cpuX */
3460                 msix->jme_msix_arg = rdata;
3461                 msix->jme_msix_func = jme_msix_rx;
3462                 msix->jme_msix_intrs = rdata->jme_rx_coal | rdata->jme_rx_empty;
3463                 msix->jme_msix_serialize = &rdata->jme_rx_serialize;
3464                 ksnprintf(msix->jme_msix_desc, sizeof(msix->jme_msix_desc),
3465                     "%s rx%d", device_get_nameunit(dev), r);
3466         }
3467
3468         KKASSERT(i == msix_count);
3469
3470         error = pci_setup_msix(dev);
3471         if (error)
3472                 return;
3473
3474         /* Setup jme_msix_cnt early, so we could cleanup */
3475         sc->jme_msix_cnt = msix_count;
3476
3477         for (i = 0; i < msix_count; ++i) {
3478                 msix = &sc->jme_msix[i];
3479
3480                 msix->jme_msix_vector = i + 1;
3481                 error = pci_alloc_msix_vector(dev, msix->jme_msix_vector,
3482                     &msix->jme_msix_rid, msix->jme_msix_cpuid);
3483                 if (error)
3484                         goto back;
3485
3486                 msix->jme_msix_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
3487                     &msix->jme_msix_rid, RF_ACTIVE);
3488                 if (msix->jme_msix_res == NULL) {
3489                         error = ENOMEM;
3490                         goto back;
3491                 }
3492         }
3493
3494         for (i = 0; i < JME_INTR_CNT; ++i) {
3495                 uint32_t intr_mask = (1 << i);
3496                 int x;
3497
3498                 if ((JME_INTRS & intr_mask) == 0)
3499                         continue;
3500
3501                 for (x = 0; x < msix_count; ++x) {
3502                         msix = &sc->jme_msix[x];
3503                         if (msix->jme_msix_intrs & intr_mask) {
3504                                 int reg, shift;
3505
3506                                 reg = i / JME_MSINUM_FACTOR;
3507                                 KKASSERT(reg < JME_MSINUM_CNT);
3508
3509                                 shift = (i % JME_MSINUM_FACTOR) * 4;
3510
3511                                 sc->jme_msinum[reg] |=
3512                                     (msix->jme_msix_vector << shift);
3513
3514                                 break;
3515                         }
3516                 }
3517         }
3518
3519         if (bootverbose) {
3520                 for (i = 0; i < JME_MSINUM_CNT; ++i) {
3521                         device_printf(dev, "MSINUM%d: %#x\n", i,
3522                             sc->jme_msinum[i]);
3523                 }
3524         }
3525
3526         pci_enable_msix(dev);
3527         sc->jme_irq_type = PCI_INTR_TYPE_MSIX;
3528
3529 back:
3530         if (error)
3531                 jme_msix_free(dev);
3532 }
3533
3534 static int
3535 jme_intr_alloc(device_t dev)
3536 {
3537         struct jme_softc *sc = device_get_softc(dev);
3538         u_int irq_flags;
3539
3540         jme_msix_try_alloc(dev);
3541
3542         if (sc->jme_irq_type != PCI_INTR_TYPE_MSIX) {
3543                 sc->jme_irq_type = pci_alloc_1intr(dev, jme_msi_enable,
3544                     &sc->jme_irq_rid, &irq_flags);
3545
3546                 sc->jme_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
3547                     &sc->jme_irq_rid, irq_flags);
3548                 if (sc->jme_irq_res == NULL) {
3549                         device_printf(dev, "can't allocate irq\n");
3550                         return ENXIO;
3551                 }
3552         }
3553         return 0;
3554 }
3555
3556 static void
3557 jme_msix_free(device_t dev)
3558 {
3559         struct jme_softc *sc = device_get_softc(dev);
3560         int i;
3561
3562         KKASSERT(sc->jme_msix_cnt > 1);
3563
3564         for (i = 0; i < sc->jme_msix_cnt; ++i) {
3565                 struct jme_msix_data *msix = &sc->jme_msix[i];
3566
3567                 if (msix->jme_msix_res != NULL) {
3568                         bus_release_resource(dev, SYS_RES_IRQ,
3569                             msix->jme_msix_rid, msix->jme_msix_res);
3570                         msix->jme_msix_res = NULL;
3571                 }
3572                 if (msix->jme_msix_rid >= 0) {
3573                         pci_release_msix_vector(dev, msix->jme_msix_rid);
3574                         msix->jme_msix_rid = -1;
3575                 }
3576         }
3577         pci_teardown_msix(dev);
3578 }
3579
3580 static void
3581 jme_intr_free(device_t dev)
3582 {
3583         struct jme_softc *sc = device_get_softc(dev);
3584
3585         if (sc->jme_irq_type != PCI_INTR_TYPE_MSIX) {
3586                 if (sc->jme_irq_res != NULL) {
3587                         bus_release_resource(dev, SYS_RES_IRQ, sc->jme_irq_rid,
3588                                              sc->jme_irq_res);
3589                 }
3590                 if (sc->jme_irq_type == PCI_INTR_TYPE_MSI)
3591                         pci_release_msi(dev);
3592         } else {
3593                 jme_msix_free(dev);
3594         }
3595 }
3596
3597 static void
3598 jme_msix_tx(void *xcd)
3599 {
3600         struct jme_chain_data *cd = xcd;
3601         struct jme_softc *sc = cd->jme_sc;
3602         struct ifnet *ifp = &sc->arpcom.ac_if;
3603
3604         ASSERT_SERIALIZED(&cd->jme_tx_serialize);
3605
3606         CSR_WRITE_4(sc, JME_INTR_MASK_CLR, INTR_TXQ_COAL | INTR_TXQ_COAL_TO);
3607
3608         CSR_WRITE_4(sc, JME_INTR_STATUS,
3609             INTR_TXQ_COAL | INTR_TXQ_COAL_TO | INTR_TXQ_COMP);
3610
3611         if (ifp->if_flags & IFF_RUNNING) {
3612                 jme_txeof(sc);
3613                 if (!ifq_is_empty(&ifp->if_snd))
3614                         if_devstart(ifp);
3615         }
3616
3617         CSR_WRITE_4(sc, JME_INTR_MASK_SET, INTR_TXQ_COAL | INTR_TXQ_COAL_TO);
3618 }
3619
3620 static void
3621 jme_msix_rx(void *xrdata)
3622 {
3623         struct jme_rxdata *rdata = xrdata;
3624         struct jme_softc *sc = rdata->jme_sc;
3625         struct ifnet *ifp = &sc->arpcom.ac_if;
3626         uint32_t status;
3627
3628         ASSERT_SERIALIZED(&rdata->jme_rx_serialize);
3629
3630         CSR_WRITE_4(sc, JME_INTR_MASK_CLR,
3631             (rdata->jme_rx_coal | rdata->jme_rx_empty));
3632
3633         status = CSR_READ_4(sc, JME_INTR_STATUS);
3634         status &= (rdata->jme_rx_coal | rdata->jme_rx_empty);
3635
3636         if (status & rdata->jme_rx_coal)
3637                 status |= (rdata->jme_rx_coal | rdata->jme_rx_comp);
3638         CSR_WRITE_4(sc, JME_INTR_STATUS, status);
3639
3640         if (ifp->if_flags & IFF_RUNNING) {
3641                 if (status & rdata->jme_rx_coal)
3642                         jme_rxeof(rdata, -1);
3643
3644                 if (status & rdata->jme_rx_empty) {
3645                         CSR_WRITE_4(sc, JME_RXCSR, sc->jme_rxcsr |
3646                             RXCSR_RX_ENB | RXCSR_RXQ_START);
3647                 }
3648         }
3649
3650         CSR_WRITE_4(sc, JME_INTR_MASK_SET,
3651             (rdata->jme_rx_coal | rdata->jme_rx_empty));
3652 }
3653
3654 static void
3655 jme_set_msinum(struct jme_softc *sc)
3656 {
3657         int i;
3658
3659         for (i = 0; i < JME_MSINUM_CNT; ++i)
3660                 CSR_WRITE_4(sc, JME_MSINUM(i), sc->jme_msinum[i]);
3661 }
3662
3663 static int
3664 jme_intr_setup(device_t dev)
3665 {
3666         struct jme_softc *sc = device_get_softc(dev);
3667         struct ifnet *ifp = &sc->arpcom.ac_if;
3668         int error;
3669
3670         if (sc->jme_irq_type == PCI_INTR_TYPE_MSIX)
3671                 return jme_msix_setup(dev);
3672
3673         error = bus_setup_intr(dev, sc->jme_irq_res, INTR_MPSAFE,
3674             jme_intr, sc, &sc->jme_irq_handle, &sc->jme_serialize);
3675         if (error) {
3676                 device_printf(dev, "could not set up interrupt handler.\n");
3677                 return error;
3678         }
3679
3680         ifp->if_cpuid = rman_get_cpuid(sc->jme_irq_res);
3681         KKASSERT(ifp->if_cpuid >= 0 && ifp->if_cpuid < ncpus);
3682         return 0;
3683 }
3684
3685 static void
3686 jme_intr_teardown(device_t dev)
3687 {
3688         struct jme_softc *sc = device_get_softc(dev);
3689
3690         if (sc->jme_irq_type == PCI_INTR_TYPE_MSIX)
3691                 jme_msix_teardown(dev, sc->jme_msix_cnt);
3692         else
3693                 bus_teardown_intr(dev, sc->jme_irq_res, sc->jme_irq_handle);
3694 }
3695
3696 static int
3697 jme_msix_setup(device_t dev)
3698 {
3699         struct jme_softc *sc = device_get_softc(dev);
3700         struct ifnet *ifp = &sc->arpcom.ac_if;
3701         int x;
3702
3703         for (x = 0; x < sc->jme_msix_cnt; ++x) {
3704                 struct jme_msix_data *msix = &sc->jme_msix[x];
3705                 int error;
3706
3707                 error = bus_setup_intr_descr(dev, msix->jme_msix_res,
3708                     INTR_MPSAFE, msix->jme_msix_func, msix->jme_msix_arg,
3709                     &msix->jme_msix_handle, msix->jme_msix_serialize,
3710                     msix->jme_msix_desc);
3711                 if (error) {
3712                         device_printf(dev, "could not set up %s "
3713                             "interrupt handler.\n", msix->jme_msix_desc);
3714                         jme_msix_teardown(dev, x);
3715                         return error;
3716                 }
3717         }
3718         ifp->if_cpuid = 0; /* XXX */
3719         return 0;
3720 }
3721
3722 static void
3723 jme_msix_teardown(device_t dev, int msix_count)
3724 {
3725         struct jme_softc *sc = device_get_softc(dev);
3726         int x;
3727
3728         for (x = 0; x < msix_count; ++x) {
3729                 struct jme_msix_data *msix = &sc->jme_msix[x];
3730
3731                 bus_teardown_intr(dev, msix->jme_msix_res,
3732                     msix->jme_msix_handle);
3733         }
3734 }
3735
3736 static void
3737 jme_serialize_skipmain(struct jme_softc *sc)
3738 {
3739         lwkt_serialize_array_enter(sc->jme_serialize_arr,
3740             sc->jme_serialize_cnt, 1);
3741 }
3742
3743 static void
3744 jme_deserialize_skipmain(struct jme_softc *sc)
3745 {
3746         lwkt_serialize_array_exit(sc->jme_serialize_arr,
3747             sc->jme_serialize_cnt, 1);
3748 }