sys/dev/netif/ath/ath/if_ath_rx_edma.c

   1 /*-
   2  * Copyright (c) 2012 Adrian Chadd <adrian@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31
  32 /*
  33  * Driver for the Atheros Wireless LAN controller.
  34  *
  35  * This software is derived from work of Atsushi Onoe; his contribution
  36  * is greatly appreciated.
  37  */
  38
  39 #include "opt_inet.h"
  40 #include "opt_ath.h"
  41 /*
  42  * This is needed for register operations which are performed
  43  * by the driver - eg, calls to ath_hal_gettsf32().
  44  *
  45  * It's also required for any AH_DEBUG checks in here, eg the
  46  * module dependencies.
  47  */
  48 #include "opt_ah.h"
  49 #include "opt_wlan.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/mbuf.h>
  55 #include <sys/malloc.h>
  56 #include <sys/lock.h>
  57 #include <sys/mutex.h>
  58 #include <sys/kernel.h>
  59 #include <sys/socket.h>
  60 #include <sys/sockio.h>
  61 #include <sys/errno.h>
  62 #include <sys/callout.h>
  63 #include <sys/bus.h>
  64 #include <sys/endian.h>
  65 #include <sys/kthread.h>
  66 #include <sys/taskqueue.h>
  67 #include <sys/priv.h>
  68 #include <sys/module.h>
  69 #include <sys/ktr.h>
  70 #include <machine/pmap.h>
  71
  72 #include <net/if.h>
  73 #include <net/if_var.h>
  74 #include <net/if_dl.h>
  75 #include <net/if_media.h>
  76 #include <net/if_types.h>
  77 #include <net/if_arp.h>
  78 #include <net/ethernet.h>
  79 #include <net/if_llc.h>
  80 #include <net/ifq_var.h>
  81
  82 #include <netproto/802_11/ieee80211_var.h>
  83 #include <netproto/802_11/ieee80211_regdomain.h>
  84 #ifdef IEEE80211_SUPPORT_SUPERG
  85 #include <netproto/802_11/ieee80211_superg.h>
  86 #endif
  87 #ifdef IEEE80211_SUPPORT_TDMA
  88 #include <netproto/802_11/ieee80211_tdma.h>
  89 #endif
  90
  91 #include <net/bpf.h>
  92
  93 #ifdef INET
  94 #include <netinet/in.h>
  95 #include <netinet/if_ether.h>
  96 #endif
  97
  98 #include <dev/netif/ath/ath/if_athvar.h>
  99 #include <dev/netif/ath/ath_hal/ah_devid.h>             /* XXX for softled */
 100 #include <dev/netif/ath/ath_hal/ah_diagcodes.h>
 101
 102 #include <dev/netif/ath/ath/if_ath_debug.h>
 103 #include <dev/netif/ath/ath/if_ath_misc.h>
 104 #include <dev/netif/ath/ath/if_ath_tsf.h>
 105 #include <dev/netif/ath/ath/if_ath_tx.h>
 106 #include <dev/netif/ath/ath/if_ath_sysctl.h>
 107 #include <dev/netif/ath/ath/if_ath_led.h>
 108 #include <dev/netif/ath/ath/if_ath_keycache.h>
 109 #include <dev/netif/ath/ath/if_ath_rx.h>
 110 #include <dev/netif/ath/ath/if_ath_beacon.h>
 111 #include <dev/netif/ath/ath/if_athdfs.h>
 112
 113 #ifdef ATH_TX99_DIAG
 114 #include <dev/netif/ath/ath_tx99/ath_tx99.h>
 115 #endif
 116
 117 #include <dev/netif/ath/ath/if_ath_rx_edma.h>
 118
 119 #ifdef  ATH_DEBUG_ALQ
 120 #include <dev/netif/ath/ath/if_ath_alq.h>
 121 #endif
 122
 123 /*
 124  * some general macros
 125   */
 126 #define INCR(_l, _sz)           (_l) ++; (_l) &= ((_sz) - 1)
 127 #define DECR(_l, _sz)           (_l) --; (_l) &= ((_sz) - 1)
 128
 129 MALLOC_DECLARE(M_ATHDEV);
 130
 131 /*
 132  * XXX TODO:
 133  *
 134  * + Make sure the FIFO is correctly flushed and reinitialised
 135  *   through a reset;
 136  * + Verify multi-descriptor frames work!
 137  * + There's a "memory use after free" which needs to be tracked down
 138  *   and fixed ASAP.  I've seen this in the legacy path too, so it
 139  *   may be a generic RX path issue.
 140  */
 141
 142 /*
 143  * XXX shuffle the function orders so these pre-declarations aren't
 144  * required!
 145  */
 146 static  int ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 147             int nbufs);
 148 static  int ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype);
 149 static  void ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf);
 150 static  int ath_edma_recv_proc_queue(struct ath_softc *sc,
 151             HAL_RX_QUEUE qtype, int dosched);
 152 static  int ath_edma_recv_proc_deferred_queue(struct ath_softc *sc,
 153             HAL_RX_QUEUE qtype, int dosched);
 154
 155 static void
 156 ath_edma_stoprecv(struct ath_softc *sc, int dodelay)
 157 {
 158         struct ath_hal *ah = sc->sc_ah;
 159
 160         ATH_RX_LOCK(sc);
 161         ath_hal_stoppcurecv(ah);
 162         ath_hal_setrxfilter(ah, 0);
 163         ath_hal_stopdmarecv(ah);
 164
 165         DELAY(3000);
 166
 167         /* Flush RX pending for each queue */
 168         /* XXX should generic-ify this */
 169         if (sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending) {
 170                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending);
 171                 sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending = NULL;
 172         }
 173
 174         if (sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending) {
 175                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending);
 176                 sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending = NULL;
 177         }
 178         ATH_RX_UNLOCK(sc);
 179 }
 180
 181 /*
 182  * Re-initialise the FIFO given the current buffer contents.
 183  * Specifically, walk from head -> tail, pushing the FIFO contents
 184  * back into the FIFO.
 185  */
 186 static void
 187 ath_edma_reinit_fifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 188 {
 189         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 190         struct ath_buf *bf;
 191         int i, j;
 192
 193         ATH_RX_LOCK_ASSERT(sc);
 194
 195         i = re->m_fifo_head;
 196         for (j = 0; j < re->m_fifo_depth; j++) {
 197                 bf = re->m_fifo[i];
 198                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 199                     "%s: Q%d: pos=%i, addr=0x%jx\n",
 200                     __func__,
 201                     qtype,
 202                     i,
 203                     (uintmax_t)bf->bf_daddr);
 204                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
 205                 INCR(i, re->m_fifolen);
 206         }
 207
 208         /* Ensure this worked out right */
 209         if (i != re->m_fifo_tail) {
 210                 device_printf(sc->sc_dev, "%s: i (%d) != tail! (%d)\n",
 211                     __func__,
 212                     i,
 213                     re->m_fifo_tail);
 214         }
 215 }
 216
 217 static void
 218 ath_edma_handle_rxfifo_reset(struct ath_softc *sc)
 219 {
 220         if (sc->sc_rxfifo_state == ATH_RXFIFO_RESET) {
 221                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 222                     "%s: Re-initing HP FIFO\n", __func__);
 223                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_HP);
 224                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 225                     "%s: Re-initing LP FIFO\n", __func__);
 226                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_LP);
 227                 sc->sc_rxfifo_state = ATH_RXFIFO_OK;
 228         }
 229 }
 230
 231 /*
 232  * Start receive.
 233  *
 234  * XXX TODO: this needs to reallocate the FIFO entries when a reset
 235  * occurs, in case the FIFO is filled up and no new descriptors get
 236  * thrown into the FIFO.
 237  */
 238 static int
 239 ath_edma_startrecv(struct ath_softc *sc)
 240 {
 241         struct ath_hal *ah = sc->sc_ah;
 242
 243         ATH_RX_LOCK(sc);
 244
 245         /* Enable RX FIFO */
 246         ath_hal_rxena(ah);
 247         ath_edma_handle_rxfifo_reset(sc);
 248
 249         /* Add up to m_fifolen entries in each queue */
 250         /*
 251          * These must occur after the above write so the FIFO buffers
 252          * are pushed/tracked in the same order as the hardware will
 253          * process them.
 254          */
 255         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_HP,
 256             sc->sc_rxedma[HAL_RX_QUEUE_HP].m_fifolen);
 257
 258         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_LP,
 259             sc->sc_rxedma[HAL_RX_QUEUE_LP].m_fifolen);
 260
 261         ath_mode_init(sc);
 262         ath_hal_startpcurecv(ah);
 263
 264         ATH_RX_UNLOCK(sc);
 265
 266         return (0);
 267 }
 268
 269 static void
 270 ath_edma_recv_sched_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 271     int dosched)
 272 {
 273         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 274         if (dosched)
 275                 ath_edma_handle_rxfifo_reset(sc);
 276         ath_edma_recv_proc_queue(sc, qtype, dosched);
 277         ath_power_restore_power_state(sc);
 278
 279         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
 280 }
 281
 282 static void
 283 ath_edma_recv_sched(struct ath_softc *sc, int dosched)
 284 {
 285         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 286         if (dosched)
 287                 ath_edma_handle_rxfifo_reset(sc);
 288         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, dosched);
 289         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, dosched);
 290         ath_power_restore_power_state(sc);
 291
 292         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
 293 }
 294
 295 static void
 296 ath_edma_recv_flush(struct ath_softc *sc)
 297 {
 298
 299         DPRINTF(sc, ATH_DEBUG_RECV, "%s: called\n", __func__);
 300
 301         ATH_PCU_LOCK(sc);
 302         sc->sc_rxproc_cnt++;
 303         ATH_PCU_UNLOCK(sc);
 304
 305         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 306
 307         /*
 308          * Flush any active frames from FIFO -> deferred list
 309          */
 310         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 0);
 311         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 0);
 312
 313         /*
 314          * Process what's in the deferred queue
 315          */
 316         /*
 317          * XXX: If we read the tsf/channoise here and then pass it in,
 318          * we could restore the power state before processing
 319          * the deferred queue.
 320          */
 321         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 0);
 322         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 0);
 323
 324         ath_power_restore_power_state(sc);
 325
 326         ATH_PCU_LOCK(sc);
 327         sc->sc_rxproc_cnt--;
 328         ATH_PCU_UNLOCK(sc);
 329 }
 330
 331 /*
 332  * Process frames from the current queue into the deferred queue.
 333  */
 334 static int
 335 ath_edma_recv_proc_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 336     int dosched)
 337 {
 338         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 339         struct ath_rx_status *rs;
 340         struct ath_desc *ds;
 341         struct ath_buf *bf;
 342         struct mbuf *m;
 343         struct ath_hal *ah = sc->sc_ah;
 344         uint64_t tsf;
 345         uint16_t nf;
 346         int npkts = 0;
 347         int n = 0;
 348
 349         tsf = ath_hal_gettsf64(ah);
 350         nf = ath_hal_getchannoise(ah, sc->sc_curchan);
 351         sc->sc_stats.ast_rx_noise = nf;
 352
 353         ATH_RX_LOCK(sc);
 354
 355         do {
 356                 bf = re->m_fifo[re->m_fifo_head];
 357                 /* This shouldn't occur! */
 358                 if (bf == NULL) {
 359                         device_printf(sc->sc_dev, "%s: Q%d: NULL bf?\n",
 360                             __func__,
 361                             qtype);
 362                         break;
 363                 }
 364                 m = bf->bf_m;
 365                 ds = bf->bf_desc;
 366
 367                 /*
 368                  * Sync descriptor memory - this also syncs the buffer for us.
 369                  * EDMA descriptors are in cached memory.
 370                  */
 371                 cpu_lfence();
 372                 bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
 373                                 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 374                 rs = &bf->bf_status.ds_rxstat;
 375                 bf->bf_rxstatus = ath_hal_rxprocdesc(ah, ds, bf->bf_daddr,
 376                     NULL, rs);
 377 #ifdef  ATH_DEBUG
 378                 if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
 379                         ath_printrxbuf(sc, bf, 0, bf->bf_rxstatus == HAL_OK);
 380 #endif /* ATH_DEBUG */
 381 #ifdef  ATH_DEBUG_ALQ
 382                 if (if_ath_alq_checkdebug(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS))
 383                         if_ath_alq_post(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS,
 384                             sc->sc_rx_statuslen, (char *) ds);
 385 #endif /* ATH_DEBUG */
 386                 if (bf->bf_rxstatus == HAL_EINPROGRESS) {
 387                         DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 388                             "%s: Q%d: still in-prog!\n", __func__, qtype);
 389                         break;
 390                 }
 391
 392                 /*
 393                  * Completed descriptor.
 394                  */
 395                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 396                     "%s: Q%d: completed!\n", __func__, qtype);
 397                 npkts++;
 398
 399                 /*
 400                  * We've been synced already, so unmap.
 401                  */
 402                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 403
 404                 /*
 405                  * Remove the FIFO entry and place it on the completion
 406                  * queue.
 407                  */
 408                 re->m_fifo[re->m_fifo_head] = NULL;
 409                 TAILQ_INSERT_TAIL(&sc->sc_rx_rxlist[qtype], bf, bf_list);
 410
 411                 /* Bump the descriptor FIFO stats */
 412                 INCR(re->m_fifo_head, re->m_fifolen);
 413                 re->m_fifo_depth--;
 414                 /* XXX check it doesn't fall below 0 */
 415         } while (re->m_fifo_depth > 0);
 416
 417         /* Append some more fresh frames to the FIFO */
 418         /* (kick already handled when dosched != 0) */
 419         if (dosched)
 420                 n = ath_edma_rxfifo_alloc(sc, qtype, re->m_fifolen);
 421
 422         ATH_RX_UNLOCK(sc);
 423
 424         /* rx signal state monitoring */
 425         ath_hal_rxmonitor(ah, &sc->sc_halstats, sc->sc_curchan);
 426
 427         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
 428             "ath edma rx proc: npkts=%d\n",
 429             npkts);
 430
 431         return n;
 432 }
 433
 434 /*
 435  * Flush the deferred queue.
 436  *
 437  * This destructively flushes the deferred queue - it doesn't
 438  * call the wireless stack on each mbuf.
 439  */
 440 static void
 441 ath_edma_flush_deferred_queue(struct ath_softc *sc)
 442 {
 443         struct ath_buf *bf;
 444
 445         ATH_RX_LOCK_ASSERT(sc);
 446
 447         /* Free in one set, inside the lock */
 448         while ((bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP])) != NULL) {
 449                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP], bf, bf_list);
 450                 ath_edma_rxbuf_free(sc, bf);
 451         }
 452         while ((bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP])) != NULL) {
 453                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP], bf, bf_list);
 454                 ath_edma_rxbuf_free(sc, bf);
 455         }
 456 }
 457
 458 static int
 459 ath_edma_recv_proc_deferred_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 460     int dosched)
 461 {
 462         int ngood = 0;
 463         uint64_t tsf;
 464         struct ath_buf *bf;
 465         struct ath_buf *next;
 466         struct ath_rx_status *rs;
 467         int16_t nf;
 468         ath_bufhead rxlist;
 469         struct mbuf *m;
 470
 471         TAILQ_INIT(&rxlist);
 472
 473         nf = ath_hal_getchannoise(sc->sc_ah, sc->sc_curchan);
 474         /*
 475          * XXX TODO: the NF/TSF should be stamped on the bufs themselves,
 476          * otherwise we may end up adding in the wrong values if this
 477          * is delayed too far..
 478          */
 479         tsf = ath_hal_gettsf64(sc->sc_ah);
 480
 481         /* Copy the list over */
 482         ATH_RX_LOCK(sc);
 483         TAILQ_CONCAT(&rxlist, &sc->sc_rx_rxlist[qtype], bf_list);
 484         ATH_RX_UNLOCK(sc);
 485
 486         /* Handle the completed descriptors */
 487         TAILQ_FOREACH_MUTABLE(bf, &rxlist, bf_list, next) {
 488                 /*
 489                  * Skip the RX descriptor status - start at the data offset
 490                  */
 491                 m_adj(bf->bf_m, sc->sc_rx_statuslen);
 492
 493                 /* Handle the frame */
 494
 495                 rs = &bf->bf_status.ds_rxstat;
 496                 m = bf->bf_m;
 497                 bf->bf_m = NULL;
 498                 if (ath_rx_pkt(sc, rs, bf->bf_rxstatus, tsf, nf, qtype, bf, m))
 499                         ngood++;
 500         }
 501
 502         if (ngood) {
 503                 sc->sc_lastrx = tsf;
 504         }
 505
 506         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
 507             "ath edma rx deferred proc: ngood=%d\n",
 508             ngood);
 509
 510         /* Free in one set, inside the lock */
 511         ATH_RX_LOCK(sc);
 512
 513         while ((bf = TAILQ_FIRST(&rxlist)) != NULL) {
 514                 /* Free the buffer/mbuf */
 515                 TAILQ_REMOVE(&rxlist, bf, bf_list);
 516                 ath_edma_rxbuf_free(sc, bf);
 517         }
 518         ATH_RX_UNLOCK(sc);
 519
 520         return (ngood);
 521 }
 522
 523 static void
 524 ath_edma_recv_tasklet(void *arg, int npending)
 525 {
 526         struct ath_softc *sc = (struct ath_softc *) arg;
 527         struct ifnet *ifp = sc->sc_ifp;
 528 #ifdef  IEEE80211_SUPPORT_SUPERG
 529         struct ieee80211com *ic = ifp->if_l2com;
 530 #endif
 531         int n1;
 532         int n2;
 533
 534         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: called; npending=%d\n",
 535             __func__,
 536             npending);
 537
 538         wlan_serialize_enter();
 539         ATH_PCU_LOCK(sc);
 540         if (sc->sc_inreset_cnt > 0) {
 541                 device_printf(sc->sc_dev, "%s: sc_inreset_cnt > 0; skipping\n",
 542                     __func__);
 543                 ATH_PCU_UNLOCK(sc);
 544                 wlan_serialize_exit();
 545                 return;
 546         }
 547         sc->sc_rxproc_cnt++;
 548         ATH_PCU_UNLOCK(sc);
 549
 550         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 551
 552         ath_edma_handle_rxfifo_reset(sc);
 553         n1 = ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 1);
 554         n2 = ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 1);
 555
 556         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 1);
 557         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 1);
 558
 559         /* Handle resched and kickpcu appropriately */
 560         ATH_PCU_LOCK(sc);
 561         if (sc->sc_kickpcu) {
 562                 kprintf("k(%d,%d)", n1, n2);
 563                 sc->sc_kickpcu = 0;
 564                 /* reload imask XXX */
 565                 if (n1 || n2)
 566                         ath_hal_intrset(sc->sc_ah, sc->sc_imask);
 567         }
 568         ATH_PCU_UNLOCK(sc);
 569
 570         /*
 571          * XXX: If we read the tsf/channoise here and then pass it in,
 572          * we could restore the power state before processing
 573          * the deferred queue.
 574          */
 575         ath_power_restore_power_state(sc);
 576
 577         /* XXX inside IF_LOCK ? */
 578         if (!ifq_is_oactive(&ifp->if_snd)) {
 579 #ifdef  IEEE80211_SUPPORT_SUPERG
 580                 ieee80211_ff_age_all(ic, 100);
 581 #endif
 582                 if (!ifq_is_empty(&ifp->if_snd))
 583                         ath_tx_kick(sc);
 584         }
 585         if (ath_dfs_tasklet_needed(sc, sc->sc_curchan))
 586                 taskqueue_enqueue(sc->sc_tq, &sc->sc_dfstask);
 587
 588         ATH_PCU_LOCK(sc);
 589         sc->sc_rxproc_cnt--;
 590         ATH_PCU_UNLOCK(sc);
 591         wlan_serialize_exit();
 592 }
 593
 594 /*
 595  * Allocate an RX mbuf for the given ath_buf and initialise
 596  * it for EDMA.
 597  *
 598  * + Allocate a 4KB mbuf;
 599  * + Setup the DMA map for the given buffer;
 600  * + Return that.
 601  */
 602 static int
 603 ath_edma_rxbuf_init(struct ath_softc *sc, struct ath_buf *bf)
 604 {
 605
 606         struct mbuf *m;
 607         int error;
 608         int len;
 609
 610         ATH_RX_LOCK_ASSERT(sc);
 611
 612         m = m_getjcl(MB_DONTWAIT, MT_DATA, M_PKTHDR, sc->sc_edma_bufsize);
 613 /*      m = m_getcl(MB_WAIT, MT_DATA, M_PKTHDR);*/
 614 /*      m = m_getm(NULL, sc->sc_edma_bufsize, MB_WAIT, MT_DATA);*/
 615         if (! m)
 616                 return (ENOBUFS);               /* XXX ?*/
 617
 618         /* XXX warn/enforce alignment */
 619
 620         len = m->m_ext.ext_size;
 621 #if 0
 622         device_printf(sc->sc_dev, "%s: called: m=%p, size=%d, mtod=%p\n",
 623             __func__,
 624             m,
 625             len,
 626             mtod(m, char *));
 627 #endif
 628
 629         m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
 630
 631         /*
 632          * Populate ath_buf fields.
 633          */
 634         KKASSERT(m->m_len >= sc->sc_rx_statuslen);
 635         bf->bf_desc = mtod(m, struct ath_desc *);
 636         bf->bf_lastds = bf->bf_desc;    /* XXX only really for TX? */
 637         bf->bf_m = m;
 638
 639         /*
 640          * Zero the descriptor and ensure it makes it out to the
 641          * bounce buffer if one is required.
 642          *
 643          * XXX PREWRITE will copy the whole buffer; we only needed it
 644          * to sync the first 32 DWORDS.  Oh well.
 645          */
 646         memset(bf->bf_desc, '\0', sc->sc_rx_statuslen);
 647
 648         /*
 649          * Create DMA mapping.
 650          */
 651         error = bus_dmamap_load_mbuf_segment(sc->sc_dmat,
 652             bf->bf_dmamap, m, bf->bf_segs, 1, &bf->bf_nseg, BUS_DMA_NOWAIT);
 653
 654         if (error != 0) {
 655                 device_printf(sc->sc_dev, "%s: failed; error=%d\n",
 656                     __func__,
 657                     error);
 658                 m_freem(m);
 659                 return (error);
 660         }
 661
 662         /*
 663          * Set daddr to the physical mapping page.
 664          */
 665         bf->bf_daddr = bf->bf_segs[0].ds_addr;
 666
 667         /*
 668          * Prepare for the upcoming read.
 669          *
 670          * We need to both sync some data into the buffer (the zero'ed
 671          * descriptor payload) and also prepare for the read that's going
 672          * to occur.
 673          */
 674         bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
 675                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 676
 677         /* Finish! */
 678         return (0);
 679 }
 680
 681 /*
 682  * Allocate a RX buffer.
 683  */
 684 static struct ath_buf *
 685 ath_edma_rxbuf_alloc(struct ath_softc *sc)
 686 {
 687         struct ath_buf *bf;
 688         int error;
 689
 690         ATH_RX_LOCK_ASSERT(sc);
 691
 692         /* Allocate buffer */
 693         bf = TAILQ_FIRST(&sc->sc_rxbuf);
 694         /* XXX shouldn't happen upon startup? */
 695         if (bf == NULL) {
 696                 device_printf(sc->sc_dev, "%s: nothing on rxbuf?!\n",
 697                     __func__);
 698                 return (NULL);
 699         }
 700
 701         /* Remove it from the free list */
 702         TAILQ_REMOVE(&sc->sc_rxbuf, bf, bf_list);
 703
 704         /* Assign RX mbuf to it */
 705         error = ath_edma_rxbuf_init(sc, bf);
 706         if (error != 0) {
 707                 device_printf(sc->sc_dev,
 708                     "%s: bf=%p, rxbuf alloc failed! error=%d\n",
 709                     __func__,
 710                     bf,
 711                     error);
 712                 TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 713                 return (NULL);
 714         }
 715
 716         return (bf);
 717 }
 718
 719 static void
 720 ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf)
 721 {
 722
 723         ATH_RX_LOCK_ASSERT(sc);
 724
 725         /*
 726          * Only unload the frame if we haven't consumed
 727          * the mbuf via ath_rx_pkt().
 728          */
 729         if (bf->bf_m) {
 730                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 731                 m_freem(bf->bf_m);
 732                 bf->bf_m = NULL;
 733         }
 734
 735         /* XXX lock? */
 736         TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 737 }
 738
 739 /*
 740  * Allocate up to 'n' entries and push them onto the hardware FIFO.
 741  *
 742  * Return how many entries were successfully pushed onto the
 743  * FIFO.
 744  */
 745 static int
 746 ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype, int nbufs)
 747 {
 748         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 749         struct ath_buf *bf;
 750         int i;
 751
 752         ATH_RX_LOCK_ASSERT(sc);
 753
 754         /*
 755          * Allocate buffers until the FIFO is full or nbufs is reached.
 756          */
 757         for (i = 0; i < nbufs && re->m_fifo_depth < re->m_fifolen; i++) {
 758                 /* Ensure the FIFO is already blank, complain loudly! */
 759                 if (re->m_fifo[re->m_fifo_tail] != NULL) {
 760                         device_printf(sc->sc_dev,
 761                             "%s: Q%d: fifo[%d] != NULL (%p)\n",
 762                             __func__,
 763                             qtype,
 764                             re->m_fifo_tail,
 765                             re->m_fifo[re->m_fifo_tail]);
 766
 767                         /* Free the slot */
 768                         ath_edma_rxbuf_free(sc, re->m_fifo[re->m_fifo_tail]);
 769                         re->m_fifo_depth--;
 770                         /* XXX check it's not < 0 */
 771                         re->m_fifo[re->m_fifo_tail] = NULL;
 772                 }
 773
 774                 bf = ath_edma_rxbuf_alloc(sc);
 775                 /* XXX should ensure the FIFO is not NULL? */
 776                 if (bf == NULL) {
 777                         device_printf(sc->sc_dev,
 778                             "%s: Q%d: alloc failed: i=%d, nbufs=%d?\n",
 779                             __func__,
 780                             qtype,
 781                             i,
 782                             nbufs);
 783                         break;
 784                 }
 785
 786                 re->m_fifo[re->m_fifo_tail] = bf;
 787
 788                 /* Write to the RX FIFO */
 789                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 790                     "%s: Q%d: putrxbuf=%p (0x%jx)\n",
 791                     __func__,
 792                     qtype,
 793                     bf->bf_desc,
 794                     (uintmax_t) bf->bf_daddr);
 795                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
 796
 797                 re->m_fifo_depth++;
 798                 INCR(re->m_fifo_tail, re->m_fifolen);
 799         }
 800
 801         /*
 802          * Return how many were allocated.
 803          */
 804         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: Q%d: nbufs=%d, nalloced=%d\n",
 805             __func__,
 806             qtype,
 807             nbufs,
 808             i);
 809         return (i);
 810 }
 811
 812 static int
 813 ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 814 {
 815         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 816         int i;
 817
 818         ATH_RX_LOCK_ASSERT(sc);
 819
 820         for (i = 0; i < re->m_fifolen; i++) {
 821                 if (re->m_fifo[i] != NULL) {
 822 #ifdef  ATH_DEBUG
 823                         struct ath_buf *bf = re->m_fifo[i];
 824
 825                         if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
 826                                 ath_printrxbuf(sc, bf, 0, HAL_OK);
 827 #endif
 828                         ath_edma_rxbuf_free(sc, re->m_fifo[i]);
 829                         re->m_fifo[i] = NULL;
 830                         re->m_fifo_depth--;
 831                 }
 832         }
 833
 834         if (re->m_rxpending != NULL) {
 835                 m_freem(re->m_rxpending);
 836                 re->m_rxpending = NULL;
 837         }
 838         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
 839
 840         return (0);
 841 }
 842
 843 /*
 844  * Setup the initial RX FIFO structure.
 845  */
 846 static int
 847 ath_edma_setup_rxfifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 848 {
 849         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 850
 851         ATH_RX_LOCK_ASSERT(sc);
 852
 853         if (! ath_hal_getrxfifodepth(sc->sc_ah, qtype, &re->m_fifolen)) {
 854                 device_printf(sc->sc_dev, "%s: qtype=%d, failed\n",
 855                     __func__,
 856                     qtype);
 857                 return (-EINVAL);
 858         }
 859         device_printf(sc->sc_dev, "%s: type=%d, FIFO depth = %d entries\n",
 860             __func__,
 861             qtype,
 862             re->m_fifolen);
 863
 864         /* Allocate ath_buf FIFO array, pre-zero'ed */
 865         re->m_fifo = kmalloc(sizeof(struct ath_buf *) * re->m_fifolen,
 866             M_ATHDEV,
 867             M_INTWAIT | M_ZERO);
 868         if (re->m_fifo == NULL) {
 869                 device_printf(sc->sc_dev, "%s: malloc failed\n",
 870                     __func__);
 871                 return (-ENOMEM);
 872         }
 873
 874         /*
 875          * Set initial "empty" state.
 876          */
 877         re->m_rxpending = NULL;
 878         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
 879
 880         return (0);
 881 }
 882
 883 static int
 884 ath_edma_rxfifo_free(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 885 {
 886         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 887
 888         device_printf(sc->sc_dev, "%s: called; qtype=%d\n",
 889             __func__,
 890             qtype);
 891
 892         kfree(re->m_fifo, M_ATHDEV);
 893
 894         return (0);
 895 }
 896
 897 static int
 898 ath_edma_dma_rxsetup(struct ath_softc *sc)
 899 {
 900         int error;
 901
 902         /*
 903          * Create RX DMA tag and buffers.
 904          */
 905         error = ath_descdma_setup_rx_edma(sc, &sc->sc_rxdma, &sc->sc_rxbuf,
 906             "rx", ath_rxbuf, sc->sc_rx_statuslen);
 907         if (error != 0)
 908                 return error;
 909
 910         ATH_RX_LOCK(sc);
 911         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_HP);
 912         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_LP);
 913         ATH_RX_UNLOCK(sc);
 914
 915         return (0);
 916 }
 917
 918 static int
 919 ath_edma_dma_rxteardown(struct ath_softc *sc)
 920 {
 921
 922         ATH_RX_LOCK(sc);
 923         ath_edma_flush_deferred_queue(sc);
 924         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_HP);
 925         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_HP);
 926
 927         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_LP);
 928         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_LP);
 929         ATH_RX_UNLOCK(sc);
 930
 931         /* Free RX ath_buf */
 932         /* Free RX DMA tag */
 933         if (sc->sc_rxdma.dd_desc_len != 0)
 934                 ath_descdma_cleanup(sc, &sc->sc_rxdma, &sc->sc_rxbuf);
 935
 936         return (0);
 937 }
 938
 939 void
 940 ath_recv_setup_edma(struct ath_softc *sc)
 941 {
 942
 943         /* Set buffer size to 4k */
 944         sc->sc_edma_bufsize = 4096;
 945
 946         /* Fetch EDMA field and buffer sizes */
 947         (void) ath_hal_getrxstatuslen(sc->sc_ah, &sc->sc_rx_statuslen);
 948
 949         /* Configure the hardware with the RX buffer size */
 950         (void) ath_hal_setrxbufsize(sc->sc_ah, sc->sc_edma_bufsize -
 951             sc->sc_rx_statuslen);
 952
 953         device_printf(sc->sc_dev, "RX status length: %d\n",
 954             sc->sc_rx_statuslen);
 955         device_printf(sc->sc_dev, "RX buffer size: %d\n",
 956             sc->sc_edma_bufsize);
 957
 958         sc->sc_rx.recv_stop = ath_edma_stoprecv;
 959         sc->sc_rx.recv_start = ath_edma_startrecv;
 960         sc->sc_rx.recv_flush = ath_edma_recv_flush;
 961         sc->sc_rx.recv_tasklet = ath_edma_recv_tasklet;
 962         sc->sc_rx.recv_rxbuf_init = ath_edma_rxbuf_init;
 963
 964         sc->sc_rx.recv_setup = ath_edma_dma_rxsetup;
 965         sc->sc_rx.recv_teardown = ath_edma_dma_rxteardown;
 966
 967         sc->sc_rx.recv_sched = ath_edma_recv_sched;
 968         sc->sc_rx.recv_sched_queue = ath_edma_recv_sched_queue;
 969 }