sys/dev/netif/ath/ath/if_ath_rx_edma.c

   1 /*-
   2  * Copyright (c) 2012 Adrian Chadd <adrian@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  13  *    redistribution must be conditioned upon including a substantially
  14  *    similar Disclaimer requirement for further binary redistribution.
  15  *
  16  * NO WARRANTY
  17  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  20  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  22  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  23  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  24  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  25  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  26  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  27  * THE POSSIBILITY OF SUCH DAMAGES.
  28  */
  29
  30 #include <sys/cdefs.h>
  31
  32 /*
  33  * Driver for the Atheros Wireless LAN controller.
  34  *
  35  * This software is derived from work of Atsushi Onoe; his contribution
  36  * is greatly appreciated.
  37  */
  38
  39 #include "opt_inet.h"
  40 #include "opt_ath.h"
  41 /*
  42  * This is needed for register operations which are performed
  43  * by the driver - eg, calls to ath_hal_gettsf32().
  44  *
  45  * It's also required for any AH_DEBUG checks in here, eg the
  46  * module dependencies.
  47  */
  48 #include "opt_ah.h"
  49 #include "opt_wlan.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/sysctl.h>
  54 #include <sys/mbuf.h>
  55 #include <sys/malloc.h>
  56 #include <sys/lock.h>
  57 #include <sys/mutex.h>
  58 #include <sys/kernel.h>
  59 #include <sys/socket.h>
  60 #include <sys/sockio.h>
  61 #include <sys/errno.h>
  62 #include <sys/callout.h>
  63 #include <sys/bus.h>
  64 #include <sys/endian.h>
  65 #include <sys/kthread.h>
  66 #include <sys/taskqueue.h>
  67 #include <sys/priv.h>
  68 #include <sys/module.h>
  69 #include <sys/ktr.h>
  70 #include <machine/pmap.h>
  71
  72 #include <net/if.h>
  73 #include <net/if_var.h>
  74 #include <net/if_dl.h>
  75 #include <net/if_media.h>
  76 #include <net/if_types.h>
  77 #include <net/if_arp.h>
  78 #include <net/ethernet.h>
  79 #include <net/if_llc.h>
  80 #include <net/ifq_var.h>
  81
  82 #include <netproto/802_11/ieee80211_var.h>
  83 #include <netproto/802_11/ieee80211_regdomain.h>
  84 #ifdef IEEE80211_SUPPORT_SUPERG
  85 #include <netproto/802_11/ieee80211_superg.h>
  86 #endif
  87 #ifdef IEEE80211_SUPPORT_TDMA
  88 #include <netproto/802_11/ieee80211_tdma.h>
  89 #endif
  90
  91 #include <net/bpf.h>
  92
  93 #ifdef INET
  94 #include <netinet/in.h>
  95 #include <netinet/if_ether.h>
  96 #endif
  97
  98 #include <dev/netif/ath/ath/if_athvar.h>
  99 #include <dev/netif/ath/ath_hal/ah_devid.h>             /* XXX for softled */
 100 #include <dev/netif/ath/ath_hal/ah_diagcodes.h>
 101
 102 #include <dev/netif/ath/ath/if_ath_debug.h>
 103 #include <dev/netif/ath/ath/if_ath_misc.h>
 104 #include <dev/netif/ath/ath/if_ath_tsf.h>
 105 #include <dev/netif/ath/ath/if_ath_tx.h>
 106 #include <dev/netif/ath/ath/if_ath_sysctl.h>
 107 #include <dev/netif/ath/ath/if_ath_led.h>
 108 #include <dev/netif/ath/ath/if_ath_keycache.h>
 109 #include <dev/netif/ath/ath/if_ath_rx.h>
 110 #include <dev/netif/ath/ath/if_ath_beacon.h>
 111 #include <dev/netif/ath/ath/if_athdfs.h>
 112
 113 #ifdef ATH_TX99_DIAG
 114 #include <dev/netif/ath/ath_tx99/ath_tx99.h>
 115 #endif
 116
 117 #include <dev/netif/ath/ath/if_ath_rx_edma.h>
 118
 119 #ifdef  ATH_DEBUG_ALQ
 120 #include <dev/netif/ath/ath/if_ath_alq.h>
 121 #endif
 122
 123 /*
 124  * some general macros
 125   */
 126 #define INCR(_l, _sz)           (_l) ++; (_l) &= ((_sz) - 1)
 127 #define DECR(_l, _sz)           (_l) --; (_l) &= ((_sz) - 1)
 128
 129 MALLOC_DECLARE(M_ATHDEV);
 130
 131 /*
 132  * XXX TODO:
 133  *
 134  * + Make sure the FIFO is correctly flushed and reinitialised
 135  *   through a reset;
 136  * + Verify multi-descriptor frames work!
 137  * + There's a "memory use after free" which needs to be tracked down
 138  *   and fixed ASAP.  I've seen this in the legacy path too, so it
 139  *   may be a generic RX path issue.
 140  */
 141
 142 /*
 143  * XXX shuffle the function orders so these pre-declarations aren't
 144  * required!
 145  */
 146 static  int ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 147             int nbufs);
 148 static  int ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype);
 149 static  void ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf);
 150 static  int ath_edma_recv_proc_queue(struct ath_softc *sc,
 151             HAL_RX_QUEUE qtype, int dosched);
 152 static  int ath_edma_recv_proc_deferred_queue(struct ath_softc *sc,
 153             HAL_RX_QUEUE qtype, int dosched);
 154
 155 static void
 156 ath_edma_stoprecv(struct ath_softc *sc, int dodelay)
 157 {
 158         struct ath_hal *ah = sc->sc_ah;
 159
 160         ATH_RX_LOCK(sc);
 161         ath_hal_stoppcurecv(ah);
 162         ath_hal_setrxfilter(ah, 0);
 163         ath_hal_stopdmarecv(ah);
 164
 165         DELAY(3000);
 166
 167         /* Flush RX pending for each queue */
 168         /* XXX should generic-ify this */
 169         if (sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending) {
 170                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending);
 171                 sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending = NULL;
 172         }
 173
 174         if (sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending) {
 175                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending);
 176                 sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending = NULL;
 177         }
 178         ATH_RX_UNLOCK(sc);
 179 }
 180
 181 /*
 182  * Re-initialise the FIFO given the current buffer contents.
 183  * Specifically, walk from head -> tail, pushing the FIFO contents
 184  * back into the FIFO.
 185  */
 186 static void
 187 ath_edma_reinit_fifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 188 {
 189         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 190         struct ath_buf *bf;
 191         int i, j;
 192
 193         ATH_RX_LOCK_ASSERT(sc);
 194
 195         i = re->m_fifo_head;
 196         for (j = 0; j < re->m_fifo_depth; j++) {
 197                 bf = re->m_fifo[i];
 198                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 199                     "%s: Q%d: pos=%i, addr=0x%jx\n",
 200                     __func__,
 201                     qtype,
 202                     i,
 203                     (uintmax_t)bf->bf_daddr);
 204                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
 205                 INCR(i, re->m_fifolen);
 206         }
 207
 208         /* Ensure this worked out right */
 209         if (i != re->m_fifo_tail) {
 210                 device_printf(sc->sc_dev, "%s: i (%d) != tail! (%d)\n",
 211                     __func__,
 212                     i,
 213                     re->m_fifo_tail);
 214         }
 215 }
 216
 217 static void
 218 ath_edma_handle_rxfifo_reset(struct ath_softc *sc)
 219 {
 220         if (sc->sc_rxfifo_state == ATH_RXFIFO_RESET) {
 221                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 222                     "%s: Re-initing HP FIFO\n", __func__);
 223                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_HP);
 224                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 225                     "%s: Re-initing LP FIFO\n", __func__);
 226                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_LP);
 227                 sc->sc_rxfifo_state = ATH_RXFIFO_OK;
 228         }
 229 }
 230
 231 /*
 232  * Start receive.
 233  *
 234  * XXX TODO: this needs to reallocate the FIFO entries when a reset
 235  * occurs, in case the FIFO is filled up and no new descriptors get
 236  * thrown into the FIFO.
 237  */
 238 static int
 239 ath_edma_startrecv(struct ath_softc *sc)
 240 {
 241         struct ath_hal *ah = sc->sc_ah;
 242
 243         ATH_RX_LOCK(sc);
 244
 245         /* Enable RX FIFO */
 246         ath_hal_rxena(ah);
 247         ath_edma_handle_rxfifo_reset(sc);
 248
 249         /* Add up to m_fifolen entries in each queue */
 250         /*
 251          * These must occur after the above write so the FIFO buffers
 252          * are pushed/tracked in the same order as the hardware will
 253          * process them.
 254          */
 255         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_HP,
 256             sc->sc_rxedma[HAL_RX_QUEUE_HP].m_fifolen);
 257
 258         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_LP,
 259             sc->sc_rxedma[HAL_RX_QUEUE_LP].m_fifolen);
 260
 261         ath_mode_init(sc);
 262         ath_hal_startpcurecv(ah);
 263
 264         ATH_RX_UNLOCK(sc);
 265
 266         return (0);
 267 }
 268
 269 static void
 270 ath_edma_recv_sched_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 271     int dosched)
 272 {
 273         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 274         if (dosched)
 275                 ath_edma_handle_rxfifo_reset(sc);
 276         ath_edma_recv_proc_queue(sc, qtype, dosched);
 277         ath_power_restore_power_state(sc);
 278
 279         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
 280 }
 281
 282 static void
 283 ath_edma_recv_sched(struct ath_softc *sc, int dosched)
 284 {
 285         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 286         if (dosched)
 287                 ath_edma_handle_rxfifo_reset(sc);
 288         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, dosched);
 289         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, dosched);
 290         ath_power_restore_power_state(sc);
 291
 292         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
 293 }
 294
 295 static void
 296 ath_edma_recv_flush(struct ath_softc *sc)
 297 {
 298
 299         DPRINTF(sc, ATH_DEBUG_RECV, "%s: called\n", __func__);
 300
 301         ATH_PCU_LOCK(sc);
 302         sc->sc_rxproc_cnt++;
 303         ATH_PCU_UNLOCK(sc);
 304
 305         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 306
 307         /*
 308          * Flush any active frames from FIFO -> deferred list
 309          */
 310         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 0);
 311         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 0);
 312
 313         /*
 314          * Process what's in the deferred queue
 315          */
 316         /*
 317          * XXX: If we read the tsf/channoise here and then pass it in,
 318          * we could restore the power state before processing
 319          * the deferred queue.
 320          */
 321         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 0);
 322         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 0);
 323
 324         ath_power_restore_power_state(sc);
 325
 326         ATH_PCU_LOCK(sc);
 327         sc->sc_rxproc_cnt--;
 328         ATH_PCU_UNLOCK(sc);
 329 }
 330
 331 /*
 332  * Process frames from the current queue into the deferred queue.
 333  */
 334 static int
 335 ath_edma_recv_proc_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 336     int dosched)
 337 {
 338         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 339         struct ath_rx_status *rs;
 340         struct ath_desc *ds;
 341         struct ath_buf *bf;
 342         struct mbuf *m;
 343         struct ath_hal *ah = sc->sc_ah;
 344         uint64_t tsf;
 345         uint16_t nf;
 346         int npkts = 0;
 347         int n = 0;
 348
 349         tsf = ath_hal_gettsf64(ah);
 350         nf = ath_hal_getchannoise(ah, sc->sc_curchan);
 351         sc->sc_stats.ast_rx_noise = nf;
 352
 353         ATH_RX_LOCK(sc);
 354
 355         do {
 356                 bf = re->m_fifo[re->m_fifo_head];
 357                 /* This shouldn't occur! */
 358                 if (bf == NULL) {
 359                         device_printf(sc->sc_dev, "%s: Q%d: NULL bf?\n",
 360                             __func__,
 361                             qtype);
 362                         break;
 363                 }
 364                 m = bf->bf_m;
 365                 ds = bf->bf_desc;
 366
 367                 /*
 368                  * Sync descriptor memory - this also syncs the buffer for us.
 369                  * EDMA descriptors are in cached memory.
 370                  */
 371                 cpu_lfence();
 372                 bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
 373                                 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 374                 rs = &bf->bf_status.ds_rxstat;
 375                 bf->bf_rxstatus = ath_hal_rxprocdesc(ah, ds, bf->bf_daddr,
 376                     NULL, rs);
 377 #ifdef  ATH_DEBUG
 378                 if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
 379                         ath_printrxbuf(sc, bf, 0, bf->bf_rxstatus == HAL_OK);
 380 #endif /* ATH_DEBUG */
 381 #ifdef  ATH_DEBUG_ALQ
 382                 if (if_ath_alq_checkdebug(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS))
 383                         if_ath_alq_post(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS,
 384                             sc->sc_rx_statuslen, (char *) ds);
 385 #endif /* ATH_DEBUG */
 386                 if (bf->bf_rxstatus == HAL_EINPROGRESS) {
 387                         DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 388                             "%s: Q%d: still in-prog!\n", __func__, qtype);
 389                         break;
 390                 }
 391
 392                 /*
 393                  * Completed descriptor.
 394                  */
 395                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 396                     "%s: Q%d: completed!\n", __func__, qtype);
 397                 npkts++;
 398
 399                 /*
 400                  * We've been synced already, so unmap.
 401                  */
 402                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 403
 404                 /*
 405                  * Remove the FIFO entry and place it on the completion
 406                  * queue.
 407                  */
 408                 re->m_fifo[re->m_fifo_head] = NULL;
 409                 TAILQ_INSERT_TAIL(&sc->sc_rx_rxlist[qtype], bf, bf_list);
 410
 411                 /* Bump the descriptor FIFO stats */
 412                 INCR(re->m_fifo_head, re->m_fifolen);
 413                 re->m_fifo_depth--;
 414                 /* XXX check it doesn't fall below 0 */
 415         } while (re->m_fifo_depth > 0);
 416
 417         /* Append some more fresh frames to the FIFO */
 418         /* (kick already handled when dosched != 0) */
 419         if (dosched)
 420                 n = ath_edma_rxfifo_alloc(sc, qtype, re->m_fifolen);
 421
 422         ATH_RX_UNLOCK(sc);
 423
 424         /* rx signal state monitoring */
 425         ath_hal_rxmonitor(ah, &sc->sc_halstats, sc->sc_curchan);
 426
 427         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
 428             "ath edma rx proc: npkts=%d\n",
 429             npkts);
 430
 431         return n;
 432 }
 433
 434 /*
 435  * Flush the deferred queue.
 436  *
 437  * This destructively flushes the deferred queue - it doesn't
 438  * call the wireless stack on each mbuf.
 439  */
 440 static void
 441 ath_edma_flush_deferred_queue(struct ath_softc *sc)
 442 {
 443         struct ath_buf *bf;
 444
 445         ATH_RX_LOCK_ASSERT(sc);
 446
 447         /* Free in one set, inside the lock */
 448         while ((bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP])) != NULL) {
 449                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP], bf, bf_list);
 450                 ath_edma_rxbuf_free(sc, bf);
 451         }
 452         while ((bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP])) != NULL) {
 453                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP], bf, bf_list);
 454                 ath_edma_rxbuf_free(sc, bf);
 455         }
 456 }
 457
 458 static int
 459 ath_edma_recv_proc_deferred_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
 460     int dosched)
 461 {
 462         int ngood = 0;
 463         uint64_t tsf;
 464         struct ath_buf *bf;
 465         struct ath_buf *next;
 466         struct ath_rx_status *rs;
 467         int16_t nf;
 468         ath_bufhead rxlist;
 469         struct mbuf *m;
 470
 471         TAILQ_INIT(&rxlist);
 472
 473         nf = ath_hal_getchannoise(sc->sc_ah, sc->sc_curchan);
 474         /*
 475          * XXX TODO: the NF/TSF should be stamped on the bufs themselves,
 476          * otherwise we may end up adding in the wrong values if this
 477          * is delayed too far..
 478          */
 479         tsf = ath_hal_gettsf64(sc->sc_ah);
 480
 481         /* Copy the list over */
 482         ATH_RX_LOCK(sc);
 483         TAILQ_CONCAT(&rxlist, &sc->sc_rx_rxlist[qtype], bf_list);
 484         ATH_RX_UNLOCK(sc);
 485
 486         /* Handle the completed descriptors */
 487         TAILQ_FOREACH_MUTABLE(bf, &rxlist, bf_list, next) {
 488                 /*
 489                  * Skip the RX descriptor status - start at the data offset
 490                  */
 491                 m_adj(bf->bf_m, sc->sc_rx_statuslen);
 492
 493                 /* Handle the frame */
 494
 495                 rs = &bf->bf_status.ds_rxstat;
 496                 m = bf->bf_m;
 497                 bf->bf_m = NULL;
 498                 if (ath_rx_pkt(sc, rs, bf->bf_rxstatus, tsf, nf, qtype, bf, m))
 499                         ngood++;
 500         }
 501
 502         if (ngood) {
 503                 sc->sc_lastrx = tsf;
 504         }
 505
 506         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
 507             "ath edma rx deferred proc: ngood=%d\n",
 508             ngood);
 509
 510         /* Free in one set, inside the lock */
 511         ATH_RX_LOCK(sc);
 512
 513         while ((bf = TAILQ_FIRST(&rxlist)) != NULL) {
 514                 /* Free the buffer/mbuf */
 515                 TAILQ_REMOVE(&rxlist, bf, bf_list);
 516                 ath_edma_rxbuf_free(sc, bf);
 517         }
 518         ATH_RX_UNLOCK(sc);
 519
 520         return (ngood);
 521 }
 522
 523 static void
 524 ath_edma_recv_tasklet(void *arg, int npending)
 525 {
 526         struct ath_softc *sc = (struct ath_softc *) arg;
 527         struct ifnet *ifp = sc->sc_ifp;
 528 #ifdef  IEEE80211_SUPPORT_SUPERG
 529         struct ieee80211com *ic = ifp->if_l2com;
 530 #endif
 531         int n1;
 532         int n2;
 533
 534         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: called; npending=%d\n",
 535             __func__,
 536             npending);
 537
 538         wlan_serialize_enter();
 539         ATH_PCU_LOCK(sc);
 540         if (sc->sc_inreset_cnt > 0) {
 541                 device_printf(sc->sc_dev, "%s: sc_inreset_cnt > 0; skipping\n",
 542                     __func__);
 543                 ATH_PCU_UNLOCK(sc);
 544                 wlan_serialize_exit();
 545                 return;
 546         }
 547         sc->sc_rxproc_cnt++;
 548         ATH_PCU_UNLOCK(sc);
 549
 550         ath_power_set_power_state(sc, HAL_PM_AWAKE);
 551
 552         ath_edma_handle_rxfifo_reset(sc);
 553         n1 = ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 1);
 554         n2 = ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 1);
 555
 556         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 1);
 557         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 1);
 558
 559         /* Handle resched and kickpcu appropriately */
 560         ATH_PCU_LOCK(sc);
 561         if (sc->sc_kickpcu) {
 562 #ifdef ATH_DEBUG
 563                 if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
 564                         kprintf("k(%d,%d)", n1, n2);
 565 #endif
 566                 sc->sc_kickpcu = 0;
 567                 /* reload imask XXX */
 568                 if (n1 || n2)
 569                         ath_hal_intrset(sc->sc_ah, sc->sc_imask);
 570         }
 571         ATH_PCU_UNLOCK(sc);
 572
 573         /*
 574          * XXX: If we read the tsf/channoise here and then pass it in,
 575          * we could restore the power state before processing
 576          * the deferred queue.
 577          */
 578         ath_power_restore_power_state(sc);
 579
 580         /* XXX inside IF_LOCK ? */
 581         if (!ifq_is_oactive(&ifp->if_snd)) {
 582 #ifdef  IEEE80211_SUPPORT_SUPERG
 583                 ieee80211_ff_age_all(ic, 100);
 584 #endif
 585                 if (!ifq_is_empty(&ifp->if_snd))
 586                         ath_tx_kick(sc);
 587         }
 588         if (ath_dfs_tasklet_needed(sc, sc->sc_curchan))
 589                 taskqueue_enqueue(sc->sc_tq, &sc->sc_dfstask);
 590
 591         ATH_PCU_LOCK(sc);
 592         sc->sc_rxproc_cnt--;
 593         ATH_PCU_UNLOCK(sc);
 594         wlan_serialize_exit();
 595 }
 596
 597 /*
 598  * Allocate an RX mbuf for the given ath_buf and initialise
 599  * it for EDMA.
 600  *
 601  * + Allocate a 4KB mbuf;
 602  * + Setup the DMA map for the given buffer;
 603  * + Return that.
 604  */
 605 static int
 606 ath_edma_rxbuf_init(struct ath_softc *sc, struct ath_buf *bf)
 607 {
 608
 609         struct mbuf *m;
 610         int error;
 611         int len;
 612
 613         ATH_RX_LOCK_ASSERT(sc);
 614
 615         m = m_getjcl(MB_DONTWAIT, MT_DATA, M_PKTHDR, sc->sc_edma_bufsize);
 616 /*      m = m_getcl(MB_WAIT, MT_DATA, M_PKTHDR);*/
 617 /*      m = m_getm(NULL, sc->sc_edma_bufsize, MB_WAIT, MT_DATA);*/
 618         if (! m)
 619                 return (ENOBUFS);               /* XXX ?*/
 620
 621         /* XXX warn/enforce alignment */
 622
 623         len = m->m_ext.ext_size;
 624 #if 0
 625         device_printf(sc->sc_dev, "%s: called: m=%p, size=%d, mtod=%p\n",
 626             __func__,
 627             m,
 628             len,
 629             mtod(m, char *));
 630 #endif
 631
 632         m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
 633
 634         /*
 635          * Populate ath_buf fields.
 636          */
 637         KKASSERT(m->m_len >= sc->sc_rx_statuslen);
 638         bf->bf_desc = mtod(m, struct ath_desc *);
 639         bf->bf_lastds = bf->bf_desc;    /* XXX only really for TX? */
 640         bf->bf_m = m;
 641
 642         /*
 643          * Zero the descriptor and ensure it makes it out to the
 644          * bounce buffer if one is required.
 645          *
 646          * XXX PREWRITE will copy the whole buffer; we only needed it
 647          * to sync the first 32 DWORDS.  Oh well.
 648          */
 649         memset(bf->bf_desc, '\0', sc->sc_rx_statuslen);
 650
 651         /*
 652          * Create DMA mapping.
 653          */
 654         error = bus_dmamap_load_mbuf_segment(sc->sc_dmat,
 655             bf->bf_dmamap, m, bf->bf_segs, 1, &bf->bf_nseg, BUS_DMA_NOWAIT);
 656
 657         if (error != 0) {
 658                 device_printf(sc->sc_dev, "%s: failed; error=%d\n",
 659                     __func__,
 660                     error);
 661                 m_freem(m);
 662                 return (error);
 663         }
 664
 665         /*
 666          * Set daddr to the physical mapping page.
 667          */
 668         bf->bf_daddr = bf->bf_segs[0].ds_addr;
 669
 670         /*
 671          * Prepare for the upcoming read.
 672          *
 673          * We need to both sync some data into the buffer (the zero'ed
 674          * descriptor payload) and also prepare for the read that's going
 675          * to occur.
 676          */
 677         bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
 678                         BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 679
 680         /* Finish! */
 681         return (0);
 682 }
 683
 684 /*
 685  * Allocate a RX buffer.
 686  */
 687 static struct ath_buf *
 688 ath_edma_rxbuf_alloc(struct ath_softc *sc)
 689 {
 690         struct ath_buf *bf;
 691         int error;
 692
 693         ATH_RX_LOCK_ASSERT(sc);
 694
 695         /* Allocate buffer */
 696         bf = TAILQ_FIRST(&sc->sc_rxbuf);
 697         /* XXX shouldn't happen upon startup? */
 698         if (bf == NULL) {
 699                 device_printf(sc->sc_dev, "%s: nothing on rxbuf?!\n",
 700                     __func__);
 701                 return (NULL);
 702         }
 703
 704         /* Remove it from the free list */
 705         TAILQ_REMOVE(&sc->sc_rxbuf, bf, bf_list);
 706
 707         /* Assign RX mbuf to it */
 708         error = ath_edma_rxbuf_init(sc, bf);
 709         if (error != 0) {
 710                 device_printf(sc->sc_dev,
 711                     "%s: bf=%p, rxbuf alloc failed! error=%d\n",
 712                     __func__,
 713                     bf,
 714                     error);
 715                 TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 716                 return (NULL);
 717         }
 718
 719         return (bf);
 720 }
 721
 722 static void
 723 ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf)
 724 {
 725
 726         ATH_RX_LOCK_ASSERT(sc);
 727
 728         /*
 729          * Only unload the frame if we haven't consumed
 730          * the mbuf via ath_rx_pkt().
 731          */
 732         if (bf->bf_m) {
 733                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 734                 m_freem(bf->bf_m);
 735                 bf->bf_m = NULL;
 736         }
 737
 738         /* XXX lock? */
 739         TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 740 }
 741
 742 /*
 743  * Allocate up to 'n' entries and push them onto the hardware FIFO.
 744  *
 745  * Return how many entries were successfully pushed onto the
 746  * FIFO.
 747  */
 748 static int
 749 ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype, int nbufs)
 750 {
 751         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 752         struct ath_buf *bf;
 753         int i;
 754
 755         ATH_RX_LOCK_ASSERT(sc);
 756
 757         /*
 758          * Allocate buffers until the FIFO is full or nbufs is reached.
 759          */
 760         for (i = 0; i < nbufs && re->m_fifo_depth < re->m_fifolen; i++) {
 761                 /* Ensure the FIFO is already blank, complain loudly! */
 762                 if (re->m_fifo[re->m_fifo_tail] != NULL) {
 763                         device_printf(sc->sc_dev,
 764                             "%s: Q%d: fifo[%d] != NULL (%p)\n",
 765                             __func__,
 766                             qtype,
 767                             re->m_fifo_tail,
 768                             re->m_fifo[re->m_fifo_tail]);
 769
 770                         /* Free the slot */
 771                         ath_edma_rxbuf_free(sc, re->m_fifo[re->m_fifo_tail]);
 772                         re->m_fifo_depth--;
 773                         /* XXX check it's not < 0 */
 774                         re->m_fifo[re->m_fifo_tail] = NULL;
 775                 }
 776
 777                 bf = ath_edma_rxbuf_alloc(sc);
 778                 /* XXX should ensure the FIFO is not NULL? */
 779                 if (bf == NULL) {
 780                         device_printf(sc->sc_dev,
 781                             "%s: Q%d: alloc failed: i=%d, nbufs=%d?\n",
 782                             __func__,
 783                             qtype,
 784                             i,
 785                             nbufs);
 786                         break;
 787                 }
 788
 789                 re->m_fifo[re->m_fifo_tail] = bf;
 790
 791                 /* Write to the RX FIFO */
 792                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
 793                     "%s: Q%d: putrxbuf=%p (0x%jx)\n",
 794                     __func__,
 795                     qtype,
 796                     bf->bf_desc,
 797                     (uintmax_t) bf->bf_daddr);
 798                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
 799
 800                 re->m_fifo_depth++;
 801                 INCR(re->m_fifo_tail, re->m_fifolen);
 802         }
 803
 804         /*
 805          * Return how many were allocated.
 806          */
 807         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: Q%d: nbufs=%d, nalloced=%d\n",
 808             __func__,
 809             qtype,
 810             nbufs,
 811             i);
 812         return (i);
 813 }
 814
 815 static int
 816 ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 817 {
 818         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 819         int i;
 820
 821         ATH_RX_LOCK_ASSERT(sc);
 822
 823         for (i = 0; i < re->m_fifolen; i++) {
 824                 if (re->m_fifo[i] != NULL) {
 825 #ifdef  ATH_DEBUG
 826                         struct ath_buf *bf = re->m_fifo[i];
 827
 828                         if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
 829                                 ath_printrxbuf(sc, bf, 0, HAL_OK);
 830 #endif
 831                         ath_edma_rxbuf_free(sc, re->m_fifo[i]);
 832                         re->m_fifo[i] = NULL;
 833                         re->m_fifo_depth--;
 834                 }
 835         }
 836
 837         if (re->m_rxpending != NULL) {
 838                 m_freem(re->m_rxpending);
 839                 re->m_rxpending = NULL;
 840         }
 841         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
 842
 843         return (0);
 844 }
 845
 846 /*
 847  * Setup the initial RX FIFO structure.
 848  */
 849 static int
 850 ath_edma_setup_rxfifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 851 {
 852         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 853
 854         ATH_RX_LOCK_ASSERT(sc);
 855
 856         if (! ath_hal_getrxfifodepth(sc->sc_ah, qtype, &re->m_fifolen)) {
 857                 device_printf(sc->sc_dev, "%s: qtype=%d, failed\n",
 858                     __func__,
 859                     qtype);
 860                 return (-EINVAL);
 861         }
 862         device_printf(sc->sc_dev, "%s: type=%d, FIFO depth = %d entries\n",
 863             __func__,
 864             qtype,
 865             re->m_fifolen);
 866
 867         /* Allocate ath_buf FIFO array, pre-zero'ed */
 868         re->m_fifo = kmalloc(sizeof(struct ath_buf *) * re->m_fifolen,
 869             M_ATHDEV,
 870             M_INTWAIT | M_ZERO);
 871         if (re->m_fifo == NULL) {
 872                 device_printf(sc->sc_dev, "%s: malloc failed\n",
 873                     __func__);
 874                 return (-ENOMEM);
 875         }
 876
 877         /*
 878          * Set initial "empty" state.
 879          */
 880         re->m_rxpending = NULL;
 881         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
 882
 883         return (0);
 884 }
 885
 886 static int
 887 ath_edma_rxfifo_free(struct ath_softc *sc, HAL_RX_QUEUE qtype)
 888 {
 889         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
 890
 891         device_printf(sc->sc_dev, "%s: called; qtype=%d\n",
 892             __func__,
 893             qtype);
 894
 895         kfree(re->m_fifo, M_ATHDEV);
 896
 897         return (0);
 898 }
 899
 900 static int
 901 ath_edma_dma_rxsetup(struct ath_softc *sc)
 902 {
 903         int error;
 904
 905         /*
 906          * Create RX DMA tag and buffers.
 907          */
 908         error = ath_descdma_setup_rx_edma(sc, &sc->sc_rxdma, &sc->sc_rxbuf,
 909             "rx", ath_rxbuf, sc->sc_rx_statuslen);
 910         if (error != 0)
 911                 return error;
 912
 913         ATH_RX_LOCK(sc);
 914         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_HP);
 915         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_LP);
 916         ATH_RX_UNLOCK(sc);
 917
 918         return (0);
 919 }
 920
 921 static int
 922 ath_edma_dma_rxteardown(struct ath_softc *sc)
 923 {
 924
 925         ATH_RX_LOCK(sc);
 926         ath_edma_flush_deferred_queue(sc);
 927         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_HP);
 928         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_HP);
 929
 930         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_LP);
 931         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_LP);
 932         ATH_RX_UNLOCK(sc);
 933
 934         /* Free RX ath_buf */
 935         /* Free RX DMA tag */
 936         if (sc->sc_rxdma.dd_desc_len != 0)
 937                 ath_descdma_cleanup(sc, &sc->sc_rxdma, &sc->sc_rxbuf);
 938
 939         return (0);
 940 }
 941
 942 void
 943 ath_recv_setup_edma(struct ath_softc *sc)
 944 {
 945
 946         /* Set buffer size to 4k */
 947         sc->sc_edma_bufsize = 4096;
 948
 949         /* Fetch EDMA field and buffer sizes */
 950         (void) ath_hal_getrxstatuslen(sc->sc_ah, &sc->sc_rx_statuslen);
 951
 952         /* Configure the hardware with the RX buffer size */
 953         (void) ath_hal_setrxbufsize(sc->sc_ah, sc->sc_edma_bufsize -
 954             sc->sc_rx_statuslen);
 955
 956         device_printf(sc->sc_dev, "RX status length: %d\n",
 957             sc->sc_rx_statuslen);
 958         device_printf(sc->sc_dev, "RX buffer size: %d\n",
 959             sc->sc_edma_bufsize);
 960
 961         sc->sc_rx.recv_stop = ath_edma_stoprecv;
 962         sc->sc_rx.recv_start = ath_edma_startrecv;
 963         sc->sc_rx.recv_flush = ath_edma_recv_flush;
 964         sc->sc_rx.recv_tasklet = ath_edma_recv_tasklet;
 965         sc->sc_rx.recv_rxbuf_init = ath_edma_rxbuf_init;
 966
 967         sc->sc_rx.recv_setup = ath_edma_dma_rxsetup;
 968         sc->sc_rx.recv_teardown = ath_edma_dma_rxteardown;
 969
 970         sc->sc_rx.recv_sched = ath_edma_recv_sched;
 971         sc->sc_rx.recv_sched_queue = ath_edma_recv_sched_queue;
 972 }