sys/bus/u4b/usb_busdma.c

   1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */
   2 /*-
   3  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/stdint.h>
  28 #include <sys/param.h>
  29 #include <sys/queue.h>
  30 #include <sys/types.h>
  31 #include <sys/systm.h>
  32 #include <sys/kernel.h>
  33 #include <sys/bus.h>
  34 #include <sys/module.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/mutex2.h>
  38 #include <sys/condvar.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/unistd.h>
  41 #include <sys/callout.h>
  42 #include <sys/malloc.h>
  43 #include <sys/priv.h>
  44
  45 #include <bus/u4b/usb.h>
  46 #include <bus/u4b/usbdi.h>
  47 #include <bus/u4b/usbdi_util.h>
  48
  49 #define USB_DEBUG_VAR usb_debug
  50
  51 #include <bus/u4b/usb_core.h>
  52 #include <bus/u4b/usb_busdma.h>
  53 #include <bus/u4b/usb_process.h>
  54 #include <bus/u4b/usb_transfer.h>
  55 #include <bus/u4b/usb_device.h>
  56 #include <bus/u4b/usb_util.h>
  57 #include <bus/u4b/usb_debug.h>
  58
  59 #include <bus/u4b/usb_controller.h>
  60 #include <bus/u4b/usb_bus.h>
  61
  62 #if USB_HAVE_BUSDMA
  63 static void     usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
  64 static void     usb_dma_tag_destroy(struct usb_dma_tag *);
  65 #if 0
  66 static void     usb_dma_lock_cb(void *, bus_dma_lock_op_t);
  67 #endif
  68 static void     usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
  69 static void     usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
  70 static void     usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
  71                     uint8_t);
  72 #endif
  73
  74 /*------------------------------------------------------------------------*
  75  *  usbd_get_page - lookup DMA-able memory for the given offset
  76  *
  77  * NOTE: Only call this function when the "page_cache" structure has
  78  * been properly initialized !
  79  *------------------------------------------------------------------------*/
  80 void
  81 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
  82     struct usb_page_search *res)
  83 {
  84 #if USB_HAVE_BUSDMA
  85         struct usb_page *page;
  86
  87         if (pc->page_start) {
  88
  89                 /* Case 1 - something has been loaded into DMA */
  90
  91                 if (pc->buffer) {
  92
  93                         /* Case 1a - Kernel Virtual Address */
  94
  95                         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
  96                 }
  97                 offset += pc->page_offset_buf;
  98
  99                 /* compute destination page */
 100
 101                 page = pc->page_start;
 102
 103                 if (pc->ismultiseg) {
 104
 105                         page += (offset / USB_PAGE_SIZE);
 106
 107                         offset %= USB_PAGE_SIZE;
 108
 109                         res->length = USB_PAGE_SIZE - offset;
 110                         res->physaddr = page->physaddr + offset;
 111                 } else {
 112                         res->length = (usb_size_t)-1;
 113                         res->physaddr = page->physaddr + offset;
 114                 }
 115                 if (!pc->buffer) {
 116
 117                         /* Case 1b - Non Kernel Virtual Address */
 118
 119                         res->buffer = USB_ADD_BYTES(page->buffer, offset);
 120                 }
 121                 return;
 122         }
 123 #endif
 124         /* Case 2 - Plain PIO */
 125
 126         res->buffer = USB_ADD_BYTES(pc->buffer, offset);
 127         res->length = (usb_size_t)-1;
 128 #if USB_HAVE_BUSDMA
 129         res->physaddr = 0;
 130 #endif
 131 }
 132
 133 /*------------------------------------------------------------------------*
 134  *  usbd_copy_in - copy directly to DMA-able memory
 135  *------------------------------------------------------------------------*/
 136 void
 137 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
 138     const void *ptr, usb_frlength_t len)
 139 {
 140         struct usb_page_search buf_res;
 141
 142         while (len != 0) {
 143
 144                 usbd_get_page(cache, offset, &buf_res);
 145
 146                 if (buf_res.length > len) {
 147                         buf_res.length = len;
 148                 }
 149                 memcpy(buf_res.buffer, ptr, buf_res.length);
 150
 151                 offset += buf_res.length;
 152                 len -= buf_res.length;
 153                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 154         }
 155 }
 156
 157 /*------------------------------------------------------------------------*
 158  *  usbd_copy_in_user - copy directly to DMA-able memory from userland
 159  *
 160  * Return values:
 161  *    0: Success
 162  * Else: Failure
 163  *------------------------------------------------------------------------*/
 164 #if USB_HAVE_USER_IO
 165 int
 166 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
 167     const void *ptr, usb_frlength_t len)
 168 {
 169         struct usb_page_search buf_res;
 170         int error;
 171
 172         while (len != 0) {
 173
 174                 usbd_get_page(cache, offset, &buf_res);
 175
 176                 if (buf_res.length > len) {
 177                         buf_res.length = len;
 178                 }
 179                 error = copyin(ptr, buf_res.buffer, buf_res.length);
 180                 if (error)
 181                         return (error);
 182
 183                 offset += buf_res.length;
 184                 len -= buf_res.length;
 185                 ptr = USB_ADD_BYTES(ptr, buf_res.length);
 186         }
 187         return (0);                     /* success */
 188 }
 189 #endif
 190
 191 /*------------------------------------------------------------------------*
 192  *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
 193  *------------------------------------------------------------------------*/
 194 #if USB_HAVE_MBUF
 195 struct usb_m_copy_in_arg {
 196         struct usb_page_cache *cache;
 197         usb_frlength_t dst_offset;
 198 };
 199
 200 static int
 201 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
 202 {
 203         register struct usb_m_copy_in_arg *ua = arg;
 204
 205         usbd_copy_in(ua->cache, ua->dst_offset, src, count);
 206         ua->dst_offset += count;
 207         return (0);
 208 }
 209
 210 void
 211 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
 212     struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
 213 {
 214         struct usb_m_copy_in_arg arg = {cache, dst_offset};
 215         (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
 216 }
 217 #endif
 218
 219 /*------------------------------------------------------------------------*
 220  *  usb_uiomove - factored out code
 221  *------------------------------------------------------------------------*/
 222 #if USB_HAVE_USER_IO
 223 int
 224 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
 225     usb_frlength_t pc_offset, usb_frlength_t len)
 226 {
 227         struct usb_page_search res;
 228         int error = 0;
 229
 230         while (len != 0) {
 231
 232                 usbd_get_page(pc, pc_offset, &res);
 233
 234                 if (res.length > len) {
 235                         res.length = len;
 236                 }
 237                 /*
 238                  * "uiomove()" can sleep so one needs to make a wrapper,
 239                  * exiting the mutex and checking things
 240                  */
 241                 error = uiomove(res.buffer, res.length, uio);
 242
 243                 if (error) {
 244                         break;
 245                 }
 246                 pc_offset += res.length;
 247                 len -= res.length;
 248         }
 249         return (error);
 250 }
 251 #endif
 252
 253 /*------------------------------------------------------------------------*
 254  *  usbd_copy_out - copy directly from DMA-able memory
 255  *------------------------------------------------------------------------*/
 256 void
 257 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
 258     void *ptr, usb_frlength_t len)
 259 {
 260         struct usb_page_search res;
 261
 262         while (len != 0) {
 263
 264                 usbd_get_page(cache, offset, &res);
 265
 266                 if (res.length > len) {
 267                         res.length = len;
 268                 }
 269                 memcpy(ptr, res.buffer, res.length);
 270
 271                 offset += res.length;
 272                 len -= res.length;
 273                 ptr = USB_ADD_BYTES(ptr, res.length);
 274         }
 275 }
 276
 277 /*------------------------------------------------------------------------*
 278  *  usbd_copy_out_user - copy directly from DMA-able memory to userland
 279  *
 280  * Return values:
 281  *    0: Success
 282  * Else: Failure
 283  *------------------------------------------------------------------------*/
 284 #if USB_HAVE_USER_IO
 285 int
 286 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
 287     void *ptr, usb_frlength_t len)
 288 {
 289         struct usb_page_search res;
 290         int error;
 291
 292         while (len != 0) {
 293
 294                 usbd_get_page(cache, offset, &res);
 295
 296                 if (res.length > len) {
 297                         res.length = len;
 298                 }
 299                 error = copyout(res.buffer, ptr, res.length);
 300                 if (error)
 301                         return (error);
 302
 303                 offset += res.length;
 304                 len -= res.length;
 305                 ptr = USB_ADD_BYTES(ptr, res.length);
 306         }
 307         return (0);                     /* success */
 308 }
 309 #endif
 310
 311 /*------------------------------------------------------------------------*
 312  *  usbd_frame_zero - zero DMA-able memory
 313  *------------------------------------------------------------------------*/
 314 void
 315 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
 316     usb_frlength_t len)
 317 {
 318         struct usb_page_search res;
 319
 320         while (len != 0) {
 321
 322                 usbd_get_page(cache, offset, &res);
 323
 324                 if (res.length > len) {
 325                         res.length = len;
 326                 }
 327                 memset(res.buffer, 0, res.length);
 328
 329                 offset += res.length;
 330                 len -= res.length;
 331         }
 332 }
 333
 334 #if USB_HAVE_BUSDMA
 335
 336 /*------------------------------------------------------------------------*
 337  *      usb_dma_lock_cb - dummy callback
 338  *------------------------------------------------------------------------*/
 339 #if 0
 340 static void
 341 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
 342 {
 343         /* we use "mtx_owned()" instead of this function */
 344 }
 345 #endif
 346
 347 /*------------------------------------------------------------------------*
 348  *      usb_dma_tag_create - allocate a DMA tag
 349  *
 350  * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
 351  * allow multi-segment mappings. Else all mappings are single-segment.
 352  *------------------------------------------------------------------------*/
 353 static void
 354 usb_dma_tag_create(struct usb_dma_tag *udt,
 355     usb_size_t size, usb_size_t align)
 356 {
 357         bus_dma_tag_t tag;
 358
 359         if (bus_dma_tag_create
 360             ( /* parent    */ udt->tag_parent->tag,
 361              /* alignment */ align,
 362              /* boundary  */ 0,
 363              /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
 364              /* highaddr  */ BUS_SPACE_MAXADDR,
 365              /* filter    */ NULL,
 366              /* filterarg */ NULL,
 367              /* maxsize   */ size,
 368              /* nsegments */ (align == 1 && size > 1) ? (2 + (size / USB_PAGE_SIZE)) : 1,
 369              /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ? USB_PAGE_SIZE : size,
 370              /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
 371             &tag)) {
 372                 tag = NULL;
 373         }
 374         udt->tag = tag;
 375 }
 376
 377 /*------------------------------------------------------------------------*
 378  *      usb_dma_tag_free - free a DMA tag
 379  *------------------------------------------------------------------------*/
 380 static void
 381 usb_dma_tag_destroy(struct usb_dma_tag *udt)
 382 {
 383         bus_dma_tag_destroy(udt->tag);
 384 }
 385
 386 /*------------------------------------------------------------------------*
 387  *      usb_pc_alloc_mem_cb - BUS-DMA callback function
 388  *------------------------------------------------------------------------*/
 389 static void
 390 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
 391     int nseg, int error)
 392 {
 393         usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
 394 }
 395
 396 /*------------------------------------------------------------------------*
 397  *      usb_pc_load_mem_cb - BUS-DMA callback function
 398  *------------------------------------------------------------------------*/
 399 static void
 400 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
 401     int nseg, int error)
 402 {
 403         usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
 404 }
 405
 406 /*------------------------------------------------------------------------*
 407  *      usb_pc_common_mem_cb - BUS-DMA callback function
 408  *------------------------------------------------------------------------*/
 409 static void
 410 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
 411     int nseg, int error, uint8_t isload)
 412 {
 413         struct usb_dma_parent_tag *uptag;
 414         struct usb_page_cache *pc;
 415         struct usb_page *pg;
 416         usb_size_t rem;
 417         bus_size_t off;
 418         uint8_t owned;
 419
 420         pc = arg;
 421         uptag = pc->tag_parent;
 422
 423         /*
 424          * XXX There is sometimes recursive locking here.
 425          * XXX We should try to find a better solution.
 426          * XXX Until further the "owned" variable does
 427          * XXX the trick.
 428          */
 429
 430         if (error) {
 431                 goto done;
 432         }
 433
 434         off = 0;
 435         pg = pc->page_start;
 436         pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
 437         rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
 438         pc->page_offset_buf = rem;
 439         pc->page_offset_end += rem;
 440 #ifdef USB_DEBUG
 441         if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
 442                 /*
 443                  * This check verifies that the physical address is correct:
 444                  */
 445                 DPRINTFN(0, "Page offset was not preserved\n");
 446                 error = 1;
 447                 goto done;
 448         }
 449 #endif
 450         while (pc->ismultiseg) {
 451                 off += USB_PAGE_SIZE;
 452                 if (off >= (segs->ds_len + rem)) {
 453                         /* page crossing */
 454                         nseg--;
 455                         segs++;
 456                         off = 0;
 457                         rem = 0;
 458                         if (nseg == 0)
 459                                 break;
 460                 }
 461                 pg++;
 462                 pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
 463         }
 464
 465 done:
 466         owned = lockowned(uptag->lock);
 467         if (!owned)
 468                 lockmgr(uptag->lock, LK_EXCLUSIVE);
 469
 470         uptag->dma_error = (error ? 1 : 0);
 471         if (isload) {
 472                 (uptag->func) (uptag);
 473         } else {
 474                 cv_broadcast(uptag->cv);
 475         }
 476         if (!owned)
 477                 lockmgr(uptag->lock, LK_RELEASE);
 478 }
 479
 480 /*------------------------------------------------------------------------*
 481  *      usb_pc_alloc_mem - allocate DMA'able memory
 482  *
 483  * Returns:
 484  *    0: Success
 485  * Else: Failure
 486  *------------------------------------------------------------------------*/
 487 uint8_t
 488 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
 489     usb_size_t size, usb_size_t align)
 490 {
 491         struct usb_dma_parent_tag *uptag;
 492         struct usb_dma_tag *utag;
 493         bus_dmamap_t map;
 494         void *ptr;
 495         int err;
 496
 497         uptag = pc->tag_parent;
 498
 499         if (align != 1) {
 500                 /*
 501                  * The alignment must be greater or equal to the
 502                  * "size" else the object can be split between two
 503                  * memory pages and we get a problem!
 504                  */
 505                 while (align < size) {
 506                         align *= 2;
 507                         if (align == 0) {
 508                                 goto error;
 509                         }
 510                 }
 511 #if 1
 512                 /*
 513                  * XXX BUS-DMA workaround - FIXME later:
 514                  *
 515                  * We assume that that the aligment at this point of
 516                  * the code is greater than or equal to the size and
 517                  * less than two times the size, so that if we double
 518                  * the size, the size will be greater than the
 519                  * alignment.
 520                  *
 521                  * The bus-dma system has a check for "alignment"
 522                  * being less than "size". If that check fails we end
 523                  * up using contigmalloc which is page based even for
 524                  * small allocations. Try to avoid that to save
 525                  * memory, hence we sometimes to a large number of
 526                  * small allocations!
 527                  */
 528                 if (size <= (USB_PAGE_SIZE / 2)) {
 529                         size *= 2;
 530                 }
 531 #endif
 532         }
 533         /* get the correct DMA tag */
 534         utag = usb_dma_tag_find(uptag, size, align);
 535         if (utag == NULL) {
 536                 goto error;
 537         }
 538         /* allocate memory */
 539         if (bus_dmamem_alloc(
 540             utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
 541                 goto error;
 542         }
 543         /* setup page cache */
 544         pc->buffer = ptr;
 545         pc->page_start = pg;
 546         pc->page_offset_buf = 0;
 547         pc->page_offset_end = size;
 548         pc->map = map;
 549         pc->tag = utag->tag;
 550         pc->ismultiseg = (align == 1);
 551
 552         lockmgr(uptag->lock, LK_EXCLUSIVE);
 553
 554         /* load memory into DMA */
 555         err = bus_dmamap_load(
 556             utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
 557             pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
 558
 559         if (err == EINPROGRESS) {
 560                 cv_wait(uptag->cv, uptag->lock);
 561                 err = 0;
 562         }
 563         lockmgr(uptag->lock, LK_RELEASE);
 564
 565         if (err || uptag->dma_error) {
 566                 bus_dmamem_free(utag->tag, ptr, map);
 567                 goto error;
 568         }
 569         memset(ptr, 0, size);
 570
 571         usb_pc_cpu_flush(pc);
 572
 573         return (0);
 574
 575 error:
 576         /* reset most of the page cache */
 577         pc->buffer = NULL;
 578         pc->page_start = NULL;
 579         pc->page_offset_buf = 0;
 580         pc->page_offset_end = 0;
 581         pc->map = NULL;
 582         pc->tag = NULL;
 583         return (1);
 584 }
 585
 586 /*------------------------------------------------------------------------*
 587  *      usb_pc_free_mem - free DMA memory
 588  *
 589  * This function is NULL safe.
 590  *------------------------------------------------------------------------*/
 591 void
 592 usb_pc_free_mem(struct usb_page_cache *pc)
 593 {
 594         if (pc && pc->buffer) {
 595
 596                 bus_dmamap_unload(pc->tag, pc->map);
 597
 598                 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
 599
 600                 pc->buffer = NULL;
 601         }
 602 }
 603
 604 /*------------------------------------------------------------------------*
 605  *      usb_pc_load_mem - load virtual memory into DMA
 606  *
 607  * Return values:
 608  * 0: Success
 609  * Else: Error
 610  *------------------------------------------------------------------------*/
 611 uint8_t
 612 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
 613 {
 614         /* setup page cache */
 615         pc->page_offset_buf = 0;
 616         pc->page_offset_end = size;
 617         pc->ismultiseg = 1;
 618
 619         KKASSERT(lockowned(pc->tag_parent->lock));
 620
 621         if (size > 0) {
 622                 if (sync) {
 623                         struct usb_dma_parent_tag *uptag;
 624                         int err;
 625
 626                         uptag = pc->tag_parent;
 627
 628                         /*
 629                          * We have to unload the previous loaded DMA
 630                          * pages before trying to load a new one!
 631                          */
 632                         bus_dmamap_unload(pc->tag, pc->map);
 633
 634                         /*
 635                          * Try to load memory into DMA.
 636                          */
 637                         err = bus_dmamap_load(
 638                             pc->tag, pc->map, pc->buffer, size,
 639                             &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
 640                         if (err == EINPROGRESS) {
 641                                 cv_wait(uptag->cv, uptag->lock);
 642                                 err = 0;
 643                         }
 644                         if (err || uptag->dma_error) {
 645                                 return (1);
 646                         }
 647                 } else {
 648
 649                         /*
 650                          * We have to unload the previous loaded DMA
 651                          * pages before trying to load a new one!
 652                          */
 653                         bus_dmamap_unload(pc->tag, pc->map);
 654
 655                         /*
 656                          * Try to load memory into DMA. The callback
 657                          * will be called in all cases:
 658                          */
 659                         if (bus_dmamap_load(
 660                             pc->tag, pc->map, pc->buffer, size,
 661                             &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
 662                         }
 663                 }
 664         } else {
 665                 if (!sync) {
 666                         /*
 667                          * Call callback so that refcount is decremented
 668                          * properly:
 669                          */
 670                         pc->tag_parent->dma_error = 0;
 671                         (pc->tag_parent->func) (pc->tag_parent);
 672                 }
 673         }
 674         return (0);
 675 }
 676
 677 /*------------------------------------------------------------------------*
 678  *      usb_pc_cpu_invalidate - invalidate CPU cache
 679  *------------------------------------------------------------------------*/
 680 void
 681 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
 682 {
 683         if (pc->page_offset_end == pc->page_offset_buf) {
 684                 /* nothing has been loaded into this page cache! */
 685                 return;
 686         }
 687
 688         /*
 689          * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
 690          * same time, but in the future we should try to isolate the
 691          * different cases to optimise the code. --HPS
 692          */
 693         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
 694         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
 695 }
 696
 697 /*------------------------------------------------------------------------*
 698  *      usb_pc_cpu_flush - flush CPU cache
 699  *------------------------------------------------------------------------*/
 700 void
 701 usb_pc_cpu_flush(struct usb_page_cache *pc)
 702 {
 703         if (pc->page_offset_end == pc->page_offset_buf) {
 704                 /* nothing has been loaded into this page cache! */
 705                 return;
 706         }
 707         bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
 708 }
 709
 710 /*------------------------------------------------------------------------*
 711  *      usb_pc_dmamap_create - create a DMA map
 712  *
 713  * Returns:
 714  *    0: Success
 715  * Else: Failure
 716  *------------------------------------------------------------------------*/
 717 uint8_t
 718 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
 719 {
 720         struct usb_xfer_root *info;
 721         struct usb_dma_tag *utag;
 722
 723         /* get info */
 724         info = USB_DMATAG_TO_XROOT(pc->tag_parent);
 725
 726         /* sanity check */
 727         if (info == NULL) {
 728                 goto error;
 729         }
 730         utag = usb_dma_tag_find(pc->tag_parent, size, 1);
 731         if (utag == NULL) {
 732                 goto error;
 733         }
 734         /* create DMA map */
 735         if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
 736                 goto error;
 737         }
 738         pc->tag = utag->tag;
 739         return 0;                       /* success */
 740
 741 error:
 742         pc->map = NULL;
 743         pc->tag = NULL;
 744         return 1;                       /* failure */
 745 }
 746
 747 /*------------------------------------------------------------------------*
 748  *      usb_pc_dmamap_destroy
 749  *
 750  * This function is NULL safe.
 751  *------------------------------------------------------------------------*/
 752 void
 753 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
 754 {
 755         if (pc && pc->tag) {
 756                 bus_dmamap_destroy(pc->tag, pc->map);
 757                 pc->tag = NULL;
 758                 pc->map = NULL;
 759         }
 760 }
 761
 762 /*------------------------------------------------------------------------*
 763  *      usb_dma_tag_find - factored out code
 764  *------------------------------------------------------------------------*/
 765 struct usb_dma_tag *
 766 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
 767     usb_size_t size, usb_size_t align)
 768 {
 769         struct usb_dma_tag *udt;
 770         uint8_t nudt;
 771
 772         USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
 773         USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
 774
 775         udt = udpt->utag_first;
 776         nudt = udpt->utag_max;
 777
 778         while (nudt--) {
 779
 780                 if (udt->align == 0) {
 781                         usb_dma_tag_create(udt, size, align);
 782                         if (udt->tag == NULL) {
 783                                 return (NULL);
 784                         }
 785                         udt->align = align;
 786                         udt->size = size;
 787                         return (udt);
 788                 }
 789                 if ((udt->align == align) && (udt->size == size)) {
 790                         return (udt);
 791                 }
 792                 udt++;
 793         }
 794         return (NULL);
 795 }
 796
 797 /*------------------------------------------------------------------------*
 798  *      usb_dma_tag_setup - initialise USB DMA tags
 799  *------------------------------------------------------------------------*/
 800 void
 801 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
 802     struct usb_dma_tag *udt, bus_dma_tag_t dmat,
 803     struct lock *lock, usb_dma_callback_t *func,
 804     uint8_t ndmabits, uint8_t nudt)
 805 {
 806         memset(udpt, 0, sizeof(*udpt));
 807
 808         /* sanity checking */
 809         if ((nudt == 0) ||
 810             (ndmabits == 0) ||
 811             (lock == NULL)) {
 812                 /* something is corrupt */
 813                 return;
 814         }
 815         /* initialise condition variable */
 816         cv_init(udpt->cv, "USB DMA CV");
 817
 818         /* store some information */
 819         udpt->lock = lock;
 820         udpt->func = func;
 821         udpt->tag = dmat;
 822         udpt->utag_first = udt;
 823         udpt->utag_max = nudt;
 824         udpt->dma_bits = ndmabits;
 825
 826         while (nudt--) {
 827                 memset(udt, 0, sizeof(*udt));
 828                 udt->tag_parent = udpt;
 829                 udt++;
 830         }
 831 }
 832
 833 /*------------------------------------------------------------------------*
 834  *      usb_bus_tag_unsetup - factored out code
 835  *------------------------------------------------------------------------*/
 836 void
 837 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
 838 {
 839         struct usb_dma_tag *udt;
 840         uint8_t nudt;
 841
 842         udt = udpt->utag_first;
 843         nudt = udpt->utag_max;
 844
 845         while (nudt--) {
 846
 847                 if (udt->align) {
 848                         /* destroy the USB DMA tag */
 849                         usb_dma_tag_destroy(udt);
 850                         udt->align = 0;
 851                 }
 852                 udt++;
 853         }
 854
 855         if (udpt->utag_max) {
 856                 /* destroy the condition variable */
 857                 cv_destroy(udpt->cv);
 858         }
 859 }
 860
 861 /*------------------------------------------------------------------------*
 862  *      usb_bdma_work_loop
 863  *
 864  * This function handles loading of virtual buffers into DMA and is
 865  * only called when "dma_refcount" is zero.
 866  *------------------------------------------------------------------------*/
 867 void
 868 usb_bdma_work_loop(struct usb_xfer_queue *pq)
 869 {
 870         struct usb_xfer_root *info;
 871         struct usb_xfer *xfer;
 872         usb_frcount_t nframes;
 873
 874         xfer = pq->curr;
 875         info = xfer->xroot;
 876
 877         KKASSERT(lockowned(info->xfer_lock));
 878
 879         if (xfer->error) {
 880                 /* some error happened */
 881                 USB_BUS_LOCK(info->bus);
 882                 usbd_transfer_done(xfer, 0);
 883                 USB_BUS_UNLOCK(info->bus);
 884                 return;
 885         }
 886         if (!xfer->flags_int.bdma_setup) {
 887                 struct usb_page *pg;
 888                 usb_frlength_t frlength_0;
 889                 uint8_t isread;
 890
 891                 xfer->flags_int.bdma_setup = 1;
 892
 893                 /* reset BUS-DMA load state */
 894
 895                 info->dma_error = 0;
 896
 897                 if (xfer->flags_int.isochronous_xfr) {
 898                         /* only one frame buffer */
 899                         nframes = 1;
 900                         frlength_0 = xfer->sumlen;
 901                 } else {
 902                         /* can be multiple frame buffers */
 903                         nframes = xfer->nframes;
 904                         frlength_0 = xfer->frlengths[0];
 905                 }
 906
 907                 /*
 908                  * Set DMA direction first. This is needed to
 909                  * select the correct cache invalidate and cache
 910                  * flush operations.
 911                  */
 912                 isread = USB_GET_DATA_ISREAD(xfer);
 913                 pg = xfer->dma_page_ptr;
 914
 915                 if (xfer->flags_int.control_xfr &&
 916                     xfer->flags_int.control_hdr) {
 917                         /* special case */
 918                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
 919                                 /* The device controller writes to memory */
 920                                 xfer->frbuffers[0].isread = 1;
 921                         } else {
 922                                 /* The host controller reads from memory */
 923                                 xfer->frbuffers[0].isread = 0;
 924                         }
 925                 } else {
 926                         /* default case */
 927                         xfer->frbuffers[0].isread = isread;
 928                 }
 929
 930                 /*
 931                  * Setup the "page_start" pointer which points to an array of
 932                  * USB pages where information about the physical address of a
 933                  * page will be stored. Also initialise the "isread" field of
 934                  * the USB page caches.
 935                  */
 936                 xfer->frbuffers[0].page_start = pg;
 937
 938                 info->dma_nframes = nframes;
 939                 info->dma_currframe = 0;
 940                 info->dma_frlength_0 = frlength_0;
 941
 942                 pg += (frlength_0 / USB_PAGE_SIZE);
 943                 pg += 2;
 944
 945                 while (--nframes > 0) {
 946                         xfer->frbuffers[nframes].isread = isread;
 947                         xfer->frbuffers[nframes].page_start = pg;
 948
 949                         pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
 950                         pg += 2;
 951                 }
 952
 953         }
 954         if (info->dma_error) {
 955                 USB_BUS_LOCK(info->bus);
 956                 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
 957                 USB_BUS_UNLOCK(info->bus);
 958                 return;
 959         }
 960         if (info->dma_currframe != info->dma_nframes) {
 961
 962                 if (info->dma_currframe == 0) {
 963                         /* special case */
 964                         usb_pc_load_mem(xfer->frbuffers,
 965                             info->dma_frlength_0, 0);
 966                 } else {
 967                         /* default case */
 968                         nframes = info->dma_currframe;
 969                         usb_pc_load_mem(xfer->frbuffers + nframes,
 970                             xfer->frlengths[nframes], 0);
 971                 }
 972
 973                 /* advance frame index */
 974                 info->dma_currframe++;
 975
 976                 return;
 977         }
 978         /* go ahead */
 979         usb_bdma_pre_sync(xfer);
 980
 981         /* start loading next USB transfer, if any */
 982         usb_command_wrapper(pq, NULL);
 983
 984         /* finally start the hardware */
 985         usbd_pipe_enter(xfer);
 986 }
 987
 988 /*------------------------------------------------------------------------*
 989  *      usb_bdma_done_event
 990  *
 991  * This function is called when the BUS-DMA has loaded virtual memory
 992  * into DMA, if any.
 993  *------------------------------------------------------------------------*/
 994 void
 995 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
 996 {
 997         struct usb_xfer_root *info;
 998
 999         info = USB_DMATAG_TO_XROOT(udpt);
1000
1001         KKASSERT(lockowned(info->xfer_lock));
1002
1003         /* copy error */
1004         info->dma_error = udpt->dma_error;
1005
1006         /* enter workloop again */
1007         usb_command_wrapper(&info->dma_q,
1008             info->dma_q.curr);
1009 }
1010
1011 /*------------------------------------------------------------------------*
1012  *      usb_bdma_pre_sync
1013  *
1014  * This function handles DMA synchronisation that must be done before
1015  * an USB transfer is started.
1016  *------------------------------------------------------------------------*/
1017 void
1018 usb_bdma_pre_sync(struct usb_xfer *xfer)
1019 {
1020         struct usb_page_cache *pc;
1021         usb_frcount_t nframes;
1022
1023         if (xfer->flags_int.isochronous_xfr) {
1024                 /* only one frame buffer */
1025                 nframes = 1;
1026         } else {
1027                 /* can be multiple frame buffers */
1028                 nframes = xfer->nframes;
1029         }
1030
1031         pc = xfer->frbuffers;
1032
1033         while (nframes--) {
1034
1035                 if (pc->isread) {
1036                         usb_pc_cpu_invalidate(pc);
1037                 } else {
1038                         usb_pc_cpu_flush(pc);
1039                 }
1040                 pc++;
1041         }
1042 }
1043
1044 /*------------------------------------------------------------------------*
1045  *      usb_bdma_post_sync
1046  *
1047  * This function handles DMA synchronisation that must be done after
1048  * an USB transfer is complete.
1049  *------------------------------------------------------------------------*/
1050 void
1051 usb_bdma_post_sync(struct usb_xfer *xfer)
1052 {
1053         struct usb_page_cache *pc;
1054         usb_frcount_t nframes;
1055
1056         if (xfer->flags_int.isochronous_xfr) {
1057                 /* only one frame buffer */
1058                 nframes = 1;
1059         } else {
1060                 /* can be multiple frame buffers */
1061                 nframes = xfer->nframes;
1062         }
1063
1064         pc = xfer->frbuffers;
1065
1066         while (nframes--) {
1067                 if (pc->isread) {
1068                         usb_pc_cpu_invalidate(pc);
1069                 }
1070                 pc++;
1071         }
1072 }
1073
1074 #endif