sys/bus/u4b/usb_transfer.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  */
  25
  26 #include <sys/stdint.h>
  27 #include <sys/param.h>
  28 #include <sys/queue.h>
  29 #include <sys/types.h>
  30 #include <sys/systm.h>
  31 #include <sys/kernel.h>
  32 #include <sys/bus.h>
  33 #include <sys/thread.h>
  34 #include <sys/module.h>
  35 #include <sys/lock.h>
  36 #include <sys/mutex.h>
  37 #include <sys/condvar.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/unistd.h>
  40 #include <sys/callout.h>
  41 #include <sys/malloc.h>
  42 #include <sys/priv.h>
  43 #include <sys/proc.h>
  44
  45 #include <sys/thread2.h>
  46
  47 #include <bus/u4b/usb.h>
  48 #include <bus/u4b/usbdi.h>
  49 #include <bus/u4b/usbdi_util.h>
  50
  51 #define USB_DEBUG_VAR usb_debug
  52
  53 #include <bus/u4b/usb_core.h>
  54 #include <bus/u4b/usb_busdma.h>
  55 #include <bus/u4b/usb_process.h>
  56 #include <bus/u4b/usb_transfer.h>
  57 #include <bus/u4b/usb_device.h>
  58 #include <bus/u4b/usb_debug.h>
  59 #include <bus/u4b/usb_util.h>
  60
  61 #include <bus/u4b/usb_controller.h>
  62 #include <bus/u4b/usb_bus.h>
  63 #include <bus/u4b/usb_pf.h>
  64
  65 struct usb_std_packet_size {
  66         struct {
  67                 uint16_t min;           /* inclusive */
  68                 uint16_t max;           /* inclusive */
  69         }       range;
  70
  71         uint16_t fixed[4];
  72 };
  73
  74 static usb_callback_t usb_request_callback;
  75
  76 static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
  77
  78         /* This transfer is used for generic control endpoint transfers */
  79
  80         [0] = {
  81                 .type = UE_CONTROL,
  82                 .endpoint = 0x00,       /* Control endpoint */
  83                 .direction = UE_DIR_ANY,
  84                 .bufsize = USB_EP0_BUFSIZE,     /* bytes */
  85                 .flags = {.proxy_buffer = 1,},
  86                 .callback = &usb_request_callback,
  87                 .usb_mode = USB_MODE_DUAL,      /* both modes */
  88         },
  89
  90         /* This transfer is used for generic clear stall only */
  91
  92         [1] = {
  93                 .type = UE_CONTROL,
  94                 .endpoint = 0x00,       /* Control pipe */
  95                 .direction = UE_DIR_ANY,
  96                 .bufsize = sizeof(struct usb_device_request),
  97                 .callback = &usb_do_clear_stall_callback,
  98                 .timeout = 1000,        /* 1 second */
  99                 .interval = 50, /* 50ms */
 100                 .usb_mode = USB_MODE_HOST,
 101         },
 102 };
 103
 104 /* function prototypes */
 105
 106 static void     usbd_update_max_frame_size(struct usb_xfer *);
 107 static void     usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
 108 static void     usbd_delayed_free(void *data, struct malloc_type *mtype);
 109 static void     usbd_control_transfer_init(struct usb_xfer *);
 110 static int      usbd_setup_ctrl_transfer(struct usb_xfer *);
 111 static void     usb_callback_proc(struct usb_proc_msg *);
 112 static void     usbd_callback_ss_done_defer(struct usb_xfer *);
 113 static void     usbd_callback_wrapper(struct usb_xfer_queue *);
 114 static void     usbd_transfer_start_cb(void *);
 115 static uint8_t  usbd_callback_wrapper_sub(struct usb_xfer *);
 116 static void     usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
 117                     uint8_t type, enum usb_dev_speed speed);
 118
 119 /*------------------------------------------------------------------------*
 120  *      usb_request_callback
 121  *------------------------------------------------------------------------*/
 122 static void
 123 usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
 124 {
 125         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
 126                 usb_handle_request_callback(xfer, error);
 127         else
 128                 usbd_do_request_callback(xfer, error);
 129 }
 130
 131 /*------------------------------------------------------------------------*
 132  *      usbd_update_max_frame_size
 133  *
 134  * This function updates the maximum frame size, hence high speed USB
 135  * can transfer multiple consecutive packets.
 136  *------------------------------------------------------------------------*/
 137 static void
 138 usbd_update_max_frame_size(struct usb_xfer *xfer)
 139 {
 140         /* compute maximum frame size */
 141         /* this computation should not overflow 16-bit */
 142         /* max = 15 * 1024 */
 143
 144         xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
 145 }
 146
 147 /*------------------------------------------------------------------------*
 148  *      usbd_get_dma_delay
 149  *
 150  * The following function is called when we need to
 151  * synchronize with DMA hardware.
 152  *
 153  * Returns:
 154  *    0: no DMA delay required
 155  * Else: milliseconds of DMA delay
 156  *------------------------------------------------------------------------*/
 157 usb_timeout_t
 158 usbd_get_dma_delay(struct usb_device *udev)
 159 {
 160         const struct usb_bus_methods *mtod;
 161         uint32_t temp;
 162
 163         mtod = udev->bus->methods;
 164         temp = 0;
 165
 166         if (mtod->get_dma_delay) {
 167                 (mtod->get_dma_delay) (udev, &temp);
 168                 /*
 169                  * Round up and convert to milliseconds. Note that we use
 170                  * 1024 milliseconds per second. to save a division.
 171                  */
 172                 temp += 0x3FF;
 173                 temp /= 0x400;
 174         }
 175         return (temp);
 176 }
 177
 178 /*------------------------------------------------------------------------*
 179  *      usbd_transfer_setup_sub_malloc
 180  *
 181  * This function will allocate one or more DMA'able memory chunks
 182  * according to "size", "align" and "count" arguments. "ppc" is
 183  * pointed to a linear array of USB page caches afterwards.
 184  *
 185  * If the "align" argument is equal to "1" a non-contiguous allocation
 186  * can happen. Else if the "align" argument is greater than "1", the
 187  * allocation will always be contiguous in memory.
 188  *
 189  * Returns:
 190  *    0: Success
 191  * Else: Failure
 192  *------------------------------------------------------------------------*/
 193 #if USB_HAVE_BUSDMA
 194 uint8_t
 195 usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
 196     struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
 197     usb_size_t count)
 198 {
 199         struct usb_page_cache *pc;
 200         struct usb_page *pg;
 201         void *buf;
 202         usb_size_t n_dma_pc;
 203         usb_size_t n_dma_pg;
 204         usb_size_t n_obj;
 205         usb_size_t x;
 206         usb_size_t y;
 207         usb_size_t r;
 208         usb_size_t z;
 209
 210         USB_ASSERT(align > 0, ("Invalid alignment, 0x%08x\n",
 211             align));
 212         USB_ASSERT(size > 0, ("Invalid size = 0\n"));
 213
 214         if (count == 0) {
 215                 return (0);             /* nothing to allocate */
 216         }
 217         /*
 218          * Make sure that the size is aligned properly.
 219          */
 220         size = -((-size) & (-align));
 221
 222         /*
 223          * Try multi-allocation chunks to reduce the number of DMA
 224          * allocations, hence DMA allocations are slow.
 225          */
 226         if (align == 1) {
 227                 /* special case - non-cached multi page DMA memory */
 228                 n_dma_pc = count;
 229                 n_dma_pg = (2 + (size / USB_PAGE_SIZE));
 230                 n_obj = 1;
 231         } else if (size >= USB_PAGE_SIZE) {
 232                 n_dma_pc = count;
 233                 n_dma_pg = 1;
 234                 n_obj = 1;
 235         } else {
 236                 /* compute number of objects per page */
 237                 n_obj = (USB_PAGE_SIZE / size);
 238                 /*
 239                  * Compute number of DMA chunks, rounded up
 240                  * to nearest one:
 241                  */
 242                 n_dma_pc = ((count + n_obj - 1) / n_obj);
 243                 n_dma_pg = 1;
 244         }
 245
 246         /*
 247          * DMA memory is allocated once, but mapped twice. That's why
 248          * there is one list for auto-free and another list for
 249          * non-auto-free which only holds the mapping and not the
 250          * allocation.
 251          */
 252         if (parm->buf == NULL) {
 253                 /* reserve memory (auto-free) */
 254                 parm->dma_page_ptr += n_dma_pc * n_dma_pg;
 255                 parm->dma_page_cache_ptr += n_dma_pc;
 256
 257                 /* reserve memory (no-auto-free) */
 258                 parm->dma_page_ptr += count * n_dma_pg;
 259                 parm->xfer_page_cache_ptr += count;
 260                 return (0);
 261         }
 262         for (x = 0; x != n_dma_pc; x++) {
 263                 /* need to initialize the page cache */
 264                 parm->dma_page_cache_ptr[x].tag_parent =
 265                     &parm->curr_xfer->xroot->dma_parent_tag;
 266         }
 267         for (x = 0; x != count; x++) {
 268                 /* need to initialize the page cache */
 269                 parm->xfer_page_cache_ptr[x].tag_parent =
 270                     &parm->curr_xfer->xroot->dma_parent_tag;
 271         }
 272
 273         if (ppc) {
 274                 *ppc = parm->xfer_page_cache_ptr;
 275         }
 276         r = count;                      /* set remainder count */
 277         z = n_obj * size;               /* set allocation size */
 278         pc = parm->xfer_page_cache_ptr;
 279         pg = parm->dma_page_ptr;
 280
 281         for (x = 0; x != n_dma_pc; x++) {
 282
 283                 if (r < n_obj) {
 284                         /* compute last remainder */
 285                         z = r * size;
 286                         n_obj = r;
 287                 }
 288                 if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
 289                     pg, z, align)) {
 290                         return (1);     /* failure */
 291                 }
 292                 /* Set beginning of current buffer */
 293                 buf = parm->dma_page_cache_ptr->buffer;
 294                 /* Make room for one DMA page cache and one page */
 295                 parm->dma_page_cache_ptr++;
 296                 pg += n_dma_pg;
 297
 298                 for (y = 0; (y != n_obj); y++, r--, pc++, pg += n_dma_pg) {
 299
 300                         /* Load sub-chunk into DMA */
 301                         if (usb_pc_dmamap_create(pc, size)) {
 302                                 return (1);     /* failure */
 303                         }
 304                         pc->buffer = USB_ADD_BYTES(buf, y * size);
 305                         pc->page_start = pg;
 306
 307                         lockmgr(pc->tag_parent->lock, LK_EXCLUSIVE);
 308                         if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
 309                                 lockmgr(pc->tag_parent->lock, LK_RELEASE);
 310                                 return (1);     /* failure */
 311                         }
 312                         lockmgr(pc->tag_parent->lock, LK_RELEASE);
 313                 }
 314         }
 315
 316         parm->xfer_page_cache_ptr = pc;
 317         parm->dma_page_ptr = pg;
 318         return (0);
 319 }
 320 #endif
 321
 322 /*------------------------------------------------------------------------*
 323  *      usbd_transfer_setup_sub - transfer setup subroutine
 324  *
 325  * This function must be called from the "xfer_setup" callback of the
 326  * USB Host or Device controller driver when setting up an USB
 327  * transfer. This function will setup correct packet sizes, buffer
 328  * sizes, flags and more, that are stored in the "usb_xfer"
 329  * structure.
 330  *------------------------------------------------------------------------*/
 331 void
 332 usbd_transfer_setup_sub(struct usb_setup_params *parm)
 333 {
 334         enum {
 335                 REQ_SIZE = 8,
 336                 MIN_PKT = 8,
 337         };
 338         struct usb_xfer *xfer = parm->curr_xfer;
 339         const struct usb_config *setup = parm->curr_setup;
 340         struct usb_endpoint_ss_comp_descriptor *ecomp;
 341         struct usb_endpoint_descriptor *edesc;
 342         struct usb_std_packet_size std_size;
 343         usb_frcount_t n_frlengths;
 344         usb_frcount_t n_frbuffers;
 345         usb_frcount_t x;
 346         uint8_t type;
 347         uint8_t zmps;
 348
 349         /*
 350          * Sanity check. The following parameters must be initialized before
 351          * calling this function.
 352          */
 353         if ((parm->hc_max_packet_size == 0) ||
 354             (parm->hc_max_packet_count == 0) ||
 355             (parm->hc_max_frame_size == 0)) {
 356                 parm->err = USB_ERR_INVAL;
 357                 goto done;
 358         }
 359         edesc = xfer->endpoint->edesc;
 360         ecomp = xfer->endpoint->ecomp;
 361
 362         type = (edesc->bmAttributes & UE_XFERTYPE);
 363
 364         xfer->flags = setup->flags;
 365         xfer->nframes = setup->frames;
 366         xfer->timeout = setup->timeout;
 367         xfer->callback = setup->callback;
 368         xfer->interval = setup->interval;
 369         xfer->endpointno = edesc->bEndpointAddress;
 370         xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
 371         xfer->max_packet_count = 1;
 372         /* make a shadow copy: */
 373         xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
 374
 375         parm->bufsize = setup->bufsize;
 376
 377         switch (parm->speed) {
 378         case USB_SPEED_HIGH:
 379                 switch (type) {
 380                 case UE_ISOCHRONOUS:
 381                 case UE_INTERRUPT:
 382                         xfer->max_packet_count +=
 383                             (xfer->max_packet_size >> 11) & 3;
 384
 385                         /* check for invalid max packet count */
 386                         if (xfer->max_packet_count > 3)
 387                                 xfer->max_packet_count = 3;
 388                         break;
 389                 default:
 390                         break;
 391                 }
 392                 xfer->max_packet_size &= 0x7FF;
 393                 break;
 394         case USB_SPEED_SUPER:
 395                 xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
 396
 397                 if (ecomp != NULL)
 398                         xfer->max_packet_count += ecomp->bMaxBurst;
 399
 400                 if ((xfer->max_packet_count == 0) ||
 401                     (xfer->max_packet_count > 16))
 402                         xfer->max_packet_count = 16;
 403
 404                 switch (type) {
 405                 case UE_CONTROL:
 406                         xfer->max_packet_count = 1;
 407                         break;
 408                 case UE_ISOCHRONOUS:
 409                         if (ecomp != NULL) {
 410                                 uint8_t mult;
 411
 412                                 mult = UE_GET_SS_ISO_MULT(
 413                                     ecomp->bmAttributes) + 1;
 414                                 if (mult > 3)
 415                                         mult = 3;
 416
 417                                 xfer->max_packet_count *= mult;
 418                         }
 419                         break;
 420                 default:
 421                         break;
 422                 }
 423                 xfer->max_packet_size &= 0x7FF;
 424                 break;
 425         default:
 426                 break;
 427         }
 428         /* range check "max_packet_count" */
 429
 430         if (xfer->max_packet_count > parm->hc_max_packet_count) {
 431                 xfer->max_packet_count = parm->hc_max_packet_count;
 432         }
 433         /* filter "wMaxPacketSize" according to HC capabilities */
 434
 435         if ((xfer->max_packet_size > parm->hc_max_packet_size) ||
 436             (xfer->max_packet_size == 0)) {
 437                 xfer->max_packet_size = parm->hc_max_packet_size;
 438         }
 439         /* filter "wMaxPacketSize" according to standard sizes */
 440
 441         usbd_get_std_packet_size(&std_size, type, parm->speed);
 442
 443         if (std_size.range.min || std_size.range.max) {
 444
 445                 if (xfer->max_packet_size < std_size.range.min) {
 446                         xfer->max_packet_size = std_size.range.min;
 447                 }
 448                 if (xfer->max_packet_size > std_size.range.max) {
 449                         xfer->max_packet_size = std_size.range.max;
 450                 }
 451         } else {
 452
 453                 if (xfer->max_packet_size >= std_size.fixed[3]) {
 454                         xfer->max_packet_size = std_size.fixed[3];
 455                 } else if (xfer->max_packet_size >= std_size.fixed[2]) {
 456                         xfer->max_packet_size = std_size.fixed[2];
 457                 } else if (xfer->max_packet_size >= std_size.fixed[1]) {
 458                         xfer->max_packet_size = std_size.fixed[1];
 459                 } else {
 460                         /* only one possibility left */
 461                         xfer->max_packet_size = std_size.fixed[0];
 462                 }
 463         }
 464
 465         /* compute "max_frame_size" */
 466
 467         usbd_update_max_frame_size(xfer);
 468
 469         /* check interrupt interval and transfer pre-delay */
 470
 471         if (type == UE_ISOCHRONOUS) {
 472
 473                 uint16_t frame_limit;
 474
 475                 xfer->interval = 0;     /* not used, must be zero */
 476                 xfer->flags_int.isochronous_xfr = 1;    /* set flag */
 477
 478                 if (xfer->timeout == 0) {
 479                         /*
 480                          * set a default timeout in
 481                          * case something goes wrong!
 482                          */
 483                         xfer->timeout = 1000 / 4;
 484                 }
 485                 switch (parm->speed) {
 486                 case USB_SPEED_LOW:
 487                 case USB_SPEED_FULL:
 488                         frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
 489                         xfer->fps_shift = 0;
 490                         break;
 491                 default:
 492                         frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
 493                         xfer->fps_shift = edesc->bInterval;
 494                         if (xfer->fps_shift > 0)
 495                                 xfer->fps_shift--;
 496                         if (xfer->fps_shift > 3)
 497                                 xfer->fps_shift = 3;
 498                         if (xfer->flags.pre_scale_frames != 0)
 499                                 xfer->nframes <<= (3 - xfer->fps_shift);
 500                         break;
 501                 }
 502
 503                 if (xfer->nframes > frame_limit) {
 504                         /*
 505                          * this is not going to work
 506                          * cross hardware
 507                          */
 508                         parm->err = USB_ERR_INVAL;
 509                         goto done;
 510                 }
 511                 if (xfer->nframes == 0) {
 512                         /*
 513                          * this is not a valid value
 514                          */
 515                         parm->err = USB_ERR_ZERO_NFRAMES;
 516                         goto done;
 517                 }
 518         } else {
 519
 520                 /*
 521                  * If a value is specified use that else check the
 522                  * endpoint descriptor!
 523                  */
 524                 if (type == UE_INTERRUPT) {
 525
 526                         uint32_t temp;
 527
 528                         if (xfer->interval == 0) {
 529
 530                                 xfer->interval = edesc->bInterval;
 531
 532                                 switch (parm->speed) {
 533                                 case USB_SPEED_LOW:
 534                                 case USB_SPEED_FULL:
 535                                         break;
 536                                 default:
 537                                         /* 125us -> 1ms */
 538                                         if (xfer->interval < 4)
 539                                                 xfer->interval = 1;
 540                                         else if (xfer->interval > 16)
 541                                                 xfer->interval = (1 << (16 - 4));
 542                                         else
 543                                                 xfer->interval =
 544                                                     (1 << (xfer->interval - 4));
 545                                         break;
 546                                 }
 547                         }
 548
 549                         if (xfer->interval == 0) {
 550                                 /*
 551                                  * One millisecond is the smallest
 552                                  * interval we support:
 553                                  */
 554                                 xfer->interval = 1;
 555                         }
 556
 557                         xfer->fps_shift = 0;
 558                         temp = 1;
 559
 560                         while ((temp != 0) && (temp < xfer->interval)) {
 561                                 xfer->fps_shift++;
 562                                 temp *= 2;
 563                         }
 564
 565                         switch (parm->speed) {
 566                         case USB_SPEED_LOW:
 567                         case USB_SPEED_FULL:
 568                                 break;
 569                         default:
 570                                 xfer->fps_shift += 3;
 571                                 break;
 572                         }
 573                 }
 574         }
 575
 576         /*
 577          * NOTE: we do not allow "max_packet_size" or "max_frame_size"
 578          * to be equal to zero when setting up USB transfers, hence
 579          * this leads to alot of extra code in the USB kernel.
 580          */
 581
 582         if ((xfer->max_frame_size == 0) ||
 583             (xfer->max_packet_size == 0)) {
 584
 585                 zmps = 1;
 586
 587                 if ((parm->bufsize <= MIN_PKT) &&
 588                     (type != UE_CONTROL) &&
 589                     (type != UE_BULK)) {
 590
 591                         /* workaround */
 592                         xfer->max_packet_size = MIN_PKT;
 593                         xfer->max_packet_count = 1;
 594                         parm->bufsize = 0;      /* automatic setup length */
 595                         usbd_update_max_frame_size(xfer);
 596
 597                 } else {
 598                         parm->err = USB_ERR_ZERO_MAXP;
 599                         goto done;
 600                 }
 601
 602         } else {
 603                 zmps = 0;
 604         }
 605
 606         /*
 607          * check if we should setup a default
 608          * length:
 609          */
 610
 611         if (parm->bufsize == 0) {
 612
 613                 parm->bufsize = xfer->max_frame_size;
 614
 615                 if (type == UE_ISOCHRONOUS) {
 616                         parm->bufsize *= xfer->nframes;
 617                 }
 618         }
 619         /*
 620          * check if we are about to setup a proxy
 621          * type of buffer:
 622          */
 623
 624         if (xfer->flags.proxy_buffer) {
 625
 626                 /* round bufsize up */
 627
 628                 parm->bufsize += (xfer->max_frame_size - 1);
 629
 630                 if (parm->bufsize < xfer->max_frame_size) {
 631                         /* length wrapped around */
 632                         parm->err = USB_ERR_INVAL;
 633                         goto done;
 634                 }
 635                 /* subtract remainder */
 636
 637                 parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
 638
 639                 /* add length of USB device request structure, if any */
 640
 641                 if (type == UE_CONTROL) {
 642                         parm->bufsize += REQ_SIZE;      /* SETUP message */
 643                 }
 644         }
 645         xfer->max_data_length = parm->bufsize;
 646
 647         /* Setup "n_frlengths" and "n_frbuffers" */
 648
 649         if (type == UE_ISOCHRONOUS) {
 650                 n_frlengths = xfer->nframes;
 651                 n_frbuffers = 1;
 652         } else {
 653
 654                 if (type == UE_CONTROL) {
 655                         xfer->flags_int.control_xfr = 1;
 656                         if (xfer->nframes == 0) {
 657                                 if (parm->bufsize <= REQ_SIZE) {
 658                                         /*
 659                                          * there will never be any data
 660                                          * stage
 661                                          */
 662                                         xfer->nframes = 1;
 663                                 } else {
 664                                         xfer->nframes = 2;
 665                                 }
 666                         }
 667                 } else {
 668                         if (xfer->nframes == 0) {
 669                                 xfer->nframes = 1;
 670                         }
 671                 }
 672
 673                 n_frlengths = xfer->nframes;
 674                 n_frbuffers = xfer->nframes;
 675         }
 676
 677         /*
 678          * check if we have room for the
 679          * USB device request structure:
 680          */
 681
 682         if (type == UE_CONTROL) {
 683
 684                 if (xfer->max_data_length < REQ_SIZE) {
 685                         /* length wrapped around or too small bufsize */
 686                         parm->err = USB_ERR_INVAL;
 687                         goto done;
 688                 }
 689                 xfer->max_data_length -= REQ_SIZE;
 690         }
 691         /*
 692          * Setup "frlengths" and shadow "frlengths" for keeping the
 693          * initial frame lengths when a USB transfer is complete. This
 694          * information is useful when computing isochronous offsets.
 695          */
 696         xfer->frlengths = parm->xfer_length_ptr;
 697         parm->xfer_length_ptr += 2 * n_frlengths;
 698
 699         /* setup "frbuffers" */
 700         xfer->frbuffers = parm->xfer_page_cache_ptr;
 701         parm->xfer_page_cache_ptr += n_frbuffers;
 702
 703         /* initialize max frame count */
 704         xfer->max_frame_count = xfer->nframes;
 705
 706         /*
 707          * check if we need to setup
 708          * a local buffer:
 709          */
 710
 711         if (!xfer->flags.ext_buffer) {
 712 #if USB_HAVE_BUSDMA
 713                 struct usb_page_search page_info;
 714                 struct usb_page_cache *pc;
 715
 716                 if (usbd_transfer_setup_sub_malloc(parm,
 717                     &pc, parm->bufsize, 1, 1)) {
 718                         parm->err = USB_ERR_NOMEM;
 719                 } else if (parm->buf != NULL) {
 720
 721                         usbd_get_page(pc, 0, &page_info);
 722
 723                         xfer->local_buffer = page_info.buffer;
 724
 725                         usbd_xfer_set_frame_offset(xfer, 0, 0);
 726
 727                         if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
 728                                 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
 729                         }
 730                 }
 731 #else
 732                 /* align data */
 733                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 734
 735                 if (parm->buf != NULL) {
 736                         xfer->local_buffer =
 737                             USB_ADD_BYTES(parm->buf, parm->size[0]);
 738
 739                         usbd_xfer_set_frame_offset(xfer, 0, 0);
 740
 741                         if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
 742                                 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
 743                         }
 744                 }
 745                 parm->size[0] += parm->bufsize;
 746
 747                 /* align data again */
 748                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
 749 #endif
 750         }
 751         /*
 752          * Compute maximum buffer size
 753          */
 754
 755         if (parm->bufsize_max < parm->bufsize) {
 756                 parm->bufsize_max = parm->bufsize;
 757         }
 758 #if USB_HAVE_BUSDMA
 759         if (xfer->flags_int.bdma_enable) {
 760                 /*
 761                  * Setup "dma_page_ptr".
 762                  *
 763                  * Proof for formula below:
 764                  *
 765                  * Assume there are three USB frames having length "a", "b" and
 766                  * "c". These USB frames will at maximum need "z"
 767                  * "usb_page" structures. "z" is given by:
 768                  *
 769                  * z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
 770                  * ((c / USB_PAGE_SIZE) + 2);
 771                  *
 772                  * Constraining "a", "b" and "c" like this:
 773                  *
 774                  * (a + b + c) <= parm->bufsize
 775                  *
 776                  * We know that:
 777                  *
 778                  * z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
 779                  *
 780                  * Here is the general formula:
 781                  */
 782                 xfer->dma_page_ptr = parm->dma_page_ptr;
 783                 parm->dma_page_ptr += (2 * n_frbuffers);
 784                 parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
 785         }
 786 #endif
 787         if (zmps) {
 788                 /* correct maximum data length */
 789                 xfer->max_data_length = 0;
 790         }
 791         /* subtract USB frame remainder from "hc_max_frame_size" */
 792
 793         xfer->max_hc_frame_size =
 794             (parm->hc_max_frame_size -
 795             (parm->hc_max_frame_size % xfer->max_frame_size));
 796
 797         if (xfer->max_hc_frame_size == 0) {
 798                 parm->err = USB_ERR_INVAL;
 799                 goto done;
 800         }
 801
 802         /* initialize frame buffers */
 803
 804         if (parm->buf) {
 805                 for (x = 0; x != n_frbuffers; x++) {
 806                         xfer->frbuffers[x].tag_parent =
 807                             &xfer->xroot->dma_parent_tag;
 808 #if USB_HAVE_BUSDMA
 809                         if (xfer->flags_int.bdma_enable &&
 810                             (parm->bufsize_max > 0)) {
 811
 812                                 if (usb_pc_dmamap_create(
 813                                     xfer->frbuffers + x,
 814                                     parm->bufsize_max)) {
 815                                         parm->err = USB_ERR_NOMEM;
 816                                         goto done;
 817                                 }
 818                         }
 819 #endif
 820                 }
 821         }
 822 done:
 823         if (parm->err) {
 824                 /*
 825                  * Set some dummy values so that we avoid division by zero:
 826                  */
 827                 xfer->max_hc_frame_size = 1;
 828                 xfer->max_frame_size = 1;
 829                 xfer->max_packet_size = 1;
 830                 xfer->max_data_length = 0;
 831                 xfer->nframes = 0;
 832                 xfer->max_frame_count = 0;
 833         }
 834 }
 835
 836 /*------------------------------------------------------------------------*
 837  *      usbd_transfer_setup - setup an array of USB transfers
 838  *
 839  * NOTE: You must always call "usbd_transfer_unsetup" after calling
 840  * "usbd_transfer_setup" if success was returned.
 841  *
 842  * The idea is that the USB device driver should pre-allocate all its
 843  * transfers by one call to this function.
 844  *
 845  * Return values:
 846  *    0: Success
 847  * Else: Failure
 848  *------------------------------------------------------------------------*/
 849 usb_error_t
 850 usbd_transfer_setup(struct usb_device *udev,
 851     const uint8_t *ifaces, struct usb_xfer **ppxfer,
 852     const struct usb_config *setup_start, uint16_t n_setup,
 853     void *priv_sc, struct lock *xfer_lock)
 854 {
 855         const struct usb_config *setup_end = setup_start + n_setup;
 856         const struct usb_config *setup;
 857         struct usb_setup_params *parm;
 858         struct usb_endpoint *ep;
 859         struct usb_xfer_root *info;
 860         struct usb_xfer *xfer;
 861         void *buf = NULL;
 862         usb_error_t error = 0;
 863         uint16_t n;
 864         uint16_t refcount;
 865         uint8_t do_unlock;
 866
 867 #if 0
 868         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
 869             "usbd_transfer_setup can sleep!");
 870 #endif
 871
 872         /* do some checking first */
 873
 874         if (n_setup == 0) {
 875                 DPRINTFN(6, "setup array has zero length!\n");
 876                 return (USB_ERR_INVAL);
 877         }
 878         if (ifaces == NULL) {
 879                 DPRINTFN(6, "ifaces array is NULL!\n");
 880                 return (USB_ERR_INVAL);
 881         }
 882         if (xfer_lock == NULL) {
 883                 panic("xfer without lock!\n");
 884                 DPRINTFN(6, "using global lock\n");
 885         }
 886
 887         /* more sanity checks */
 888
 889         for (setup = setup_start, n = 0;
 890             setup != setup_end; setup++, n++) {
 891                 if (setup->bufsize == (usb_frlength_t)-1) {
 892                         error = USB_ERR_BAD_BUFSIZE;
 893                         DPRINTF("invalid bufsize\n");
 894                 }
 895                 if (setup->callback == NULL) {
 896                         error = USB_ERR_NO_CALLBACK;
 897                         DPRINTF("no callback\n");
 898                 }
 899                 ppxfer[n] = NULL;
 900         }
 901
 902         if (error)
 903                 return (error);
 904
 905         /* Protect scratch area */
 906         do_unlock = usbd_enum_lock(udev);
 907
 908         refcount = 0;
 909         info = NULL;
 910
 911         parm = &udev->scratch.xfer_setup[0].parm;
 912         memset(parm, 0, sizeof(*parm));
 913
 914         parm->udev = udev;
 915         parm->speed = usbd_get_speed(udev);
 916         parm->hc_max_packet_count = 1;
 917
 918         if (parm->speed >= USB_SPEED_MAX) {
 919                 parm->err = USB_ERR_INVAL;
 920                 goto done;
 921         }
 922         /* setup all transfers */
 923
 924         while (1) {
 925
 926                 if (buf) {
 927                         /*
 928                          * Initialize the "usb_xfer_root" structure,
 929                          * which is common for all our USB transfers.
 930                          */
 931                         info = USB_ADD_BYTES(buf, 0);
 932
 933                         info->memory_base = buf;
 934                         info->memory_size = parm->size[0];
 935
 936 #if USB_HAVE_BUSDMA
 937                         info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
 938                         info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
 939 #endif
 940                         info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
 941                         info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
 942
 943                         cv_init(&info->cv_drain, "WDRAIN");
 944
 945                         info->xfer_lock = xfer_lock;
 946 #if USB_HAVE_BUSDMA
 947                         usb_dma_tag_setup(&info->dma_parent_tag,
 948                             parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
 949                             xfer_lock, &usb_bdma_done_event, 32, parm->dma_tag_max);
 950 #endif
 951
 952                         info->bus = udev->bus;
 953                         info->udev = udev;
 954
 955                         TAILQ_INIT(&info->done_q.head);
 956                         info->done_q.command = &usbd_callback_wrapper;
 957 #if USB_HAVE_BUSDMA
 958                         TAILQ_INIT(&info->dma_q.head);
 959                         info->dma_q.command = &usb_bdma_work_loop;
 960 #endif
 961                         info->done_m[0].hdr.pm_callback = &usb_callback_proc;
 962                         info->done_m[0].xroot = info;
 963                         info->done_m[1].hdr.pm_callback = &usb_callback_proc;
 964                         info->done_m[1].xroot = info;
 965
 966                         /*
 967                          * In device side mode control endpoint
 968                          * requests need to run from a separate
 969                          * context, else there is a chance of
 970                          * deadlock!
 971                          */
 972                         if (setup_start == usb_control_ep_cfg)
 973                                 info->done_p =
 974                                     USB_BUS_CONTROL_XFER_PROC(udev->bus);
 975                         else
 976                                 info->done_p =
 977                                     USB_BUS_NON_GIANT_PROC(udev->bus);
 978                 }
 979                 /* reset sizes */
 980
 981                 parm->size[0] = 0;
 982                 parm->buf = buf;
 983                 parm->size[0] += sizeof(info[0]);
 984
 985                 for (setup = setup_start, n = 0;
 986                     setup != setup_end; setup++, n++) {
 987
 988                         /* skip USB transfers without callbacks: */
 989                         if (setup->callback == NULL) {
 990                                 continue;
 991                         }
 992                         /* see if there is a matching endpoint */
 993                         ep = usbd_get_endpoint(udev,
 994                             ifaces[setup->if_index], setup);
 995
 996                         /*
 997                          * Check that the USB PIPE is valid and that
 998                          * the endpoint mode is proper.
 999                          *
1000                          * Make sure we don't allocate a streams
1001                          * transfer when such a combination is not
1002                          * valid.
1003                          */
1004                         if ((ep == NULL) || (ep->methods == NULL) ||
1005                             ((ep->ep_mode != USB_EP_MODE_STREAMS) &&
1006                             (ep->ep_mode != USB_EP_MODE_DEFAULT)) ||
1007                             (setup->stream_id != 0 &&
1008                             (setup->stream_id >= USB_MAX_EP_STREAMS ||
1009                             (ep->ep_mode != USB_EP_MODE_STREAMS)))) {
1010                                 if (setup->flags.no_pipe_ok)
1011                                         continue;
1012                                 if ((setup->usb_mode != USB_MODE_DUAL) &&
1013                                     (setup->usb_mode != udev->flags.usb_mode))
1014                                         continue;
1015                                 parm->err = USB_ERR_NO_PIPE;
1016                                 goto done;
1017                         }
1018
1019                         /* align data properly */
1020                         parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1021
1022                         /* store current setup pointer */
1023                         parm->curr_setup = setup;
1024
1025                         if (buf) {
1026                                 /*
1027                                  * Common initialization of the
1028                                  * "usb_xfer" structure.
1029                                  */
1030                                 xfer = USB_ADD_BYTES(buf, parm->size[0]);
1031                                 xfer->address = udev->address;
1032                                 xfer->priv_sc = priv_sc;
1033                                 xfer->xroot = info;
1034
1035                                 usb_callout_init_mtx(&xfer->timeout_handle,
1036                                     &udev->bus->bus_lock, 0);
1037                         } else {
1038                                 /*
1039                                  * Setup a dummy xfer, hence we are
1040                                  * writing to the "usb_xfer"
1041                                  * structure pointed to by "xfer"
1042                                  * before we have allocated any
1043                                  * memory:
1044                                  */
1045                                 xfer = &udev->scratch.xfer_setup[0].dummy;
1046                                 memset(xfer, 0, sizeof(*xfer));
1047                                 refcount++;
1048                         }
1049
1050                         /* set transfer endpoint pointer */
1051                         xfer->endpoint = ep;
1052
1053                         /* set transfer stream ID */
1054                         xfer->stream_id = setup->stream_id;
1055
1056                         parm->size[0] += sizeof(xfer[0]);
1057                         parm->methods = xfer->endpoint->methods;
1058                         parm->curr_xfer = xfer;
1059
1060                         /*
1061                          * Call the Host or Device controller transfer
1062                          * setup routine:
1063                          */
1064                         (udev->bus->methods->xfer_setup) (parm);
1065
1066                         /* check for error */
1067                         if (parm->err)
1068                                 goto done;
1069
1070                         if (buf) {
1071                                 /*
1072                                  * Increment the endpoint refcount. This
1073                                  * basically prevents setting a new
1074                                  * configuration and alternate setting
1075                                  * when USB transfers are in use on
1076                                  * the given interface. Search the USB
1077                                  * code for "endpoint->refcount_alloc" if you
1078                                  * want more information.
1079                                  */
1080                                 USB_BUS_LOCK(info->bus);
1081                                 if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1082                                         parm->err = USB_ERR_INVAL;
1083
1084                                 xfer->endpoint->refcount_alloc++;
1085
1086                                 if (xfer->endpoint->refcount_alloc == 0)
1087                                         panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1088                                 USB_BUS_UNLOCK(info->bus);
1089
1090                                 /*
1091                                  * Whenever we set ppxfer[] then we
1092                                  * also need to increment the
1093                                  * "setup_refcount":
1094                                  */
1095                                 info->setup_refcount++;
1096
1097                                 /*
1098                                  * Transfer is successfully setup and
1099                                  * can be used:
1100                                  */
1101                                 ppxfer[n] = xfer;
1102                         }
1103
1104                         /* check for error */
1105                         if (parm->err)
1106                                 goto done;
1107                 }
1108
1109                 if (buf != NULL || parm->err != 0)
1110                         goto done;
1111
1112                 /* if no transfers, nothing to do */
1113                 if (refcount == 0)
1114                         goto done;
1115
1116                 /* align data properly */
1117                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1118
1119                 /* store offset temporarily */
1120                 parm->size[1] = parm->size[0];
1121
1122                 /*
1123                  * The number of DMA tags required depends on
1124                  * the number of endpoints. The current estimate
1125                  * for maximum number of DMA tags per endpoint
1126                  * is three:
1127                  * 1) for loading memory
1128                  * 2) for allocating memory
1129                  * 3) for fixing memory [UHCI]
1130                  */
1131                 parm->dma_tag_max += 3 * MIN(n_setup, USB_EP_MAX);
1132
1133                 /*
1134                  * DMA tags for QH, TD, Data and more.
1135                  */
1136                 parm->dma_tag_max += 8;
1137
1138                 parm->dma_tag_p += parm->dma_tag_max;
1139
1140                 parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1141                     ((uint8_t *)0);
1142
1143                 /* align data properly */
1144                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1145
1146                 /* store offset temporarily */
1147                 parm->size[3] = parm->size[0];
1148
1149                 parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1150                     ((uint8_t *)0);
1151
1152                 /* align data properly */
1153                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1154
1155                 /* store offset temporarily */
1156                 parm->size[4] = parm->size[0];
1157
1158                 parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1159                     ((uint8_t *)0);
1160
1161                 /* store end offset temporarily */
1162                 parm->size[5] = parm->size[0];
1163
1164                 parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1165                     ((uint8_t *)0);
1166
1167                 /* store end offset temporarily */
1168
1169                 parm->size[2] = parm->size[0];
1170
1171                 /* align data properly */
1172                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1173
1174                 parm->size[6] = parm->size[0];
1175
1176                 parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1177                     ((uint8_t *)0);
1178
1179                 /* align data properly */
1180                 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1181
1182                 /* allocate zeroed memory */
1183                 buf = kmalloc(parm->size[0], M_USB, M_WAITOK | M_ZERO);
1184
1185                 if (buf == NULL) {
1186                         parm->err = USB_ERR_NOMEM;
1187                         DPRINTFN(0, "cannot allocate memory block for "
1188                             "configuration (%d bytes)\n",
1189                             parm->size[0]);
1190                         goto done;
1191                 }
1192                 parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1193                 parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1194                 parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1195                 parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1196                 parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1197         }
1198
1199 done:
1200         if (buf) {
1201                 if (info->setup_refcount == 0) {
1202                         /*
1203                          * "usbd_transfer_unsetup_sub" will unlock
1204                          * the bus mutex before returning !
1205                          */
1206                         USB_BUS_LOCK(info->bus);
1207
1208                         /* something went wrong */
1209                         usbd_transfer_unsetup_sub(info, 0);
1210                 }
1211         }
1212
1213         /* check if any errors happened */
1214         if (parm->err)
1215                 usbd_transfer_unsetup(ppxfer, n_setup);
1216
1217         error = parm->err;
1218
1219         if (do_unlock)
1220                 usbd_enum_unlock(udev);
1221
1222         return (error);
1223 }
1224
1225 /*------------------------------------------------------------------------*
1226  *      usbd_transfer_unsetup_sub - factored out code
1227  *------------------------------------------------------------------------*/
1228 static void
1229 usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1230 {
1231 #if USB_HAVE_BUSDMA
1232         struct usb_page_cache *pc;
1233 #endif
1234
1235         USB_BUS_LOCK_ASSERT(info->bus);
1236
1237         /* wait for any outstanding DMA operations */
1238         /* This is insane */
1239         if (needs_delay) {
1240                 usb_timeout_t temp;
1241                 temp = usbd_get_dma_delay(info->udev);
1242                 if (temp != 0) {
1243                         usb_pause_mtx(&info->bus->bus_lock,
1244                             USB_MS_TO_TICKS(temp));
1245                 }
1246         }
1247
1248         /* make sure that our done messages are not queued anywhere */
1249         usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1250
1251         USB_BUS_UNLOCK(info->bus);
1252
1253 #if USB_HAVE_BUSDMA
1254         /* free DMA'able memory, if any */
1255         pc = info->dma_page_cache_start;
1256         while (pc != info->dma_page_cache_end) {
1257                 usb_pc_free_mem(pc);
1258                 pc++;
1259         }
1260
1261         /* free DMA maps in all "xfer->frbuffers" */
1262         pc = info->xfer_page_cache_start;
1263         while (pc != info->xfer_page_cache_end) {
1264                 usb_pc_dmamap_destroy(pc);
1265                 pc++;
1266         }
1267
1268         /* free all DMA tags */
1269         usb_dma_tag_unsetup(&info->dma_parent_tag);
1270 #endif
1271
1272         cv_destroy(&info->cv_drain);
1273
1274         /*
1275          * free the "memory_base" last, hence the "info" structure is
1276          * contained within the "memory_base"!
1277          */
1278         usbd_delayed_free(info->memory_base, M_USB);
1279 }
1280
1281 /*
1282  * This is a horrible hack and workaround to a very bad decision by
1283  * the original U4B coder to integrate the QH/TD structures into the
1284  * xfer and then free the whole mess all at once.
1285  *
1286  * The problem is that the controller may still be accessing the QHs,
1287  * because it might have gotten side-tracked onto the removed QHs
1288  * chain link.  They have to remain intact long enough for the
1289  * controller to get out.
1290  *
1291  * This horrible hack basically just delays freeing by 256 slots.
1292  * It's not even time-based or door-bell based (which is the way
1293  * the linux driver does it)... but to fix it properly requires rewriting
1294  * too much of this driver.
1295  */
1296 #define DFREE_SLOTS     256
1297 #define DFREE_MASK      (DFREE_SLOTS - 1)
1298
1299 static struct dfree_slot {
1300         void *data;
1301         struct malloc_type *mtype;
1302 } dfree_slots[DFREE_SLOTS];
1303 static int dfree_index;
1304
1305 static void
1306 usbd_delayed_free(void *data, struct malloc_type *mtype)
1307 {
1308         struct dfree_slot slot;
1309         int index;
1310
1311         crit_enter();
1312         index = atomic_fetchadd_int(&dfree_index, 1);
1313         index &= DFREE_MASK;
1314         slot = dfree_slots[index];
1315         dfree_slots[index].data = data;
1316         dfree_slots[index].mtype = mtype;
1317         crit_exit();
1318         if (slot.data)
1319                 kfree(slot.data, slot.mtype);
1320 }
1321
1322 /*------------------------------------------------------------------------*
1323  *      usbd_transfer_unsetup - unsetup/free an array of USB transfers
1324  *
1325  * NOTE: All USB transfers in progress will get called back passing
1326  * the error code "USB_ERR_CANCELLED" before this function
1327  * returns.
1328  *------------------------------------------------------------------------*/
1329 void
1330 usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1331 {
1332         struct usb_xfer *xfer;
1333         struct usb_xfer_root *info;
1334         uint8_t needs_delay = 0;
1335
1336 #if 0
1337         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1338             "usbd_transfer_unsetup can sleep!");
1339 #endif
1340
1341         while (n_setup--) {
1342                 xfer = pxfer[n_setup];
1343
1344                 if (xfer == NULL)
1345                         continue;
1346
1347                 info = xfer->xroot;
1348
1349                 USB_XFER_LOCK(xfer);
1350                 USB_BUS_LOCK(info->bus);
1351
1352                 /*
1353                  * HINT: when you start/stop a transfer, it might be a
1354                  * good idea to directly use the "pxfer[]" structure:
1355                  *
1356                  * usbd_transfer_start(sc->pxfer[0]);
1357                  * usbd_transfer_stop(sc->pxfer[0]);
1358                  *
1359                  * That way, if your code has many parts that will not
1360                  * stop running under the same lock, in other words
1361                  * "xfer_mtx", the usbd_transfer_start and
1362                  * usbd_transfer_stop functions will simply return
1363                  * when they detect a NULL pointer argument.
1364                  *
1365                  * To avoid any races we clear the "pxfer[]" pointer
1366                  * while holding the private mutex of the driver:
1367                  */
1368                 pxfer[n_setup] = NULL;
1369
1370                 USB_BUS_UNLOCK(info->bus);
1371                 USB_XFER_UNLOCK(xfer);
1372
1373                 usbd_transfer_drain(xfer);
1374
1375 #if USB_HAVE_BUSDMA
1376                 if (xfer->flags_int.bdma_enable)
1377                         needs_delay = 1;
1378 #endif
1379                 /*
1380                  * NOTE: default endpoint does not have an
1381                  * interface, even if endpoint->iface_index == 0
1382                  */
1383                 USB_BUS_LOCK(info->bus);
1384                 xfer->endpoint->refcount_alloc--;
1385                 USB_BUS_UNLOCK(info->bus);
1386
1387                 usb_callout_drain(&xfer->timeout_handle);
1388
1389                 USB_BUS_LOCK(info->bus);
1390
1391                 USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1392                     "reference count\n"));
1393
1394                 info->setup_refcount--;
1395
1396                 if (info->setup_refcount == 0) {
1397                         usbd_transfer_unsetup_sub(info,
1398                             needs_delay);
1399                 } else {
1400                         USB_BUS_UNLOCK(info->bus);
1401                 }
1402         }
1403 }
1404
1405 /*------------------------------------------------------------------------*
1406  *      usbd_control_transfer_init - factored out code
1407  *
1408  * In USB Device Mode we have to wait for the SETUP packet which
1409  * containst the "struct usb_device_request" structure, before we can
1410  * transfer any data. In USB Host Mode we already have the SETUP
1411  * packet at the moment the USB transfer is started. This leads us to
1412  * having to setup the USB transfer at two different places in
1413  * time. This function just contains factored out control transfer
1414  * initialisation code, so that we don't duplicate the code.
1415  *------------------------------------------------------------------------*/
1416 static void
1417 usbd_control_transfer_init(struct usb_xfer *xfer)
1418 {
1419         struct usb_device_request req;
1420
1421         /* copy out the USB request header */
1422
1423         usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1424
1425         /* setup remainder */
1426
1427         xfer->flags_int.control_rem = UGETW(req.wLength);
1428
1429         /* copy direction to endpoint variable */
1430
1431         xfer->endpointno &= ~(UE_DIR_IN | UE_DIR_OUT);
1432         xfer->endpointno |=
1433             (req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1434 }
1435
1436 /*------------------------------------------------------------------------*
1437  *      usbd_setup_ctrl_transfer
1438  *
1439  * This function handles initialisation of control transfers. Control
1440  * transfers are special in that regard that they can both transmit
1441  * and receive data.
1442  *
1443  * Return values:
1444  *    0: Success
1445  * Else: Failure
1446  *------------------------------------------------------------------------*/
1447 static int
1448 usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1449 {
1450         usb_frlength_t len;
1451
1452         /* Check for control endpoint stall */
1453         if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1454                 /* the control transfer is no longer active */
1455                 xfer->flags_int.control_stall = 1;
1456                 xfer->flags_int.control_act = 0;
1457         } else {
1458                 /* don't stall control transfer by default */
1459                 xfer->flags_int.control_stall = 0;
1460         }
1461
1462         /* Check for invalid number of frames */
1463         if (xfer->nframes > 2) {
1464                 /*
1465                  * If you need to split a control transfer, you
1466                  * have to do one part at a time. Only with
1467                  * non-control transfers you can do multiple
1468                  * parts a time.
1469                  */
1470                 DPRINTFN(0, "Too many frames: %u\n",
1471                     (unsigned int)xfer->nframes);
1472                 goto error;
1473         }
1474
1475         /*
1476          * Check if there is a control
1477          * transfer in progress:
1478          */
1479         if (xfer->flags_int.control_act) {
1480
1481                 if (xfer->flags_int.control_hdr) {
1482
1483                         /* clear send header flag */
1484
1485                         xfer->flags_int.control_hdr = 0;
1486
1487                         /* setup control transfer */
1488                         if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1489                                 usbd_control_transfer_init(xfer);
1490                         }
1491                 }
1492                 /* get data length */
1493
1494                 len = xfer->sumlen;
1495
1496         } else {
1497
1498                 /* the size of the SETUP structure is hardcoded ! */
1499
1500                 if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1501                         DPRINTFN(0, "Wrong framelength %u != %zu\n",
1502                             xfer->frlengths[0], sizeof(struct
1503                             usb_device_request));
1504                         goto error;
1505                 }
1506                 /* check USB mode */
1507                 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1508
1509                         /* check number of frames */
1510                         if (xfer->nframes != 1) {
1511                                 /*
1512                                  * We need to receive the setup
1513                                  * message first so that we know the
1514                                  * data direction!
1515                                  */
1516                                 DPRINTF("Misconfigured transfer\n");
1517                                 goto error;
1518                         }
1519                         /*
1520                          * Set a dummy "control_rem" value.  This
1521                          * variable will be overwritten later by a
1522                          * call to "usbd_control_transfer_init()" !
1523                          */
1524                         xfer->flags_int.control_rem = 0xFFFF;
1525                 } else {
1526
1527                         /* setup "endpoint" and "control_rem" */
1528
1529                         usbd_control_transfer_init(xfer);
1530                 }
1531
1532                 /* set transfer-header flag */
1533
1534                 xfer->flags_int.control_hdr = 1;
1535
1536                 /* get data length */
1537
1538                 len = (xfer->sumlen - sizeof(struct usb_device_request));
1539         }
1540
1541         /* check if there is a length mismatch */
1542
1543         if (len > xfer->flags_int.control_rem) {
1544                 DPRINTFN(0, "Length (%d) greater than "
1545                     "remaining length (%d)\n", len,
1546                     xfer->flags_int.control_rem);
1547                 goto error;
1548         }
1549         /* check if we are doing a short transfer */
1550
1551         if (xfer->flags.force_short_xfer) {
1552                 xfer->flags_int.control_rem = 0;
1553         } else {
1554                 if ((len != xfer->max_data_length) &&
1555                     (len != xfer->flags_int.control_rem) &&
1556                     (xfer->nframes != 1)) {
1557                         DPRINTFN(0, "Short control transfer without "
1558                             "force_short_xfer set\n");
1559                         goto error;
1560                 }
1561                 xfer->flags_int.control_rem -= len;
1562         }
1563
1564         /* the status part is executed when "control_act" is 0 */
1565
1566         if ((xfer->flags_int.control_rem > 0) ||
1567             (xfer->flags.manual_status)) {
1568                 /* don't execute the STATUS stage yet */
1569                 xfer->flags_int.control_act = 1;
1570
1571                 /* sanity check */
1572                 if ((!xfer->flags_int.control_hdr) &&
1573                     (xfer->nframes == 1)) {
1574                         /*
1575                          * This is not a valid operation!
1576                          */
1577                         DPRINTFN(0, "Invalid parameter "
1578                             "combination\n");
1579                         goto error;
1580                 }
1581         } else {
1582                 /* time to execute the STATUS stage */
1583                 xfer->flags_int.control_act = 0;
1584         }
1585         return (0);                     /* success */
1586
1587 error:
1588         return (1);                     /* failure */
1589 }
1590
1591 /*------------------------------------------------------------------------*
1592  *      usbd_transfer_submit - start USB hardware for the given transfer
1593  *
1594  * This function should only be called from the USB callback.
1595  *------------------------------------------------------------------------*/
1596 void
1597 usbd_transfer_submit(struct usb_xfer *xfer)
1598 {
1599         struct usb_xfer_root *info;
1600         struct usb_bus *bus;
1601         usb_frcount_t x;
1602
1603         info = xfer->xroot;
1604         bus = info->bus;
1605
1606         DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1607             xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1608             "read" : "write");
1609
1610 #ifdef USB_DEBUG
1611         if (USB_DEBUG_VAR > 0) {
1612                 USB_BUS_LOCK(bus);
1613
1614                 usb_dump_endpoint(xfer->endpoint);
1615
1616                 USB_BUS_UNLOCK(bus);
1617         }
1618 #endif
1619
1620         USB_XFER_LOCK_ASSERT(xfer);
1621         USB_BUS_LOCK_ASSERT_NOTOWNED(bus);
1622
1623         /* Only open the USB transfer once! */
1624         if (!xfer->flags_int.open) {
1625                 xfer->flags_int.open = 1;
1626
1627                 DPRINTF("open\n");
1628
1629                 USB_BUS_LOCK(bus);
1630                 (xfer->endpoint->methods->open) (xfer);
1631                 USB_BUS_UNLOCK(bus);
1632         }
1633         /* set "transferring" flag */
1634         xfer->flags_int.transferring = 1;
1635
1636 #if USB_HAVE_POWERD
1637         /* increment power reference */
1638         usbd_transfer_power_ref(xfer, 1);
1639 #endif
1640         /*
1641          * Check if the transfer is waiting on a queue, most
1642          * frequently the "done_q":
1643          */
1644         if (xfer->wait_queue) {
1645                 USB_BUS_LOCK(bus);
1646                 usbd_transfer_dequeue(xfer);
1647                 USB_BUS_UNLOCK(bus);
1648         }
1649         /* clear "did_dma_delay" flag */
1650         xfer->flags_int.did_dma_delay = 0;
1651
1652         /* clear "did_close" flag */
1653         xfer->flags_int.did_close = 0;
1654
1655 #if USB_HAVE_BUSDMA
1656         /* clear "bdma_setup" flag */
1657         xfer->flags_int.bdma_setup = 0;
1658 #endif
1659         /* by default we cannot cancel any USB transfer immediately */
1660         xfer->flags_int.can_cancel_immed = 0;
1661
1662         /* clear lengths and frame counts by default */
1663         xfer->sumlen = 0;
1664         xfer->actlen = 0;
1665         xfer->aframes = 0;
1666
1667         /* clear any previous errors */
1668         xfer->error = 0;
1669
1670         /* Check if the device is still alive */
1671         if (info->udev->state < USB_STATE_POWERED) {
1672                 USB_BUS_LOCK(bus);
1673                 /*
1674                  * Must return cancelled error code else
1675                  * device drivers can hang.
1676                  */
1677                 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1678                 USB_BUS_UNLOCK(bus);
1679                 return;
1680         }
1681
1682         /* sanity check */
1683         if (xfer->nframes == 0) {
1684                 if (xfer->flags.stall_pipe) {
1685                         /*
1686                          * Special case - want to stall without transferring
1687                          * any data:
1688                          */
1689                         DPRINTF("xfer=%p nframes=0: stall "
1690                             "or clear stall!\n", xfer);
1691                         USB_BUS_LOCK(bus);
1692                         xfer->flags_int.can_cancel_immed = 1;
1693                         /* start the transfer */
1694                         usb_command_wrapper(&xfer->endpoint->
1695                             endpoint_q[xfer->stream_id], xfer);
1696                         USB_BUS_UNLOCK(bus);
1697                         return;
1698                 }
1699                 USB_BUS_LOCK(bus);
1700                 usbd_transfer_done(xfer, USB_ERR_INVAL);
1701                 USB_BUS_UNLOCK(bus);
1702                 return;
1703         }
1704         /* compute some variables */
1705
1706         for (x = 0; x != xfer->nframes; x++) {
1707                 /* make a copy of the frlenghts[] */
1708                 xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1709                 /* compute total transfer length */
1710                 xfer->sumlen += xfer->frlengths[x];
1711                 if (xfer->sumlen < xfer->frlengths[x]) {
1712                         /* length wrapped around */
1713                         USB_BUS_LOCK(bus);
1714                         usbd_transfer_done(xfer, USB_ERR_INVAL);
1715                         USB_BUS_UNLOCK(bus);
1716                         return;
1717                 }
1718         }
1719
1720         /* clear some internal flags */
1721
1722         xfer->flags_int.short_xfer_ok = 0;
1723         xfer->flags_int.short_frames_ok = 0;
1724
1725         /* check if this is a control transfer */
1726
1727         if (xfer->flags_int.control_xfr) {
1728
1729                 if (usbd_setup_ctrl_transfer(xfer)) {
1730                         USB_BUS_LOCK(bus);
1731                         usbd_transfer_done(xfer, USB_ERR_STALLED);
1732                         USB_BUS_UNLOCK(bus);
1733                         return;
1734                 }
1735         }
1736         /*
1737          * Setup filtered version of some transfer flags,
1738          * in case of data read direction
1739          */
1740         if (USB_GET_DATA_ISREAD(xfer)) {
1741
1742                 if (xfer->flags.short_frames_ok) {
1743                         xfer->flags_int.short_xfer_ok = 1;
1744                         xfer->flags_int.short_frames_ok = 1;
1745                 } else if (xfer->flags.short_xfer_ok) {
1746                         xfer->flags_int.short_xfer_ok = 1;
1747
1748                         /* check for control transfer */
1749                         if (xfer->flags_int.control_xfr) {
1750                                 /*
1751                                  * 1) Control transfers do not support
1752                                  * reception of multiple short USB
1753                                  * frames in host mode and device side
1754                                  * mode, with exception of:
1755                                  *
1756                                  * 2) Due to sometimes buggy device
1757                                  * side firmware we need to do a
1758                                  * STATUS stage in case of short
1759                                  * control transfers in USB host mode.
1760                                  * The STATUS stage then becomes the
1761                                  * "alt_next" to the DATA stage.
1762                                  */
1763                                 xfer->flags_int.short_frames_ok = 1;
1764                         }
1765                 }
1766         }
1767         /*
1768          * Check if BUS-DMA support is enabled and try to load virtual
1769          * buffers into DMA, if any:
1770          */
1771 #if USB_HAVE_BUSDMA
1772         if (xfer->flags_int.bdma_enable) {
1773                 /* insert the USB transfer last in the BUS-DMA queue */
1774                 usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1775                 return;
1776         }
1777 #endif
1778         /*
1779          * Enter the USB transfer into the Host Controller or
1780          * Device Controller schedule:
1781          */
1782         usbd_pipe_enter(xfer);
1783 }
1784
1785 /*------------------------------------------------------------------------*
1786  *      usbd_pipe_enter - factored out code
1787  *------------------------------------------------------------------------*/
1788 void
1789 usbd_pipe_enter(struct usb_xfer *xfer)
1790 {
1791         struct usb_endpoint *ep;
1792
1793         USB_XFER_LOCK_ASSERT(xfer);
1794
1795         USB_BUS_LOCK(xfer->xroot->bus);
1796
1797         ep = xfer->endpoint;
1798
1799         DPRINTF("enter\n");
1800
1801         /* the transfer can now be cancelled */
1802         xfer->flags_int.can_cancel_immed = 1;
1803
1804         /* enter the transfer */
1805         (ep->methods->enter) (xfer);
1806
1807         /* check for transfer error */
1808         if (xfer->error) {
1809                 /* some error has happened */
1810                 usbd_transfer_done(xfer, 0);
1811                 USB_BUS_UNLOCK(xfer->xroot->bus);
1812                 return;
1813         }
1814
1815         /* start the transfer */
1816         usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], xfer);
1817         USB_BUS_UNLOCK(xfer->xroot->bus);
1818 }
1819
1820 /*------------------------------------------------------------------------*
1821  *      usbd_transfer_start - start an USB transfer
1822  *
1823  * NOTE: Calling this function more than one time will only
1824  *       result in a single transfer start, until the USB transfer
1825  *       completes.
1826  *------------------------------------------------------------------------*/
1827 void
1828 usbd_transfer_start(struct usb_xfer *xfer)
1829 {
1830         if (xfer == NULL) {
1831                 /* transfer is gone */
1832                 return;
1833         }
1834         USB_XFER_LOCK_ASSERT(xfer);
1835
1836         /* mark the USB transfer started */
1837
1838         if (!xfer->flags_int.started) {
1839                 /* lock the BUS lock to avoid races updating flags_int */
1840                 USB_BUS_LOCK(xfer->xroot->bus);
1841                 xfer->flags_int.started = 1;
1842                 USB_BUS_UNLOCK(xfer->xroot->bus);
1843         }
1844         /* check if the USB transfer callback is already transferring */
1845
1846         if (xfer->flags_int.transferring) {
1847                 return;
1848         }
1849         USB_BUS_LOCK(xfer->xroot->bus);
1850         /* call the USB transfer callback */
1851         usbd_callback_ss_done_defer(xfer);
1852         USB_BUS_UNLOCK(xfer->xroot->bus);
1853 }
1854
1855 /*------------------------------------------------------------------------*
1856  *      usbd_transfer_stop - stop an USB transfer
1857  *
1858  * NOTE: Calling this function more than one time will only
1859  *       result in a single transfer stop.
1860  * NOTE: When this function returns it is not safe to free nor
1861  *       reuse any DMA buffers. See "usbd_transfer_drain()".
1862  *------------------------------------------------------------------------*/
1863 void
1864 usbd_transfer_stop(struct usb_xfer *xfer)
1865 {
1866         struct usb_endpoint *ep;
1867
1868         if (xfer == NULL) {
1869                 /* transfer is gone */
1870                 return;
1871         }
1872 #if 0
1873         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1874 #endif
1875
1876         /* check if the USB transfer was ever opened */
1877
1878         if (!xfer->flags_int.open) {
1879                 if (xfer->flags_int.started) {
1880                         /* nothing to do except clearing the "started" flag */
1881                         /* lock the BUS lock to avoid races updating flags_int */
1882                         USB_BUS_LOCK(xfer->xroot->bus);
1883                         xfer->flags_int.started = 0;
1884                         USB_BUS_UNLOCK(xfer->xroot->bus);
1885                 }
1886                 return;
1887         }
1888         /* try to stop the current USB transfer */
1889
1890         USB_BUS_LOCK(xfer->xroot->bus);
1891         /* override any previous error */
1892         xfer->error = USB_ERR_CANCELLED;
1893
1894         /*
1895          * Clear "open" and "started" when both private and USB lock
1896          * is locked so that we don't get a race updating "flags_int"
1897          */
1898         xfer->flags_int.open = 0;
1899         xfer->flags_int.started = 0;
1900
1901         /*
1902          * Check if we can cancel the USB transfer immediately.
1903          */
1904         if (xfer->flags_int.transferring) {
1905                 if (xfer->flags_int.can_cancel_immed &&
1906                     (!xfer->flags_int.did_close)) {
1907                         DPRINTF("close\n");
1908                         /*
1909                          * The following will lead to an USB_ERR_CANCELLED
1910                          * error code being passed to the USB callback.
1911                          */
1912                         (xfer->endpoint->methods->close) (xfer);
1913                         /* only close once */
1914                         xfer->flags_int.did_close = 1;
1915                 } else {
1916                         /* need to wait for the next done callback */
1917                 }
1918         } else {
1919                 DPRINTF("close\n");
1920
1921                 /* close here and now */
1922                 (xfer->endpoint->methods->close) (xfer);
1923
1924                 /*
1925                  * Any additional DMA delay is done by
1926                  * "usbd_transfer_unsetup()".
1927                  */
1928
1929                 /*
1930                  * Special case. Check if we need to restart a blocked
1931                  * endpoint.
1932                  */
1933                 ep = xfer->endpoint;
1934
1935                 /*
1936                  * If the current USB transfer is completing we need
1937                  * to start the next one:
1938                  */
1939                 if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
1940                         usb_command_wrapper(
1941                             &ep->endpoint_q[xfer->stream_id], NULL);
1942                 }
1943         }
1944
1945         USB_BUS_UNLOCK(xfer->xroot->bus);
1946 }
1947
1948 /*------------------------------------------------------------------------*
1949  *      usbd_transfer_pending
1950  *
1951  * This function will check if an USB transfer is pending which is a
1952  * little bit complicated!
1953  * Return values:
1954  * 0: Not pending
1955  * 1: Pending: The USB transfer will receive a callback in the future.
1956  *------------------------------------------------------------------------*/
1957 uint8_t
1958 usbd_transfer_pending(struct usb_xfer *xfer)
1959 {
1960         struct usb_xfer_root *info;
1961         struct usb_xfer_queue *pq;
1962
1963         if (xfer == NULL) {
1964                 /* transfer is gone */
1965                 return (0);
1966         }
1967 #if 0
1968         USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1969 #endif
1970
1971         if (xfer->flags_int.transferring) {
1972                 /* trivial case */
1973                 return (1);
1974         }
1975         USB_BUS_LOCK(xfer->xroot->bus);
1976         if (xfer->wait_queue) {
1977                 /* we are waiting on a queue somewhere */
1978                 USB_BUS_UNLOCK(xfer->xroot->bus);
1979                 return (1);
1980         }
1981         info = xfer->xroot;
1982         pq = &info->done_q;
1983
1984         if (pq->curr == xfer) {
1985                 /* we are currently scheduled for callback */
1986                 USB_BUS_UNLOCK(xfer->xroot->bus);
1987                 return (1);
1988         }
1989         /* we are not pending */
1990         USB_BUS_UNLOCK(xfer->xroot->bus);
1991         return (0);
1992 }
1993
1994 /*------------------------------------------------------------------------*
1995  *      usbd_transfer_drain
1996  *
1997  * This function will stop the USB transfer and wait for any
1998  * additional BUS-DMA and HW-DMA operations to complete. Buffers that
1999  * are loaded into DMA can safely be freed or reused after that this
2000  * function has returned.
2001  *------------------------------------------------------------------------*/
2002 void
2003 usbd_transfer_drain(struct usb_xfer *xfer)
2004 {
2005 #if 0
2006         WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2007             "usbd_transfer_drain can sleep!");
2008 #endif
2009
2010         if (xfer == NULL) {
2011                 /* transfer is gone */
2012                 return;
2013         }
2014         USB_XFER_LOCK_ASSERT_NOTOWNED(xfer);
2015         USB_XFER_LOCK(xfer);
2016
2017         usbd_transfer_stop(xfer);
2018
2019         /*
2020          * It is allowed that the callback can drop its
2021          * transfer mutex. In that case checking only
2022          * "usbd_transfer_pending()" is not enough to tell if
2023          * the USB transfer is fully drained. We also need to
2024          * check the internal "doing_callback" flag.
2025          */
2026         xfer->flags_int.draining = 1;
2027
2028         /*
2029          * XXX hack, the wakeup of xfer can race conditions which
2030          *     clear the pending status of the xfer.
2031          */
2032         while (usbd_transfer_pending(xfer) ||
2033             xfer->flags_int.doing_callback) {
2034
2035                 /*
2036                  * Wait until the current outstanding USB
2037                  * transfer is complete !
2038                  */
2039                 /* cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_lock); */
2040                 lksleep(xfer, xfer->xroot->xfer_lock, 0, "DRAIN", hz);
2041         }
2042         xfer->flags_int.draining = 0;
2043         USB_XFER_UNLOCK(xfer);
2044 }
2045
2046 struct usb_page_cache *
2047 usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
2048 {
2049         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2050
2051         return (&xfer->frbuffers[frindex]);
2052 }
2053
2054 void *
2055 usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
2056 {
2057         struct usb_page_search page_info;
2058
2059         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2060
2061         usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
2062         return (page_info.buffer);
2063 }
2064
2065 /*------------------------------------------------------------------------*
2066  *      usbd_xfer_get_fps_shift
2067  *
2068  * The following function is only useful for isochronous transfers. It
2069  * returns how many times the frame execution rate has been shifted
2070  * down.
2071  *
2072  * Return value:
2073  * Success: 0..3
2074  * Failure: 0
2075  *------------------------------------------------------------------------*/
2076 uint8_t
2077 usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
2078 {
2079         return (xfer->fps_shift);
2080 }
2081
2082 usb_frlength_t
2083 usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
2084 {
2085         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2086
2087         return (xfer->frlengths[frindex]);
2088 }
2089
2090 /*------------------------------------------------------------------------*
2091  *      usbd_xfer_set_frame_data
2092  *
2093  * This function sets the pointer of the buffer that should
2094  * loaded directly into DMA for the given USB frame. Passing "ptr"
2095  * equal to NULL while the corresponding "frlength" is greater
2096  * than zero gives undefined results!
2097  *------------------------------------------------------------------------*/
2098 void
2099 usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2100     void *ptr, usb_frlength_t len)
2101 {
2102         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2103
2104         /* set virtual address to load and length */
2105         xfer->frbuffers[frindex].buffer = ptr;
2106         usbd_xfer_set_frame_len(xfer, frindex, len);
2107 }
2108
2109 void
2110 usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2111     void **ptr, int *len)
2112 {
2113         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2114
2115         if (ptr != NULL)
2116                 *ptr = xfer->frbuffers[frindex].buffer;
2117         if (len != NULL)
2118                 *len = xfer->frlengths[frindex];
2119 }
2120
2121 /*------------------------------------------------------------------------*
2122  *      usbd_xfer_old_frame_length
2123  *
2124  * This function returns the framelength of the given frame at the
2125  * time the transfer was submitted. This function can be used to
2126  * compute the starting data pointer of the next isochronous frame
2127  * when an isochronous transfer has completed.
2128  *------------------------------------------------------------------------*/
2129 usb_frlength_t
2130 usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2131 {
2132         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2133
2134         return (xfer->frlengths[frindex + xfer->max_frame_count]);
2135 }
2136
2137 void
2138 usbd_xfer_status(struct usb_xfer *xfer, int *actlen, int *sumlen, int *aframes,
2139     int *nframes)
2140 {
2141         if (actlen != NULL)
2142                 *actlen = xfer->actlen;
2143         if (sumlen != NULL)
2144                 *sumlen = xfer->sumlen;
2145         if (aframes != NULL)
2146                 *aframes = xfer->aframes;
2147         if (nframes != NULL)
2148                 *nframes = xfer->nframes;
2149 }
2150
2151 /*------------------------------------------------------------------------*
2152  *      usbd_xfer_set_frame_offset
2153  *
2154  * This function sets the frame data buffer offset relative to the beginning
2155  * of the USB DMA buffer allocated for this USB transfer.
2156  *------------------------------------------------------------------------*/
2157 void
2158 usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2159     usb_frcount_t frindex)
2160 {
2161         KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2162             "when the USB buffer is external\n"));
2163         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2164
2165         /* set virtual address to load */
2166         xfer->frbuffers[frindex].buffer =
2167             USB_ADD_BYTES(xfer->local_buffer, offset);
2168 }
2169
2170 void
2171 usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2172 {
2173         xfer->interval = i;
2174 }
2175
2176 void
2177 usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2178 {
2179         xfer->timeout = t;
2180 }
2181
2182 void
2183 usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2184 {
2185         xfer->nframes = n;
2186 }
2187
2188 usb_frcount_t
2189 usbd_xfer_max_frames(struct usb_xfer *xfer)
2190 {
2191         return (xfer->max_frame_count);
2192 }
2193
2194 usb_frlength_t
2195 usbd_xfer_max_len(struct usb_xfer *xfer)
2196 {
2197         return (xfer->max_data_length);
2198 }
2199
2200 usb_frlength_t
2201 usbd_xfer_max_framelen(struct usb_xfer *xfer)
2202 {
2203         return (xfer->max_frame_size);
2204 }
2205
2206 void
2207 usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2208     usb_frlength_t len)
2209 {
2210         KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2211
2212         xfer->frlengths[frindex] = len;
2213 }
2214
2215 /*------------------------------------------------------------------------*
2216  *      usb_callback_proc - factored out code
2217  *
2218  * This function performs USB callbacks.
2219  *------------------------------------------------------------------------*/
2220 static void
2221 usb_callback_proc(struct usb_proc_msg *_pm)
2222 {
2223         struct usb_done_msg *pm = (void *)_pm;
2224         struct usb_xfer_root *info = pm->xroot;
2225
2226         /* Change locking order */
2227         USB_BUS_UNLOCK(info->bus);
2228
2229         /*
2230          * We exploit the fact that the mutex is the same for all
2231          * callbacks that will be called from this thread:
2232          */
2233         lockmgr(info->xfer_lock, LK_EXCLUSIVE);
2234         USB_BUS_LOCK(info->bus);
2235
2236         /* Continue where we lost track */
2237         usb_command_wrapper(&info->done_q,
2238             info->done_q.curr);
2239
2240         lockmgr(info->xfer_lock, LK_RELEASE);
2241 }
2242
2243 /*------------------------------------------------------------------------*
2244  *      usbd_callback_ss_done_defer
2245  *
2246  * This function will defer the start, stop and done callback to the
2247  * correct thread.
2248  *------------------------------------------------------------------------*/
2249 static void
2250 usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2251 {
2252         struct usb_xfer_root *info = xfer->xroot;
2253         struct usb_xfer_queue *pq = &info->done_q;
2254
2255         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2256
2257         if (pq->curr != xfer) {
2258                 usbd_transfer_enqueue(pq, xfer);
2259         }
2260         if (!pq->recurse_1) {
2261
2262                 /*
2263                  * We have to postpone the callback due to the fact we
2264                  * will have a Lock Order Reversal, LOR, if we try to
2265                  * proceed !
2266                  */
2267                 if (usb_proc_msignal(info->done_p,
2268                     &info->done_m[0], &info->done_m[1])) {
2269                         /* ignore */
2270                 }
2271         } else {
2272                 /* clear second recurse flag */
2273                 pq->recurse_2 = 0;
2274         }
2275         return;
2276
2277 }
2278
2279 /*------------------------------------------------------------------------*
2280  *      usbd_callback_wrapper
2281  *
2282  * This is a wrapper for USB callbacks. This wrapper does some
2283  * auto-magic things like figuring out if we can call the callback
2284  * directly from the current context or if we need to wakeup the
2285  * interrupt process.
2286  *------------------------------------------------------------------------*/
2287 static void
2288 usbd_callback_wrapper(struct usb_xfer_queue *pq)
2289 {
2290         struct usb_xfer *xfer = pq->curr;
2291         struct usb_xfer_root *info = xfer->xroot;
2292
2293         USB_BUS_LOCK_ASSERT(info->bus);
2294         if (!lockowned(info->xfer_lock)) {
2295                 /*
2296                  * Cases that end up here:
2297                  *
2298                  * 5) HW interrupt done callback or other source.
2299                  */
2300                 DPRINTFN(3, "case 5\n");
2301
2302                 /*
2303                  * We have to postpone the callback due to the fact we
2304                  * will have a Lock Order Reversal, LOR, if we try to
2305                  * proceed !
2306                  */
2307                 if (usb_proc_msignal(info->done_p,
2308                     &info->done_m[0], &info->done_m[1])) {
2309                         /* ignore */
2310                 }
2311                 return;
2312         }
2313         /*
2314          * Cases that end up here:
2315          *
2316          * 1) We are starting a transfer
2317          * 2) We are prematurely calling back a transfer
2318          * 3) We are stopping a transfer
2319          * 4) We are doing an ordinary callback
2320          */
2321         DPRINTFN(3, "case 1-4\n");
2322         /* get next USB transfer in the queue */
2323         info->done_q.curr = NULL;
2324
2325         /* set flag in case of drain */
2326         xfer->flags_int.doing_callback = 1;
2327
2328         USB_BUS_UNLOCK(info->bus);
2329         USB_BUS_LOCK_ASSERT_NOTOWNED(info->bus);
2330
2331         /* set correct USB state for callback */
2332         if (!xfer->flags_int.transferring) {
2333                 xfer->usb_state = USB_ST_SETUP;
2334                 if (!xfer->flags_int.started) {
2335                         /* we got stopped before we even got started */
2336                         USB_BUS_LOCK(info->bus);
2337                         goto done;
2338                 }
2339         } else {
2340
2341                 if (usbd_callback_wrapper_sub(xfer)) {
2342                         /* the callback has been deferred */
2343                         USB_BUS_LOCK(info->bus);
2344                         goto done;
2345                 }
2346 #if USB_HAVE_POWERD
2347                 /* decrement power reference */
2348                 usbd_transfer_power_ref(xfer, -1);
2349 #endif
2350                 xfer->flags_int.transferring = 0;
2351
2352                 if (xfer->error) {
2353                         xfer->usb_state = USB_ST_ERROR;
2354                 } else {
2355                         /* set transferred state */
2356                         xfer->usb_state = USB_ST_TRANSFERRED;
2357 #if USB_HAVE_BUSDMA
2358                         /* sync DMA memory, if any */
2359                         if (xfer->flags_int.bdma_enable &&
2360                             (!xfer->flags_int.bdma_no_post_sync)) {
2361                                 usb_bdma_post_sync(xfer);
2362                         }
2363 #endif
2364                 }
2365         }
2366
2367 #if USB_HAVE_PF
2368         if (xfer->usb_state != USB_ST_SETUP)
2369                 usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2370 #endif
2371         /* call processing routine */
2372         (xfer->callback) (xfer, xfer->error);
2373
2374         /* pickup the USB mutex again */
2375         USB_BUS_LOCK(info->bus);
2376
2377         /*
2378          * Check if we got started after that we got cancelled, but
2379          * before we managed to do the callback.
2380          */
2381         if ((!xfer->flags_int.open) &&
2382             (xfer->flags_int.started) &&
2383             (xfer->usb_state == USB_ST_ERROR)) {
2384                 /* clear flag in case of drain */
2385                 xfer->flags_int.doing_callback = 0;
2386                 /* try to loop, but not recursivly */
2387                 usb_command_wrapper(&info->done_q, xfer);
2388                 return;
2389         }
2390
2391 done:
2392         /* clear flag in case of drain */
2393         xfer->flags_int.doing_callback = 0;
2394
2395         /*
2396          * Check if we are draining.
2397          */
2398         if (xfer->flags_int.draining &&
2399             (!xfer->flags_int.transferring)) {
2400                 /* "usbd_transfer_drain()" is waiting for end of transfer */
2401                 xfer->flags_int.draining = 0;
2402                 /* cv_broadcast(&info->cv_drain); */
2403                 wakeup(xfer);
2404         }
2405
2406         /* do the next callback, if any */
2407         usb_command_wrapper(&info->done_q,
2408             info->done_q.curr);
2409 }
2410
2411 /*------------------------------------------------------------------------*
2412  *      usb_dma_delay_done_cb
2413  *
2414  * This function is called when the DMA delay has been exectuded, and
2415  * will make sure that the callback is called to complete the USB
2416  * transfer. This code path is ususally only used when there is an USB
2417  * error like USB_ERR_CANCELLED.
2418  *------------------------------------------------------------------------*/
2419 void
2420 usb_dma_delay_done_cb(struct usb_xfer *xfer)
2421 {
2422         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2423
2424         DPRINTFN(3, "Completed %p\n", xfer);
2425
2426         /* queue callback for execution, again */
2427         usbd_transfer_done(xfer, 0);
2428 }
2429
2430 /*------------------------------------------------------------------------*
2431  *      usbd_transfer_dequeue
2432  *
2433  *  - This function is used to remove an USB transfer from a USB
2434  *  transfer queue.
2435  *
2436  *  - This function can be called multiple times in a row.
2437  *------------------------------------------------------------------------*/
2438 void
2439 usbd_transfer_dequeue(struct usb_xfer *xfer)
2440 {
2441         struct usb_xfer_queue *pq;
2442
2443         pq = xfer->wait_queue;
2444         if (pq) {
2445                 TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2446                 xfer->wait_queue = NULL;
2447         }
2448 }
2449
2450 /*------------------------------------------------------------------------*
2451  *      usbd_transfer_enqueue
2452  *
2453  *  - This function is used to insert an USB transfer into a USB *
2454  *  transfer queue.
2455  *
2456  *  - This function can be called multiple times in a row.
2457  *------------------------------------------------------------------------*/
2458 void
2459 usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2460 {
2461         /*
2462          * Insert the USB transfer into the queue, if it is not
2463          * already on a USB transfer queue:
2464          */
2465         /* mpf ?
2466         KKASSERT(xfer->wait_queue == NULL);
2467         */
2468         if (xfer->wait_queue == NULL) {
2469                 xfer->wait_queue = pq;
2470                 TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2471         }
2472 }
2473
2474 /*------------------------------------------------------------------------*
2475  *      usbd_transfer_done
2476  *
2477  *  - This function is used to remove an USB transfer from the busdma,
2478  *  pipe or interrupt queue.
2479  *
2480  *  - This function is used to queue the USB transfer on the done
2481  *  queue.
2482  *
2483  *  - This function is used to stop any USB transfer timeouts.
2484  *------------------------------------------------------------------------*/
2485 void
2486 usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2487 {
2488         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2489
2490         DPRINTF("err=%s\n", usbd_errstr(error));
2491
2492         /*
2493          * If we are not transferring then just return.
2494          * This can happen during transfer cancel.
2495          */
2496         if (!xfer->flags_int.transferring) {
2497                 DPRINTF("not transferring\n");
2498                 /* end of control transfer, if any */
2499                 xfer->flags_int.control_act = 0;
2500                 return;
2501         }
2502         /* only set transfer error if not already set */
2503         if (!xfer->error) {
2504                 xfer->error = error;
2505         }
2506         /* stop any callouts */
2507         usb_callout_stop(&xfer->timeout_handle);
2508
2509         /*
2510          * If we are waiting on a queue, just remove the USB transfer
2511          * from the queue, if any. We should have the required locks
2512          * locked to do the remove when this function is called.
2513          */
2514         usbd_transfer_dequeue(xfer);
2515
2516 #if USB_HAVE_BUSDMA
2517         if (lockowned(xfer->xroot->xfer_lock)) {
2518                 struct usb_xfer_queue *pq;
2519
2520                 /*
2521                  * If the private USB lock is not locked, then we assume
2522                  * that the BUS-DMA load stage has been passed:
2523                  */
2524                 pq = &xfer->xroot->dma_q;
2525
2526                 if (pq->curr == xfer) {
2527                         /* start the next BUS-DMA load, if any */
2528                         usb_command_wrapper(pq, NULL);
2529                 }
2530         }
2531 #endif
2532         /* keep some statistics */
2533         if (xfer->error) {
2534                 xfer->xroot->bus->stats_err.uds_requests
2535                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2536         } else {
2537                 xfer->xroot->bus->stats_ok.uds_requests
2538                     [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2539         }
2540
2541         /* call the USB transfer callback */
2542         usbd_callback_ss_done_defer(xfer);
2543 }
2544
2545 /*------------------------------------------------------------------------*
2546  *      usbd_transfer_start_cb
2547  *
2548  * This function is called to start the USB transfer when
2549  * "xfer->interval" is greater than zero, and and the endpoint type is
2550  * BULK or CONTROL.
2551  *------------------------------------------------------------------------*/
2552 static void
2553 usbd_transfer_start_cb(void *arg)
2554 {
2555         struct usb_xfer *xfer = arg;
2556         struct usb_endpoint *ep = xfer->endpoint;
2557
2558         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2559
2560         DPRINTF("start\n");
2561
2562 #if USB_HAVE_PF
2563         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2564 #endif
2565
2566         /* the transfer can now be cancelled */
2567         xfer->flags_int.can_cancel_immed = 1;
2568
2569         /* start USB transfer, if no error */
2570         if (xfer->error == 0)
2571                 (ep->methods->start) (xfer);
2572
2573         /* check for transfer error */
2574         if (xfer->error) {
2575                 /* some error has happened */
2576                 usbd_transfer_done(xfer, 0);
2577         }
2578 }
2579
2580 /*------------------------------------------------------------------------*
2581  *      usbd_xfer_set_stall
2582  *
2583  * This function is used to set the stall flag outside the
2584  * callback. This function is NULL safe.
2585  *------------------------------------------------------------------------*/
2586 void
2587 usbd_xfer_set_stall(struct usb_xfer *xfer)
2588 {
2589         if (xfer == NULL) {
2590                 /* tearing down */
2591                 return;
2592         }
2593         USB_XFER_LOCK_ASSERT(xfer);
2594
2595         /* avoid any races by locking the USB mutex */
2596         USB_BUS_LOCK(xfer->xroot->bus);
2597         xfer->flags.stall_pipe = 1;
2598         USB_BUS_UNLOCK(xfer->xroot->bus);
2599 }
2600
2601 int
2602 usbd_xfer_is_stalled(struct usb_xfer *xfer)
2603 {
2604         return (xfer->endpoint->is_stalled);
2605 }
2606
2607 /*------------------------------------------------------------------------*
2608  *      usbd_transfer_clear_stall
2609  *
2610  * This function is used to clear the stall flag outside the
2611  * callback. This function is NULL safe.
2612  *------------------------------------------------------------------------*/
2613 void
2614 usbd_transfer_clear_stall(struct usb_xfer *xfer)
2615 {
2616         if (xfer == NULL) {
2617                 /* tearing down */
2618                 return;
2619         }
2620         USB_XFER_LOCK_ASSERT(xfer);
2621
2622         /* avoid any races by locking the USB mutex */
2623         USB_BUS_LOCK(xfer->xroot->bus);
2624
2625         xfer->flags.stall_pipe = 0;
2626
2627         USB_BUS_UNLOCK(xfer->xroot->bus);
2628 }
2629
2630 /*------------------------------------------------------------------------*
2631  *      usbd_pipe_start
2632  *
2633  * This function is used to add an USB transfer to the pipe transfer list.
2634  *------------------------------------------------------------------------*/
2635 void
2636 usbd_pipe_start(struct usb_xfer_queue *pq)
2637 {
2638         struct usb_endpoint *ep;
2639         struct usb_xfer *xfer;
2640         uint8_t type;
2641
2642         xfer = pq->curr;
2643         ep = xfer->endpoint;
2644
2645         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2646
2647         /*
2648          * If the endpoint is already stalled we do nothing !
2649          */
2650         if (ep->is_stalled) {
2651                 return;
2652         }
2653         /*
2654          * Check if we are supposed to stall the endpoint:
2655          */
2656         if (xfer->flags.stall_pipe) {
2657                 struct usb_device *udev;
2658                 struct usb_xfer_root *info;
2659
2660                 /* clear stall command */
2661                 xfer->flags.stall_pipe = 0;
2662
2663                 /* get pointer to USB device */
2664                 info = xfer->xroot;
2665                 udev = info->udev;
2666
2667                 /*
2668                  * Only stall BULK and INTERRUPT endpoints.
2669                  */
2670                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2671                 if ((type == UE_BULK) ||
2672                     (type == UE_INTERRUPT)) {
2673                         uint8_t did_stall;
2674
2675                         did_stall = 1;
2676
2677                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2678                                 (udev->bus->methods->set_stall) (
2679                                     udev, ep, &did_stall);
2680                         } else if (udev->ctrl_xfer[1]) {
2681                                 info = udev->ctrl_xfer[1]->xroot;
2682                                 usb_proc_msignal(
2683                                     USB_BUS_NON_GIANT_PROC(info->bus),
2684                                     &udev->cs_msg[0], &udev->cs_msg[1]);
2685                         } else {
2686                                 /* should not happen */
2687                                 DPRINTFN(0, "No stall handler\n");
2688                         }
2689                         /*
2690                          * Check if we should stall. Some USB hardware
2691                          * handles set- and clear-stall in hardware.
2692                          */
2693                         if (did_stall) {
2694                                 /*
2695                                  * The transfer will be continued when
2696                                  * the clear-stall control endpoint
2697                                  * message is received.
2698                                  */
2699                                 ep->is_stalled = 1;
2700                                 return;
2701                         }
2702                 } else if (type == UE_ISOCHRONOUS) {
2703
2704                         /*
2705                          * Make sure any FIFO overflow or other FIFO
2706                          * error conditions go away by resetting the
2707                          * endpoint FIFO through the clear stall
2708                          * method.
2709                          */
2710                         if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2711                                 (udev->bus->methods->clear_stall) (udev, ep);
2712                         }
2713                 }
2714         }
2715         /* Set or clear stall complete - special case */
2716         if (xfer->nframes == 0) {
2717                 /* we are complete */
2718                 xfer->aframes = 0;
2719                 usbd_transfer_done(xfer, 0);
2720                 return;
2721         }
2722         /*
2723          * Handled cases:
2724          *
2725          * 1) Start the first transfer queued.
2726          *
2727          * 2) Re-start the current USB transfer.
2728          */
2729         /*
2730          * Check if there should be any
2731          * pre transfer start delay:
2732          */
2733         if (xfer->interval > 0) {
2734                 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2735                 if ((type == UE_BULK) ||
2736                     (type == UE_CONTROL)) {
2737                         usbd_transfer_timeout_ms(xfer,
2738                             &usbd_transfer_start_cb,
2739                             xfer->interval);
2740                         return;
2741                 }
2742         }
2743         DPRINTF("start\n");
2744
2745 #if USB_HAVE_PF
2746         usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2747 #endif
2748         /* the transfer can now be cancelled */
2749         xfer->flags_int.can_cancel_immed = 1;
2750
2751         /* start USB transfer, if no error */
2752         if (xfer->error == 0)
2753                 (ep->methods->start) (xfer);
2754
2755         /* check for transfer error */
2756         if (xfer->error) {
2757                 /* some error has happened */
2758                 usbd_transfer_done(xfer, 0);
2759         }
2760 }
2761
2762 /*------------------------------------------------------------------------*
2763  *      usbd_transfer_timeout_ms
2764  *
2765  * This function is used to setup a timeout on the given USB
2766  * transfer. If the timeout has been deferred the callback given by
2767  * "cb" will get called after "ms" milliseconds.
2768  *------------------------------------------------------------------------*/
2769 void
2770 usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2771     void (*cb) (void *arg), usb_timeout_t ms)
2772 {
2773         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2774
2775         /* defer delay */
2776         usb_callout_reset(&xfer->timeout_handle,
2777             USB_MS_TO_TICKS(ms), cb, xfer);
2778 }
2779
2780 /*------------------------------------------------------------------------*
2781  *      usbd_callback_wrapper_sub
2782  *
2783  *  - This function will update variables in an USB transfer after
2784  *  that the USB transfer is complete.
2785  *
2786  *  - This function is used to start the next USB transfer on the
2787  *  ep transfer queue, if any.
2788  *
2789  * NOTE: In some special cases the USB transfer will not be removed from
2790  * the pipe queue, but remain first. To enforce USB transfer removal call
2791  * this function passing the error code "USB_ERR_CANCELLED".
2792  *
2793  * Return values:
2794  * 0: Success.
2795  * Else: The callback has been deferred.
2796  *------------------------------------------------------------------------*/
2797 static uint8_t
2798 usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2799 {
2800         struct usb_endpoint *ep;
2801         struct usb_bus *bus;
2802         usb_frcount_t x;
2803
2804         bus = xfer->xroot->bus;
2805
2806         if ((!xfer->flags_int.open) &&
2807             (!xfer->flags_int.did_close)) {
2808                 DPRINTF("close\n");
2809                 USB_BUS_LOCK(bus);
2810                 (xfer->endpoint->methods->close) (xfer);
2811                 USB_BUS_UNLOCK(bus);
2812                 /* only close once */
2813                 xfer->flags_int.did_close = 1;
2814                 return (1);             /* wait for new callback */
2815         }
2816         /*
2817          * If we have a non-hardware induced error we
2818          * need to do the DMA delay!
2819          */
2820         if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2821             (xfer->error == USB_ERR_CANCELLED ||
2822             xfer->error == USB_ERR_TIMEOUT ||
2823             bus->methods->start_dma_delay != NULL)) {
2824
2825                 usb_timeout_t temp;
2826
2827                 /* only delay once */
2828                 xfer->flags_int.did_dma_delay = 1;
2829
2830                 /* we can not cancel this delay */
2831                 xfer->flags_int.can_cancel_immed = 0;
2832
2833                 temp = usbd_get_dma_delay(xfer->xroot->udev);
2834
2835                 DPRINTFN(3, "DMA delay, %u ms, "
2836                     "on %p\n", temp, xfer);
2837
2838                 if (temp != 0) {
2839                         USB_BUS_LOCK(bus);
2840                         /*
2841                          * Some hardware solutions have dedicated
2842                          * events when it is safe to free DMA'ed
2843                          * memory. For the other hardware platforms we
2844                          * use a static delay.
2845                          */
2846                         if (bus->methods->start_dma_delay != NULL) {
2847                                 (bus->methods->start_dma_delay) (xfer);
2848                         } else {
2849                                 usbd_transfer_timeout_ms(xfer,
2850                                         (void (*)(void *))&usb_dma_delay_done_cb,
2851                                         temp);
2852                         }
2853                         USB_BUS_UNLOCK(bus);
2854                         return (1);     /* wait for new callback */
2855                 }
2856         }
2857         /* check actual number of frames */
2858         if (xfer->aframes > xfer->nframes) {
2859                 if (xfer->error == 0) {
2860                         panic("%s: actual number of frames, %d, is "
2861                             "greater than initial number of frames, %d\n",
2862                             __func__, xfer->aframes, xfer->nframes);
2863                 } else {
2864                         /* just set some valid value */
2865                         xfer->aframes = xfer->nframes;
2866                 }
2867         }
2868         /* compute actual length */
2869         xfer->actlen = 0;
2870
2871         for (x = 0; x != xfer->aframes; x++) {
2872                 xfer->actlen += xfer->frlengths[x];
2873         }
2874
2875         /*
2876          * Frames that were not transferred get zero actual length in
2877          * case the USB device driver does not check the actual number
2878          * of frames transferred, "xfer->aframes":
2879          */
2880         for (; x < xfer->nframes; x++) {
2881                 usbd_xfer_set_frame_len(xfer, x, 0);
2882         }
2883
2884         /* check actual length */
2885         if (xfer->actlen > xfer->sumlen) {
2886                 if (xfer->error == 0) {
2887                         panic("%s: actual length, %d, is greater than "
2888                             "initial length, %d\n",
2889                             __func__, xfer->actlen, xfer->sumlen);
2890                 } else {
2891                         /* just set some valid value */
2892                         xfer->actlen = xfer->sumlen;
2893                 }
2894         }
2895         DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2896             xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2897             xfer->aframes, xfer->nframes);
2898
2899         if (xfer->error) {
2900                 /* end of control transfer, if any */
2901                 xfer->flags_int.control_act = 0;
2902
2903                 /* check if we should block the execution queue */
2904                 if ((xfer->error != USB_ERR_CANCELLED) &&
2905                     (xfer->flags.pipe_bof)) {
2906                         DPRINTFN(2, "xfer=%p: Block On Failure "
2907                             "on endpoint=%p\n", xfer, xfer->endpoint);
2908                         goto done;
2909                 }
2910         } else {
2911                 /* check for short transfers */
2912                 if (xfer->actlen < xfer->sumlen) {
2913
2914                         /* end of control transfer, if any */
2915                         xfer->flags_int.control_act = 0;
2916
2917                         if (!xfer->flags_int.short_xfer_ok) {
2918                                 xfer->error = USB_ERR_SHORT_XFER;
2919                                 if (xfer->flags.pipe_bof) {
2920                                         DPRINTFN(2, "xfer=%p: Block On Failure on "
2921                                             "Short Transfer on endpoint %p.\n",
2922                                             xfer, xfer->endpoint);
2923                                         goto done;
2924                                 }
2925                         }
2926                 } else {
2927                         /*
2928                          * Check if we are in the middle of a
2929                          * control transfer:
2930                          */
2931                         if (xfer->flags_int.control_act) {
2932                                 DPRINTFN(5, "xfer=%p: Control transfer "
2933                                     "active on endpoint=%p\n", xfer, xfer->endpoint);
2934                                 goto done;
2935                         }
2936                 }
2937         }
2938
2939         ep = xfer->endpoint;
2940
2941         /*
2942          * If the current USB transfer is completing we need to start the
2943          * next one:
2944          */
2945         USB_BUS_LOCK(bus);
2946         if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
2947                 usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], NULL);
2948
2949                 if (ep->endpoint_q[xfer->stream_id].curr != NULL ||
2950                     TAILQ_FIRST(&ep->endpoint_q[xfer->stream_id].head) != NULL) {
2951                         /* there is another USB transfer waiting */
2952                 } else {
2953                         /* this is the last USB transfer */
2954                         /* clear isochronous sync flag */
2955                         xfer->endpoint->is_synced = 0;
2956                 }
2957         }
2958         USB_BUS_UNLOCK(bus);
2959 done:
2960         return (0);
2961 }
2962
2963 /*------------------------------------------------------------------------*
2964  *      usb_command_wrapper
2965  *
2966  * This function is used to execute commands non-recursivly on an USB
2967  * transfer.
2968  *------------------------------------------------------------------------*/
2969 void
2970 usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2971 {
2972         if (xfer) {
2973                 /*
2974                  * If the transfer is not already processing,
2975                  * queue it!
2976                  */
2977                 if (pq->curr != xfer) {
2978                         usbd_transfer_enqueue(pq, xfer);
2979                         if (pq->curr != NULL) {
2980                                 /* something is already processing */
2981                                 DPRINTFN(6, "busy %p\n", pq->curr);
2982                                 return;
2983                         }
2984                 }
2985         } else {
2986                 /* Get next element in queue */
2987                 pq->curr = NULL;
2988         }
2989
2990         if (!pq->recurse_1) {
2991
2992                 do {
2993
2994                         /* set both recurse flags */
2995                         pq->recurse_1 = 1;
2996                         pq->recurse_2 = 1;
2997
2998                         if (pq->curr == NULL) {
2999                                 xfer = TAILQ_FIRST(&pq->head);
3000                                 if (xfer) {
3001                                         TAILQ_REMOVE(&pq->head, xfer,
3002                                             wait_entry);
3003                                         xfer->wait_queue = NULL;
3004                                         pq->curr = xfer;
3005                                 } else {
3006                                         break;
3007                                 }
3008                         }
3009                         DPRINTFN(6, "cb %p (enter)\n", pq->curr);
3010                         (pq->command) (pq);
3011                         DPRINTFN(6, "cb %p (leave)\n", pq->curr);
3012
3013                 } while (!pq->recurse_2);
3014
3015                 /* clear first recurse flag */
3016                 pq->recurse_1 = 0;
3017
3018         } else {
3019                 /* clear second recurse flag */
3020                 pq->recurse_2 = 0;
3021         }
3022 }
3023
3024 /*------------------------------------------------------------------------*
3025  *      usbd_ctrl_transfer_setup
3026  *
3027  * This function is used to setup the default USB control endpoint
3028  * transfer.
3029  *------------------------------------------------------------------------*/
3030 void
3031 usbd_ctrl_transfer_setup(struct usb_device *udev)
3032 {
3033         struct usb_xfer *xfer;
3034         uint8_t no_resetup;
3035         uint8_t iface_index;
3036
3037         /* check for root HUB */
3038         if (udev->parent_hub == NULL)
3039                 return;
3040 repeat:
3041
3042         xfer = udev->ctrl_xfer[0];
3043         if (xfer) {
3044                 USB_XFER_LOCK(xfer);
3045                 no_resetup =
3046                     ((xfer->address == udev->address) &&
3047                     (udev->ctrl_ep_desc.wMaxPacketSize[0] ==
3048                     udev->ddesc.bMaxPacketSize));
3049                 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
3050                         if (no_resetup) {
3051                                 /*
3052                                  * NOTE: checking "xfer->address" and
3053                                  * starting the USB transfer must be
3054                                  * atomic!
3055                                  */
3056                                 usbd_transfer_start(xfer);
3057                         }
3058                 }
3059                 USB_XFER_UNLOCK(xfer);
3060         } else {
3061                 no_resetup = 0;
3062         }
3063
3064         if (no_resetup) {
3065                 /*
3066                  * All parameters are exactly the same like before.
3067                  * Just return.
3068                  */
3069                 return;
3070         }
3071         /*
3072          * Update wMaxPacketSize for the default control endpoint:
3073          */
3074         udev->ctrl_ep_desc.wMaxPacketSize[0] =
3075             udev->ddesc.bMaxPacketSize;
3076
3077         /*
3078          * Unsetup any existing USB transfer:
3079          */
3080         usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
3081
3082         /*
3083          * Reset clear stall error counter.
3084          */
3085         udev->clear_stall_errors = 0;
3086
3087         /*
3088          * Try to setup a new USB transfer for the
3089          * default control endpoint:
3090          */
3091         iface_index = 0;
3092         if (usbd_transfer_setup(udev, &iface_index,
3093             udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
3094             &udev->device_lock)) {
3095                 DPRINTFN(0, "could not setup default "
3096                     "USB transfer\n");
3097         } else {
3098                 goto repeat;
3099         }
3100 }
3101
3102 /*------------------------------------------------------------------------*
3103  *      usbd_clear_data_toggle - factored out code
3104  *
3105  * NOTE: the intention of this function is not to reset the hardware
3106  * data toggle.
3107  *------------------------------------------------------------------------*/
3108 void
3109 usbd_clear_stall_locked(struct usb_device *udev, struct usb_endpoint *ep)
3110 {
3111         USB_BUS_LOCK_ASSERT(udev->bus);
3112
3113         /* check that we have a valid case */
3114         if (udev->flags.usb_mode == USB_MODE_HOST &&
3115             udev->parent_hub != NULL &&
3116             udev->bus->methods->clear_stall != NULL &&
3117             ep->methods != NULL) {
3118                 (udev->bus->methods->clear_stall) (udev, ep);
3119         }
3120 }
3121
3122 /*------------------------------------------------------------------------*
3123  *      usbd_clear_data_toggle - factored out code
3124  *
3125  * NOTE: the intention of this function is not to reset the hardware
3126  * data toggle on the USB device side.
3127  *------------------------------------------------------------------------*/
3128 void
3129 usbd_clear_data_toggle(struct usb_device *udev, struct usb_endpoint *ep)
3130 {
3131         DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3132
3133         USB_BUS_LOCK(udev->bus);
3134         ep->toggle_next = 0;
3135         /* some hardware needs a callback to clear the data toggle */
3136         usbd_clear_stall_locked(udev, ep);
3137         USB_BUS_UNLOCK(udev->bus);
3138 }
3139
3140 /*------------------------------------------------------------------------*
3141  *      usbd_clear_stall_callback - factored out clear stall callback
3142  *
3143  * Input parameters:
3144  *  xfer1: Clear Stall Control Transfer
3145  *  xfer2: Stalled USB Transfer
3146  *
3147  * This function is NULL safe.
3148  *
3149  * Return values:
3150  *   0: In progress
3151  *   Else: Finished
3152  *
3153  * Clear stall config example:
3154  *
3155  * static const struct usb_config my_clearstall =  {
3156  *      .type = UE_CONTROL,
3157  *      .endpoint = 0,
3158  *      .direction = UE_DIR_ANY,
3159  *      .interval = 50, //50 milliseconds
3160  *      .bufsize = sizeof(struct usb_device_request),
3161  *      .timeout = 1000, //1.000 seconds
3162  *      .callback = &my_clear_stall_callback, // **
3163  *      .usb_mode = USB_MODE_HOST,
3164  * };
3165  *
3166  * ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3167  * passing the correct parameters.
3168  *------------------------------------------------------------------------*/
3169 uint8_t
3170 usbd_clear_stall_callback(struct usb_xfer *xfer1,
3171     struct usb_xfer *xfer2)
3172 {
3173         struct usb_device_request req;
3174
3175         if (xfer2 == NULL) {
3176                 /* looks like we are tearing down */
3177                 DPRINTF("NULL input parameter\n");
3178                 return (0);
3179         }
3180         USB_XFER_LOCK_ASSERT(xfer1);
3181         USB_XFER_LOCK_ASSERT(xfer2);
3182
3183         switch (USB_GET_STATE(xfer1)) {
3184         case USB_ST_SETUP:
3185
3186                 /*
3187                  * pre-clear the data toggle to DATA0 ("umass.c" and
3188                  * "ata-usb.c" depends on this)
3189                  */
3190
3191                 usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3192
3193                 /* setup a clear-stall packet */
3194
3195                 req.bmRequestType = UT_WRITE_ENDPOINT;
3196                 req.bRequest = UR_CLEAR_FEATURE;
3197                 USETW(req.wValue, UF_ENDPOINT_HALT);
3198                 req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3199                 req.wIndex[1] = 0;
3200                 USETW(req.wLength, 0);
3201
3202                 /*
3203                  * "usbd_transfer_setup_sub()" will ensure that
3204                  * we have sufficient room in the buffer for
3205                  * the request structure!
3206                  */
3207
3208                 /* copy in the transfer */
3209
3210                 usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3211
3212                 /* set length */
3213                 xfer1->frlengths[0] = sizeof(req);
3214                 xfer1->nframes = 1;
3215
3216                 usbd_transfer_submit(xfer1);
3217                 return (0);
3218
3219         case USB_ST_TRANSFERRED:
3220                 break;
3221
3222         default:                        /* Error */
3223                 if (xfer1->error == USB_ERR_CANCELLED) {
3224                         return (0);
3225                 }
3226                 break;
3227         }
3228         return (1);                     /* Clear Stall Finished */
3229 }
3230
3231 /*------------------------------------------------------------------------*
3232  *      usbd_transfer_poll
3233  *
3234  * The following function gets called from the USB keyboard driver and
3235  * UMASS when the system has paniced.
3236  *
3237  * NOTE: It is currently not possible to resume normal operation on
3238  * the USB controller which has been polled, due to clearing of the
3239  * "up_dsleep" and "up_msleep" flags.
3240  *------------------------------------------------------------------------*/
3241 void
3242 usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3243 {
3244         struct usb_xfer *xfer;
3245         struct usb_xfer_root *xroot;
3246         struct usb_device *udev;
3247         struct usb_proc_msg *pm;
3248         uint16_t n;
3249         uint16_t drop_bus;
3250         uint16_t drop_xfer;
3251
3252         for (n = 0; n != max; n++) {
3253                 /* Extra checks to avoid panic */
3254                 xfer = ppxfer[n];
3255                 if (xfer == NULL)
3256                         continue;       /* no USB transfer */
3257                 xroot = xfer->xroot;
3258                 if (xroot == NULL)
3259                         continue;       /* no USB root */
3260                 udev = xroot->udev;
3261                 if (udev == NULL)
3262                         continue;       /* no USB device */
3263                 if (udev->bus == NULL)
3264                         continue;       /* no BUS structure */
3265                 if (udev->bus->methods == NULL)
3266                         continue;       /* no BUS methods */
3267                 if (udev->bus->methods->xfer_poll == NULL)
3268                         continue;       /* no poll method */
3269
3270                 /* make sure that the BUS mutex is not locked */
3271                 drop_bus = 0;
3272                 while (lockowned(&xroot->udev->bus->bus_lock)) {
3273                         lockmgr(&xroot->udev->bus->bus_lock, LK_RELEASE);
3274                         drop_bus++;
3275                 }
3276
3277                 /* make sure that the transfer mutex is not locked */
3278                 drop_xfer = 0;
3279                 while (lockowned(xroot->xfer_lock)) {
3280                         lockmgr(xroot->xfer_lock, LK_RELEASE);
3281                         drop_xfer++;
3282                 }
3283
3284                 /* Make sure cv_signal() and cv_broadcast() is not called */
3285                 USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0;
3286                 USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0;
3287                 USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0;
3288                 USB_BUS_NON_GIANT_PROC(udev->bus)->up_msleep = 0;
3289
3290                 /* poll USB hardware */
3291                 (udev->bus->methods->xfer_poll) (udev->bus);
3292
3293                 USB_BUS_LOCK(xroot->bus);
3294
3295                 /* check for clear stall */
3296                 if (udev->ctrl_xfer[1] != NULL) {
3297
3298                         /* poll clear stall start */
3299                         pm = &udev->cs_msg[0].hdr;
3300                         (pm->pm_callback) (pm);
3301                         /* poll clear stall done thread */
3302                         pm = &udev->ctrl_xfer[1]->
3303                             xroot->done_m[0].hdr;
3304                         (pm->pm_callback) (pm);
3305                 }
3306
3307                 /* poll done thread */
3308                 pm = &xroot->done_m[0].hdr;
3309                 (pm->pm_callback) (pm);
3310
3311                 USB_BUS_UNLOCK(xroot->bus);
3312
3313                 /* restore transfer mutex */
3314                 while (drop_xfer--)
3315                         lockmgr(xroot->xfer_lock, LK_EXCLUSIVE);
3316
3317                 /* restore BUS mutex */
3318                 while (drop_bus--)
3319                         lockmgr(&xroot->udev->bus->bus_lock, LK_EXCLUSIVE);
3320         }
3321 }
3322
3323 static void
3324 usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3325     uint8_t type, enum usb_dev_speed speed)
3326 {
3327         static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3328                 [USB_SPEED_LOW] = 8,
3329                 [USB_SPEED_FULL] = 64,
3330                 [USB_SPEED_HIGH] = 1024,
3331                 [USB_SPEED_VARIABLE] = 1024,
3332                 [USB_SPEED_SUPER] = 1024,
3333         };
3334
3335         static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3336                 [USB_SPEED_LOW] = 0,    /* invalid */
3337                 [USB_SPEED_FULL] = 1023,
3338                 [USB_SPEED_HIGH] = 1024,
3339                 [USB_SPEED_VARIABLE] = 3584,
3340                 [USB_SPEED_SUPER] = 1024,
3341         };
3342
3343         static const uint16_t control_min[USB_SPEED_MAX] = {
3344                 [USB_SPEED_LOW] = 8,
3345                 [USB_SPEED_FULL] = 8,
3346                 [USB_SPEED_HIGH] = 64,
3347                 [USB_SPEED_VARIABLE] = 512,
3348                 [USB_SPEED_SUPER] = 512,
3349         };
3350
3351         static const uint16_t bulk_min[USB_SPEED_MAX] = {
3352                 [USB_SPEED_LOW] = 8,
3353                 [USB_SPEED_FULL] = 8,
3354                 [USB_SPEED_HIGH] = 512,
3355                 [USB_SPEED_VARIABLE] = 512,
3356                 [USB_SPEED_SUPER] = 1024,
3357         };
3358
3359         uint16_t temp;
3360
3361         memset(ptr, 0, sizeof(*ptr));
3362
3363         switch (type) {
3364         case UE_INTERRUPT:
3365                 ptr->range.max = intr_range_max[speed];
3366                 break;
3367         case UE_ISOCHRONOUS:
3368                 ptr->range.max = isoc_range_max[speed];
3369                 break;
3370         default:
3371                 if (type == UE_BULK)
3372                         temp = bulk_min[speed];
3373                 else /* UE_CONTROL */
3374                         temp = control_min[speed];
3375
3376                 /* default is fixed */
3377                 ptr->fixed[0] = temp;
3378                 ptr->fixed[1] = temp;
3379                 ptr->fixed[2] = temp;
3380                 ptr->fixed[3] = temp;
3381
3382                 if (speed == USB_SPEED_FULL) {
3383                         /* multiple sizes */
3384                         ptr->fixed[1] = 16;
3385                         ptr->fixed[2] = 32;
3386                         ptr->fixed[3] = 64;
3387                 }
3388                 if ((speed == USB_SPEED_VARIABLE) &&
3389                     (type == UE_BULK)) {
3390                         /* multiple sizes */
3391                         ptr->fixed[2] = 1024;
3392                         ptr->fixed[3] = 1536;
3393                 }
3394                 break;
3395         }
3396 }
3397
3398 void    *
3399 usbd_xfer_softc(struct usb_xfer *xfer)
3400 {
3401         return (xfer->priv_sc);
3402 }
3403
3404 void *
3405 usbd_xfer_get_priv(struct usb_xfer *xfer)
3406 {
3407         return (xfer->priv_fifo);
3408 }
3409
3410 void
3411 usbd_xfer_set_priv(struct usb_xfer *xfer, void *ptr)
3412 {
3413         xfer->priv_fifo = ptr;
3414 }
3415
3416 uint8_t
3417 usbd_xfer_state(struct usb_xfer *xfer)
3418 {
3419         return (xfer->usb_state);
3420 }
3421
3422 void
3423 usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3424 {
3425         switch (flag) {
3426                 case USB_FORCE_SHORT_XFER:
3427                         xfer->flags.force_short_xfer = 1;
3428                         break;
3429                 case USB_SHORT_XFER_OK:
3430                         xfer->flags.short_xfer_ok = 1;
3431                         break;
3432                 case USB_MULTI_SHORT_OK:
3433                         xfer->flags.short_frames_ok = 1;
3434                         break;
3435                 case USB_MANUAL_STATUS:
3436                         xfer->flags.manual_status = 1;
3437                         break;
3438         }
3439 }
3440
3441 void
3442 usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3443 {
3444         switch (flag) {
3445                 case USB_FORCE_SHORT_XFER:
3446                         xfer->flags.force_short_xfer = 0;
3447                         break;
3448                 case USB_SHORT_XFER_OK:
3449                         xfer->flags.short_xfer_ok = 0;
3450                         break;
3451                 case USB_MULTI_SHORT_OK:
3452                         xfer->flags.short_frames_ok = 0;
3453                         break;
3454                 case USB_MANUAL_STATUS:
3455                         xfer->flags.manual_status = 0;
3456                         break;
3457         }
3458 }
3459
3460 /*
3461  * The following function returns in milliseconds when the isochronous
3462  * transfer was completed by the hardware. The returned value wraps
3463  * around 65536 milliseconds.
3464  */
3465 uint16_t
3466 usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3467 {
3468         return (xfer->isoc_time_complete);
3469 }