1 /* $FreeBSD: head/sys/dev/usb/usb_transfer.c 276717 2015-01-05 20:22:18Z hselasky $ */
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/stdint.h>
28 #include <sys/param.h>
29 #include <sys/queue.h>
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
34 #include <sys/thread.h>
35 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/condvar.h>
39 #include <sys/sysctl.h>
40 #include <sys/unistd.h>
41 #include <sys/callout.h>
42 #include <sys/malloc.h>
46 #include <sys/thread2.h>
48 #include <bus/u4b/usb.h>
49 #include <bus/u4b/usbdi.h>
50 #include <bus/u4b/usbdi_util.h>
52 #define USB_DEBUG_VAR usb_debug
54 #include <bus/u4b/usb_core.h>
55 #include <bus/u4b/usb_busdma.h>
56 #include <bus/u4b/usb_process.h>
57 #include <bus/u4b/usb_transfer.h>
58 #include <bus/u4b/usb_device.h>
59 #include <bus/u4b/usb_debug.h>
60 #include <bus/u4b/usb_util.h>
62 #include <bus/u4b/usb_controller.h>
63 #include <bus/u4b/usb_bus.h>
64 #include <bus/u4b/usb_pf.h>
66 struct usb_std_packet_size {
68 uint16_t min; /* inclusive */
69 uint16_t max; /* inclusive */
75 static usb_callback_t usb_request_callback;
77 static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
79 /* This transfer is used for generic control endpoint transfers */
83 .endpoint = 0x00, /* Control endpoint */
84 .direction = UE_DIR_ANY,
85 .bufsize = USB_EP0_BUFSIZE, /* bytes */
86 .flags = {.proxy_buffer = 1,},
87 .callback = &usb_request_callback,
88 .usb_mode = USB_MODE_DUAL, /* both modes */
91 /* This transfer is used for generic clear stall only */
95 .endpoint = 0x00, /* Control pipe */
96 .direction = UE_DIR_ANY,
97 .bufsize = sizeof(struct usb_device_request),
98 .callback = &usb_do_clear_stall_callback,
99 .timeout = 1000, /* 1 second */
100 .interval = 50, /* 50ms */
101 .usb_mode = USB_MODE_HOST,
105 /* function prototypes */
107 static void usbd_update_max_frame_size(struct usb_xfer *);
108 static void usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
109 static void usbd_delayed_free(void *data, struct malloc_type *mtype);
110 static void usbd_control_transfer_init(struct usb_xfer *);
111 static int usbd_setup_ctrl_transfer(struct usb_xfer *);
112 static void usb_callback_proc(struct usb_proc_msg *);
113 static void usbd_callback_ss_done_defer(struct usb_xfer *);
114 static void usbd_callback_wrapper(struct usb_xfer_queue *);
115 static void usbd_transfer_start_cb(void *);
116 static uint8_t usbd_callback_wrapper_sub(struct usb_xfer *);
117 static void usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
118 uint8_t type, enum usb_dev_speed speed);
120 /*------------------------------------------------------------------------*
121 * usb_request_callback
122 *------------------------------------------------------------------------*/
124 usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
126 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
127 usb_handle_request_callback(xfer, error);
129 usbd_do_request_callback(xfer, error);
132 /*------------------------------------------------------------------------*
133 * usbd_update_max_frame_size
135 * This function updates the maximum frame size, hence high speed USB
136 * can transfer multiple consecutive packets.
137 *------------------------------------------------------------------------*/
139 usbd_update_max_frame_size(struct usb_xfer *xfer)
141 /* compute maximum frame size */
142 /* this computation should not overflow 16-bit */
143 /* max = 15 * 1024 */
145 xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
148 /*------------------------------------------------------------------------*
151 * The following function is called when we need to
152 * synchronize with DMA hardware.
155 * 0: no DMA delay required
156 * Else: milliseconds of DMA delay
157 *------------------------------------------------------------------------*/
159 usbd_get_dma_delay(struct usb_device *udev)
161 const struct usb_bus_methods *mtod;
164 mtod = udev->bus->methods;
167 if (mtod->get_dma_delay) {
168 (mtod->get_dma_delay) (udev, &temp);
170 * Round up and convert to milliseconds. Note that we use
171 * 1024 milliseconds per second. to save a division.
179 /*------------------------------------------------------------------------*
180 * usbd_transfer_setup_sub_malloc
182 * This function will allocate one or more DMA'able memory chunks
183 * according to "size", "align" and "count" arguments. "ppc" is
184 * pointed to a linear array of USB page caches afterwards.
186 * If the "align" argument is equal to "1" a non-contiguous allocation
187 * can happen. Else if the "align" argument is greater than "1", the
188 * allocation will always be contiguous in memory.
193 *------------------------------------------------------------------------*/
196 usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
197 struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
200 struct usb_page_cache *pc;
211 USB_ASSERT(align > 0, ("Invalid alignment, 0x%08x\n",
213 USB_ASSERT(size > 0, ("Invalid size = 0\n"));
216 return (0); /* nothing to allocate */
219 * Make sure that the size is aligned properly.
221 size = -((-size) & (-align));
224 * Try multi-allocation chunks to reduce the number of DMA
225 * allocations, hence DMA allocations are slow.
228 /* special case - non-cached multi page DMA memory */
230 n_dma_pg = (2 + (size / USB_PAGE_SIZE));
232 } else if (size >= USB_PAGE_SIZE) {
237 /* compute number of objects per page */
238 #ifdef USB_DMA_SINGLE_ALLOC
241 n_obj = (USB_PAGE_SIZE / size);
244 * Compute number of DMA chunks, rounded up
247 n_dma_pc = ((count + n_obj - 1) / n_obj);
252 * DMA memory is allocated once, but mapped twice. That's why
253 * there is one list for auto-free and another list for
254 * non-auto-free which only holds the mapping and not the
257 if (parm->buf == NULL) {
258 /* reserve memory (auto-free) */
259 parm->dma_page_ptr += n_dma_pc * n_dma_pg;
260 parm->dma_page_cache_ptr += n_dma_pc;
262 /* reserve memory (no-auto-free) */
263 parm->dma_page_ptr += count * n_dma_pg;
264 parm->xfer_page_cache_ptr += count;
267 for (x = 0; x != n_dma_pc; x++) {
268 /* need to initialize the page cache */
269 parm->dma_page_cache_ptr[x].tag_parent =
270 &parm->curr_xfer->xroot->dma_parent_tag;
272 for (x = 0; x != count; x++) {
273 /* need to initialize the page cache */
274 parm->xfer_page_cache_ptr[x].tag_parent =
275 &parm->curr_xfer->xroot->dma_parent_tag;
280 *ppc = parm->xfer_page_cache_ptr;
282 *ppc = parm->dma_page_cache_ptr;
284 r = count; /* set remainder count */
285 z = n_obj * size; /* set allocation size */
286 pc = parm->xfer_page_cache_ptr;
287 pg = parm->dma_page_ptr;
291 * Avoid mapping memory twice if only a single object
292 * should be allocated per page cache:
294 for (x = 0; x != n_dma_pc; x++) {
295 if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
297 return (1); /* failure */
299 /* Make room for one DMA page cache and "n_dma_pg" pages */
300 parm->dma_page_cache_ptr++;
304 for (x = 0; x != n_dma_pc; x++) {
307 /* compute last remainder */
311 if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
313 return (1); /* failure */
315 /* Set beginning of current buffer */
316 buf = parm->dma_page_cache_ptr->buffer;
317 /* Make room for one DMA page cache and "n_dma_pg" pages */
318 parm->dma_page_cache_ptr++;
321 for (y = 0; (y != n_obj); y++, r--, pc++, pg += n_dma_pg) {
323 /* Load sub-chunk into DMA */
324 if (usb_pc_dmamap_create(pc, size)) {
325 return (1); /* failure */
327 pc->buffer = USB_ADD_BYTES(buf, y * size);
330 lockmgr(pc->tag_parent->lock, LK_EXCLUSIVE);
331 if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
332 lockmgr(pc->tag_parent->lock, LK_RELEASE);
333 return (1); /* failure */
335 lockmgr(pc->tag_parent->lock, LK_RELEASE);
340 parm->xfer_page_cache_ptr = pc;
341 parm->dma_page_ptr = pg;
346 /*------------------------------------------------------------------------*
347 * usbd_transfer_setup_sub - transfer setup subroutine
349 * This function must be called from the "xfer_setup" callback of the
350 * USB Host or Device controller driver when setting up an USB
351 * transfer. This function will setup correct packet sizes, buffer
352 * sizes, flags and more, that are stored in the "usb_xfer"
354 *------------------------------------------------------------------------*/
356 usbd_transfer_setup_sub(struct usb_setup_params *parm)
362 struct usb_xfer *xfer = parm->curr_xfer;
363 const struct usb_config *setup = parm->curr_setup;
364 struct usb_endpoint_ss_comp_descriptor *ecomp;
365 struct usb_endpoint_descriptor *edesc;
366 struct usb_std_packet_size std_size;
367 usb_frcount_t n_frlengths;
368 usb_frcount_t n_frbuffers;
375 * Sanity check. The following parameters must be initialized before
376 * calling this function.
378 if ((parm->hc_max_packet_size == 0) ||
379 (parm->hc_max_packet_count == 0) ||
380 (parm->hc_max_frame_size == 0)) {
381 parm->err = USB_ERR_INVAL;
384 edesc = xfer->endpoint->edesc;
385 ecomp = xfer->endpoint->ecomp;
387 type = (edesc->bmAttributes & UE_XFERTYPE);
389 xfer->flags = setup->flags;
390 xfer->nframes = setup->frames;
391 xfer->timeout = setup->timeout;
392 xfer->callback = setup->callback;
393 xfer->interval = setup->interval;
394 xfer->endpointno = edesc->bEndpointAddress;
395 xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
396 xfer->max_packet_count = 1;
397 /* make a shadow copy: */
398 xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
400 parm->bufsize = setup->bufsize;
402 switch (parm->speed) {
407 xfer->max_packet_count +=
408 (xfer->max_packet_size >> 11) & 3;
410 /* check for invalid max packet count */
411 if (xfer->max_packet_count > 3)
412 xfer->max_packet_count = 3;
417 xfer->max_packet_size &= 0x7FF;
419 case USB_SPEED_SUPER:
420 xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
423 xfer->max_packet_count += ecomp->bMaxBurst;
425 if ((xfer->max_packet_count == 0) ||
426 (xfer->max_packet_count > 16))
427 xfer->max_packet_count = 16;
431 xfer->max_packet_count = 1;
437 mult = UE_GET_SS_ISO_MULT(
438 ecomp->bmAttributes) + 1;
442 xfer->max_packet_count *= mult;
448 xfer->max_packet_size &= 0x7FF;
453 /* range check "max_packet_count" */
455 if (xfer->max_packet_count > parm->hc_max_packet_count) {
456 xfer->max_packet_count = parm->hc_max_packet_count;
459 /* store max packet size value before filtering */
461 maxp_old = xfer->max_packet_size;
463 /* filter "wMaxPacketSize" according to HC capabilities */
465 if ((xfer->max_packet_size > parm->hc_max_packet_size) ||
466 (xfer->max_packet_size == 0)) {
467 xfer->max_packet_size = parm->hc_max_packet_size;
469 /* filter "wMaxPacketSize" according to standard sizes */
471 usbd_get_std_packet_size(&std_size, type, parm->speed);
473 if (std_size.range.min || std_size.range.max) {
475 if (xfer->max_packet_size < std_size.range.min) {
476 xfer->max_packet_size = std_size.range.min;
478 if (xfer->max_packet_size > std_size.range.max) {
479 xfer->max_packet_size = std_size.range.max;
483 if (xfer->max_packet_size >= std_size.fixed[3]) {
484 xfer->max_packet_size = std_size.fixed[3];
485 } else if (xfer->max_packet_size >= std_size.fixed[2]) {
486 xfer->max_packet_size = std_size.fixed[2];
487 } else if (xfer->max_packet_size >= std_size.fixed[1]) {
488 xfer->max_packet_size = std_size.fixed[1];
490 /* only one possibility left */
491 xfer->max_packet_size = std_size.fixed[0];
496 * Check if the max packet size was outside its allowed range
497 * and clamped to a valid value:
499 if (maxp_old != xfer->max_packet_size)
500 xfer->flags_int.maxp_was_clamped = 1;
502 /* compute "max_frame_size" */
504 usbd_update_max_frame_size(xfer);
506 /* check interrupt interval and transfer pre-delay */
508 if (type == UE_ISOCHRONOUS) {
510 uint16_t frame_limit;
512 xfer->interval = 0; /* not used, must be zero */
513 xfer->flags_int.isochronous_xfr = 1; /* set flag */
515 if (xfer->timeout == 0) {
517 * set a default timeout in
518 * case something goes wrong!
520 xfer->timeout = 1000 / 4;
522 switch (parm->speed) {
525 frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
529 frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
530 xfer->fps_shift = edesc->bInterval;
531 if (xfer->fps_shift > 0)
533 if (xfer->fps_shift > 3)
535 if (xfer->flags.pre_scale_frames != 0)
536 xfer->nframes <<= (3 - xfer->fps_shift);
540 if (xfer->nframes > frame_limit) {
542 * this is not going to work
545 parm->err = USB_ERR_INVAL;
548 if (xfer->nframes == 0) {
550 * this is not a valid value
552 parm->err = USB_ERR_ZERO_NFRAMES;
558 * If a value is specified use that else check the
559 * endpoint descriptor!
561 if (type == UE_INTERRUPT) {
565 if (xfer->interval == 0) {
567 xfer->interval = edesc->bInterval;
569 switch (parm->speed) {
575 if (xfer->interval < 4)
577 else if (xfer->interval > 16)
578 xfer->interval = (1 << (16 - 4));
581 (1 << (xfer->interval - 4));
586 if (xfer->interval == 0) {
588 * One millisecond is the smallest
589 * interval we support:
597 while ((temp != 0) && (temp < xfer->interval)) {
602 switch (parm->speed) {
607 xfer->fps_shift += 3;
614 * NOTE: we do not allow "max_packet_size" or "max_frame_size"
615 * to be equal to zero when setting up USB transfers, hence
616 * this leads to alot of extra code in the USB kernel.
619 if ((xfer->max_frame_size == 0) ||
620 (xfer->max_packet_size == 0)) {
624 if ((parm->bufsize <= MIN_PKT) &&
625 (type != UE_CONTROL) &&
629 xfer->max_packet_size = MIN_PKT;
630 xfer->max_packet_count = 1;
631 parm->bufsize = 0; /* automatic setup length */
632 usbd_update_max_frame_size(xfer);
635 parm->err = USB_ERR_ZERO_MAXP;
644 * check if we should setup a default
648 if (parm->bufsize == 0) {
650 parm->bufsize = xfer->max_frame_size;
652 if (type == UE_ISOCHRONOUS) {
653 parm->bufsize *= xfer->nframes;
657 * check if we are about to setup a proxy
661 if (xfer->flags.proxy_buffer) {
663 /* round bufsize up */
665 parm->bufsize += (xfer->max_frame_size - 1);
667 if (parm->bufsize < xfer->max_frame_size) {
668 /* length wrapped around */
669 parm->err = USB_ERR_INVAL;
672 /* subtract remainder */
674 parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
676 /* add length of USB device request structure, if any */
678 if (type == UE_CONTROL) {
679 parm->bufsize += REQ_SIZE; /* SETUP message */
682 xfer->max_data_length = parm->bufsize;
684 /* Setup "n_frlengths" and "n_frbuffers" */
686 if (type == UE_ISOCHRONOUS) {
687 n_frlengths = xfer->nframes;
691 if (type == UE_CONTROL) {
692 xfer->flags_int.control_xfr = 1;
693 if (xfer->nframes == 0) {
694 if (parm->bufsize <= REQ_SIZE) {
696 * there will never be any data
705 if (xfer->nframes == 0) {
710 n_frlengths = xfer->nframes;
711 n_frbuffers = xfer->nframes;
715 * check if we have room for the
716 * USB device request structure:
719 if (type == UE_CONTROL) {
721 if (xfer->max_data_length < REQ_SIZE) {
722 /* length wrapped around or too small bufsize */
723 parm->err = USB_ERR_INVAL;
726 xfer->max_data_length -= REQ_SIZE;
729 * Setup "frlengths" and shadow "frlengths" for keeping the
730 * initial frame lengths when a USB transfer is complete. This
731 * information is useful when computing isochronous offsets.
733 xfer->frlengths = parm->xfer_length_ptr;
734 parm->xfer_length_ptr += 2 * n_frlengths;
736 /* setup "frbuffers" */
737 xfer->frbuffers = parm->xfer_page_cache_ptr;
738 parm->xfer_page_cache_ptr += n_frbuffers;
740 /* initialize max frame count */
741 xfer->max_frame_count = xfer->nframes;
744 * check if we need to setup
748 if (!xfer->flags.ext_buffer) {
750 struct usb_page_search page_info;
751 struct usb_page_cache *pc;
753 if (usbd_transfer_setup_sub_malloc(parm,
754 &pc, parm->bufsize, 1, 1)) {
755 parm->err = USB_ERR_NOMEM;
756 } else if (parm->buf != NULL) {
758 usbd_get_page(pc, 0, &page_info);
760 xfer->local_buffer = page_info.buffer;
762 usbd_xfer_set_frame_offset(xfer, 0, 0);
764 if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
765 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
770 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
772 if (parm->buf != NULL) {
774 USB_ADD_BYTES(parm->buf, parm->size[0]);
776 usbd_xfer_set_frame_offset(xfer, 0, 0);
778 if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
779 usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
782 parm->size[0] += parm->bufsize;
784 /* align data again */
785 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
789 * Compute maximum buffer size
792 if (parm->bufsize_max < parm->bufsize) {
793 parm->bufsize_max = parm->bufsize;
796 if (xfer->flags_int.bdma_enable) {
798 * Setup "dma_page_ptr".
800 * Proof for formula below:
802 * Assume there are three USB frames having length "a", "b" and
803 * "c". These USB frames will at maximum need "z"
804 * "usb_page" structures. "z" is given by:
806 * z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
807 * ((c / USB_PAGE_SIZE) + 2);
809 * Constraining "a", "b" and "c" like this:
811 * (a + b + c) <= parm->bufsize
815 * z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
817 * Here is the general formula:
819 xfer->dma_page_ptr = parm->dma_page_ptr;
820 parm->dma_page_ptr += (2 * n_frbuffers);
821 parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
825 /* correct maximum data length */
826 xfer->max_data_length = 0;
828 /* subtract USB frame remainder from "hc_max_frame_size" */
830 xfer->max_hc_frame_size =
831 (parm->hc_max_frame_size -
832 (parm->hc_max_frame_size % xfer->max_frame_size));
834 if (xfer->max_hc_frame_size == 0) {
835 parm->err = USB_ERR_INVAL;
839 /* initialize frame buffers */
842 for (x = 0; x != n_frbuffers; x++) {
843 xfer->frbuffers[x].tag_parent =
844 &xfer->xroot->dma_parent_tag;
846 if (xfer->flags_int.bdma_enable &&
847 (parm->bufsize_max > 0)) {
849 if (usb_pc_dmamap_create(
851 parm->bufsize_max)) {
852 parm->err = USB_ERR_NOMEM;
862 * Set some dummy values so that we avoid division by zero:
864 xfer->max_hc_frame_size = 1;
865 xfer->max_frame_size = 1;
866 xfer->max_packet_size = 1;
867 xfer->max_data_length = 0;
869 xfer->max_frame_count = 0;
873 /*------------------------------------------------------------------------*
874 * usbd_transfer_setup - setup an array of USB transfers
876 * NOTE: You must always call "usbd_transfer_unsetup" after calling
877 * "usbd_transfer_setup" if success was returned.
879 * The idea is that the USB device driver should pre-allocate all its
880 * transfers by one call to this function.
885 *------------------------------------------------------------------------*/
887 usbd_transfer_setup(struct usb_device *udev,
888 const uint8_t *ifaces, struct usb_xfer **ppxfer,
889 const struct usb_config *setup_start, uint16_t n_setup,
890 void *priv_sc, struct lock *xfer_lock)
892 const struct usb_config *setup_end = setup_start + n_setup;
893 const struct usb_config *setup;
894 struct usb_setup_params *parm;
895 struct usb_endpoint *ep;
896 struct usb_xfer_root *info;
897 struct usb_xfer *xfer;
899 usb_error_t error = 0;
905 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
906 "usbd_transfer_setup can sleep!");
909 /* do some checking first */
912 DPRINTFN(6, "setup array has zero length!\n");
913 return (USB_ERR_INVAL);
915 if (ifaces == NULL) {
916 DPRINTFN(6, "ifaces array is NULL!\n");
917 return (USB_ERR_INVAL);
919 if (xfer_lock == NULL) {
920 panic("xfer without lock!\n");
921 DPRINTFN(6, "using global lock\n");
924 /* more sanity checks */
926 for (setup = setup_start, n = 0;
927 setup != setup_end; setup++, n++) {
928 if (setup->bufsize == (usb_frlength_t)-1) {
929 error = USB_ERR_BAD_BUFSIZE;
930 DPRINTF("invalid bufsize\n");
932 if (setup->callback == NULL) {
933 error = USB_ERR_NO_CALLBACK;
934 DPRINTF("no callback\n");
942 /* Protect scratch area */
943 do_unlock = usbd_enum_lock(udev);
948 parm = &udev->scratch.xfer_setup[0].parm;
949 memset(parm, 0, sizeof(*parm));
952 parm->speed = usbd_get_speed(udev);
953 parm->hc_max_packet_count = 1;
955 if (parm->speed >= USB_SPEED_MAX) {
956 parm->err = USB_ERR_INVAL;
959 /* setup all transfers */
965 * Initialize the "usb_xfer_root" structure,
966 * which is common for all our USB transfers.
968 info = USB_ADD_BYTES(buf, 0);
970 info->memory_base = buf;
971 info->memory_size = parm->size[0];
974 info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
975 info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
977 info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
978 info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
980 cv_init(&info->cv_drain, "WDRAIN");
982 info->xfer_lock = xfer_lock;
984 usb_dma_tag_setup(&info->dma_parent_tag,
985 parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
986 xfer_lock, &usb_bdma_done_event,
987 udev->bus->dma_bits, parm->dma_tag_max);
990 info->bus = udev->bus;
993 TAILQ_INIT(&info->done_q.head);
994 info->done_q.command = &usbd_callback_wrapper;
996 TAILQ_INIT(&info->dma_q.head);
997 info->dma_q.command = &usb_bdma_work_loop;
999 info->done_m[0].hdr.pm_callback = &usb_callback_proc;
1000 info->done_m[0].xroot = info;
1001 info->done_m[1].hdr.pm_callback = &usb_callback_proc;
1002 info->done_m[1].xroot = info;
1005 * In device side mode control endpoint
1006 * requests need to run from a separate
1007 * context, else there is a chance of
1010 if (setup_start == usb_control_ep_cfg)
1012 USB_BUS_CONTROL_XFER_PROC(udev->bus);
1015 USB_BUS_NON_GIANT_PROC(udev->bus);
1021 parm->size[0] += sizeof(info[0]);
1023 for (setup = setup_start, n = 0;
1024 setup != setup_end; setup++, n++) {
1026 /* skip USB transfers without callbacks: */
1027 if (setup->callback == NULL) {
1030 /* see if there is a matching endpoint */
1031 ep = usbd_get_endpoint(udev,
1032 ifaces[setup->if_index], setup);
1035 * Check that the USB PIPE is valid and that
1036 * the endpoint mode is proper.
1038 * Make sure we don't allocate a streams
1039 * transfer when such a combination is not
1042 if ((ep == NULL) || (ep->methods == NULL) ||
1043 ((ep->ep_mode != USB_EP_MODE_STREAMS) &&
1044 (ep->ep_mode != USB_EP_MODE_DEFAULT)) ||
1045 (setup->stream_id != 0 &&
1046 (setup->stream_id >= USB_MAX_EP_STREAMS ||
1047 (ep->ep_mode != USB_EP_MODE_STREAMS)))) {
1048 if (setup->flags.no_pipe_ok)
1050 if ((setup->usb_mode != USB_MODE_DUAL) &&
1051 (setup->usb_mode != udev->flags.usb_mode))
1053 parm->err = USB_ERR_NO_PIPE;
1057 /* align data properly */
1058 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1060 /* store current setup pointer */
1061 parm->curr_setup = setup;
1065 * Common initialization of the
1066 * "usb_xfer" structure.
1068 xfer = USB_ADD_BYTES(buf, parm->size[0]);
1069 xfer->address = udev->address;
1070 xfer->priv_sc = priv_sc;
1073 usb_callout_init_mtx(&xfer->timeout_handle,
1074 &udev->bus->bus_lock, 0);
1077 * Setup a dummy xfer, hence we are
1078 * writing to the "usb_xfer"
1079 * structure pointed to by "xfer"
1080 * before we have allocated any
1083 xfer = &udev->scratch.xfer_setup[0].dummy;
1084 memset(xfer, 0, sizeof(*xfer));
1088 /* set transfer endpoint pointer */
1089 xfer->endpoint = ep;
1091 /* set transfer stream ID */
1092 xfer->stream_id = setup->stream_id;
1094 parm->size[0] += sizeof(xfer[0]);
1095 parm->methods = xfer->endpoint->methods;
1096 parm->curr_xfer = xfer;
1099 * Call the Host or Device controller transfer
1102 (udev->bus->methods->xfer_setup) (parm);
1104 /* check for error */
1110 * Increment the endpoint refcount. This
1111 * basically prevents setting a new
1112 * configuration and alternate setting
1113 * when USB transfers are in use on
1114 * the given interface. Search the USB
1115 * code for "endpoint->refcount_alloc" if you
1116 * want more information.
1118 USB_BUS_LOCK(info->bus);
1119 if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1120 parm->err = USB_ERR_INVAL;
1122 xfer->endpoint->refcount_alloc++;
1124 if (xfer->endpoint->refcount_alloc == 0)
1125 panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1126 USB_BUS_UNLOCK(info->bus);
1129 * Whenever we set ppxfer[] then we
1130 * also need to increment the
1133 info->setup_refcount++;
1136 * Transfer is successfully setup and
1142 /* check for error */
1147 if (buf != NULL || parm->err != 0)
1150 /* if no transfers, nothing to do */
1154 /* align data properly */
1155 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1157 /* store offset temporarily */
1158 parm->size[1] = parm->size[0];
1161 * The number of DMA tags required depends on
1162 * the number of endpoints. The current estimate
1163 * for maximum number of DMA tags per endpoint
1165 * 1) for loading memory
1166 * 2) for allocating memory
1167 * 3) for fixing memory [UHCI]
1169 parm->dma_tag_max += 3 * MIN(n_setup, USB_EP_MAX);
1172 * DMA tags for QH, TD, Data and more.
1174 parm->dma_tag_max += 8;
1176 parm->dma_tag_p += parm->dma_tag_max;
1178 parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1181 /* align data properly */
1182 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1184 /* store offset temporarily */
1185 parm->size[3] = parm->size[0];
1187 parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1190 /* align data properly */
1191 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1193 /* store offset temporarily */
1194 parm->size[4] = parm->size[0];
1196 parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1199 /* store end offset temporarily */
1200 parm->size[5] = parm->size[0];
1202 parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1205 /* store end offset temporarily */
1207 parm->size[2] = parm->size[0];
1209 /* align data properly */
1210 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1212 parm->size[6] = parm->size[0];
1214 parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1217 /* align data properly */
1218 parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1220 /* allocate zeroed memory */
1221 buf = kmalloc(parm->size[0], M_USB, M_WAITOK | M_ZERO);
1224 parm->err = USB_ERR_NOMEM;
1225 DPRINTFN(0, "cannot allocate memory block for "
1226 "configuration (%d bytes)\n",
1230 parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1231 parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1232 parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1233 parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1234 parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1239 if (info->setup_refcount == 0) {
1241 * "usbd_transfer_unsetup_sub" will unlock
1242 * the bus mutex before returning !
1244 USB_BUS_LOCK(info->bus);
1246 /* something went wrong */
1247 usbd_transfer_unsetup_sub(info, 0);
1251 /* check if any errors happened */
1253 usbd_transfer_unsetup(ppxfer, n_setup);
1258 usbd_enum_unlock(udev);
1263 /*------------------------------------------------------------------------*
1264 * usbd_transfer_unsetup_sub - factored out code
1265 *------------------------------------------------------------------------*/
1267 usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1270 struct usb_page_cache *pc;
1273 USB_BUS_LOCK_ASSERT(info->bus);
1275 /* wait for any outstanding DMA operations */
1276 /* This is insane */
1279 temp = usbd_get_dma_delay(info->udev);
1281 usb_pause_mtx(&info->bus->bus_lock,
1282 USB_MS_TO_TICKS(temp));
1286 /* make sure that our done messages are not queued anywhere */
1287 usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1289 USB_BUS_UNLOCK(info->bus);
1292 /* free DMA'able memory, if any */
1293 pc = info->dma_page_cache_start;
1294 while (pc != info->dma_page_cache_end) {
1295 usb_pc_free_mem(pc);
1299 /* free DMA maps in all "xfer->frbuffers" */
1300 pc = info->xfer_page_cache_start;
1301 while (pc != info->xfer_page_cache_end) {
1302 usb_pc_dmamap_destroy(pc);
1306 /* free all DMA tags */
1307 usb_dma_tag_unsetup(&info->dma_parent_tag);
1310 cv_destroy(&info->cv_drain);
1313 * free the "memory_base" last, hence the "info" structure is
1314 * contained within the "memory_base"!
1316 usbd_delayed_free(info->memory_base, M_USB);
1320 * This is a horrible hack and workaround to a very bad decision by
1321 * the original U4B coder to integrate the QH/TD structures into the
1322 * xfer and then free the whole mess all at once.
1324 * The problem is that the controller may still be accessing the QHs,
1325 * because it might have gotten side-tracked onto the removed QHs
1326 * chain link. They have to remain intact long enough for the
1327 * controller to get out.
1329 * This horrible hack basically just delays freeing by 256 slots.
1330 * It's not even time-based or door-bell based (which is the way
1331 * the linux driver does it)... but to fix it properly requires rewriting
1332 * too much of this driver.
1334 #define DFREE_SLOTS 256
1335 #define DFREE_MASK (DFREE_SLOTS - 1)
1337 static struct dfree_slot {
1339 struct malloc_type *mtype;
1340 } dfree_slots[DFREE_SLOTS];
1341 static int dfree_index;
1344 usbd_delayed_free(void *data, struct malloc_type *mtype)
1346 struct dfree_slot slot;
1350 index = atomic_fetchadd_int(&dfree_index, 1);
1351 index &= DFREE_MASK;
1352 slot = dfree_slots[index];
1353 dfree_slots[index].data = data;
1354 dfree_slots[index].mtype = mtype;
1357 kfree(slot.data, slot.mtype);
1360 /*------------------------------------------------------------------------*
1361 * usbd_transfer_unsetup - unsetup/free an array of USB transfers
1363 * NOTE: All USB transfers in progress will get called back passing
1364 * the error code "USB_ERR_CANCELLED" before this function
1366 *------------------------------------------------------------------------*/
1368 usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1370 struct usb_xfer *xfer;
1371 struct usb_xfer_root *info;
1372 uint8_t needs_delay = 0;
1375 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1376 "usbd_transfer_unsetup can sleep!");
1380 xfer = pxfer[n_setup];
1387 USB_XFER_LOCK(xfer);
1388 USB_BUS_LOCK(info->bus);
1391 * HINT: when you start/stop a transfer, it might be a
1392 * good idea to directly use the "pxfer[]" structure:
1394 * usbd_transfer_start(sc->pxfer[0]);
1395 * usbd_transfer_stop(sc->pxfer[0]);
1397 * That way, if your code has many parts that will not
1398 * stop running under the same lock, in other words
1399 * "xfer_mtx", the usbd_transfer_start and
1400 * usbd_transfer_stop functions will simply return
1401 * when they detect a NULL pointer argument.
1403 * To avoid any races we clear the "pxfer[]" pointer
1404 * while holding the private mutex of the driver:
1406 pxfer[n_setup] = NULL;
1408 USB_BUS_UNLOCK(info->bus);
1409 USB_XFER_UNLOCK(xfer);
1411 usbd_transfer_drain(xfer);
1414 if (xfer->flags_int.bdma_enable)
1418 * NOTE: default endpoint does not have an
1419 * interface, even if endpoint->iface_index == 0
1421 USB_BUS_LOCK(info->bus);
1422 xfer->endpoint->refcount_alloc--;
1423 USB_BUS_UNLOCK(info->bus);
1425 usb_callout_drain(&xfer->timeout_handle);
1427 USB_BUS_LOCK(info->bus);
1429 USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1430 "reference count\n"));
1432 info->setup_refcount--;
1434 if (info->setup_refcount == 0) {
1435 usbd_transfer_unsetup_sub(info,
1438 USB_BUS_UNLOCK(info->bus);
1443 /*------------------------------------------------------------------------*
1444 * usbd_control_transfer_init - factored out code
1446 * In USB Device Mode we have to wait for the SETUP packet which
1447 * containst the "struct usb_device_request" structure, before we can
1448 * transfer any data. In USB Host Mode we already have the SETUP
1449 * packet at the moment the USB transfer is started. This leads us to
1450 * having to setup the USB transfer at two different places in
1451 * time. This function just contains factored out control transfer
1452 * initialisation code, so that we don't duplicate the code.
1453 *------------------------------------------------------------------------*/
1455 usbd_control_transfer_init(struct usb_xfer *xfer)
1457 struct usb_device_request req;
1459 /* copy out the USB request header */
1461 usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1463 /* setup remainder */
1465 xfer->flags_int.control_rem = UGETW(req.wLength);
1467 /* copy direction to endpoint variable */
1469 xfer->endpointno &= ~(UE_DIR_IN | UE_DIR_OUT);
1471 (req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1474 /*------------------------------------------------------------------------*
1475 * usbd_control_transfer_did_data
1477 * This function returns non-zero if a control endpoint has
1478 * transferred the first DATA packet after the SETUP packet.
1479 * Else it returns zero.
1480 *------------------------------------------------------------------------*/
1482 usbd_control_transfer_did_data(struct usb_xfer *xfer)
1484 struct usb_device_request req;
1486 /* SETUP packet is not yet sent */
1487 if (xfer->flags_int.control_hdr != 0)
1490 /* copy out the USB request header */
1491 usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1493 /* compare remainder to the initial value */
1494 return (xfer->flags_int.control_rem != UGETW(req.wLength));
1497 /*------------------------------------------------------------------------*
1498 * usbd_setup_ctrl_transfer
1500 * This function handles initialisation of control transfers. Control
1501 * transfers are special in that regard that they can both transmit
1507 *------------------------------------------------------------------------*/
1509 usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1513 /* Check for control endpoint stall */
1514 if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1515 /* the control transfer is no longer active */
1516 xfer->flags_int.control_stall = 1;
1517 xfer->flags_int.control_act = 0;
1519 /* don't stall control transfer by default */
1520 xfer->flags_int.control_stall = 0;
1523 /* Check for invalid number of frames */
1524 if (xfer->nframes > 2) {
1526 * If you need to split a control transfer, you
1527 * have to do one part at a time. Only with
1528 * non-control transfers you can do multiple
1531 DPRINTFN(0, "Too many frames: %u\n",
1532 (unsigned int)xfer->nframes);
1537 * Check if there is a control
1538 * transfer in progress:
1540 if (xfer->flags_int.control_act) {
1542 if (xfer->flags_int.control_hdr) {
1544 /* clear send header flag */
1546 xfer->flags_int.control_hdr = 0;
1548 /* setup control transfer */
1549 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1550 usbd_control_transfer_init(xfer);
1553 /* get data length */
1559 /* the size of the SETUP structure is hardcoded ! */
1561 if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1562 DPRINTFN(0, "Wrong framelength %u != %zu\n",
1563 xfer->frlengths[0], sizeof(struct
1564 usb_device_request));
1567 /* check USB mode */
1568 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1570 /* check number of frames */
1571 if (xfer->nframes != 1) {
1573 * We need to receive the setup
1574 * message first so that we know the
1577 DPRINTF("Misconfigured transfer\n");
1581 * Set a dummy "control_rem" value. This
1582 * variable will be overwritten later by a
1583 * call to "usbd_control_transfer_init()" !
1585 xfer->flags_int.control_rem = 0xFFFF;
1588 /* setup "endpoint" and "control_rem" */
1590 usbd_control_transfer_init(xfer);
1593 /* set transfer-header flag */
1595 xfer->flags_int.control_hdr = 1;
1597 /* get data length */
1599 len = (xfer->sumlen - sizeof(struct usb_device_request));
1602 /* update did data flag */
1604 xfer->flags_int.control_did_data =
1605 usbd_control_transfer_did_data(xfer);
1607 /* check if there is a length mismatch */
1609 if (len > xfer->flags_int.control_rem) {
1610 DPRINTFN(0, "Length (%d) greater than "
1611 "remaining length (%d)\n", len,
1612 xfer->flags_int.control_rem);
1615 /* check if we are doing a short transfer */
1617 if (xfer->flags.force_short_xfer) {
1618 xfer->flags_int.control_rem = 0;
1620 if ((len != xfer->max_data_length) &&
1621 (len != xfer->flags_int.control_rem) &&
1622 (xfer->nframes != 1)) {
1623 DPRINTFN(0, "Short control transfer without "
1624 "force_short_xfer set\n");
1627 xfer->flags_int.control_rem -= len;
1630 /* the status part is executed when "control_act" is 0 */
1632 if ((xfer->flags_int.control_rem > 0) ||
1633 (xfer->flags.manual_status)) {
1634 /* don't execute the STATUS stage yet */
1635 xfer->flags_int.control_act = 1;
1638 if ((!xfer->flags_int.control_hdr) &&
1639 (xfer->nframes == 1)) {
1641 * This is not a valid operation!
1643 DPRINTFN(0, "Invalid parameter "
1648 /* time to execute the STATUS stage */
1649 xfer->flags_int.control_act = 0;
1651 return (0); /* success */
1654 return (1); /* failure */
1657 /*------------------------------------------------------------------------*
1658 * usbd_transfer_submit - start USB hardware for the given transfer
1660 * This function should only be called from the USB callback.
1661 *------------------------------------------------------------------------*/
1663 usbd_transfer_submit(struct usb_xfer *xfer)
1665 struct usb_xfer_root *info;
1666 struct usb_bus *bus;
1672 DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1673 xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1677 if (USB_DEBUG_VAR > 0) {
1680 usb_dump_endpoint(xfer->endpoint);
1682 USB_BUS_UNLOCK(bus);
1686 USB_XFER_LOCK_ASSERT(xfer);
1687 USB_BUS_LOCK_ASSERT_NOTOWNED(bus);
1689 /* Only open the USB transfer once! */
1690 if (!xfer->flags_int.open) {
1691 xfer->flags_int.open = 1;
1696 (xfer->endpoint->methods->open) (xfer);
1697 USB_BUS_UNLOCK(bus);
1699 /* set "transferring" flag */
1700 xfer->flags_int.transferring = 1;
1703 /* increment power reference */
1704 usbd_transfer_power_ref(xfer, 1);
1707 * Check if the transfer is waiting on a queue, most
1708 * frequently the "done_q":
1710 if (xfer->wait_queue) {
1712 usbd_transfer_dequeue(xfer);
1713 USB_BUS_UNLOCK(bus);
1715 /* clear "did_dma_delay" flag */
1716 xfer->flags_int.did_dma_delay = 0;
1718 /* clear "did_close" flag */
1719 xfer->flags_int.did_close = 0;
1722 /* clear "bdma_setup" flag */
1723 xfer->flags_int.bdma_setup = 0;
1725 /* by default we cannot cancel any USB transfer immediately */
1726 xfer->flags_int.can_cancel_immed = 0;
1728 /* clear lengths and frame counts by default */
1733 /* clear any previous errors */
1736 /* Check if the device is still alive */
1737 if (info->udev->state < USB_STATE_POWERED) {
1740 * Must return cancelled error code else
1741 * device drivers can hang.
1743 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1744 USB_BUS_UNLOCK(bus);
1749 if (xfer->nframes == 0) {
1750 if (xfer->flags.stall_pipe) {
1752 * Special case - want to stall without transferring
1755 DPRINTF("xfer=%p nframes=0: stall "
1756 "or clear stall!\n", xfer);
1758 xfer->flags_int.can_cancel_immed = 1;
1759 /* start the transfer */
1760 usb_command_wrapper(&xfer->endpoint->
1761 endpoint_q[xfer->stream_id], xfer);
1762 USB_BUS_UNLOCK(bus);
1766 usbd_transfer_done(xfer, USB_ERR_INVAL);
1767 USB_BUS_UNLOCK(bus);
1770 /* compute some variables */
1772 for (x = 0; x != xfer->nframes; x++) {
1773 /* make a copy of the frlenghts[] */
1774 xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1775 /* compute total transfer length */
1776 xfer->sumlen += xfer->frlengths[x];
1777 if (xfer->sumlen < xfer->frlengths[x]) {
1778 /* length wrapped around */
1780 usbd_transfer_done(xfer, USB_ERR_INVAL);
1781 USB_BUS_UNLOCK(bus);
1786 /* clear some internal flags */
1788 xfer->flags_int.short_xfer_ok = 0;
1789 xfer->flags_int.short_frames_ok = 0;
1791 /* check if this is a control transfer */
1793 if (xfer->flags_int.control_xfr) {
1795 if (usbd_setup_ctrl_transfer(xfer)) {
1797 usbd_transfer_done(xfer, USB_ERR_STALLED);
1798 USB_BUS_UNLOCK(bus);
1803 * Setup filtered version of some transfer flags,
1804 * in case of data read direction
1806 if (USB_GET_DATA_ISREAD(xfer)) {
1808 if (xfer->flags.short_frames_ok) {
1809 xfer->flags_int.short_xfer_ok = 1;
1810 xfer->flags_int.short_frames_ok = 1;
1811 } else if (xfer->flags.short_xfer_ok) {
1812 xfer->flags_int.short_xfer_ok = 1;
1814 /* check for control transfer */
1815 if (xfer->flags_int.control_xfr) {
1817 * 1) Control transfers do not support
1818 * reception of multiple short USB
1819 * frames in host mode and device side
1820 * mode, with exception of:
1822 * 2) Due to sometimes buggy device
1823 * side firmware we need to do a
1824 * STATUS stage in case of short
1825 * control transfers in USB host mode.
1826 * The STATUS stage then becomes the
1827 * "alt_next" to the DATA stage.
1829 xfer->flags_int.short_frames_ok = 1;
1834 * Check if BUS-DMA support is enabled and try to load virtual
1835 * buffers into DMA, if any:
1838 if (xfer->flags_int.bdma_enable) {
1839 /* insert the USB transfer last in the BUS-DMA queue */
1840 usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1845 * Enter the USB transfer into the Host Controller or
1846 * Device Controller schedule:
1848 usbd_pipe_enter(xfer);
1851 /*------------------------------------------------------------------------*
1852 * usbd_pipe_enter - factored out code
1853 *------------------------------------------------------------------------*/
1855 usbd_pipe_enter(struct usb_xfer *xfer)
1857 struct usb_endpoint *ep;
1859 USB_XFER_LOCK_ASSERT(xfer);
1861 USB_BUS_LOCK(xfer->xroot->bus);
1863 ep = xfer->endpoint;
1867 /* the transfer can now be cancelled */
1868 xfer->flags_int.can_cancel_immed = 1;
1870 /* enter the transfer */
1871 (ep->methods->enter) (xfer);
1873 /* check for transfer error */
1875 /* some error has happened */
1876 usbd_transfer_done(xfer, 0);
1877 USB_BUS_UNLOCK(xfer->xroot->bus);
1881 /* start the transfer */
1882 usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], xfer);
1883 USB_BUS_UNLOCK(xfer->xroot->bus);
1886 /*------------------------------------------------------------------------*
1887 * usbd_transfer_start - start an USB transfer
1889 * NOTE: Calling this function more than one time will only
1890 * result in a single transfer start, until the USB transfer
1892 *------------------------------------------------------------------------*/
1894 usbd_transfer_start(struct usb_xfer *xfer)
1897 /* transfer is gone */
1900 USB_XFER_LOCK_ASSERT(xfer);
1902 /* mark the USB transfer started */
1904 if (!xfer->flags_int.started) {
1905 /* lock the BUS lock to avoid races updating flags_int */
1906 USB_BUS_LOCK(xfer->xroot->bus);
1907 xfer->flags_int.started = 1;
1908 USB_BUS_UNLOCK(xfer->xroot->bus);
1910 /* check if the USB transfer callback is already transferring */
1912 if (xfer->flags_int.transferring) {
1915 USB_BUS_LOCK(xfer->xroot->bus);
1916 /* call the USB transfer callback */
1917 usbd_callback_ss_done_defer(xfer);
1918 USB_BUS_UNLOCK(xfer->xroot->bus);
1921 /*------------------------------------------------------------------------*
1922 * usbd_transfer_stop - stop an USB transfer
1924 * NOTE: Calling this function more than one time will only
1925 * result in a single transfer stop.
1926 * NOTE: When this function returns it is not safe to free nor
1927 * reuse any DMA buffers. See "usbd_transfer_drain()".
1928 *------------------------------------------------------------------------*/
1930 usbd_transfer_stop(struct usb_xfer *xfer)
1932 struct usb_endpoint *ep;
1935 /* transfer is gone */
1939 USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1942 /* check if the USB transfer was ever opened */
1944 if (!xfer->flags_int.open) {
1945 if (xfer->flags_int.started) {
1946 /* nothing to do except clearing the "started" flag */
1947 /* lock the BUS lock to avoid races updating flags_int */
1948 USB_BUS_LOCK(xfer->xroot->bus);
1949 xfer->flags_int.started = 0;
1950 USB_BUS_UNLOCK(xfer->xroot->bus);
1954 /* try to stop the current USB transfer */
1956 USB_BUS_LOCK(xfer->xroot->bus);
1957 /* override any previous error */
1958 xfer->error = USB_ERR_CANCELLED;
1961 * Clear "open" and "started" when both private and USB lock
1962 * is locked so that we don't get a race updating "flags_int"
1964 xfer->flags_int.open = 0;
1965 xfer->flags_int.started = 0;
1968 * Check if we can cancel the USB transfer immediately.
1970 if (xfer->flags_int.transferring) {
1971 if (xfer->flags_int.can_cancel_immed &&
1972 (!xfer->flags_int.did_close)) {
1975 * The following will lead to an USB_ERR_CANCELLED
1976 * error code being passed to the USB callback.
1978 (xfer->endpoint->methods->close) (xfer);
1979 /* only close once */
1980 xfer->flags_int.did_close = 1;
1982 /* need to wait for the next done callback */
1987 /* close here and now */
1988 (xfer->endpoint->methods->close) (xfer);
1991 * Any additional DMA delay is done by
1992 * "usbd_transfer_unsetup()".
1996 * Special case. Check if we need to restart a blocked
1999 ep = xfer->endpoint;
2002 * If the current USB transfer is completing we need
2003 * to start the next one:
2005 if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
2006 usb_command_wrapper(
2007 &ep->endpoint_q[xfer->stream_id], NULL);
2011 USB_BUS_UNLOCK(xfer->xroot->bus);
2014 /*------------------------------------------------------------------------*
2015 * usbd_transfer_pending
2017 * This function will check if an USB transfer is pending which is a
2018 * little bit complicated!
2021 * 1: Pending: The USB transfer will receive a callback in the future.
2022 *------------------------------------------------------------------------*/
2024 usbd_transfer_pending(struct usb_xfer *xfer)
2026 struct usb_xfer_root *info;
2027 struct usb_xfer_queue *pq;
2030 /* transfer is gone */
2034 USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2037 if (xfer->flags_int.transferring) {
2041 USB_BUS_LOCK(xfer->xroot->bus);
2042 if (xfer->wait_queue) {
2043 /* we are waiting on a queue somewhere */
2044 USB_BUS_UNLOCK(xfer->xroot->bus);
2050 if (pq->curr == xfer) {
2051 /* we are currently scheduled for callback */
2052 USB_BUS_UNLOCK(xfer->xroot->bus);
2055 /* we are not pending */
2056 USB_BUS_UNLOCK(xfer->xroot->bus);
2060 /*------------------------------------------------------------------------*
2061 * usbd_transfer_drain
2063 * This function will stop the USB transfer and wait for any
2064 * additional BUS-DMA and HW-DMA operations to complete. Buffers that
2065 * are loaded into DMA can safely be freed or reused after that this
2066 * function has returned.
2067 *------------------------------------------------------------------------*/
2069 usbd_transfer_drain(struct usb_xfer *xfer)
2072 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
2073 "usbd_transfer_drain can sleep!");
2077 /* transfer is gone */
2080 USB_XFER_LOCK_ASSERT_NOTOWNED(xfer);
2081 USB_XFER_LOCK(xfer);
2083 usbd_transfer_stop(xfer);
2086 * It is allowed that the callback can drop its
2087 * transfer mutex. In that case checking only
2088 * "usbd_transfer_pending()" is not enough to tell if
2089 * the USB transfer is fully drained. We also need to
2090 * check the internal "doing_callback" flag.
2092 xfer->flags_int.draining = 1;
2095 * XXX hack, the wakeup of xfer can race conditions which
2096 * clear the pending status of the xfer.
2098 while (usbd_transfer_pending(xfer) ||
2099 xfer->flags_int.doing_callback) {
2102 * Wait until the current outstanding USB
2103 * transfer is complete !
2105 /* cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_lock); */
2106 lksleep(xfer, xfer->xroot->xfer_lock, 0, "DRAIN", hz);
2108 xfer->flags_int.draining = 0;
2109 USB_XFER_UNLOCK(xfer);
2112 struct usb_page_cache *
2113 usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
2115 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2117 return (&xfer->frbuffers[frindex]);
2121 usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
2123 struct usb_page_search page_info;
2125 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2127 usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
2128 return (page_info.buffer);
2131 /*------------------------------------------------------------------------*
2132 * usbd_xfer_get_fps_shift
2134 * The following function is only useful for isochronous transfers. It
2135 * returns how many times the frame execution rate has been shifted
2141 *------------------------------------------------------------------------*/
2143 usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
2145 return (xfer->fps_shift);
2149 usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
2151 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2153 return (xfer->frlengths[frindex]);
2156 /*------------------------------------------------------------------------*
2157 * usbd_xfer_set_frame_data
2159 * This function sets the pointer of the buffer that should
2160 * loaded directly into DMA for the given USB frame. Passing "ptr"
2161 * equal to NULL while the corresponding "frlength" is greater
2162 * than zero gives undefined results!
2163 *------------------------------------------------------------------------*/
2165 usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2166 void *ptr, usb_frlength_t len)
2168 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2170 /* set virtual address to load and length */
2171 xfer->frbuffers[frindex].buffer = ptr;
2172 usbd_xfer_set_frame_len(xfer, frindex, len);
2176 usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2177 void **ptr, int *len)
2179 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2182 *ptr = xfer->frbuffers[frindex].buffer;
2184 *len = xfer->frlengths[frindex];
2187 /*------------------------------------------------------------------------*
2188 * usbd_xfer_old_frame_length
2190 * This function returns the framelength of the given frame at the
2191 * time the transfer was submitted. This function can be used to
2192 * compute the starting data pointer of the next isochronous frame
2193 * when an isochronous transfer has completed.
2194 *------------------------------------------------------------------------*/
2196 usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2198 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2200 return (xfer->frlengths[frindex + xfer->max_frame_count]);
2204 usbd_xfer_status(struct usb_xfer *xfer, int *actlen, int *sumlen, int *aframes,
2208 *actlen = xfer->actlen;
2210 *sumlen = xfer->sumlen;
2211 if (aframes != NULL)
2212 *aframes = xfer->aframes;
2213 if (nframes != NULL)
2214 *nframes = xfer->nframes;
2217 /*------------------------------------------------------------------------*
2218 * usbd_xfer_set_frame_offset
2220 * This function sets the frame data buffer offset relative to the beginning
2221 * of the USB DMA buffer allocated for this USB transfer.
2222 *------------------------------------------------------------------------*/
2224 usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2225 usb_frcount_t frindex)
2227 KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2228 "when the USB buffer is external\n"));
2229 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2231 /* set virtual address to load */
2232 xfer->frbuffers[frindex].buffer =
2233 USB_ADD_BYTES(xfer->local_buffer, offset);
2237 usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2243 usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2249 usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2255 usbd_xfer_max_frames(struct usb_xfer *xfer)
2257 return (xfer->max_frame_count);
2261 usbd_xfer_max_len(struct usb_xfer *xfer)
2263 return (xfer->max_data_length);
2267 usbd_xfer_max_framelen(struct usb_xfer *xfer)
2269 return (xfer->max_frame_size);
2273 usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2276 KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2278 xfer->frlengths[frindex] = len;
2281 /*------------------------------------------------------------------------*
2282 * usb_callback_proc - factored out code
2284 * This function performs USB callbacks.
2285 *------------------------------------------------------------------------*/
2287 usb_callback_proc(struct usb_proc_msg *_pm)
2289 struct usb_done_msg *pm = (void *)_pm;
2290 struct usb_xfer_root *info = pm->xroot;
2292 /* Change locking order */
2293 USB_BUS_UNLOCK(info->bus);
2296 * We exploit the fact that the mutex is the same for all
2297 * callbacks that will be called from this thread:
2299 lockmgr(info->xfer_lock, LK_EXCLUSIVE);
2300 USB_BUS_LOCK(info->bus);
2302 /* Continue where we lost track */
2303 usb_command_wrapper(&info->done_q,
2306 lockmgr(info->xfer_lock, LK_RELEASE);
2309 /*------------------------------------------------------------------------*
2310 * usbd_callback_ss_done_defer
2312 * This function will defer the start, stop and done callback to the
2314 *------------------------------------------------------------------------*/
2316 usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2318 struct usb_xfer_root *info = xfer->xroot;
2319 struct usb_xfer_queue *pq = &info->done_q;
2321 USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2323 if (pq->curr != xfer) {
2324 usbd_transfer_enqueue(pq, xfer);
2326 if (!pq->recurse_1) {
2329 * We have to postpone the callback due to the fact we
2330 * will have a Lock Order Reversal, LOR, if we try to
2333 if (usb_proc_msignal(info->done_p,
2334 &info->done_m[0], &info->done_m[1])) {
2338 /* clear second recurse flag */
2345 /*------------------------------------------------------------------------*
2346 * usbd_callback_wrapper
2348 * This is a wrapper for USB callbacks. This wrapper does some
2349 * auto-magic things like figuring out if we can call the callback
2350 * directly from the current context or if we need to wakeup the
2351 * interrupt process.
2352 *------------------------------------------------------------------------*/
2354 usbd_callback_wrapper(struct usb_xfer_queue *pq)
2356 struct usb_xfer *xfer = pq->curr;
2357 struct usb_xfer_root *info = xfer->xroot;
2359 USB_BUS_LOCK_ASSERT(info->bus);
2360 if (!lockowned(info->xfer_lock)) {
2362 * Cases that end up here:
2364 * 5) HW interrupt done callback or other source.
2366 DPRINTFN(3, "case 5\n");
2369 * We have to postpone the callback due to the fact we
2370 * will have a Lock Order Reversal, LOR, if we try to
2373 if (usb_proc_msignal(info->done_p,
2374 &info->done_m[0], &info->done_m[1])) {
2380 * Cases that end up here:
2382 * 1) We are starting a transfer
2383 * 2) We are prematurely calling back a transfer
2384 * 3) We are stopping a transfer
2385 * 4) We are doing an ordinary callback
2387 DPRINTFN(3, "case 1-4\n");
2388 /* get next USB transfer in the queue */
2389 info->done_q.curr = NULL;
2391 /* set flag in case of drain */
2392 xfer->flags_int.doing_callback = 1;
2394 USB_BUS_UNLOCK(info->bus);
2395 USB_BUS_LOCK_ASSERT_NOTOWNED(info->bus);
2397 /* set correct USB state for callback */
2398 if (!xfer->flags_int.transferring) {
2399 xfer->usb_state = USB_ST_SETUP;
2400 if (!xfer->flags_int.started) {
2401 /* we got stopped before we even got started */
2402 USB_BUS_LOCK(info->bus);
2407 if (usbd_callback_wrapper_sub(xfer)) {
2408 /* the callback has been deferred */
2409 USB_BUS_LOCK(info->bus);
2413 /* decrement power reference */
2414 usbd_transfer_power_ref(xfer, -1);
2416 xfer->flags_int.transferring = 0;
2419 xfer->usb_state = USB_ST_ERROR;
2421 /* set transferred state */
2422 xfer->usb_state = USB_ST_TRANSFERRED;
2424 /* sync DMA memory, if any */
2425 if (xfer->flags_int.bdma_enable &&
2426 (!xfer->flags_int.bdma_no_post_sync)) {
2427 usb_bdma_post_sync(xfer);
2434 if (xfer->usb_state != USB_ST_SETUP)
2435 usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2437 /* call processing routine */
2438 (xfer->callback) (xfer, xfer->error);
2440 /* pickup the USB mutex again */
2441 USB_BUS_LOCK(info->bus);
2444 * Check if we got started after that we got cancelled, but
2445 * before we managed to do the callback.
2447 if ((!xfer->flags_int.open) &&
2448 (xfer->flags_int.started) &&
2449 (xfer->usb_state == USB_ST_ERROR)) {
2450 /* clear flag in case of drain */
2451 xfer->flags_int.doing_callback = 0;
2452 /* try to loop, but not recursivly */
2453 usb_command_wrapper(&info->done_q, xfer);
2458 /* clear flag in case of drain */
2459 xfer->flags_int.doing_callback = 0;
2462 * Check if we are draining.
2464 if (xfer->flags_int.draining &&
2465 (!xfer->flags_int.transferring)) {
2466 /* "usbd_transfer_drain()" is waiting for end of transfer */
2467 xfer->flags_int.draining = 0;
2468 /* cv_broadcast(&info->cv_drain); */
2472 /* do the next callback, if any */
2473 usb_command_wrapper(&info->done_q,
2477 /*------------------------------------------------------------------------*
2478 * usb_dma_delay_done_cb
2480 * This function is called when the DMA delay has been exectuded, and
2481 * will make sure that the callback is called to complete the USB
2482 * transfer. This code path is ususally only used when there is an USB
2483 * error like USB_ERR_CANCELLED.
2484 *------------------------------------------------------------------------*/
2486 usb_dma_delay_done_cb(struct usb_xfer *xfer)
2488 USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2490 DPRINTFN(3, "Completed %p\n", xfer);
2492 /* queue callback for execution, again */
2493 usbd_transfer_done(xfer, 0);
2496 /*------------------------------------------------------------------------*
2497 * usbd_transfer_dequeue
2499 * - This function is used to remove an USB transfer from a USB
2502 * - This function can be called multiple times in a row.
2503 *------------------------------------------------------------------------*/
2505 usbd_transfer_dequeue(struct usb_xfer *xfer)
2507 struct usb_xfer_queue *pq;
2509 pq = xfer->wait_queue;
2511 TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2512 xfer->wait_queue = NULL;
2516 /*------------------------------------------------------------------------*
2517 * usbd_transfer_enqueue
2519 * - This function is used to insert an USB transfer into a USB *
2522 * - This function can be called multiple times in a row.
2523 *------------------------------------------------------------------------*/
2525 usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2528 * Insert the USB transfer into the queue, if it is not
2529 * already on a USB transfer queue:
2532 KKASSERT(xfer->wait_queue == NULL);
2534 if (xfer->wait_queue == NULL) {
2535 xfer->wait_queue = pq;
2536 TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2540 /*------------------------------------------------------------------------*
2541 * usbd_transfer_done
2543 * - This function is used to remove an USB transfer from the busdma,
2544 * pipe or interrupt queue.
2546 * - This function is used to queue the USB transfer on the done
2549 * - This function is used to stop any USB transfer timeouts.
2550 *------------------------------------------------------------------------*/
2552 usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2554 struct usb_xfer_root *info = xfer->xroot;
2556 USB_BUS_LOCK_ASSERT(info->bus);
2558 DPRINTF("err=%s\n", usbd_errstr(error));
2561 * If we are not transferring then just return.
2562 * This can happen during transfer cancel.
2564 if (!xfer->flags_int.transferring) {
2565 DPRINTF("not transferring\n");
2566 /* end of control transfer, if any */
2567 xfer->flags_int.control_act = 0;
2570 /* only set transfer error, if not already set */
2571 if (xfer->error == USB_ERR_NORMAL_COMPLETION)
2572 xfer->error = error;
2574 /* stop any callouts */
2575 usb_callout_stop(&xfer->timeout_handle);
2578 * If we are waiting on a queue, just remove the USB transfer
2579 * from the queue, if any. We should have the required locks
2580 * locked to do the remove when this function is called.
2582 usbd_transfer_dequeue(xfer);
2585 if (lockowned(xfer->xroot->xfer_lock)) {
2586 struct usb_xfer_queue *pq;
2589 * If the private USB lock is not locked, then we assume
2590 * that the BUS-DMA load stage has been passed:
2594 if (pq->curr == xfer) {
2595 /* start the next BUS-DMA load, if any */
2596 usb_command_wrapper(pq, NULL);
2600 /* keep some statistics */
2602 info->bus->stats_err.uds_requests
2603 [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2605 info->bus->stats_ok.uds_requests
2606 [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2609 /* call the USB transfer callback */
2610 usbd_callback_ss_done_defer(xfer);
2613 /*------------------------------------------------------------------------*
2614 * usbd_transfer_start_cb
2616 * This function is called to start the USB transfer when
2617 * "xfer->interval" is greater than zero, and and the endpoint type is
2619 *------------------------------------------------------------------------*/
2621 usbd_transfer_start_cb(void *arg)
2623 struct usb_xfer *xfer = arg;
2624 struct usb_endpoint *ep = xfer->endpoint;
2626 USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2631 usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2634 /* the transfer can now be cancelled */
2635 xfer->flags_int.can_cancel_immed = 1;
2637 /* start USB transfer, if no error */
2638 if (xfer->error == 0)
2639 (ep->methods->start) (xfer);
2641 /* check for transfer error */
2643 /* some error has happened */
2644 usbd_transfer_done(xfer, 0);
2648 /*------------------------------------------------------------------------*
2649 * usbd_xfer_set_stall
2651 * This function is used to set the stall flag outside the
2652 * callback. This function is NULL safe.
2653 *------------------------------------------------------------------------*/
2655 usbd_xfer_set_stall(struct usb_xfer *xfer)
2661 USB_XFER_LOCK_ASSERT(xfer);
2663 /* avoid any races by locking the USB mutex */
2664 USB_BUS_LOCK(xfer->xroot->bus);
2665 xfer->flags.stall_pipe = 1;
2666 USB_BUS_UNLOCK(xfer->xroot->bus);
2670 usbd_xfer_is_stalled(struct usb_xfer *xfer)
2672 return (xfer->endpoint->is_stalled);
2675 /*------------------------------------------------------------------------*
2676 * usbd_transfer_clear_stall
2678 * This function is used to clear the stall flag outside the
2679 * callback. This function is NULL safe.
2680 *------------------------------------------------------------------------*/
2682 usbd_transfer_clear_stall(struct usb_xfer *xfer)
2688 USB_XFER_LOCK_ASSERT(xfer);
2690 /* avoid any races by locking the USB mutex */
2691 USB_BUS_LOCK(xfer->xroot->bus);
2693 xfer->flags.stall_pipe = 0;
2695 USB_BUS_UNLOCK(xfer->xroot->bus);
2698 /*------------------------------------------------------------------------*
2701 * This function is used to add an USB transfer to the pipe transfer list.
2702 *------------------------------------------------------------------------*/
2704 usbd_pipe_start(struct usb_xfer_queue *pq)
2706 struct usb_endpoint *ep;
2707 struct usb_xfer *xfer;
2711 ep = xfer->endpoint;
2713 USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2716 * If the endpoint is already stalled we do nothing !
2718 if (ep->is_stalled) {
2722 * Check if we are supposed to stall the endpoint:
2724 if (xfer->flags.stall_pipe) {
2725 struct usb_device *udev;
2726 struct usb_xfer_root *info;
2728 /* clear stall command */
2729 xfer->flags.stall_pipe = 0;
2731 /* get pointer to USB device */
2736 * Only stall BULK and INTERRUPT endpoints.
2738 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2739 if ((type == UE_BULK) ||
2740 (type == UE_INTERRUPT)) {
2745 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2746 (udev->bus->methods->set_stall) (
2747 udev, ep, &did_stall);
2748 } else if (udev->ctrl_xfer[1]) {
2749 info = udev->ctrl_xfer[1]->xroot;
2751 USB_BUS_NON_GIANT_PROC(info->bus),
2752 &udev->cs_msg[0], &udev->cs_msg[1]);
2754 /* should not happen */
2755 DPRINTFN(0, "No stall handler\n");
2758 * Check if we should stall. Some USB hardware
2759 * handles set- and clear-stall in hardware.
2763 * The transfer will be continued when
2764 * the clear-stall control endpoint
2765 * message is received.
2770 } else if (type == UE_ISOCHRONOUS) {
2773 * Make sure any FIFO overflow or other FIFO
2774 * error conditions go away by resetting the
2775 * endpoint FIFO through the clear stall
2778 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2779 (udev->bus->methods->clear_stall) (udev, ep);
2783 /* Set or clear stall complete - special case */
2784 if (xfer->nframes == 0) {
2785 /* we are complete */
2787 usbd_transfer_done(xfer, 0);
2793 * 1) Start the first transfer queued.
2795 * 2) Re-start the current USB transfer.
2798 * Check if there should be any
2799 * pre transfer start delay:
2801 if (xfer->interval > 0) {
2802 type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2803 if ((type == UE_BULK) ||
2804 (type == UE_CONTROL)) {
2805 usbd_transfer_timeout_ms(xfer,
2806 &usbd_transfer_start_cb,
2814 usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2816 /* the transfer can now be cancelled */
2817 xfer->flags_int.can_cancel_immed = 1;
2819 /* start USB transfer, if no error */
2820 if (xfer->error == 0)
2821 (ep->methods->start) (xfer);
2823 /* check for transfer error */
2825 /* some error has happened */
2826 usbd_transfer_done(xfer, 0);
2830 /*------------------------------------------------------------------------*
2831 * usbd_transfer_timeout_ms
2833 * This function is used to setup a timeout on the given USB
2834 * transfer. If the timeout has been deferred the callback given by
2835 * "cb" will get called after "ms" milliseconds.
2836 *------------------------------------------------------------------------*/
2838 usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2839 void (*cb) (void *arg), usb_timeout_t ms)
2841 USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
2844 usb_callout_reset(&xfer->timeout_handle,
2845 USB_MS_TO_TICKS(ms) + USB_CALLOUT_ZERO_TICKS, cb, xfer);
2848 /*------------------------------------------------------------------------*
2849 * usbd_callback_wrapper_sub
2851 * - This function will update variables in an USB transfer after
2852 * that the USB transfer is complete.
2854 * - This function is used to start the next USB transfer on the
2855 * ep transfer queue, if any.
2857 * NOTE: In some special cases the USB transfer will not be removed from
2858 * the pipe queue, but remain first. To enforce USB transfer removal call
2859 * this function passing the error code "USB_ERR_CANCELLED".
2863 * Else: The callback has been deferred.
2864 *------------------------------------------------------------------------*/
2866 usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2868 struct usb_endpoint *ep;
2869 struct usb_bus *bus;
2872 bus = xfer->xroot->bus;
2874 if ((!xfer->flags_int.open) &&
2875 (!xfer->flags_int.did_close)) {
2878 (xfer->endpoint->methods->close) (xfer);
2879 USB_BUS_UNLOCK(bus);
2880 /* only close once */
2881 xfer->flags_int.did_close = 1;
2882 return (1); /* wait for new callback */
2885 * If we have a non-hardware induced error we
2886 * need to do the DMA delay!
2888 if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2889 (xfer->error == USB_ERR_CANCELLED ||
2890 xfer->error == USB_ERR_TIMEOUT ||
2891 bus->methods->start_dma_delay != NULL)) {
2895 /* only delay once */
2896 xfer->flags_int.did_dma_delay = 1;
2898 /* we can not cancel this delay */
2899 xfer->flags_int.can_cancel_immed = 0;
2901 temp = usbd_get_dma_delay(xfer->xroot->udev);
2903 DPRINTFN(3, "DMA delay, %u ms, "
2904 "on %p\n", temp, xfer);
2909 * Some hardware solutions have dedicated
2910 * events when it is safe to free DMA'ed
2911 * memory. For the other hardware platforms we
2912 * use a static delay.
2914 if (bus->methods->start_dma_delay != NULL) {
2915 (bus->methods->start_dma_delay) (xfer);
2917 usbd_transfer_timeout_ms(xfer,
2918 (void (*)(void *))&usb_dma_delay_done_cb,
2921 USB_BUS_UNLOCK(bus);
2922 return (1); /* wait for new callback */
2925 /* check actual number of frames */
2926 if (xfer->aframes > xfer->nframes) {
2927 if (xfer->error == 0) {
2928 panic("%s: actual number of frames, %d, is "
2929 "greater than initial number of frames, %d\n",
2930 __func__, xfer->aframes, xfer->nframes);
2932 /* just set some valid value */
2933 xfer->aframes = xfer->nframes;
2936 /* compute actual length */
2939 for (x = 0; x != xfer->aframes; x++) {
2940 xfer->actlen += xfer->frlengths[x];
2944 * Frames that were not transferred get zero actual length in
2945 * case the USB device driver does not check the actual number
2946 * of frames transferred, "xfer->aframes":
2948 for (; x < xfer->nframes; x++) {
2949 usbd_xfer_set_frame_len(xfer, x, 0);
2952 /* check actual length */
2953 if (xfer->actlen > xfer->sumlen) {
2954 if (xfer->error == 0) {
2955 panic("%s: actual length, %d, is greater than "
2956 "initial length, %d\n",
2957 __func__, xfer->actlen, xfer->sumlen);
2959 /* just set some valid value */
2960 xfer->actlen = xfer->sumlen;
2963 DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2964 xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2965 xfer->aframes, xfer->nframes);
2968 /* end of control transfer, if any */
2969 xfer->flags_int.control_act = 0;
2971 /* check if we should block the execution queue */
2972 if ((xfer->error != USB_ERR_CANCELLED) &&
2973 (xfer->flags.pipe_bof)) {
2974 DPRINTFN(2, "xfer=%p: Block On Failure "
2975 "on endpoint=%p\n", xfer, xfer->endpoint);
2979 /* check for short transfers */
2980 if (xfer->actlen < xfer->sumlen) {
2982 /* end of control transfer, if any */
2983 xfer->flags_int.control_act = 0;
2985 if (!xfer->flags_int.short_xfer_ok) {
2986 xfer->error = USB_ERR_SHORT_XFER;
2987 if (xfer->flags.pipe_bof) {
2988 DPRINTFN(2, "xfer=%p: Block On Failure on "
2989 "Short Transfer on endpoint %p.\n",
2990 xfer, xfer->endpoint);
2996 * Check if we are in the middle of a
2999 if (xfer->flags_int.control_act) {
3000 DPRINTFN(5, "xfer=%p: Control transfer "
3001 "active on endpoint=%p\n", xfer, xfer->endpoint);
3007 ep = xfer->endpoint;
3010 * If the current USB transfer is completing we need to start the
3014 if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
3015 usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], NULL);
3017 if (ep->endpoint_q[xfer->stream_id].curr != NULL ||
3018 TAILQ_FIRST(&ep->endpoint_q[xfer->stream_id].head) != NULL) {
3019 /* there is another USB transfer waiting */
3021 /* this is the last USB transfer */
3022 /* clear isochronous sync flag */
3023 xfer->endpoint->is_synced = 0;
3026 USB_BUS_UNLOCK(bus);
3031 /*------------------------------------------------------------------------*
3032 * usb_command_wrapper
3034 * This function is used to execute commands non-recursivly on an USB
3036 *------------------------------------------------------------------------*/
3038 usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
3042 * If the transfer is not already processing,
3045 if (pq->curr != xfer) {
3046 usbd_transfer_enqueue(pq, xfer);
3047 if (pq->curr != NULL) {
3048 /* something is already processing */
3049 DPRINTFN(6, "busy %p\n", pq->curr);
3054 /* Get next element in queue */
3058 if (!pq->recurse_1) {
3062 /* set both recurse flags */
3066 if (pq->curr == NULL) {
3067 xfer = TAILQ_FIRST(&pq->head);
3069 TAILQ_REMOVE(&pq->head, xfer,
3071 xfer->wait_queue = NULL;
3077 DPRINTFN(6, "cb %p (enter)\n", pq->curr);
3079 DPRINTFN(6, "cb %p (leave)\n", pq->curr);
3081 } while (!pq->recurse_2);
3083 /* clear first recurse flag */
3087 /* clear second recurse flag */
3092 /*------------------------------------------------------------------------*
3093 * usbd_ctrl_transfer_setup
3095 * This function is used to setup the default USB control endpoint
3097 *------------------------------------------------------------------------*/
3099 usbd_ctrl_transfer_setup(struct usb_device *udev)
3101 struct usb_xfer *xfer;
3103 uint8_t iface_index;
3105 /* check for root HUB */
3106 if (udev->parent_hub == NULL)
3110 xfer = udev->ctrl_xfer[0];
3112 USB_XFER_LOCK(xfer);
3114 ((xfer->address == udev->address) &&
3115 (udev->ctrl_ep_desc.wMaxPacketSize[0] ==
3116 udev->ddesc.bMaxPacketSize));
3117 if (udev->flags.usb_mode == USB_MODE_DEVICE) {
3120 * NOTE: checking "xfer->address" and
3121 * starting the USB transfer must be
3124 usbd_transfer_start(xfer);
3127 USB_XFER_UNLOCK(xfer);
3134 * All parameters are exactly the same like before.
3140 * Update wMaxPacketSize for the default control endpoint:
3142 udev->ctrl_ep_desc.wMaxPacketSize[0] =
3143 udev->ddesc.bMaxPacketSize;
3146 * Unsetup any existing USB transfer:
3148 usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
3151 * Reset clear stall error counter.
3153 udev->clear_stall_errors = 0;
3156 * Try to setup a new USB transfer for the
3157 * default control endpoint:
3160 if (usbd_transfer_setup(udev, &iface_index,
3161 udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
3162 &udev->device_lock)) {
3163 DPRINTFN(0, "could not setup default "
3170 /*------------------------------------------------------------------------*
3171 * usbd_clear_data_toggle - factored out code
3173 * NOTE: the intention of this function is not to reset the hardware
3175 *------------------------------------------------------------------------*/
3177 usbd_clear_stall_locked(struct usb_device *udev, struct usb_endpoint *ep)
3179 USB_BUS_LOCK_ASSERT(udev->bus);
3181 /* check that we have a valid case */
3182 if (udev->flags.usb_mode == USB_MODE_HOST &&
3183 udev->parent_hub != NULL &&
3184 udev->bus->methods->clear_stall != NULL &&
3185 ep->methods != NULL) {
3186 (udev->bus->methods->clear_stall) (udev, ep);
3190 /*------------------------------------------------------------------------*
3191 * usbd_clear_data_toggle - factored out code
3193 * NOTE: the intention of this function is not to reset the hardware
3194 * data toggle on the USB device side.
3195 *------------------------------------------------------------------------*/
3197 usbd_clear_data_toggle(struct usb_device *udev, struct usb_endpoint *ep)
3199 DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3201 USB_BUS_LOCK(udev->bus);
3202 ep->toggle_next = 0;
3203 /* some hardware needs a callback to clear the data toggle */
3204 usbd_clear_stall_locked(udev, ep);
3205 USB_BUS_UNLOCK(udev->bus);
3208 /*------------------------------------------------------------------------*
3209 * usbd_clear_stall_callback - factored out clear stall callback
3212 * xfer1: Clear Stall Control Transfer
3213 * xfer2: Stalled USB Transfer
3215 * This function is NULL safe.
3221 * Clear stall config example:
3223 * static const struct usb_config my_clearstall = {
3224 * .type = UE_CONTROL,
3226 * .direction = UE_DIR_ANY,
3227 * .interval = 50, //50 milliseconds
3228 * .bufsize = sizeof(struct usb_device_request),
3229 * .timeout = 1000, //1.000 seconds
3230 * .callback = &my_clear_stall_callback, // **
3231 * .usb_mode = USB_MODE_HOST,
3234 * ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3235 * passing the correct parameters.
3236 *------------------------------------------------------------------------*/
3238 usbd_clear_stall_callback(struct usb_xfer *xfer1,
3239 struct usb_xfer *xfer2)
3241 struct usb_device_request req;
3243 if (xfer2 == NULL) {
3244 /* looks like we are tearing down */
3245 DPRINTF("NULL input parameter\n");
3248 USB_XFER_LOCK_ASSERT(xfer1);
3249 USB_XFER_LOCK_ASSERT(xfer2);
3251 switch (USB_GET_STATE(xfer1)) {
3255 * pre-clear the data toggle to DATA0 ("umass.c" and
3256 * "ata-usb.c" depends on this)
3259 usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3261 /* setup a clear-stall packet */
3263 req.bmRequestType = UT_WRITE_ENDPOINT;
3264 req.bRequest = UR_CLEAR_FEATURE;
3265 USETW(req.wValue, UF_ENDPOINT_HALT);
3266 req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3268 USETW(req.wLength, 0);
3271 * "usbd_transfer_setup_sub()" will ensure that
3272 * we have sufficient room in the buffer for
3273 * the request structure!
3276 /* copy in the transfer */
3278 usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3281 xfer1->frlengths[0] = sizeof(req);
3284 usbd_transfer_submit(xfer1);
3287 case USB_ST_TRANSFERRED:
3290 default: /* Error */
3291 if (xfer1->error == USB_ERR_CANCELLED) {
3296 return (1); /* Clear Stall Finished */
3299 /*------------------------------------------------------------------------*
3300 * usbd_transfer_poll
3302 * The following function gets called from the USB keyboard driver and
3303 * UMASS when the system has paniced.
3305 * NOTE: It is currently not possible to resume normal operation on
3306 * the USB controller which has been polled, due to clearing of the
3307 * "up_dsleep" and "up_msleep" flags.
3308 *------------------------------------------------------------------------*/
3310 usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3312 struct usb_xfer *xfer;
3313 struct usb_xfer_root *xroot;
3314 struct usb_device *udev;
3315 struct usb_proc_msg *pm;
3320 for (n = 0; n != max; n++) {
3321 /* Extra checks to avoid panic */
3324 continue; /* no USB transfer */
3325 xroot = xfer->xroot;
3327 continue; /* no USB root */
3330 continue; /* no USB device */
3331 if (udev->bus == NULL)
3332 continue; /* no BUS structure */
3333 if (udev->bus->methods == NULL)
3334 continue; /* no BUS methods */
3335 if (udev->bus->methods->xfer_poll == NULL)
3336 continue; /* no poll method */
3338 /* make sure that the BUS mutex is not locked */
3340 while (lockowned(&xroot->udev->bus->bus_lock)) {
3341 lockmgr(&xroot->udev->bus->bus_lock, LK_RELEASE);
3345 /* make sure that the transfer mutex is not locked */
3347 while (lockowned(xroot->xfer_lock)) {
3348 lockmgr(xroot->xfer_lock, LK_RELEASE);
3352 /* Make sure cv_signal() and cv_broadcast() is not called */
3353 USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0;
3354 USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0;
3355 USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0;
3356 USB_BUS_NON_GIANT_PROC(udev->bus)->up_msleep = 0;
3358 /* poll USB hardware */
3359 (udev->bus->methods->xfer_poll) (udev->bus);
3361 USB_BUS_LOCK(xroot->bus);
3363 /* check for clear stall */
3364 if (udev->ctrl_xfer[1] != NULL) {
3366 /* poll clear stall start */
3367 pm = &udev->cs_msg[0].hdr;
3368 (pm->pm_callback) (pm);
3369 /* poll clear stall done thread */
3370 pm = &udev->ctrl_xfer[1]->
3371 xroot->done_m[0].hdr;
3372 (pm->pm_callback) (pm);
3375 /* poll done thread */
3376 pm = &xroot->done_m[0].hdr;
3377 (pm->pm_callback) (pm);
3379 USB_BUS_UNLOCK(xroot->bus);
3381 /* restore transfer mutex */
3383 lockmgr(xroot->xfer_lock, LK_EXCLUSIVE);
3385 /* restore BUS mutex */
3387 lockmgr(&xroot->udev->bus->bus_lock, LK_EXCLUSIVE);
3392 usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3393 uint8_t type, enum usb_dev_speed speed)
3395 static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3396 [USB_SPEED_LOW] = 8,
3397 [USB_SPEED_FULL] = 64,
3398 [USB_SPEED_HIGH] = 1024,
3399 [USB_SPEED_VARIABLE] = 1024,
3400 [USB_SPEED_SUPER] = 1024,
3403 static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3404 [USB_SPEED_LOW] = 0, /* invalid */
3405 [USB_SPEED_FULL] = 1023,
3406 [USB_SPEED_HIGH] = 1024,
3407 [USB_SPEED_VARIABLE] = 3584,
3408 [USB_SPEED_SUPER] = 1024,
3411 static const uint16_t control_min[USB_SPEED_MAX] = {
3412 [USB_SPEED_LOW] = 8,
3413 [USB_SPEED_FULL] = 8,
3414 [USB_SPEED_HIGH] = 64,
3415 [USB_SPEED_VARIABLE] = 512,
3416 [USB_SPEED_SUPER] = 512,
3419 static const uint16_t bulk_min[USB_SPEED_MAX] = {
3420 [USB_SPEED_LOW] = 8,
3421 [USB_SPEED_FULL] = 8,
3422 [USB_SPEED_HIGH] = 512,
3423 [USB_SPEED_VARIABLE] = 512,
3424 [USB_SPEED_SUPER] = 1024,
3429 memset(ptr, 0, sizeof(*ptr));
3433 ptr->range.max = intr_range_max[speed];
3435 case UE_ISOCHRONOUS:
3436 ptr->range.max = isoc_range_max[speed];
3439 if (type == UE_BULK)
3440 temp = bulk_min[speed];
3441 else /* UE_CONTROL */
3442 temp = control_min[speed];
3444 /* default is fixed */
3445 ptr->fixed[0] = temp;
3446 ptr->fixed[1] = temp;
3447 ptr->fixed[2] = temp;
3448 ptr->fixed[3] = temp;
3450 if (speed == USB_SPEED_FULL) {
3451 /* multiple sizes */
3456 if ((speed == USB_SPEED_VARIABLE) &&
3457 (type == UE_BULK)) {
3458 /* multiple sizes */
3459 ptr->fixed[2] = 1024;
3460 ptr->fixed[3] = 1536;
3467 usbd_xfer_softc(struct usb_xfer *xfer)
3469 return (xfer->priv_sc);
3473 usbd_xfer_get_priv(struct usb_xfer *xfer)
3475 return (xfer->priv_fifo);
3479 usbd_xfer_set_priv(struct usb_xfer *xfer, void *ptr)
3481 xfer->priv_fifo = ptr;
3485 usbd_xfer_state(struct usb_xfer *xfer)
3487 return (xfer->usb_state);
3491 usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3494 case USB_FORCE_SHORT_XFER:
3495 xfer->flags.force_short_xfer = 1;
3497 case USB_SHORT_XFER_OK:
3498 xfer->flags.short_xfer_ok = 1;
3500 case USB_MULTI_SHORT_OK:
3501 xfer->flags.short_frames_ok = 1;
3503 case USB_MANUAL_STATUS:
3504 xfer->flags.manual_status = 1;
3510 usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3513 case USB_FORCE_SHORT_XFER:
3514 xfer->flags.force_short_xfer = 0;
3516 case USB_SHORT_XFER_OK:
3517 xfer->flags.short_xfer_ok = 0;
3519 case USB_MULTI_SHORT_OK:
3520 xfer->flags.short_frames_ok = 0;
3522 case USB_MANUAL_STATUS:
3523 xfer->flags.manual_status = 0;
3529 * The following function returns in milliseconds when the isochronous
3530 * transfer was completed by the hardware. The returned value wraps
3531 * around 65536 milliseconds.
3534 usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3536 return (xfer->isoc_time_complete);
3540 * The following function returns non-zero if the max packet size
3541 * field was clamped to a valid value. Else it returns zero.
3544 usbd_xfer_maxp_was_clamped(struct usb_xfer *xfer)
3546 return (xfer->flags_int.maxp_was_clamped);