sys/bus/usb/ehci.c

   1 /*
   2  * $NetBSD: ehci.c,v 1.79 2004/10/23 13:36:43 augustss Exp $
   3  * $FreeBSD: src/sys/dev/usb/ehci.c,v 1.5 2003/11/10 00:20:52 joe Exp $
   4  * $DragonFly: src/sys/bus/usb/ehci.c,v 1.12 2005/08/27 19:00:49 asmodai Exp $
   5  */
   6
   7 /*
   8  * Copyright (c) 2004 The NetBSD Foundation, Inc.
   9  * All rights reserved.
  10  *
  11  * This code is derived from software contributed to The NetBSD Foundation
  12  * by Lennart Augustsson (lennart@augustsson.net) and by Charles M. Hannum.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. All advertising materials mentioning features or use of this software
  23  *    must display the following acknowledgement:
  24  *        This product includes software developed by the NetBSD
  25  *        Foundation, Inc. and its contributors.
  26  * 4. Neither the name of The NetBSD Foundation nor the names of its
  27  *    contributors may be used to endorse or promote products derived
  28  *    from this software without specific prior written permission.
  29  *
  30  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  31  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  32  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  33  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  34  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  35  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  36  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  37  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  38  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  39  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  40  * POSSIBILITY OF SUCH DAMAGE.
  41  */
  42
  43 /*
  44  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  45  *
  46  * The EHCI 1.0 spec can be found at
  47  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  48  * and the USB 2.0 spec at
  49  * http://www.usb.org/developers/docs/usb_20.zip
  50  *
  51  */
  52
  53 /*
  54  * TODO:
  55  * 1) hold off explorations by companion controllers until ehci has started.
  56  *
  57  * 2) The EHCI driver lacks support for interrupt isochronous transfers, so
  58  *    devices using them don't work.
  59  *    Interrupt transfers are not difficult, it's just not done.
  60  *
  61  * 3) The meaty part to implement is the support for USB 2.0 hubs.
  62  *    They are quite complicated since the need to be able to do
  63  *    "transaction translation", i.e., converting to/from USB 2 and USB 1.
  64  *    So the hub driver needs to handle and schedule these things, to
  65  *    assign place in frame where different devices get to go. See chapter
  66  *    on hubs in USB 2.0 for details.
  67  *
  68  * 4) command failures are not recovered correctly
  69  */
  70
  71 #include <sys/param.h>
  72 #include <sys/systm.h>
  73 #include <sys/malloc.h>
  74 #include <sys/kernel.h>
  75 #if defined(__NetBSD__) || defined(__OpenBSD__)
  76 #include <sys/device.h>
  77 #include <sys/select.h>
  78 #elif defined(__FreeBSD__) || defined(__DragonFly__)
  79 #include <sys/endian.h>
  80 #include <sys/module.h>
  81 #include <sys/bus.h>
  82 #include <machine/bus_pio.h>
  83 #include <machine/bus_memio.h>
  84 #include <sys/lock.h>
  85 #if defined(DIAGNOSTIC) && defined(__i386__)
  86 #include <machine/cpu.h>
  87 #endif
  88 #endif
  89 #include <sys/proc.h>
  90 #include <sys/queue.h>
  91 #include <sys/sysctl.h>
  92 #include <sys/thread2.h>
  93
  94 #include <machine/bus.h>
  95 #include <machine/endian.h>
  96
  97 #include <bus/usb/usb.h>
  98 #include <bus/usb/usbdi.h>
  99 #include <bus/usb/usbdivar.h>
 100 #include <bus/usb/usb_mem.h>
 101 #include <bus/usb/usb_quirks.h>
 102
 103 #include <bus/usb/ehcireg.h>
 104 #include <bus/usb/ehcivar.h>
 105
 106 #if defined(__FreeBSD__) || defined(__DragonFly__)
 107 #include <machine/clock.h>
 108
 109 #define delay(d)                DELAY(d)
 110 #endif
 111
 112 #ifdef USB_DEBUG
 113 #define DPRINTF(x)      do { if (ehcidebug) logprintf x; } while(0)
 114 #define DPRINTFN(n,x)   do { if (ehcidebug>(n)) logprintf x; } while(0)
 115 int ehcidebug = 0;
 116 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
 117 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
 118            &ehcidebug, 0, "ehci debug level");
 119 #ifndef __NetBSD__
 120 #define bitmask_snprintf(q,f,b,l) snprintf((b), (l), "%b", (q), (f))
 121 #endif
 122 #else
 123 #define DPRINTF(x)
 124 #define DPRINTFN(n,x)
 125 #endif
 126
 127 struct ehci_pipe {
 128         struct usbd_pipe pipe;
 129         int nexttoggle;
 130
 131         ehci_soft_qh_t *sqh;
 132         union {
 133                 ehci_soft_qtd_t *qtd;
 134                 /* ehci_soft_itd_t *itd; */
 135         } tail;
 136         union {
 137                 /* Control pipe */
 138                 struct {
 139                         usb_dma_t reqdma;
 140                         u_int length;
 141                         /*ehci_soft_qtd_t *setup, *data, *stat;*/
 142                 } ctl;
 143                 /* Interrupt pipe */
 144                 struct {
 145                         u_int length;
 146                 } intr;
 147                 /* Bulk pipe */
 148                 struct {
 149                         u_int length;
 150                 } bulk;
 151                 /* Iso pipe */
 152                 /* XXX */
 153         } u;
 154 };
 155
 156 #if defined(__NetBSD__) || defined(__OpenBSD__)
 157 Static void             ehci_shutdown(void *);
 158 Static void             ehci_power(int, void *);
 159 #endif
 160
 161 Static usbd_status      ehci_open(usbd_pipe_handle);
 162 Static void             ehci_poll(struct usbd_bus *);
 163 Static void             ehci_softintr(void *);
 164 Static int              ehci_intr1(ehci_softc_t *);
 165 Static void             ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
 166 Static void             ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
 167 Static void             ehci_idone(struct ehci_xfer *);
 168 Static void             ehci_timeout(void *);
 169 Static void             ehci_timeout_task(void *);
 170
 171 Static usbd_status      ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
 172 Static void             ehci_freem(struct usbd_bus *, usb_dma_t *);
 173
 174 Static usbd_xfer_handle ehci_allocx(struct usbd_bus *);
 175 Static void             ehci_freex(struct usbd_bus *, usbd_xfer_handle);
 176
 177 Static usbd_status      ehci_root_ctrl_transfer(usbd_xfer_handle);
 178 Static usbd_status      ehci_root_ctrl_start(usbd_xfer_handle);
 179 Static void             ehci_root_ctrl_abort(usbd_xfer_handle);
 180 Static void             ehci_root_ctrl_close(usbd_pipe_handle);
 181 Static void             ehci_root_ctrl_done(usbd_xfer_handle);
 182
 183 Static usbd_status      ehci_root_intr_transfer(usbd_xfer_handle);
 184 Static usbd_status      ehci_root_intr_start(usbd_xfer_handle);
 185 Static void             ehci_root_intr_abort(usbd_xfer_handle);
 186 Static void             ehci_root_intr_close(usbd_pipe_handle);
 187 Static void             ehci_root_intr_done(usbd_xfer_handle);
 188
 189 Static usbd_status      ehci_device_ctrl_transfer(usbd_xfer_handle);
 190 Static usbd_status      ehci_device_ctrl_start(usbd_xfer_handle);
 191 Static void             ehci_device_ctrl_abort(usbd_xfer_handle);
 192 Static void             ehci_device_ctrl_close(usbd_pipe_handle);
 193 Static void             ehci_device_ctrl_done(usbd_xfer_handle);
 194
 195 Static usbd_status      ehci_device_bulk_transfer(usbd_xfer_handle);
 196 Static usbd_status      ehci_device_bulk_start(usbd_xfer_handle);
 197 Static void             ehci_device_bulk_abort(usbd_xfer_handle);
 198 Static void             ehci_device_bulk_close(usbd_pipe_handle);
 199 Static void             ehci_device_bulk_done(usbd_xfer_handle);
 200
 201 Static usbd_status      ehci_device_intr_transfer(usbd_xfer_handle);
 202 Static usbd_status      ehci_device_intr_start(usbd_xfer_handle);
 203 Static void             ehci_device_intr_abort(usbd_xfer_handle);
 204 Static void             ehci_device_intr_close(usbd_pipe_handle);
 205 Static void             ehci_device_intr_done(usbd_xfer_handle);
 206
 207 Static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle);
 208 Static usbd_status      ehci_device_isoc_start(usbd_xfer_handle);
 209 Static void             ehci_device_isoc_abort(usbd_xfer_handle);
 210 Static void             ehci_device_isoc_close(usbd_pipe_handle);
 211 Static void             ehci_device_isoc_done(usbd_xfer_handle);
 212
 213 Static void             ehci_device_clear_toggle(usbd_pipe_handle pipe);
 214 Static void             ehci_noop(usbd_pipe_handle pipe);
 215
 216 Static int              ehci_str(usb_string_descriptor_t *, int, char *);
 217 Static void             ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
 218 Static void             ehci_pcd_able(ehci_softc_t *, int);
 219 Static void             ehci_pcd_enable(void *);
 220 Static void             ehci_disown(ehci_softc_t *, int, int);
 221
 222 Static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
 223 Static void             ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
 224
 225 Static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
 226 Static void             ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
 227 Static usbd_status      ehci_alloc_sqtd_chain(struct ehci_pipe *,
 228                             ehci_softc_t *, int, int, usbd_xfer_handle,
 229                             ehci_soft_qtd_t **, ehci_soft_qtd_t **);
 230 Static void             ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qtd_t *,
 231                                             ehci_soft_qtd_t *);
 232
 233 Static usbd_status      ehci_device_request(usbd_xfer_handle xfer);
 234
 235 Static usbd_status      ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
 236                                             int ival);
 237
 238 Static void             ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
 239 Static void             ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
 240                                     ehci_soft_qh_t *);
 241 Static void             ehci_set_qh_qtd(ehci_soft_qh_t *, ehci_soft_qtd_t *);
 242 Static void             ehci_sync_hc(ehci_softc_t *);
 243
 244 Static void             ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
 245 Static void             ehci_abort_xfer(usbd_xfer_handle, usbd_status);
 246
 247 #ifdef USB_DEBUG
 248 Static void             ehci_dump_regs(ehci_softc_t *);
 249 void                    ehci_dump(void);
 250 Static ehci_softc_t     *theehci;
 251 Static void             ehci_dump_link(ehci_link_t, int);
 252 Static void             ehci_dump_sqtds(ehci_soft_qtd_t *);
 253 Static void             ehci_dump_sqtd(ehci_soft_qtd_t *);
 254 Static void             ehci_dump_qtd(ehci_qtd_t *);
 255 Static void             ehci_dump_sqh(ehci_soft_qh_t *);
 256 #ifdef DIAGNOSTIC
 257 Static void             ehci_dump_exfer(struct ehci_xfer *);
 258 #endif
 259 #endif
 260
 261 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
 262
 263 #define EHCI_INTR_ENDPT 1
 264
 265 #define ehci_add_intr_list(sc, ex) \
 266         LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext);
 267 #define ehci_del_intr_list(ex) \
 268         do { \
 269                 LIST_REMOVE((ex), inext); \
 270                 (ex)->inext.le_prev = NULL; \
 271         } while (0)
 272 #define ehci_active_intr_list(ex) ((ex)->inext.le_prev != NULL)
 273
 274 Static struct usbd_bus_methods ehci_bus_methods = {
 275         ehci_open,
 276         ehci_softintr,
 277         ehci_poll,
 278         ehci_allocm,
 279         ehci_freem,
 280         ehci_allocx,
 281         ehci_freex,
 282 };
 283
 284 Static struct usbd_pipe_methods ehci_root_ctrl_methods = {
 285         ehci_root_ctrl_transfer,
 286         ehci_root_ctrl_start,
 287         ehci_root_ctrl_abort,
 288         ehci_root_ctrl_close,
 289         ehci_noop,
 290         ehci_root_ctrl_done,
 291 };
 292
 293 Static struct usbd_pipe_methods ehci_root_intr_methods = {
 294         ehci_root_intr_transfer,
 295         ehci_root_intr_start,
 296         ehci_root_intr_abort,
 297         ehci_root_intr_close,
 298         ehci_noop,
 299         ehci_root_intr_done,
 300 };
 301
 302 Static struct usbd_pipe_methods ehci_device_ctrl_methods = {
 303         ehci_device_ctrl_transfer,
 304         ehci_device_ctrl_start,
 305         ehci_device_ctrl_abort,
 306         ehci_device_ctrl_close,
 307         ehci_noop,
 308         ehci_device_ctrl_done,
 309 };
 310
 311 Static struct usbd_pipe_methods ehci_device_intr_methods = {
 312         ehci_device_intr_transfer,
 313         ehci_device_intr_start,
 314         ehci_device_intr_abort,
 315         ehci_device_intr_close,
 316         ehci_device_clear_toggle,
 317         ehci_device_intr_done,
 318 };
 319
 320 Static struct usbd_pipe_methods ehci_device_bulk_methods = {
 321         ehci_device_bulk_transfer,
 322         ehci_device_bulk_start,
 323         ehci_device_bulk_abort,
 324         ehci_device_bulk_close,
 325         ehci_device_clear_toggle,
 326         ehci_device_bulk_done,
 327 };
 328
 329 Static struct usbd_pipe_methods ehci_device_isoc_methods = {
 330         ehci_device_isoc_transfer,
 331         ehci_device_isoc_start,
 332         ehci_device_isoc_abort,
 333         ehci_device_isoc_close,
 334         ehci_noop,
 335         ehci_device_isoc_done,
 336 };
 337
 338 usbd_status
 339 ehci_init(ehci_softc_t *sc)
 340 {
 341         u_int32_t version, sparams, cparams, hcr;
 342         u_int i;
 343         usbd_status err;
 344         ehci_soft_qh_t *sqh;
 345
 346         DPRINTF(("ehci_init: start\n"));
 347 #ifdef USB_DEBUG
 348         theehci = sc;
 349 #endif
 350
 351         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 352
 353         version = EREAD2(sc, EHCI_HCIVERSION);
 354         printf("%s: EHCI version %x.%x\n", USBDEVNAME(sc->sc_bus.bdev),
 355                version >> 8, version & 0xff);
 356
 357         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 358         DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
 359         sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
 360         if (EHCI_HCS_N_CC(sparams) != sc->sc_ncomp) {
 361                 printf("%s: wrong number of companions (%d != %d)\n",
 362                        USBDEVNAME(sc->sc_bus.bdev),
 363                        EHCI_HCS_N_CC(sparams), sc->sc_ncomp);
 364                 return (USBD_IOERROR);
 365         }
 366         if (sc->sc_ncomp > 0) {
 367                 printf("%s: companion controller%s, %d port%s each:",
 368                     USBDEVNAME(sc->sc_bus.bdev), sc->sc_ncomp!=1 ? "s" : "",
 369                     EHCI_HCS_N_PCC(sparams),
 370                     EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
 371                 for (i = 0; i < sc->sc_ncomp; i++)
 372                         printf(" %s", USBDEVNAME(sc->sc_comps[i]->bdev));
 373                 printf("\n");
 374         }
 375         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 376         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 377         DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
 378
 379         if (EHCI_HCC_64BIT(cparams)) {
 380                 /* MUST clear segment register if 64 bit capable. */
 381                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 382         }
 383
 384         sc->sc_bus.usbrev = USBREV_2_0;
 385
 386         /* Reset the controller */
 387         EOWRITE4(sc, EHCI_USBINTR, 0);  /* disable interrupts */
 388         DPRINTF(("%s: resetting\n", USBDEVNAME(sc->sc_bus.bdev)));
 389         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 390         usb_delay_ms(&sc->sc_bus, 1);
 391         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 392         for (i = 0; i < 100; i++) {
 393                 usb_delay_ms(&sc->sc_bus, 1);
 394                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 395                 if (!hcr)
 396                         break;
 397         }
 398         if (hcr) {
 399                 printf("%s: reset timeout\n",
 400                     USBDEVNAME(sc->sc_bus.bdev));
 401                 return (USBD_IOERROR);
 402         }
 403
 404         /* XXX need proper intr scheduling */
 405         sc->sc_rand = 96;
 406
 407         /* frame list size at default, read back what we got and use that */
 408         switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
 409         case 0: sc->sc_flsize = 1024; break;
 410         case 1: sc->sc_flsize = 512; break;
 411         case 2: sc->sc_flsize = 256; break;
 412         case 3: return (USBD_IOERROR);
 413         }
 414         err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 415                            EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 416         if (err)
 417                 return (err);
 418         DPRINTF(("%s: flsize=%d\n", USBDEVNAME(sc->sc_bus.bdev),sc->sc_flsize));
 419         sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
 420         EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 421
 422         /* Set up the bus struct. */
 423         sc->sc_bus.methods = &ehci_bus_methods;
 424         sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
 425
 426 #if defined(__NetBSD__) || defined(__OpenBSD__)
 427         sc->sc_powerhook = powerhook_establish(ehci_power, sc);
 428         sc->sc_shutdownhook = shutdownhook_establish(ehci_shutdown, sc);
 429 #endif
 430
 431         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 432
 433         /*
 434          * Allocate the interrupt dummy QHs. These are arranged to give poll
 435          * intervals that are powers of 2 times 1ms.
 436          */
 437         for (i = 0; i < EHCI_INTRQHS; i++) {
 438                 sqh = ehci_alloc_sqh(sc);
 439                 if (sqh == NULL) {
 440                         err = USBD_NOMEM;
 441                         goto bad1;
 442                 }
 443                 sc->sc_islots[i].sqh = sqh;
 444         }
 445         for (i = 0; i < EHCI_INTRQHS; i++) {
 446                 sqh = sc->sc_islots[i].sqh;
 447                 if (i == 0) {
 448                         /* The last (1ms) QH terminates. */
 449                         sqh->qh.qh_link = EHCI_NULL;
 450                         sqh->next = NULL;
 451                 } else {
 452                         /* Otherwise the next QH has half the poll interval */
 453                         sqh->next = sc->sc_islots[(i + 1) / 2 - 1].sqh;
 454                         sqh->qh.qh_link = htole32(sqh->next->physaddr |
 455                                                   EHCI_LINK_QH);
 456                 }
 457                 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 458                 sqh->qh.qh_link = EHCI_NULL;
 459                 sqh->qh.qh_curqtd = EHCI_NULL;
 460                 sqh->next = NULL;
 461                 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 462                 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 463                 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 464                 sqh->sqtd = NULL;
 465         }
 466         /* Point the frame list at the last level (128ms). */
 467         for (i = 0; i < sc->sc_flsize; i++) {
 468                 sc->sc_flist[i] = htole32(EHCI_LINK_QH |
 469                     sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
 470                     i)].sqh->physaddr);
 471         }
 472
 473         /* Allocate dummy QH that starts the async list. */
 474         sqh = ehci_alloc_sqh(sc);
 475         if (sqh == NULL) {
 476                 err = USBD_NOMEM;
 477                 goto bad1;
 478         }
 479         /* Fill the QH */
 480         sqh->qh.qh_endp =
 481             htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 482         sqh->qh.qh_link =
 483             htole32(sqh->physaddr | EHCI_LINK_QH);
 484         sqh->qh.qh_curqtd = EHCI_NULL;
 485         sqh->next = NULL;
 486         /* Fill the overlay qTD */
 487         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 488         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 489         sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 490         sqh->sqtd = NULL;
 491 #ifdef USB_DEBUG
 492         if (ehcidebug) {
 493                 ehci_dump_sqh(sqh);
 494         }
 495 #endif
 496
 497         /* Point to async list */
 498         sc->sc_async_head = sqh;
 499         EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
 500
 501         usb_callout_init(sc->sc_tmo_pcd);
 502
 503         lockinit(&sc->sc_doorbell_lock, 0, "ehcidb", 0, 0);
 504
 505         /* Turn on controller */
 506         EOWRITE4(sc, EHCI_USBCMD,
 507                  EHCI_CMD_ITC_8 | /* 8 microframes */
 508                  (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 509                  EHCI_CMD_ASE |
 510                  EHCI_CMD_PSE |
 511                  EHCI_CMD_RS);
 512
 513         /* Take over port ownership */
 514         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 515
 516         for (i = 0; i < 100; i++) {
 517                 usb_delay_ms(&sc->sc_bus, 1);
 518                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 519                 if (!hcr)
 520                         break;
 521         }
 522         if (hcr) {
 523                 printf("%s: run timeout\n", USBDEVNAME(sc->sc_bus.bdev));
 524                 return (USBD_IOERROR);
 525         }
 526
 527         return (USBD_NORMAL_COMPLETION);
 528
 529 #if 0
 530  bad2:
 531         ehci_free_sqh(sc, sc->sc_async_head);
 532 #endif
 533  bad1:
 534         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 535         return (err);
 536 }
 537
 538 void
 539 ehci_init_intrs(ehci_softc_t *sc)
 540 {
 541         /* Enable interrupts */
 542         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 543 }
 544
 545 int
 546 ehci_intr(void *v)
 547 {
 548         ehci_softc_t *sc = v;
 549
 550         if (sc == NULL || sc->sc_dying)
 551                 return (0);
 552
 553         /* If we get an interrupt while polling, then just ignore it. */
 554         if (sc->sc_bus.use_polling) {
 555                 u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 556
 557                 if (intrs)
 558                         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 559 #ifdef DIAGNOSTIC
 560                 DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
 561 #endif
 562                 return (0);
 563         }
 564
 565         return (ehci_intr1(sc));
 566 }
 567
 568 Static int
 569 ehci_intr1(ehci_softc_t *sc)
 570 {
 571         u_int32_t intrs, eintrs;
 572
 573         DPRINTFN(20,("ehci_intr1: enter\n"));
 574
 575         /* In case the interrupt occurs before initialization has completed. */
 576         if (sc == NULL) {
 577 #ifdef DIAGNOSTIC
 578                 printf("ehci_intr1: sc == NULL\n");
 579 #endif
 580                 return (0);
 581         }
 582
 583         intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 584         if (!intrs)
 585                 return (0);
 586
 587         eintrs = intrs & sc->sc_eintrs;
 588         DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
 589                      sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
 590                      (u_int)eintrs));
 591         if (!eintrs)
 592                 return (0);
 593
 594         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 595         sc->sc_bus.intr_context++;
 596         sc->sc_bus.no_intrs++;
 597         if (eintrs & EHCI_STS_IAA) {
 598                 DPRINTF(("ehci_intr1: door bell\n"));
 599                 wakeup(&sc->sc_async_head);
 600                 eintrs &= ~EHCI_STS_IAA;
 601         }
 602         if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
 603                 DPRINTFN(5,("ehci_intr1: %s %s\n",
 604                             eintrs & EHCI_STS_INT ? "INT" : "",
 605                             eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
 606                 usb_schedsoftintr(&sc->sc_bus);
 607                 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
 608         }
 609         if (eintrs & EHCI_STS_HSE) {
 610                 printf("%s: unrecoverable error, controller halted\n",
 611                        USBDEVNAME(sc->sc_bus.bdev));
 612                 /* XXX what else */
 613         }
 614         if (eintrs & EHCI_STS_PCD) {
 615                 ehci_pcd(sc, sc->sc_intrxfer);
 616                 /*
 617                  * Disable PCD interrupt for now, because it will be
 618                  * on until the port has been reset.
 619                  */
 620                 ehci_pcd_able(sc, 0);
 621                 /* Do not allow RHSC interrupts > 1 per second */
 622                 usb_callout(sc->sc_tmo_pcd, hz, ehci_pcd_enable, sc);
 623                 eintrs &= ~EHCI_STS_PCD;
 624         }
 625
 626         sc->sc_bus.intr_context--;
 627
 628         if (eintrs != 0) {
 629                 /* Block unprocessed interrupts. */
 630                 sc->sc_eintrs &= ~eintrs;
 631                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 632                 printf("%s: blocking intrs 0x%x\n",
 633                        USBDEVNAME(sc->sc_bus.bdev), eintrs);
 634         }
 635
 636         return (1);
 637 }
 638
 639 void
 640 ehci_pcd_able(ehci_softc_t *sc, int on)
 641 {
 642         DPRINTFN(4, ("ehci_pcd_able: on=%d\n", on));
 643         if (on)
 644                 sc->sc_eintrs |= EHCI_STS_PCD;
 645         else
 646                 sc->sc_eintrs &= ~EHCI_STS_PCD;
 647         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 648 }
 649
 650 void
 651 ehci_pcd_enable(void *v_sc)
 652 {
 653         ehci_softc_t *sc = v_sc;
 654
 655         ehci_pcd_able(sc, 1);
 656 }
 657
 658 void
 659 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 660 {
 661         usbd_pipe_handle pipe;
 662         u_char *p;
 663         int i, m;
 664
 665         if (xfer == NULL) {
 666                 /* Just ignore the change. */
 667                 return;
 668         }
 669
 670         pipe = xfer->pipe;
 671
 672         p = KERNADDR(&xfer->dmabuf, 0);
 673         m = min(sc->sc_noport, xfer->length * 8 - 1);
 674         memset(p, 0, xfer->length);
 675         for (i = 1; i <= m; i++) {
 676                 /* Pick out CHANGE bits from the status reg. */
 677                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
 678                         p[i/8] |= 1 << (i%8);
 679         }
 680         DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
 681         xfer->actlen = xfer->length;
 682         xfer->status = USBD_NORMAL_COMPLETION;
 683
 684         usb_transfer_complete(xfer);
 685 }
 686
 687 void
 688 ehci_softintr(void *v)
 689 {
 690         ehci_softc_t *sc = v;
 691         struct ehci_xfer *ex, *nextex;
 692
 693         DPRINTFN(10,("%s: ehci_softintr (%d)\n", USBDEVNAME(sc->sc_bus.bdev),
 694                      sc->sc_bus.intr_context));
 695
 696         sc->sc_bus.intr_context++;
 697
 698         /*
 699          * The only explanation I can think of for why EHCI is as brain dead
 700          * as UHCI interrupt-wise is that Intel was involved in both.
 701          * An interrupt just tells us that something is done, we have no
 702          * clue what, so we need to scan through all active transfers. :-(
 703          */
 704         for (ex = LIST_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
 705                 nextex = LIST_NEXT(ex, inext);
 706                 ehci_check_intr(sc, ex);
 707         }
 708
 709 #ifdef USB_USE_SOFTINTR
 710         if (sc->sc_softwake) {
 711                 sc->sc_softwake = 0;
 712                 wakeup(&sc->sc_softwake);
 713         }
 714 #endif /* USB_USE_SOFTINTR */
 715
 716         sc->sc_bus.intr_context--;
 717 }
 718
 719 /* Check for an interrupt. */
 720 void
 721 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
 722 {
 723         ehci_soft_qtd_t *sqtd, *lsqtd;
 724         u_int32_t status;
 725
 726         DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
 727
 728         if (ex->sqtdstart == NULL) {
 729                 printf("ehci_check_intr: sqtdstart=NULL\n");
 730                 return;
 731         }
 732         lsqtd = ex->sqtdend;
 733 #ifdef DIAGNOSTIC
 734         if (lsqtd == NULL) {
 735                 printf("ehci_check_intr: sqtd==0\n");
 736                 return;
 737         }
 738 #endif
 739         /*
 740          * If the last TD is still active we need to check whether there
 741          * is a an error somewhere in the middle, or whether there was a
 742          * short packet (SPD and not ACTIVE).
 743          */
 744         if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
 745                 DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
 746                 for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
 747                         status = le32toh(sqtd->qtd.qtd_status);
 748                         /* If there's an active QTD the xfer isn't done. */
 749                         if (status & EHCI_QTD_ACTIVE)
 750                                 break;
 751                         /* Any kind of error makes the xfer done. */
 752                         if (status & EHCI_QTD_HALTED)
 753                                 goto done;
 754                         /* We want short packets, and it is short: it's done */
 755                         if (EHCI_QTD_GET_BYTES(status) != 0)
 756                                 goto done;
 757                 }
 758                 DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
 759                               ex, ex->sqtdstart));
 760                 return;
 761         }
 762  done:
 763         DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
 764         usb_uncallout(ex->xfer.timeout_handle, ehci_timeout, ex);
 765         ehci_idone(ex);
 766 }
 767
 768 void
 769 ehci_idone(struct ehci_xfer *ex)
 770 {
 771         usbd_xfer_handle xfer = &ex->xfer;
 772         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
 773         ehci_soft_qtd_t *sqtd;
 774         u_int32_t status = 0, nstatus;
 775         int actlen;
 776
 777         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
 778 #ifdef DIAGNOSTIC
 779         {
 780                 crit_enter();
 781                 if (ex->isdone) {
 782                         crit_exit();
 783 #ifdef USB_DEBUG
 784                         printf("ehci_idone: ex is done!\n   ");
 785                         ehci_dump_exfer(ex);
 786 #else
 787                         printf("ehci_idone: ex=%p is done!\n", ex);
 788 #endif
 789                         return;
 790                 }
 791                 ex->isdone = 1;
 792                 crit_exit();
 793         }
 794 #endif
 795
 796         if (xfer->status == USBD_CANCELLED ||
 797             xfer->status == USBD_TIMEOUT) {
 798                 DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
 799                 return;
 800         }
 801
 802 #ifdef USB_DEBUG
 803         DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
 804         if (ehcidebug > 10)
 805                 ehci_dump_sqtds(ex->sqtdstart);
 806 #endif
 807
 808         /* The transfer is done, compute actual length and status. */
 809         actlen = 0;
 810         for (sqtd = ex->sqtdstart; sqtd != NULL; sqtd = sqtd->nextqtd) {
 811                 nstatus = le32toh(sqtd->qtd.qtd_status);
 812                 if (nstatus & EHCI_QTD_ACTIVE)
 813                         break;
 814
 815                 status = nstatus;
 816                 /* halt is ok if descriptor is last, and complete */
 817                 if(sqtd->qtd.qtd_next == EHCI_NULL &&
 818                    EHCI_QTD_GET_BYTES(status) == 0)
 819                         status &= ~EHCI_QTD_HALTED;
 820                 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
 821                         actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
 822         }
 823
 824         /* If there are left over TDs we need to update the toggle. */
 825         if (sqtd != NULL) {
 826                 DPRINTF(("ehci_idone: need toggle update status=%08x nstatus=%08x\n", status, nstatus));
 827 #if 0
 828                 ehci_dump_sqh(epipe->sqh);
 829                 ehci_dump_sqtds(ex->sqtdstart);
 830 #endif
 831                 epipe->nexttoggle = EHCI_QTD_GET_TOGGLE(nstatus);
 832         }
 833
 834         status &= EHCI_QTD_STATERRS;
 835         DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, status=0x%x\n",
 836                            xfer->length, actlen, status));
 837         xfer->actlen = actlen;
 838         if (status != 0) {
 839 #ifdef USB_DEBUG
 840                 char sbuf[128];
 841
 842                 bitmask_snprintf((u_int32_t)status,
 843                                  "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR"
 844                                  "\3MISSED", sbuf, sizeof(sbuf));
 845
 846                 DPRINTFN((status & EHCI_QTD_HALTED) ? 2 : 0,
 847                          ("ehci_idone: error, addr=%d, endpt=0x%02x, "
 848                           "status 0x%s\n",
 849                           xfer->pipe->device->address,
 850                           xfer->pipe->endpoint->edesc->bEndpointAddress,
 851                           sbuf));
 852                 if (ehcidebug > 2) {
 853                         ehci_dump_sqh(epipe->sqh);
 854                         ehci_dump_sqtds(ex->sqtdstart);
 855                 }
 856 #endif
 857                 if (status == EHCI_QTD_HALTED)
 858                         xfer->status = USBD_STALLED;
 859                 else
 860                         xfer->status = USBD_IOERROR; /* more info XXX */
 861         } else {
 862                 xfer->status = USBD_NORMAL_COMPLETION;
 863         }
 864
 865         usb_transfer_complete(xfer);
 866         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
 867 }
 868
 869 /*
 870  * Wait here until controller claims to have an interrupt.
 871  * Then call ehci_intr and return.  Use timeout to avoid waiting
 872  * too long.
 873  */
 874 void
 875 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
 876 {
 877         int timo = xfer->timeout;
 878         int usecs;
 879         u_int32_t intrs;
 880
 881         xfer->status = USBD_IN_PROGRESS;
 882         for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) {
 883                 usb_delay_ms(&sc->sc_bus, 1);
 884                 if (sc->sc_dying)
 885                         break;
 886                 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
 887                         sc->sc_eintrs;
 888                 DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
 889 #ifdef USB_DEBUG
 890                 if (ehcidebug > 15)
 891                         ehci_dump_regs(sc);
 892 #endif
 893                 if (intrs) {
 894                         ehci_intr1(sc);
 895                         if (xfer->status != USBD_IN_PROGRESS)
 896                                 return;
 897                 }
 898         }
 899
 900         /* Timeout */
 901         DPRINTF(("ehci_waitintr: timeout\n"));
 902         xfer->status = USBD_TIMEOUT;
 903         usb_transfer_complete(xfer);
 904         /* XXX should free TD */
 905 }
 906
 907 void
 908 ehci_poll(struct usbd_bus *bus)
 909 {
 910         ehci_softc_t *sc = (ehci_softc_t *)bus;
 911 #ifdef USB_DEBUG
 912         static int last;
 913         int new;
 914         new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 915         if (new != last) {
 916                 DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
 917                 last = new;
 918         }
 919 #endif
 920
 921         if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
 922                 ehci_intr1(sc);
 923 }
 924
 925 #if defined(__NetBSD__) || defined(__OpenBSD__)
 926 int
 927 ehci_detach(struct ehci_softc *sc, int flags)
 928 {
 929         int rv = 0;
 930
 931         if (sc->sc_child != NULL)
 932                 rv = config_detach(sc->sc_child, flags);
 933
 934         if (rv != 0)
 935                 return (rv);
 936
 937         usb_uncallout(sc->sc_tmo_pcd, ehci_pcd_enable, sc);
 938
 939         if (sc->sc_powerhook != NULL)
 940                 powerhook_disestablish(sc->sc_powerhook);
 941         if (sc->sc_shutdownhook != NULL)
 942                 shutdownhook_disestablish(sc->sc_shutdownhook);
 943
 944         usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
 945
 946         /* XXX free other data structures XXX */
 947
 948         return (rv);
 949 }
 950 #endif
 951
 952 #if defined(__NetBSD__) || defined(__OpenBSD__)
 953 int
 954 ehci_activate(device_ptr_t self, enum devact act)
 955 {
 956         struct ehci_softc *sc = (struct ehci_softc *)self;
 957         int rv = 0;
 958
 959         switch (act) {
 960         case DVACT_ACTIVATE:
 961                 return (EOPNOTSUPP);
 962
 963         case DVACT_DEACTIVATE:
 964                 if (sc->sc_child != NULL)
 965                         rv = config_deactivate(sc->sc_child);
 966                 sc->sc_dying = 1;
 967                 break;
 968         }
 969         return (rv);
 970 }
 971 #endif
 972
 973 /*
 974  * Handle suspend/resume.
 975  *
 976  * We need to switch to polling mode here, because this routine is
 977  * called from an interrupt context.  This is all right since we
 978  * are almost suspended anyway.
 979  */
 980 #if defined(__NetBSD__) || defined(__OpenBSD__)
 981 void
 982 ehci_power(int why, void *v)
 983 {
 984         ehci_softc_t *sc = v;
 985         u_int32_t cmd, hcr;
 986         int s, i;
 987
 988 #ifdef USB_DEBUG
 989         DPRINTF(("ehci_power: sc=%p, why=%d\n", sc, why));
 990         if (ehcidebug > 0)
 991                 ehci_dump_regs(sc);
 992 #endif
 993
 994         crit_enter();
 995
 996         switch (why) {
 997         case PWR_SUSPEND:
 998         case PWR_STANDBY:
 999                 sc->sc_bus.use_polling++;
1000
1001                 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
1002
1003                 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
1004                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1005
1006                 for (i = 0; i < 100; i++) {
1007                         hcr = EOREAD4(sc, EHCI_USBSTS) &
1008                             (EHCI_STS_ASS | EHCI_STS_PSS);
1009                         if (hcr == 0)
1010                                 break;
1011
1012                         usb_delay_ms(&sc->sc_bus, 1);
1013                 }
1014                 if (hcr != 0) {
1015                         printf("%s: reset timeout\n",
1016                                USBDEVNAME(sc->sc_bus.bdev));
1017                 }
1018
1019                 cmd &= ~EHCI_CMD_RS;
1020                 EOWRITE4(sc, EHCI_USBCMD, cmd);
1021
1022                 for (i = 0; i < 100; i++) {
1023                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1024                         if (hcr == EHCI_STS_HCH)
1025                                 break;
1026
1027                         usb_delay_ms(&sc->sc_bus, 1);
1028                 }
1029                 if (hcr != EHCI_STS_HCH) {
1030                         printf("%s: config timeout\n",
1031                                USBDEVNAME(sc->sc_bus.bdev));
1032                 }
1033
1034                 sc->sc_bus.use_polling--;
1035                 break;
1036
1037         case PWR_RESUME:
1038                 sc->sc_bus.use_polling++;
1039
1040                 /* restore things in case the bios sucks */
1041                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
1042                 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
1043                 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
1044                          sc->sc_async_head->physaddr | EHCI_LINK_QH);
1045                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1046
1047                 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
1048
1049                 for (i = 0; i < 100; i++) {
1050                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
1051                         if (hcr != EHCI_STS_HCH)
1052                                 break;
1053
1054                         usb_delay_ms(&sc->sc_bus, 1);
1055                 }
1056                 if (hcr == EHCI_STS_HCH) {
1057                         printf("%s: config timeout\n",
1058                                USBDEVNAME(sc->sc_bus.bdev));
1059                 }
1060
1061                 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
1062
1063                 sc->sc_bus.use_polling--;
1064                 break;
1065         case PWR_SOFTSUSPEND:
1066         case PWR_SOFTSTANDBY:
1067         case PWR_SOFTRESUME:
1068                 break;
1069         }
1070         crit_exit();
1071
1072 #ifdef USB_DEBUG
1073         DPRINTF(("ehci_power: sc=%p\n", sc));
1074         if (ehcidebug > 0)
1075                 ehci_dump_regs(sc);
1076 #endif
1077 }
1078 #endif
1079
1080 /*
1081  * Shut down the controller when the system is going down.
1082  */
1083 #if defined(__NetBSD__) || defined(__OpenBSD__)
1084 void
1085 ehci_shutdown(void *v)
1086 {
1087         ehci_softc_t *sc = v;
1088
1089         DPRINTF(("ehci_shutdown: stopping the HC\n"));
1090         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
1091         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1092 }
1093 #endif
1094
1095 usbd_status
1096 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1097 {
1098         usbd_status err;
1099
1100         err = usb_allocmem(bus, size, 0, dma);
1101 #ifdef USB_DEBUG
1102         if (err)
1103                 printf("ehci_allocm: usb_allocmem()=%d\n", err);
1104 #endif
1105         return (err);
1106 }
1107
1108 void
1109 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1110 {
1111         usb_freemem(bus, dma);
1112 }
1113
1114 usbd_xfer_handle
1115 ehci_allocx(struct usbd_bus *bus)
1116 {
1117         struct ehci_softc *sc = (struct ehci_softc *)bus;
1118         usbd_xfer_handle xfer;
1119
1120         xfer = SIMPLEQ_FIRST(&sc->sc_free_xfers);
1121         if (xfer != NULL) {
1122                 SIMPLEQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1123 #ifdef DIAGNOSTIC
1124                 if (xfer->busy_free != XFER_FREE) {
1125                         printf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1126                                xfer->busy_free);
1127                 }
1128 #endif
1129         } else {
1130                 xfer = malloc(sizeof(struct ehci_xfer), M_USB, M_INTWAIT);
1131         }
1132         if (xfer != NULL) {
1133                 memset(xfer, 0, sizeof(struct ehci_xfer));
1134 #ifdef DIAGNOSTIC
1135                 EXFER(xfer)->isdone = 1;
1136                 xfer->busy_free = XFER_BUSY;
1137 #endif
1138         }
1139         return (xfer);
1140 }
1141
1142 void
1143 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1144 {
1145         struct ehci_softc *sc = (struct ehci_softc *)bus;
1146
1147 #ifdef DIAGNOSTIC
1148         if (xfer->busy_free != XFER_BUSY) {
1149                 printf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1150                        xfer->busy_free);
1151                 return;
1152         }
1153         xfer->busy_free = XFER_FREE;
1154         if (!EXFER(xfer)->isdone) {
1155                 printf("ehci_freex: !isdone\n");
1156                 return;
1157         }
1158 #endif
1159         SIMPLEQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1160 }
1161
1162 Static void
1163 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1164 {
1165         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1166
1167         DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1168                  epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1169 #ifdef USB_DEBUG
1170         if (ehcidebug)
1171                 usbd_dump_pipe(pipe);
1172 #endif
1173         epipe->nexttoggle = 0;
1174 }
1175
1176 Static void
1177 ehci_noop(usbd_pipe_handle pipe)
1178 {
1179 }
1180
1181 #ifdef USB_DEBUG
1182 void
1183 ehci_dump_regs(ehci_softc_t *sc)
1184 {
1185         int i;
1186         printf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1187                EOREAD4(sc, EHCI_USBCMD),
1188                EOREAD4(sc, EHCI_USBSTS),
1189                EOREAD4(sc, EHCI_USBINTR));
1190         printf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1191                EOREAD4(sc, EHCI_FRINDEX),
1192                EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1193                EOREAD4(sc, EHCI_PERIODICLISTBASE),
1194                EOREAD4(sc, EHCI_ASYNCLISTADDR));
1195         for (i = 1; i <= sc->sc_noport; i++)
1196                 printf("port %d status=0x%08x\n", i,
1197                        EOREAD4(sc, EHCI_PORTSC(i)));
1198 }
1199
1200 /*
1201  * Unused function - this is meant to be called from a kernel
1202  * debugger.
1203  */
1204 void
1205 ehci_dump()
1206 {
1207         ehci_dump_regs(theehci);
1208 }
1209
1210 void
1211 ehci_dump_link(ehci_link_t link, int type)
1212 {
1213         link = le32toh(link);
1214         printf("0x%08x", link);
1215         if (link & EHCI_LINK_TERMINATE)
1216                 printf("<T>");
1217         else {
1218                 printf("<");
1219                 if (type) {
1220                         switch (EHCI_LINK_TYPE(link)) {
1221                         case EHCI_LINK_ITD: printf("ITD"); break;
1222                         case EHCI_LINK_QH: printf("QH"); break;
1223                         case EHCI_LINK_SITD: printf("SITD"); break;
1224                         case EHCI_LINK_FSTN: printf("FSTN"); break;
1225                         }
1226                 }
1227                 printf(">");
1228         }
1229 }
1230
1231 void
1232 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1233 {
1234         int i;
1235         u_int32_t stop;
1236
1237         stop = 0;
1238         for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1239                 ehci_dump_sqtd(sqtd);
1240                 stop = sqtd->qtd.qtd_next & EHCI_LINK_TERMINATE;
1241         }
1242         if (sqtd)
1243                 printf("dump aborted, too many TDs\n");
1244 }
1245
1246 void
1247 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1248 {
1249         printf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1250         ehci_dump_qtd(&sqtd->qtd);
1251 }
1252
1253 void
1254 ehci_dump_qtd(ehci_qtd_t *qtd)
1255 {
1256         u_int32_t s;
1257         char sbuf[128];
1258
1259         printf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1260         printf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1261         printf("\n");
1262         s = le32toh(qtd->qtd_status);
1263         bitmask_snprintf(EHCI_QTD_GET_STATUS(s),
1264                          "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1265                          "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
1266         printf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1267                s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1268                EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1269         printf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1270                EHCI_QTD_GET_PID(s), sbuf);
1271         for (s = 0; s < 5; s++)
1272                 printf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1273 }
1274
1275 void
1276 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1277 {
1278         ehci_qh_t *qh = &sqh->qh;
1279         u_int32_t endp, endphub;
1280
1281         printf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1282         printf("  link="); ehci_dump_link(qh->qh_link, 1); printf("\n");
1283         endp = le32toh(qh->qh_endp);
1284         printf("  endp=0x%08x\n", endp);
1285         printf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1286                EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1287                EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1288                EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1289         printf("    mpl=0x%x ctl=%d nrl=%d\n",
1290                EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1291                EHCI_QH_GET_NRL(endp));
1292         endphub = le32toh(qh->qh_endphub);
1293         printf("  endphub=0x%08x\n", endphub);
1294         printf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1295                EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1296                EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1297                EHCI_QH_GET_MULT(endphub));
1298         printf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); printf("\n");
1299         printf("Overlay qTD:\n");
1300         ehci_dump_qtd(&qh->qh_qtd);
1301 }
1302
1303 #ifdef DIAGNOSTIC
1304 Static void
1305 ehci_dump_exfer(struct ehci_xfer *ex)
1306 {
1307         printf("ehci_dump_exfer: ex=%p\n", ex);
1308 }
1309 #endif
1310 #endif
1311
1312 usbd_status
1313 ehci_open(usbd_pipe_handle pipe)
1314 {
1315         usbd_device_handle dev = pipe->device;
1316         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
1317         usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1318         u_int8_t addr = dev->address;
1319         u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1320         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1321         ehci_soft_qh_t *sqh;
1322         usbd_status err;
1323         int ival, speed, naks;
1324
1325         DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1326                      pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1327
1328         if (sc->sc_dying)
1329                 return (USBD_IOERROR);
1330
1331         epipe->nexttoggle = 0;
1332
1333         if (addr == sc->sc_addr) {
1334                 switch (ed->bEndpointAddress) {
1335                 case USB_CONTROL_ENDPOINT:
1336                         pipe->methods = &ehci_root_ctrl_methods;
1337                         break;
1338                 case UE_DIR_IN | EHCI_INTR_ENDPT:
1339                         pipe->methods = &ehci_root_intr_methods;
1340                         break;
1341                 default:
1342                         return (USBD_INVAL);
1343                 }
1344                 return (USBD_NORMAL_COMPLETION);
1345         }
1346
1347         /* XXX All this stuff is only valid for async. */
1348         switch (dev->speed) {
1349         case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1350         case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1351         case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1352         default: panic("ehci_open: bad device speed %d", dev->speed);
1353         }
1354         naks = 8;               /* XXX */
1355         sqh = ehci_alloc_sqh(sc);
1356         if (sqh == NULL)
1357                 goto bad0;
1358         /* qh_link filled when the QH is added */
1359         sqh->qh.qh_endp = htole32(
1360                 EHCI_QH_SET_ADDR(addr) |
1361                 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1362                 EHCI_QH_SET_EPS(speed) |
1363                 EHCI_QH_DTC |
1364                 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1365                 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1366                  EHCI_QH_CTL : 0) |
1367                 EHCI_QH_SET_NRL(naks)
1368                 );
1369         sqh->qh.qh_endphub = htole32(
1370                 EHCI_QH_SET_MULT(1) |
1371                 /* XXX TT stuff */
1372                 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x01 : 0)
1373                 );
1374         sqh->qh.qh_curqtd = EHCI_NULL;
1375         /* Fill the overlay qTD */
1376         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
1377         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1378         sqh->qh.qh_qtd.qtd_status = htole32(0);
1379
1380         epipe->sqh = sqh;
1381
1382         switch (xfertype) {
1383         case UE_CONTROL:
1384                 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1385                                    0, &epipe->u.ctl.reqdma);
1386 #ifdef USB_DEBUG
1387                 if (err)
1388                         printf("ehci_open: usb_allocmem()=%d\n", err);
1389 #endif
1390                 if (err)
1391                         goto bad1;
1392                 pipe->methods = &ehci_device_ctrl_methods;
1393                 crit_enter();
1394                 ehci_add_qh(sqh, sc->sc_async_head);
1395                 crit_exit();
1396                 break;
1397         case UE_BULK:
1398                 pipe->methods = &ehci_device_bulk_methods;
1399                 crit_enter();
1400                 ehci_add_qh(sqh, sc->sc_async_head);
1401                 crit_exit();
1402                 break;
1403         case UE_INTERRUPT:
1404                 pipe->methods = &ehci_device_intr_methods;
1405                 ival = pipe->interval;
1406                 if (ival == USBD_DEFAULT_INTERVAL)
1407                         ival = ed->bInterval;
1408                 return (ehci_device_setintr(sc, sqh, ival));
1409         case UE_ISOCHRONOUS:
1410                 pipe->methods = &ehci_device_isoc_methods;
1411                 return (USBD_INVAL);
1412         default:
1413                 return (USBD_INVAL);
1414         }
1415         return (USBD_NORMAL_COMPLETION);
1416
1417  bad1:
1418         ehci_free_sqh(sc, sqh);
1419  bad0:
1420         return (USBD_NOMEM);
1421 }
1422
1423 /*
1424  * Add an ED to the schedule.  Called while in a critical section.
1425  */
1426 void
1427 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1428 {
1429         sqh->next = head->next;
1430         sqh->qh.qh_link = head->qh.qh_link;
1431         head->next = sqh;
1432         head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1433
1434 #ifdef USB_DEBUG
1435         if (ehcidebug > 5) {
1436                 printf("ehci_add_qh:\n");
1437                 ehci_dump_sqh(sqh);
1438         }
1439 #endif
1440 }
1441
1442 /*
1443  * Remove an ED from the schedule.  Called while in a critical section.
1444  */
1445 void
1446 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1447 {
1448         ehci_soft_qh_t *p;
1449
1450         /* XXX */
1451         for (p = head; p != NULL && p->next != sqh; p = p->next)
1452                 ;
1453         if (p == NULL)
1454                 panic("ehci_rem_qh: ED not found");
1455         p->next = sqh->next;
1456         p->qh.qh_link = sqh->qh.qh_link;
1457
1458         ehci_sync_hc(sc);
1459 }
1460
1461 void
1462 ehci_set_qh_qtd(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1463 {
1464         /* Halt while we are messing. */
1465         sqh->qh.qh_qtd.qtd_status |= htole32(EHCI_QTD_HALTED);
1466         sqh->qh.qh_curqtd = 0;
1467         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1468         sqh->sqtd = sqtd;
1469         /* Clear halt */
1470         sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_HALTED);
1471 }
1472
1473 /*
1474  * Ensure that the HC has released all references to the QH.  We do this
1475  * by asking for a Async Advance Doorbell interrupt and then we wait for
1476  * the interrupt.
1477  * To make this easier we first obtain exclusive use of the doorbell.
1478  */
1479 void
1480 ehci_sync_hc(ehci_softc_t *sc)
1481 {
1482         int error;
1483
1484         if (sc->sc_dying) {
1485                 DPRINTFN(2,("ehci_sync_hc: dying\n"));
1486                 return;
1487         }
1488         DPRINTFN(2,("ehci_sync_hc: enter\n"));
1489         /* get doorbell */
1490         usb_lockmgr(&sc->sc_doorbell_lock, LK_EXCLUSIVE, NULL, NULL);
1491         crit_enter();
1492         /* ask for doorbell */
1493         EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1494         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1495                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1496         error = tsleep(&sc->sc_async_head, 0, "ehcidi", hz); /* bell wait */
1497         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1498                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1499         crit_exit();
1500         /* release doorbell */
1501         usb_lockmgr(&sc->sc_doorbell_lock, LK_RELEASE, NULL, NULL);
1502 #ifdef DIAGNOSTIC
1503         if (error)
1504                 printf("ehci_sync_hc: tsleep() = %d\n", error);
1505 #endif
1506         DPRINTFN(2,("ehci_sync_hc: exit\n"));
1507 }
1508
1509 /***********/
1510
1511 /*
1512  * Data structures and routines to emulate the root hub.
1513  */
1514 Static usb_device_descriptor_t ehci_devd = {
1515         USB_DEVICE_DESCRIPTOR_SIZE,
1516         UDESC_DEVICE,           /* type */
1517         {0x00, 0x02},           /* USB version */
1518         UDCLASS_HUB,            /* class */
1519         UDSUBCLASS_HUB,         /* subclass */
1520         UDPROTO_HSHUBSTT,       /* protocol */
1521         64,                     /* max packet */
1522         {0},{0},{0x00,0x01},    /* device id */
1523         1,2,0,                  /* string indicies */
1524         1                       /* # of configurations */
1525 };
1526
1527 Static usb_device_qualifier_t ehci_odevd = {
1528         USB_DEVICE_DESCRIPTOR_SIZE,
1529         UDESC_DEVICE_QUALIFIER, /* type */
1530         {0x00, 0x02},           /* USB version */
1531         UDCLASS_HUB,            /* class */
1532         UDSUBCLASS_HUB,         /* subclass */
1533         UDPROTO_FSHUB,          /* protocol */
1534         64,                     /* max packet */
1535         1,                      /* # of configurations */
1536         0
1537 };
1538
1539 Static usb_config_descriptor_t ehci_confd = {
1540         USB_CONFIG_DESCRIPTOR_SIZE,
1541         UDESC_CONFIG,
1542         {USB_CONFIG_DESCRIPTOR_SIZE +
1543          USB_INTERFACE_DESCRIPTOR_SIZE +
1544          USB_ENDPOINT_DESCRIPTOR_SIZE},
1545         1,
1546         1,
1547         0,
1548         UC_SELF_POWERED,
1549         0                       /* max power */
1550 };
1551
1552 Static usb_interface_descriptor_t ehci_ifcd = {
1553         USB_INTERFACE_DESCRIPTOR_SIZE,
1554         UDESC_INTERFACE,
1555         0,
1556         0,
1557         1,
1558         UICLASS_HUB,
1559         UISUBCLASS_HUB,
1560         UIPROTO_HSHUBSTT,
1561         0
1562 };
1563
1564 Static usb_endpoint_descriptor_t ehci_endpd = {
1565         USB_ENDPOINT_DESCRIPTOR_SIZE,
1566         UDESC_ENDPOINT,
1567         UE_DIR_IN | EHCI_INTR_ENDPT,
1568         UE_INTERRUPT,
1569         {8, 0},                 /* max packet */
1570         255
1571 };
1572
1573 Static usb_hub_descriptor_t ehci_hubd = {
1574         USB_HUB_DESCRIPTOR_SIZE,
1575         UDESC_HUB,
1576         0,
1577         {0,0},
1578         0,
1579         0,
1580         {0},
1581 };
1582
1583 Static int
1584 ehci_str( usb_string_descriptor_t *p, int l, char *s)
1585 {
1586         int i;
1587
1588         if (l == 0)
1589                 return (0);
1590         p->bLength = 2 * strlen(s) + 2;
1591         if (l == 1)
1592                 return (1);
1593         p->bDescriptorType = UDESC_STRING;
1594         l -= 2;
1595         for (i = 0; s[i] && l > 1; i++, l -= 2)
1596                 USETW2(p->bString[i], 0, s[i]);
1597         return (2*i+2);
1598 }
1599
1600 /*
1601  * Simulate a hardware hub by handling all the necessary requests.
1602  */
1603 Static usbd_status
1604 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1605 {
1606         usbd_status err;
1607
1608         /* Insert last in queue. */
1609         err = usb_insert_transfer(xfer);
1610         if (err)
1611                 return (err);
1612
1613         /* Pipe isn't running, start first */
1614         return (ehci_root_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
1615 }
1616
1617 Static usbd_status
1618 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1619 {
1620         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
1621         usb_device_request_t *req;
1622         void *buf = NULL;
1623         int port, i;
1624         int len, value, index, l, totlen = 0;
1625         usb_port_status_t ps;
1626         usb_hub_descriptor_t hubd;
1627         usbd_status err;
1628         u_int32_t v;
1629
1630         if (sc->sc_dying)
1631                 return (USBD_IOERROR);
1632
1633 #ifdef DIAGNOSTIC
1634         if (!(xfer->rqflags & URQ_REQUEST))
1635                 /* XXX panic */
1636                 return (USBD_INVAL);
1637 #endif
1638         req = &xfer->request;
1639
1640         DPRINTFN(4,("ehci_root_ctrl_start type=0x%02x request=%02x\n",
1641                     req->bmRequestType, req->bRequest));
1642
1643         len = UGETW(req->wLength);
1644         value = UGETW(req->wValue);
1645         index = UGETW(req->wIndex);
1646
1647         if (len != 0)
1648                 buf = KERNADDR(&xfer->dmabuf, 0);
1649
1650 #define C(x,y) ((x) | ((y) << 8))
1651         switch(C(req->bRequest, req->bmRequestType)) {
1652         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1653         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1654         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1655                 /*
1656                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1657                  * for the integrated root hub.
1658                  */
1659                 break;
1660         case C(UR_GET_CONFIG, UT_READ_DEVICE):
1661                 if (len > 0) {
1662                         *(u_int8_t *)buf = sc->sc_conf;
1663                         totlen = 1;
1664                 }
1665                 break;
1666         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1667                 DPRINTFN(8,("ehci_root_ctrl_start wValue=0x%04x\n", value));
1668                 switch(value >> 8) {
1669                 case UDESC_DEVICE:
1670                         if ((value & 0xff) != 0) {
1671                                 err = USBD_IOERROR;
1672                                 goto ret;
1673                         }
1674                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1675                         USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1676                         memcpy(buf, &ehci_devd, l);
1677                         break;
1678                 /*
1679                  * We can't really operate at another speed, but the spec says
1680                  * we need this descriptor.
1681                  */
1682                 case UDESC_DEVICE_QUALIFIER:
1683                         if ((value & 0xff) != 0) {
1684                                 err = USBD_IOERROR;
1685                                 goto ret;
1686                         }
1687                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1688                         memcpy(buf, &ehci_odevd, l);
1689                         break;
1690                 /*
1691                  * We can't really operate at another speed, but the spec says
1692                  * we need this descriptor.
1693                  */
1694                 case UDESC_OTHER_SPEED_CONFIGURATION:
1695                 case UDESC_CONFIG:
1696                         if ((value & 0xff) != 0) {
1697                                 err = USBD_IOERROR;
1698                                 goto ret;
1699                         }
1700                         totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1701                         memcpy(buf, &ehci_confd, l);
1702                         ((usb_config_descriptor_t *)buf)->bDescriptorType =
1703                                 value >> 8;
1704                         buf = (char *)buf + l;
1705                         len -= l;
1706                         l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1707                         totlen += l;
1708                         memcpy(buf, &ehci_ifcd, l);
1709                         buf = (char *)buf + l;
1710                         len -= l;
1711                         l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1712                         totlen += l;
1713                         memcpy(buf, &ehci_endpd, l);
1714                         break;
1715                 case UDESC_STRING:
1716                         if (len == 0)
1717                                 break;
1718                         *(u_int8_t *)buf = 0;
1719                         totlen = 1;
1720                         switch (value & 0xff) {
1721                         case 1: /* Vendor */
1722                                 totlen = ehci_str(buf, len, sc->sc_vendor);
1723                                 break;
1724                         case 2: /* Product */
1725                                 totlen = ehci_str(buf, len, "EHCI root hub");
1726                                 break;
1727                         }
1728                         break;
1729                 default:
1730                         err = USBD_IOERROR;
1731                         goto ret;
1732                 }
1733                 break;
1734         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1735                 if (len > 0) {
1736                         *(u_int8_t *)buf = 0;
1737                         totlen = 1;
1738                 }
1739                 break;
1740         case C(UR_GET_STATUS, UT_READ_DEVICE):
1741                 if (len > 1) {
1742                         USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1743                         totlen = 2;
1744                 }
1745                 break;
1746         case C(UR_GET_STATUS, UT_READ_INTERFACE):
1747         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1748                 if (len > 1) {
1749                         USETW(((usb_status_t *)buf)->wStatus, 0);
1750                         totlen = 2;
1751                 }
1752                 break;
1753         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1754                 if (value >= USB_MAX_DEVICES) {
1755                         err = USBD_IOERROR;
1756                         goto ret;
1757                 }
1758                 sc->sc_addr = value;
1759                 break;
1760         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1761                 if (value != 0 && value != 1) {
1762                         err = USBD_IOERROR;
1763                         goto ret;
1764                 }
1765                 sc->sc_conf = value;
1766                 break;
1767         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1768                 break;
1769         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1770         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1771         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1772                 err = USBD_IOERROR;
1773                 goto ret;
1774         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1775                 break;
1776         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1777                 break;
1778         /* Hub requests */
1779         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1780                 break;
1781         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1782                 DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1783                              "port=%d feature=%d\n",
1784                              index, value));
1785                 if (index < 1 || index > sc->sc_noport) {
1786                         err = USBD_IOERROR;
1787                         goto ret;
1788                 }
1789                 port = EHCI_PORTSC(index);
1790                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1791                 switch(value) {
1792                 case UHF_PORT_ENABLE:
1793                         EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1794                         break;
1795                 case UHF_PORT_SUSPEND:
1796                         EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1797                         break;
1798                 case UHF_PORT_POWER:
1799                         EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1800                         break;
1801                 case UHF_PORT_TEST:
1802                         DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
1803                                     "%d\n", index));
1804                         break;
1805                 case UHF_PORT_INDICATOR:
1806                         DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
1807                                     "%d\n", index));
1808                         EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1809                         break;
1810                 case UHF_C_PORT_CONNECTION:
1811                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
1812                         break;
1813                 case UHF_C_PORT_ENABLE:
1814                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
1815                         break;
1816                 case UHF_C_PORT_SUSPEND:
1817                         /* how? */
1818                         break;
1819                 case UHF_C_PORT_OVER_CURRENT:
1820                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
1821                         break;
1822                 case UHF_C_PORT_RESET:
1823                         sc->sc_isreset = 0;
1824                         break;
1825                 default:
1826                         err = USBD_IOERROR;
1827                         goto ret;
1828                 }
1829 #if 0
1830                 switch(value) {
1831                 case UHF_C_PORT_CONNECTION:
1832                 case UHF_C_PORT_ENABLE:
1833                 case UHF_C_PORT_SUSPEND:
1834                 case UHF_C_PORT_OVER_CURRENT:
1835                 case UHF_C_PORT_RESET:
1836                         /* Enable RHSC interrupt if condition is cleared. */
1837                         if ((OREAD4(sc, port) >> 16) == 0)
1838                                 ehci_pcd_able(sc, 1);
1839                         break;
1840                 default:
1841                         break;
1842                 }
1843 #endif
1844                 break;
1845         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
1846                 if ((value & 0xff) != 0) {
1847                         err = USBD_IOERROR;
1848                         goto ret;
1849                 }
1850                 hubd = ehci_hubd;
1851                 hubd.bNbrPorts = sc->sc_noport;
1852                 v = EOREAD4(sc, EHCI_HCSPARAMS);
1853                 USETW(hubd.wHubCharacteristics,
1854                     EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
1855                     EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
1856                         ? UHD_PORT_IND : 0);
1857                 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
1858                 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
1859                         hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
1860                 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
1861                 l = min(len, hubd.bDescLength);
1862                 totlen = l;
1863                 memcpy(buf, &hubd, l);
1864                 break;
1865         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
1866                 if (len != 4) {
1867                         err = USBD_IOERROR;
1868                         goto ret;
1869                 }
1870                 memset(buf, 0, len); /* ? XXX */
1871                 totlen = len;
1872                 break;
1873         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
1874                 DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
1875                             index));
1876                 if (index < 1 || index > sc->sc_noport) {
1877                         err = USBD_IOERROR;
1878                         goto ret;
1879                 }
1880                 if (len != 4) {
1881                         err = USBD_IOERROR;
1882                         goto ret;
1883                 }
1884                 v = EOREAD4(sc, EHCI_PORTSC(index));
1885                 DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n", v));
1886                 i = UPS_HIGH_SPEED;
1887                 if (v & EHCI_PS_CS)     i |= UPS_CURRENT_CONNECT_STATUS;
1888                 if (v & EHCI_PS_PE)     i |= UPS_PORT_ENABLED;
1889                 if (v & EHCI_PS_SUSP)   i |= UPS_SUSPEND;
1890                 if (v & EHCI_PS_OCA)    i |= UPS_OVERCURRENT_INDICATOR;
1891                 if (v & EHCI_PS_PR)     i |= UPS_RESET;
1892                 if (v & EHCI_PS_PP)     i |= UPS_PORT_POWER;
1893                 USETW(ps.wPortStatus, i);
1894                 i = 0;
1895                 if (v & EHCI_PS_CSC)    i |= UPS_C_CONNECT_STATUS;
1896                 if (v & EHCI_PS_PEC)    i |= UPS_C_PORT_ENABLED;
1897                 if (v & EHCI_PS_OCC)    i |= UPS_C_OVERCURRENT_INDICATOR;
1898                 if (sc->sc_isreset)     i |= UPS_C_PORT_RESET;
1899                 USETW(ps.wPortChange, i);
1900                 l = min(len, sizeof ps);
1901                 memcpy(buf, &ps, l);
1902                 totlen = l;
1903                 break;
1904         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
1905                 err = USBD_IOERROR;
1906                 goto ret;
1907         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
1908                 break;
1909         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
1910                 if (index < 1 || index > sc->sc_noport) {
1911                         err = USBD_IOERROR;
1912                         goto ret;
1913                 }
1914                 port = EHCI_PORTSC(index);
1915                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1916                 switch(value) {
1917                 case UHF_PORT_ENABLE:
1918                         EOWRITE4(sc, port, v | EHCI_PS_PE);
1919                         break;
1920                 case UHF_PORT_SUSPEND:
1921                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
1922                         break;
1923                 case UHF_PORT_RESET:
1924                         DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
1925                                     index));
1926                         if (EHCI_PS_IS_LOWSPEED(v)) {
1927                                 /* Low speed device, give up ownership. */
1928                                 ehci_disown(sc, index, 1);
1929                                 break;
1930                         }
1931                         /* Start reset sequence. */
1932                         v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
1933                         EOWRITE4(sc, port, v | EHCI_PS_PR);
1934                         /* Wait for reset to complete. */
1935                         usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
1936                         if (sc->sc_dying) {
1937                                 err = USBD_IOERROR;
1938                                 goto ret;
1939                         }
1940                         /* Terminate reset sequence. */
1941                         EOWRITE4(sc, port, v);
1942                         /* Wait for HC to complete reset. */
1943                         usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
1944                         if (sc->sc_dying) {
1945                                 err = USBD_IOERROR;
1946                                 goto ret;
1947                         }
1948                         v = EOREAD4(sc, port);
1949                         DPRINTF(("ehci after reset, status=0x%08x\n", v));
1950                         if (v & EHCI_PS_PR) {
1951                                 printf("%s: port reset timeout\n",
1952                                        USBDEVNAME(sc->sc_bus.bdev));
1953                                 return (USBD_TIMEOUT);
1954                         }
1955                         if (!(v & EHCI_PS_PE)) {
1956                                 /* Not a high speed device, give up ownership.*/
1957                                 ehci_disown(sc, index, 0);
1958                                 break;
1959                         }
1960                         sc->sc_isreset = 1;
1961                         DPRINTF(("ehci port %d reset, status = 0x%08x\n",
1962                                  index, v));
1963                         break;
1964                 case UHF_PORT_POWER:
1965                         DPRINTFN(2,("ehci_root_ctrl_start: set port power "
1966                                     "%d\n", index));
1967                         EOWRITE4(sc, port, v | EHCI_PS_PP);
1968                         break;
1969                 case UHF_PORT_TEST:
1970                         DPRINTFN(2,("ehci_root_ctrl_start: set port test "
1971                                     "%d\n", index));
1972                         break;
1973                 case UHF_PORT_INDICATOR:
1974                         DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
1975                                     "%d\n", index));
1976                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
1977                         break;
1978                 default:
1979                         err = USBD_IOERROR;
1980                         goto ret;
1981                 }
1982                 break;
1983         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
1984         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
1985         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
1986         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
1987                 break;
1988         default:
1989                 err = USBD_IOERROR;
1990                 goto ret;
1991         }
1992         xfer->actlen = totlen;
1993         err = USBD_NORMAL_COMPLETION;
1994  ret:
1995         xfer->status = err;
1996         crit_enter();
1997         usb_transfer_complete(xfer);
1998         crit_exit();
1999         return (USBD_IN_PROGRESS);
2000 }
2001
2002 void
2003 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
2004 {
2005         int port;
2006         u_int32_t v;
2007
2008         DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
2009 #ifdef DIAGNOSTIC
2010         if (sc->sc_npcomp != 0) {
2011                 int i = (index-1) / sc->sc_npcomp;
2012                 if (i >= sc->sc_ncomp)
2013                         printf("%s: strange port\n",
2014                                USBDEVNAME(sc->sc_bus.bdev));
2015                 else
2016                         printf("%s: handing over %s speed device on "
2017                                "port %d to %s\n",
2018                                USBDEVNAME(sc->sc_bus.bdev),
2019                                lowspeed ? "low" : "full",
2020                                index, USBDEVNAME(sc->sc_comps[i]->bdev));
2021         } else {
2022                 printf("%s: npcomp == 0\n", USBDEVNAME(sc->sc_bus.bdev));
2023         }
2024 #endif
2025         port = EHCI_PORTSC(index);
2026         v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
2027         EOWRITE4(sc, port, v | EHCI_PS_PO);
2028 }
2029
2030 /* Abort a root control request. */
2031 Static void
2032 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
2033 {
2034         /* Nothing to do, all transfers are synchronous. */
2035 }
2036
2037 /* Close the root pipe. */
2038 Static void
2039 ehci_root_ctrl_close(usbd_pipe_handle pipe)
2040 {
2041         DPRINTF(("ehci_root_ctrl_close\n"));
2042         /* Nothing to do. */
2043 }
2044
2045 void
2046 ehci_root_intr_done(usbd_xfer_handle xfer)
2047 {
2048         xfer->hcpriv = NULL;
2049 }
2050
2051 Static usbd_status
2052 ehci_root_intr_transfer(usbd_xfer_handle xfer)
2053 {
2054         usbd_status err;
2055
2056         /* Insert last in queue. */
2057         err = usb_insert_transfer(xfer);
2058         if (err)
2059                 return (err);
2060
2061         /* Pipe isn't running, start first */
2062         return (ehci_root_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2063 }
2064
2065 Static usbd_status
2066 ehci_root_intr_start(usbd_xfer_handle xfer)
2067 {
2068         usbd_pipe_handle pipe = xfer->pipe;
2069         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2070
2071         if (sc->sc_dying)
2072                 return (USBD_IOERROR);
2073
2074         sc->sc_intrxfer = xfer;
2075
2076         return (USBD_IN_PROGRESS);
2077 }
2078
2079 /* Abort a root interrupt request. */
2080 Static void
2081 ehci_root_intr_abort(usbd_xfer_handle xfer)
2082 {
2083         if (xfer->pipe->intrxfer == xfer) {
2084                 DPRINTF(("ehci_root_intr_abort: remove\n"));
2085                 xfer->pipe->intrxfer = NULL;
2086         }
2087         xfer->status = USBD_CANCELLED;
2088         crit_enter();
2089         usb_transfer_complete(xfer);
2090         crit_exit();
2091 }
2092
2093 /* Close the root pipe. */
2094 Static void
2095 ehci_root_intr_close(usbd_pipe_handle pipe)
2096 {
2097         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2098
2099         DPRINTF(("ehci_root_intr_close\n"));
2100
2101         sc->sc_intrxfer = NULL;
2102 }
2103
2104 void
2105 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2106 {
2107         xfer->hcpriv = NULL;
2108 }
2109
2110 /************************/
2111
2112 ehci_soft_qh_t *
2113 ehci_alloc_sqh(ehci_softc_t *sc)
2114 {
2115         ehci_soft_qh_t *sqh;
2116         usbd_status err;
2117         int i, offs;
2118         usb_dma_t dma;
2119
2120         if (sc->sc_freeqhs == NULL) {
2121                 DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2122                 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2123                           EHCI_PAGE_SIZE, &dma);
2124 #ifdef USB_DEBUG
2125                 if (err)
2126                         printf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2127 #endif
2128                 if (err)
2129                         return (NULL);
2130                 for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2131                         offs = i * EHCI_SQH_SIZE;
2132                         sqh = KERNADDR(&dma, offs);
2133                         sqh->physaddr = DMAADDR(&dma, offs);
2134                         sqh->next = sc->sc_freeqhs;
2135                         sc->sc_freeqhs = sqh;
2136                 }
2137         }
2138         sqh = sc->sc_freeqhs;
2139         sc->sc_freeqhs = sqh->next;
2140         memset(&sqh->qh, 0, sizeof(ehci_qh_t));
2141         sqh->next = NULL;
2142         return (sqh);
2143 }
2144
2145 void
2146 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2147 {
2148         sqh->next = sc->sc_freeqhs;
2149         sc->sc_freeqhs = sqh;
2150 }
2151
2152 ehci_soft_qtd_t *
2153 ehci_alloc_sqtd(ehci_softc_t *sc)
2154 {
2155         ehci_soft_qtd_t *sqtd;
2156         usbd_status err;
2157         int i, offs;
2158         usb_dma_t dma;
2159
2160         if (sc->sc_freeqtds == NULL) {
2161                 DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2162                 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2163                           EHCI_PAGE_SIZE, &dma);
2164 #ifdef USB_DEBUG
2165                 if (err)
2166                         printf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2167 #endif
2168                 if (err)
2169                         return (NULL);
2170                 crit_enter();
2171                 for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2172                         offs = i * EHCI_SQTD_SIZE;
2173                         sqtd = KERNADDR(&dma, offs);
2174                         sqtd->physaddr = DMAADDR(&dma, offs);
2175                         sqtd->nextqtd = sc->sc_freeqtds;
2176                         sc->sc_freeqtds = sqtd;
2177                 }
2178                 crit_exit();
2179         }
2180
2181         crit_enter();
2182         sqtd = sc->sc_freeqtds;
2183         sc->sc_freeqtds = sqtd->nextqtd;
2184         memset(&sqtd->qtd, 0, sizeof(ehci_qtd_t));
2185         sqtd->nextqtd = NULL;
2186         sqtd->xfer = NULL;
2187         crit_exit();
2188
2189         return (sqtd);
2190 }
2191
2192 void
2193 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2194 {
2195         crit_enter();
2196         sqtd->nextqtd = sc->sc_freeqtds;
2197         sc->sc_freeqtds = sqtd;
2198         crit_exit();
2199 }
2200
2201 usbd_status
2202 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2203                      int alen, int rd, usbd_xfer_handle xfer,
2204                      ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2205 {
2206         ehci_soft_qtd_t *next, *cur;
2207         ehci_physaddr_t dataphys, dataphyspage, dataphyslastpage, nextphys;
2208         u_int32_t qtdstatus;
2209         int len, curlen, mps;
2210         int i, tog;
2211         int offset;
2212         usb_dma_t *dma = &xfer->dmabuf;
2213
2214         DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2215
2216         offset = 0;
2217         len = alen;
2218         dataphys = DMAADDR(dma, 0);
2219         dataphyslastpage = EHCI_PAGE(DMAADDR(dma, len - 1));
2220 #if 0
2221 printf("status=%08x toggle=%d\n", epipe->sqh->qh.qh_qtd.qtd_status,
2222     epipe->nexttoggle);
2223 #endif
2224         qtdstatus = EHCI_QTD_ACTIVE |
2225             EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2226             EHCI_QTD_SET_CERR(3)
2227             /* IOC set below */
2228             /* BYTES set below */
2229             ;
2230         mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2231         tog = epipe->nexttoggle;
2232         qtdstatus |= EHCI_QTD_SET_TOGGLE(tog);
2233
2234         cur = ehci_alloc_sqtd(sc);
2235         *sp = cur;
2236         if (cur == NULL)
2237                 goto nomem;
2238         for (;;) {
2239                 dataphyspage = EHCI_PAGE(dataphys);
2240                 /* The EHCI hardware can handle at most 5 pages. */
2241 #if defined(__NetBSD__) || defined(__OpenBSD__)
2242                 if (dataphyslastpage - dataphyspage <
2243                     EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE) {
2244                         /* we can handle it in this QTD */
2245                         curlen = len;
2246 #elif defined(__FreeBSD__) || defined(__DragonFly__)
2247                 /* XXX This is pretty broken: Because we do not allocate
2248                  * a contiguous buffer (contiguous in physical pages) we
2249                  * can only transfer one page in one go.
2250                  * So check whether the start and end of the buffer are on
2251                  * the same page.
2252                  */
2253                 if (dataphyspage == dataphyslastpage) {
2254                         curlen = len;
2255 #endif
2256                 } else {
2257 #if defined(__NetBSD__) || defined(__OpenBSD__)
2258                         /* must use multiple TDs, fill as much as possible. */
2259                         curlen = EHCI_QTD_NBUFFERS * EHCI_PAGE_SIZE -
2260                                  EHCI_PAGE_OFFSET(dataphys);
2261 #ifdef DIAGNOSTIC
2262                         if (curlen > len) {
2263                                 printf("ehci_alloc_sqtd_chain: curlen=0x%x "
2264                                        "len=0x%x offs=0x%x\n", curlen, len,
2265                                        EHCI_PAGE_OFFSET(dataphys));
2266                                 printf("lastpage=0x%x page=0x%x phys=0x%x\n",
2267                                        dataphyslastpage, dataphyspage,
2268                                        dataphys);
2269                                 curlen = len;
2270                         }
2271 #endif
2272 #elif defined(__FreeBSD__) || defined(__DragonFly__)
2273                         /* See comment above (XXX) */
2274                         curlen = EHCI_PAGE_SIZE -
2275                                  EHCI_PAGE_MASK(dataphys);
2276 #endif
2277                         /* the length must be a multiple of the max size */
2278                         curlen -= curlen % mps;
2279                         DPRINTFN(1,("ehci_alloc_sqtd_chain: multiple QTDs, "
2280                                     "curlen=%d\n", curlen));
2281 #ifdef DIAGNOSTIC
2282                         if (curlen == 0)
2283                                 panic("ehci_alloc_std: curlen == 0");
2284 #endif
2285                 }
2286                 DPRINTFN(4,("ehci_alloc_sqtd_chain: dataphys=0x%08x "
2287                             "dataphyslastpage=0x%08x len=%d curlen=%d\n",
2288                             dataphys, dataphyslastpage,
2289                             len, curlen));
2290                 len -= curlen;
2291
2292                 if (len != 0) {
2293                         next = ehci_alloc_sqtd(sc);
2294                         if (next == NULL)
2295                                 goto nomem;
2296                         nextphys = htole32(next->physaddr);
2297                 } else {
2298                         next = NULL;
2299                         nextphys = EHCI_NULL;
2300                 }
2301
2302                 for (i = 0; i * EHCI_PAGE_SIZE < curlen; i++) {
2303                         ehci_physaddr_t a = dataphys + i * EHCI_PAGE_SIZE;
2304                         if (i != 0) /* use offset only in first buffer */
2305                                 a = EHCI_PAGE(a);
2306                         cur->qtd.qtd_buffer[i] = htole32(a);
2307                         cur->qtd.qtd_buffer_hi[i] = 0;
2308 #ifdef DIAGNOSTIC
2309                         if (i >= EHCI_QTD_NBUFFERS) {
2310                                 printf("ehci_alloc_sqtd_chain: i=%d\n", i);
2311                                 goto nomem;
2312                         }
2313 #endif
2314                 }
2315                 cur->nextqtd = next;
2316                 cur->qtd.qtd_next = cur->qtd.qtd_altnext = nextphys;
2317                 cur->qtd.qtd_status =
2318                     htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2319                 cur->xfer = xfer;
2320                 cur->len = curlen;
2321                 DPRINTFN(10,("ehci_alloc_sqtd_chain: cbp=0x%08x end=0x%08x\n",
2322                             dataphys, dataphys + curlen));
2323                 /* adjust the toggle based on the number of packets in this
2324                    qtd */
2325                 if (((curlen + mps - 1) / mps) & 1) {
2326                         tog ^= 1;
2327                         qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2328                 }
2329                 if (len == 0)
2330                         break;
2331                 DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2332                 offset += curlen;
2333                 dataphys = DMAADDR(dma, offset);
2334                 cur = next;
2335         }
2336         cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2337         *ep = cur;
2338         epipe->nexttoggle = tog;
2339
2340         DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2341                      *sp, *ep));
2342
2343         return (USBD_NORMAL_COMPLETION);
2344
2345  nomem:
2346         /* XXX free chain */
2347         DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2348         return (USBD_NOMEM);
2349 }
2350
2351 Static void
2352 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd,
2353                     ehci_soft_qtd_t *sqtdend)
2354 {
2355         ehci_soft_qtd_t *p;
2356         int i;
2357
2358         DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2359                      sqtd, sqtdend));
2360
2361         for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2362                 p = sqtd->nextqtd;
2363                 ehci_free_sqtd(sc, sqtd);
2364         }
2365 }
2366
2367 /****************/
2368
2369 /*
2370  * Close a reqular pipe.
2371  * Assumes that there are no pending transactions.
2372  */
2373 void
2374 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2375 {
2376         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2377         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2378         ehci_soft_qh_t *sqh = epipe->sqh;
2379
2380         crit_enter();
2381         ehci_rem_qh(sc, sqh, head);
2382         crit_exit();
2383         ehci_free_sqh(sc, epipe->sqh);
2384 }
2385
2386 /*
2387  * Abort a device request.
2388  * If this routine is called from a critical section it guarantees that the
2389  * request will be removed from the hardware scheduling and that the callback
2390  * for it will be called with USBD_CANCELLED status.
2391  * It's impossible to guarantee that the requested transfer will not
2392  * have happened since the hardware runs concurrently.
2393  * If the transaction has already happened we rely on the ordinary
2394  * interrupt processing to process it.
2395  * XXX This is most probably wrong.
2396  */
2397 void
2398 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2399 {
2400 #define exfer EXFER(xfer)
2401         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2402         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2403         ehci_soft_qh_t *sqh = epipe->sqh;
2404         ehci_soft_qtd_t *sqtd;
2405         ehci_physaddr_t cur;
2406         u_int32_t qhstatus;
2407         int hit;
2408
2409         DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2410
2411         if (sc->sc_dying) {
2412                 /* If we're dying, just do the software part. */
2413                 crit_enter();
2414                 xfer->status = status;  /* make software ignore it */
2415                 usb_uncallout(xfer->timeout_handle, ehci_timeout, xfer);
2416                 usb_transfer_complete(xfer);
2417                 crit_exit();
2418                 return;
2419         }
2420
2421         if (xfer->device->bus->intr_context /* || !curproc REMOVED DFly */)
2422                 panic("ehci_abort_xfer: not in process context");
2423
2424         /*
2425          * Step 1: Make interrupt routine and hardware ignore xfer.
2426          */
2427         crit_enter();
2428         xfer->status = status;  /* make software ignore it */
2429         usb_uncallout(xfer->timeout_handle, ehci_timeout, xfer);
2430         qhstatus = sqh->qh.qh_qtd.qtd_status;
2431         sqh->qh.qh_qtd.qtd_status = qhstatus | htole32(EHCI_QTD_HALTED);
2432         for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2433                 sqtd->qtd.qtd_status |= htole32(EHCI_QTD_HALTED);
2434                 if (sqtd == exfer->sqtdend)
2435                         break;
2436         }
2437         crit_exit();
2438
2439         /*
2440          * Step 2: Wait until we know hardware has finished any possible
2441          * use of the xfer.  Also make sure the soft interrupt routine
2442          * has run.
2443          */
2444         ehci_sync_hc(sc);
2445         crit_enter();
2446 #ifdef USB_USE_SOFTINTR
2447         sc->sc_softwake = 1;
2448 #endif /* USB_USE_SOFTINTR */
2449         usb_schedsoftintr(&sc->sc_bus);
2450 #ifdef USB_USE_SOFTINTR
2451         tsleep(&sc->sc_softwake, 0, "ehciab", 0);
2452 #endif /* USB_USE_SOFTINTR */
2453         crit_exit();
2454
2455         /*
2456          * Step 3: Remove any vestiges of the xfer from the hardware.
2457          * The complication here is that the hardware may have executed
2458          * beyond the xfer we're trying to abort.  So as we're scanning
2459          * the TDs of this xfer we check if the hardware points to
2460          * any of them.
2461          */
2462         crit_enter();           /* XXX why? */
2463         cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
2464         hit = 0;
2465         for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2466                 hit |= cur == sqtd->physaddr;
2467                 if (sqtd == exfer->sqtdend)
2468                         break;
2469         }
2470         sqtd = sqtd->nextqtd;
2471         /* Zap curqtd register if hardware pointed inside the xfer. */
2472         if (hit && sqtd != NULL) {
2473                 DPRINTFN(1,("ehci_abort_xfer: cur=0x%08x\n", sqtd->physaddr));
2474                 sqh->qh.qh_curqtd = htole32(sqtd->physaddr); /* unlink qTDs */
2475                 sqh->qh.qh_qtd.qtd_status = qhstatus;
2476         } else {
2477                 DPRINTFN(1,("ehci_abort_xfer: no hit\n"));
2478         }
2479
2480         /*
2481          * Step 4: Execute callback.
2482          */
2483 #ifdef DIAGNOSTIC
2484         exfer->isdone = 1;
2485 #endif
2486         usb_transfer_complete(xfer);
2487
2488         crit_exit();
2489 #undef exfer
2490 }
2491
2492 void
2493 ehci_timeout(void *addr)
2494 {
2495         struct ehci_xfer *exfer = addr;
2496         struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
2497         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2498
2499         DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
2500 #ifdef USB_DEBUG
2501         if (ehcidebug > 1)
2502                 usbd_dump_pipe(exfer->xfer.pipe);
2503 #endif
2504
2505         if (sc->sc_dying) {
2506                 ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
2507                 return;
2508         }
2509
2510         /* Execute the abort in a process context. */
2511         usb_init_task(&exfer->abort_task, ehci_timeout_task, addr);
2512         usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task);
2513 }
2514
2515 void
2516 ehci_timeout_task(void *addr)
2517 {
2518         usbd_xfer_handle xfer = addr;
2519
2520         DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
2521         crit_enter();
2522         ehci_abort_xfer(xfer, USBD_TIMEOUT);
2523         crit_exit();
2524 }
2525
2526 /************************/
2527
2528 Static usbd_status
2529 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
2530 {
2531         usbd_status err;
2532
2533         /* Insert last in queue. */
2534         err = usb_insert_transfer(xfer);
2535         if (err)
2536                 return (err);
2537
2538         /* Pipe isn't running, start first */
2539         return (ehci_device_ctrl_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2540 }
2541
2542 Static usbd_status
2543 ehci_device_ctrl_start(usbd_xfer_handle xfer)
2544 {
2545         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2546         usbd_status err;
2547
2548         if (sc->sc_dying)
2549                 return (USBD_IOERROR);
2550
2551 #ifdef DIAGNOSTIC
2552         if (!(xfer->rqflags & URQ_REQUEST)) {
2553                 /* XXX panic */
2554                 printf("ehci_device_ctrl_transfer: not a request\n");
2555                 return (USBD_INVAL);
2556         }
2557 #endif
2558
2559         err = ehci_device_request(xfer);
2560         if (err)
2561                 return (err);
2562
2563         if (sc->sc_bus.use_polling)
2564                 ehci_waitintr(sc, xfer);
2565         return (USBD_IN_PROGRESS);
2566 }
2567
2568 void
2569 ehci_device_ctrl_done(usbd_xfer_handle xfer)
2570 {
2571         struct ehci_xfer *ex = EXFER(xfer);
2572         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2573         /*struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;*/
2574
2575         DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
2576
2577 #ifdef DIAGNOSTIC
2578         if (!(xfer->rqflags & URQ_REQUEST)) {
2579                 panic("ehci_ctrl_done: not a request");
2580         }
2581 #endif
2582
2583         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2584                 ehci_del_intr_list(ex); /* remove from active list */
2585                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
2586         }
2587
2588         DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
2589 }
2590
2591 /* Abort a device control request. */
2592 Static void
2593 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
2594 {
2595         DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
2596         ehci_abort_xfer(xfer, USBD_CANCELLED);
2597 }
2598
2599 /* Close a device control pipe. */
2600 Static void
2601 ehci_device_ctrl_close(usbd_pipe_handle pipe)
2602 {
2603         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2604         /*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
2605
2606         DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
2607         ehci_close_pipe(pipe, sc->sc_async_head);
2608 }
2609
2610 usbd_status
2611 ehci_device_request(usbd_xfer_handle xfer)
2612 {
2613 #define exfer EXFER(xfer)
2614         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2615         usb_device_request_t *req = &xfer->request;
2616         usbd_device_handle dev = epipe->pipe.device;
2617         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2618         int addr = dev->address;
2619         ehci_soft_qtd_t *setup, *stat, *next;
2620         ehci_soft_qh_t *sqh;
2621         int isread;
2622         int len;
2623         usbd_status err;
2624
2625         isread = req->bmRequestType & UT_READ;
2626         len = UGETW(req->wLength);
2627
2628         DPRINTFN(3,("ehci_device_request type=0x%02x, request=0x%02x, "
2629                     "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
2630                     req->bmRequestType, req->bRequest, UGETW(req->wValue),
2631                     UGETW(req->wIndex), len, addr,
2632                     epipe->pipe.endpoint->edesc->bEndpointAddress));
2633
2634         setup = ehci_alloc_sqtd(sc);
2635         if (setup == NULL) {
2636                 err = USBD_NOMEM;
2637                 goto bad1;
2638         }
2639         stat = ehci_alloc_sqtd(sc);
2640         if (stat == NULL) {
2641                 err = USBD_NOMEM;
2642                 goto bad2;
2643         }
2644
2645         sqh = epipe->sqh;
2646         epipe->u.ctl.length = len;
2647
2648         /* Update device address and length since they may have changed
2649            during the setup of the control pipe in usbd_new_device(). */
2650         /* XXX This only needs to be done once, but it's too early in open. */
2651         /* XXXX Should not touch ED here! */
2652         sqh->qh.qh_endp =
2653             (sqh->qh.qh_endp & htole32(~(EHCI_QH_ADDRMASK | EHCI_QH_MPLMASK))) |
2654             htole32(
2655              EHCI_QH_SET_ADDR(addr) |
2656              EHCI_QH_SET_MPL(UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize))
2657             );
2658
2659         /* Set up data transaction */
2660         if (len != 0) {
2661                 ehci_soft_qtd_t *end;
2662
2663                 /* Start toggle at 1. */
2664                 epipe->nexttoggle = 1;
2665                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
2666                           &next, &end);
2667                 if (err)
2668                         goto bad3;
2669                 end->nextqtd = stat;
2670                 end->qtd.qtd_next =
2671                 end->qtd.qtd_altnext = htole32(stat->physaddr);
2672         } else {
2673                 next = stat;
2674         }
2675
2676         memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
2677
2678         /* Clear toggle */
2679         setup->qtd.qtd_status = htole32(
2680             EHCI_QTD_ACTIVE |
2681             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2682             EHCI_QTD_SET_CERR(3) |
2683             EHCI_QTD_SET_TOGGLE(0) |
2684             EHCI_QTD_SET_BYTES(sizeof *req)
2685             );
2686         setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2687         setup->qtd.qtd_buffer_hi[0] = 0;
2688         setup->nextqtd = next;
2689         setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
2690         setup->xfer = xfer;
2691         setup->len = sizeof *req;
2692
2693         stat->qtd.qtd_status = htole32(
2694             EHCI_QTD_ACTIVE |
2695             EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2696             EHCI_QTD_SET_CERR(3) |
2697             EHCI_QTD_SET_TOGGLE(1) |
2698             EHCI_QTD_IOC
2699             );
2700         stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2701         stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2702         stat->nextqtd = NULL;
2703         stat->qtd.qtd_next = stat->qtd.qtd_altnext = EHCI_NULL;
2704         stat->xfer = xfer;
2705         stat->len = 0;
2706
2707 #ifdef USB_DEBUG
2708         if (ehcidebug > 5) {
2709                 DPRINTF(("ehci_device_request:\n"));
2710                 ehci_dump_sqh(sqh);
2711                 ehci_dump_sqtds(setup);
2712         }
2713 #endif
2714
2715         exfer->sqtdstart = setup;
2716         exfer->sqtdend = stat;
2717 #ifdef DIAGNOSTIC
2718         if (!exfer->isdone) {
2719                 printf("ehci_device_request: not done, exfer=%p\n", exfer);
2720         }
2721         exfer->isdone = 0;
2722 #endif
2723
2724         /* Insert qTD in QH list. */
2725         crit_enter();
2726         ehci_set_qh_qtd(sqh, setup);
2727         if (xfer->timeout && !sc->sc_bus.use_polling) {
2728                 usb_callout(xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2729                             ehci_timeout, xfer);
2730         }
2731         ehci_add_intr_list(sc, exfer);
2732         xfer->status = USBD_IN_PROGRESS;
2733         crit_exit();
2734
2735 #ifdef USB_DEBUG
2736         if (ehcidebug > 10) {
2737                 DPRINTF(("ehci_device_request: status=%x\n",
2738                          EOREAD4(sc, EHCI_USBSTS)));
2739                 delay(10000);
2740                 ehci_dump_regs(sc);
2741                 ehci_dump_sqh(sc->sc_async_head);
2742                 ehci_dump_sqh(sqh);
2743                 ehci_dump_sqtds(setup);
2744         }
2745 #endif
2746
2747         return (USBD_NORMAL_COMPLETION);
2748
2749  bad3:
2750         ehci_free_sqtd(sc, stat);
2751  bad2:
2752         ehci_free_sqtd(sc, setup);
2753  bad1:
2754         DPRINTFN(-1,("ehci_device_request: no memory\n"));
2755         xfer->status = err;
2756         usb_transfer_complete(xfer);
2757         return (err);
2758 #undef exfer
2759 }
2760
2761 /************************/
2762
2763 Static usbd_status
2764 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
2765 {
2766         usbd_status err;
2767
2768         /* Insert last in queue. */
2769         err = usb_insert_transfer(xfer);
2770         if (err)
2771                 return (err);
2772
2773         /* Pipe isn't running, start first */
2774         return (ehci_device_bulk_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2775 }
2776
2777 usbd_status
2778 ehci_device_bulk_start(usbd_xfer_handle xfer)
2779 {
2780 #define exfer EXFER(xfer)
2781         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2782         usbd_device_handle dev = epipe->pipe.device;
2783         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2784         ehci_soft_qtd_t *data, *dataend;
2785         ehci_soft_qh_t *sqh;
2786         usbd_status err;
2787         int len, isread, endpt;
2788
2789         DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
2790                      xfer, xfer->length, xfer->flags));
2791
2792         if (sc->sc_dying)
2793                 return (USBD_IOERROR);
2794
2795 #ifdef DIAGNOSTIC
2796         if (xfer->rqflags & URQ_REQUEST)
2797                 panic("ehci_device_bulk_start: a request");
2798 #endif
2799
2800         len = xfer->length;
2801         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
2802         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
2803         sqh = epipe->sqh;
2804
2805         epipe->u.bulk.length = len;
2806
2807         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
2808                                    &dataend);
2809         if (err) {
2810                 DPRINTFN(-1,("ehci_device_bulk_start: no memory\n"));
2811                 xfer->status = err;
2812                 usb_transfer_complete(xfer);
2813                 return (err);
2814         }
2815
2816 #ifdef USB_DEBUG
2817         if (ehcidebug > 5) {
2818                 DPRINTF(("ehci_device_bulk_start: data(1)\n"));
2819                 ehci_dump_sqh(sqh);
2820                 ehci_dump_sqtds(data);
2821         }
2822 #endif
2823
2824         /* Set up interrupt info. */
2825         exfer->sqtdstart = data;
2826         exfer->sqtdend = dataend;
2827 #ifdef DIAGNOSTIC
2828         if (!exfer->isdone) {
2829                 printf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
2830         }
2831         exfer->isdone = 0;
2832 #endif
2833
2834         crit_enter();
2835         ehci_set_qh_qtd(sqh, data);
2836         if (xfer->timeout && !sc->sc_bus.use_polling) {
2837                 usb_callout(xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2838                             ehci_timeout, xfer);
2839         }
2840         ehci_add_intr_list(sc, exfer);
2841         xfer->status = USBD_IN_PROGRESS;
2842         crit_exit();
2843
2844 #ifdef USB_DEBUG
2845         if (ehcidebug > 10) {
2846                 DPRINTF(("ehci_device_bulk_start: data(2)\n"));
2847                 delay(10000);
2848                 DPRINTF(("ehci_device_bulk_start: data(3)\n"));
2849                 ehci_dump_regs(sc);
2850 #if 0
2851                 printf("async_head:\n");
2852                 ehci_dump_sqh(sc->sc_async_head);
2853 #endif
2854                 printf("sqh:\n");
2855                 ehci_dump_sqh(sqh);
2856                 ehci_dump_sqtds(data);
2857         }
2858 #endif
2859
2860         if (sc->sc_bus.use_polling)
2861                 ehci_waitintr(sc, xfer);
2862
2863         return (USBD_IN_PROGRESS);
2864 #undef exfer
2865 }
2866
2867 Static void
2868 ehci_device_bulk_abort(usbd_xfer_handle xfer)
2869 {
2870         DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
2871         ehci_abort_xfer(xfer, USBD_CANCELLED);
2872 }
2873
2874 /*
2875  * Close a device bulk pipe.
2876  */
2877 Static void
2878 ehci_device_bulk_close(usbd_pipe_handle pipe)
2879 {
2880         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2881
2882         DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
2883         ehci_close_pipe(pipe, sc->sc_async_head);
2884 }
2885
2886 void
2887 ehci_device_bulk_done(usbd_xfer_handle xfer)
2888 {
2889         struct ehci_xfer *ex = EXFER(xfer);
2890         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2891         /*struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;*/
2892
2893         DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
2894                      xfer, xfer->actlen));
2895
2896         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2897                 ehci_del_intr_list(ex); /* remove from active list */
2898                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
2899         }
2900
2901         DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
2902 }
2903
2904 /************************/
2905
2906 Static usbd_status
2907 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
2908 {
2909         struct ehci_soft_islot *isp;
2910         int islot, lev;
2911
2912         /* Find a poll rate that is large enough. */
2913         for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
2914                 if (EHCI_ILEV_IVAL(lev) <= ival)
2915                         break;
2916
2917         /* Pick an interrupt slot at the right level. */
2918         /* XXX could do better than picking at random */
2919         sc->sc_rand = (sc->sc_rand + 191) % sc->sc_flsize;
2920         islot = EHCI_IQHIDX(lev, sc->sc_rand);
2921
2922         sqh->islot = islot;
2923         isp = &sc->sc_islots[islot];
2924         ehci_add_qh(sqh, isp->sqh);
2925
2926         return (USBD_NORMAL_COMPLETION);
2927 }
2928
2929 Static usbd_status
2930 ehci_device_intr_transfer(usbd_xfer_handle xfer)
2931 {
2932         usbd_status err;
2933
2934         /* Insert last in queue. */
2935         err = usb_insert_transfer(xfer);
2936         if (err)
2937                 return (err);
2938
2939         /*
2940          * Pipe isn't running (otherwise err would be USBD_INPROG),
2941          * so start it first.
2942          */
2943         return (ehci_device_intr_start(SIMPLEQ_FIRST(&xfer->pipe->queue)));
2944 }
2945
2946 Static usbd_status
2947 ehci_device_intr_start(usbd_xfer_handle xfer)
2948 {
2949 #define exfer EXFER(xfer)
2950         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2951         usbd_device_handle dev = xfer->pipe->device;
2952         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2953         ehci_soft_qtd_t *data, *dataend;
2954         ehci_soft_qh_t *sqh;
2955         usbd_status err;
2956         int len, isread, endpt;
2957         int s;
2958
2959         DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
2960                      xfer, xfer->length, xfer->flags));
2961
2962         if (sc->sc_dying)
2963                 return (USBD_IOERROR);
2964
2965 #ifdef DIAGNOSTIC
2966         if (xfer->rqflags & URQ_REQUEST)
2967                 panic("ehci_device_intr_start: a request");
2968 #endif
2969
2970         len = xfer->length;
2971         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
2972         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
2973         sqh = epipe->sqh;
2974
2975         epipe->u.intr.length = len;
2976
2977         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
2978                                     &dataend);
2979         if (err) {
2980                 DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
2981                 xfer->status = err;
2982                 usb_transfer_complete(xfer);
2983                 return (err);
2984         }
2985
2986 #ifdef USB_DEBUG
2987         if (ehcidebug > 5) {
2988                 DPRINTF(("ehci_device_intr_start: data(1)\n"));
2989                 ehci_dump_sqh(sqh);
2990                 ehci_dump_sqtds(data);
2991         }
2992 #endif
2993
2994         /* Set up interrupt info. */
2995         exfer->sqtdstart = data;
2996         exfer->sqtdend = dataend;
2997 #ifdef DIAGNOSTIC
2998         if (!exfer->isdone) {
2999                 printf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3000         }
3001         exfer->isdone = 0;
3002 #endif
3003
3004          crit_enter();
3005          ehci_set_qh_qtd(sqh, data);
3006          if (xfer->timeout && !sc->sc_bus.use_polling) {
3007                  usb_callout(xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3008                              ehci_timeout, xfer);
3009          }
3010          ehci_add_intr_list(sc, exfer);
3011          xfer->status = USBD_IN_PROGRESS;
3012          crit_exit();
3013
3014 #ifdef USB_DEBUG
3015          if (ehcidebug > 10) {
3016                  DPRINTF(("ehci_device_intr_start: data(2)\n"));
3017                  delay(10000);
3018                  DPRINTF(("ehci_device_intr_start: data(3)\n"));
3019                  ehci_dump_regs(sc);
3020                  printf("sqh:\n");
3021                  ehci_dump_sqh(sqh);
3022                  ehci_dump_sqtds(data);
3023          }
3024 #endif
3025
3026          if (sc->sc_bus.use_polling)
3027                  ehci_waitintr(sc, xfer);
3028          return (USBD_IN_PROGRESS);
3029 #undef exfer
3030 }
3031
3032 Static void
3033 ehci_device_intr_abort(usbd_xfer_handle xfer)
3034 {
3035         DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3036         if (xfer->pipe->intrxfer == xfer) {
3037                 DPRINTFN(1, ("echi_device_intr_abort: remove\n"));
3038                 xfer->pipe->intrxfer = NULL;
3039         }
3040         ehci_abort_xfer(xfer, USBD_CANCELLED);
3041 }
3042
3043 Static void
3044 ehci_device_intr_close(usbd_pipe_handle pipe)
3045 {
3046         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3047         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3048         struct ehci_soft_islot *isp;
3049
3050         isp = &sc->sc_islots[epipe->sqh->islot];
3051         ehci_close_pipe(pipe, isp->sqh);
3052 }
3053
3054 Static void
3055 ehci_device_intr_done(usbd_xfer_handle xfer)
3056 {
3057 #define exfer EXFER(xfer)
3058         struct ehci_xfer *ex = EXFER(xfer);
3059         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3060         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3061         ehci_soft_qtd_t *data, *dataend;
3062         ehci_soft_qh_t *sqh;
3063         usbd_status err;
3064         int len, isread, endpt, s;
3065
3066         DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3067                       xfer, xfer->actlen));
3068
3069         if (xfer->pipe->repeat) {
3070                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3071
3072                 len = epipe->u.intr.length;
3073                 xfer->length = len;
3074                 endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3075                 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3076                 sqh = epipe->sqh;
3077
3078                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3079                                             &data, &dataend);
3080                 if (err) {
3081                         DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3082                         xfer->status = err;
3083                         return;
3084                 }
3085
3086                 /* Set up interrupt info. */
3087                 exfer->sqtdstart = data;
3088                 exfer->sqtdend = dataend;
3089 #ifdef DIAGNOSTIC
3090                 if (!exfer->isdone) {
3091                         printf("ehci_device_intr_done: not done, ex=%p\n",
3092                                exfer);
3093                 }
3094                 exfer->isdone = 0;
3095 #endif
3096
3097                 crit_enter();
3098                 ehci_set_qh_qtd(sqh, data);
3099                 if (xfer->timeout && !sc->sc_bus.use_polling) {
3100                         usb_callout(xfer->timeout_handle,
3101                                     MS_TO_TICKS(xfer->timeout), ehci_timeout,
3102                                     xfer);
3103                 }
3104                 crit_exit();
3105
3106                 xfer->status = USBD_IN_PROGRESS;
3107         } else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3108                 ehci_del_intr_list(ex); /* remove from active list */
3109                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3110         }
3111 #undef exfer
3112 }
3113
3114 /************************/
3115
3116 Static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3117 Static usbd_status      ehci_device_isoc_start(usbd_xfer_handle xfer) { return USBD_IOERROR; }
3118 Static void             ehci_device_isoc_abort(usbd_xfer_handle xfer) { }
3119 Static void             ehci_device_isoc_close(usbd_pipe_handle pipe) { }
3120 Static void             ehci_device_isoc_done(usbd_xfer_handle xfer) { }