sys/bus/usb/ehci.c

   1 /*      $NetBSD: ehci.c,v 1.91 2005/02/27 00:27:51 perry Exp $ */
   2 /*      $FreeBSD: src/sys/dev/usb/ehci.c,v 1.36.2.3 2006/09/24 13:39:04 iedowse Exp $   */
   3 /*      $DragonFly: src/sys/bus/usb/ehci.c,v 1.35 2008/06/10 10:04:05 hasso Exp $       */
   4
   5 /*
   6  * Copyright (c) 2004 The NetBSD Foundation, Inc.
   7  * All rights reserved.
   8  *
   9  * This code is derived from software contributed to The NetBSD Foundation
  10  * by Lennart Augustsson (lennart@augustsson.net) and by Charles M. Hannum.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  * 3. All advertising materials mentioning features or use of this software
  21  *    must display the following acknowledgement:
  22  *        This product includes software developed by the NetBSD
  23  *        Foundation, Inc. and its contributors.
  24  * 4. Neither the name of The NetBSD Foundation nor the names of its
  25  *    contributors may be used to endorse or promote products derived
  26  *    from this software without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  29  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  30  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  31  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  32  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  33  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  34  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  35  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  36  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  37  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGE.
  39  */
  40
  41 /*
  42  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  43  *
  44  * The EHCI 1.0 spec can be found at
  45  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  46  * and the USB 2.0 spec at
  47  * http://www.usb.org/developers/docs/usb_20.zip
  48  *
  49  */
  50
  51 /*
  52  * TODO:
  53  * 1) The EHCI driver lacks support for isochronous transfers, so
  54  *    devices using them don't work.
  55  *
  56  * 2) Interrupt transfer scheduling does not manage the time available
  57  *    in each frame, so it is possible for the transfers to overrun
  58  *    the end of the frame.
  59  *
  60  * 3) Command failures are not recovered correctly.
  61  */
  62
  63 #include <sys/param.h>
  64 #include <sys/systm.h>
  65 #include <sys/malloc.h>
  66 #include <sys/kernel.h>
  67 #include <sys/endian.h>
  68 #include <sys/module.h>
  69 #include <sys/bus.h>
  70 #include <sys/lock.h>
  71 #include <sys/proc.h>
  72 #include <sys/queue.h>
  73 #include <sys/sysctl.h>
  74 #include <sys/thread2.h>
  75
  76 #include <machine/cpu.h>
  77 #include <machine/endian.h>
  78
  79 #include <bus/usb/usb.h>
  80 #include <bus/usb/usbdi.h>
  81 #include <bus/usb/usbdivar.h>
  82 #include <bus/usb/usb_mem.h>
  83 #include <bus/usb/usb_quirks.h>
  84
  85 #include <bus/usb/ehcireg.h>
  86 #include <bus/usb/ehcivar.h>
  87
  88 #ifdef USB_DEBUG
  89 #define EHCI_DEBUG USB_DEBUG
  90 #define DPRINTF(x)      do { if (ehcidebug) kprintf x; } while (0)
  91 #define DPRINTFN(n,x)   do { if (ehcidebug>(n)) kprintf x; } while (0)
  92 int ehcidebug = 0;
  93 SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  94 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
  95            &ehcidebug, 0, "ehci debug level");
  96 #define bitmask_snprintf(q,f,b,l) ksnprintf((b), (l), "%b", (q), (f))
  97 #else
  98 #define DPRINTF(x)
  99 #define DPRINTFN(n,x)
 100 #endif
 101
 102 struct ehci_pipe {
 103         struct usbd_pipe pipe;
 104
 105         ehci_soft_qh_t *sqh;
 106         union {
 107                 ehci_soft_qtd_t *qtd;
 108                 /* ehci_soft_itd_t *itd; */
 109         } tail;
 110         union {
 111                 /* Control pipe */
 112                 struct {
 113                         usb_dma_t reqdma;
 114                         u_int length;
 115                         /*ehci_soft_qtd_t *setup, *data, *stat;*/
 116                 } ctl;
 117                 /* Interrupt pipe */
 118                 struct {
 119                         u_int length;
 120                 } intr;
 121                 /* Bulk pipe */
 122                 struct {
 123                         u_int length;
 124                 } bulk;
 125                 /* Iso pipe */
 126                 /* XXX */
 127         } u;
 128 };
 129
 130 static usbd_status      ehci_open(usbd_pipe_handle);
 131 static void             ehci_poll(struct usbd_bus *);
 132 static void             ehci_softintr(void *);
 133 static int              ehci_intr1(ehci_softc_t *);
 134 static void             ehci_waitintr(ehci_softc_t *, usbd_xfer_handle);
 135 static void             ehci_check_intr(ehci_softc_t *, struct ehci_xfer *);
 136 static void             ehci_idone(struct ehci_xfer *);
 137 static void             ehci_timeout(void *);
 138 static void             ehci_timeout_task(void *);
 139 static void             ehci_intrlist_timeout(void *);
 140
 141 static usbd_status      ehci_allocm(struct usbd_bus *, usb_dma_t *, u_int32_t);
 142 static void             ehci_freem(struct usbd_bus *, usb_dma_t *);
 143
 144 static usbd_xfer_handle ehci_allocx(struct usbd_bus *);
 145 static void             ehci_freex(struct usbd_bus *, usbd_xfer_handle);
 146
 147 static usbd_status      ehci_root_ctrl_transfer(usbd_xfer_handle);
 148 static usbd_status      ehci_root_ctrl_start(usbd_xfer_handle);
 149 static void             ehci_root_ctrl_abort(usbd_xfer_handle);
 150 static void             ehci_root_ctrl_close(usbd_pipe_handle);
 151 static void             ehci_root_ctrl_done(usbd_xfer_handle);
 152
 153 static usbd_status      ehci_root_intr_transfer(usbd_xfer_handle);
 154 static usbd_status      ehci_root_intr_start(usbd_xfer_handle);
 155 static void             ehci_root_intr_abort(usbd_xfer_handle);
 156 static void             ehci_root_intr_close(usbd_pipe_handle);
 157 static void             ehci_root_intr_done(usbd_xfer_handle);
 158
 159 static usbd_status      ehci_device_ctrl_transfer(usbd_xfer_handle);
 160 static usbd_status      ehci_device_ctrl_start(usbd_xfer_handle);
 161 static void             ehci_device_ctrl_abort(usbd_xfer_handle);
 162 static void             ehci_device_ctrl_close(usbd_pipe_handle);
 163 static void             ehci_device_ctrl_done(usbd_xfer_handle);
 164
 165 static usbd_status      ehci_device_bulk_transfer(usbd_xfer_handle);
 166 static usbd_status      ehci_device_bulk_start(usbd_xfer_handle);
 167 static void             ehci_device_bulk_abort(usbd_xfer_handle);
 168 static void             ehci_device_bulk_close(usbd_pipe_handle);
 169 static void             ehci_device_bulk_done(usbd_xfer_handle);
 170
 171 static usbd_status      ehci_device_intr_transfer(usbd_xfer_handle);
 172 static usbd_status      ehci_device_intr_start(usbd_xfer_handle);
 173 static void             ehci_device_intr_abort(usbd_xfer_handle);
 174 static void             ehci_device_intr_close(usbd_pipe_handle);
 175 static void             ehci_device_intr_done(usbd_xfer_handle);
 176
 177 static usbd_status      ehci_device_isoc_transfer(usbd_xfer_handle);
 178 static usbd_status      ehci_device_isoc_start(usbd_xfer_handle);
 179 static void             ehci_device_isoc_abort(usbd_xfer_handle);
 180 static void             ehci_device_isoc_close(usbd_pipe_handle);
 181 static void             ehci_device_isoc_done(usbd_xfer_handle);
 182
 183 static void             ehci_device_clear_toggle(usbd_pipe_handle pipe);
 184 static void             ehci_noop(usbd_pipe_handle pipe);
 185
 186 static int              ehci_str(usb_string_descriptor_t *, int, char *);
 187 static void             ehci_pcd(ehci_softc_t *, usbd_xfer_handle);
 188 static void             ehci_disown(ehci_softc_t *, int, int);
 189
 190 static ehci_soft_qh_t  *ehci_alloc_sqh(ehci_softc_t *);
 191 static void             ehci_free_sqh(ehci_softc_t *, ehci_soft_qh_t *);
 192
 193 static ehci_soft_qtd_t  *ehci_alloc_sqtd(ehci_softc_t *);
 194 static void             ehci_free_sqtd(ehci_softc_t *, ehci_soft_qtd_t *);
 195 static usbd_status      ehci_alloc_sqtd_chain(struct ehci_pipe *,
 196                             ehci_softc_t *, int, int, usbd_xfer_handle,
 197                             ehci_soft_qtd_t **, ehci_soft_qtd_t **);
 198 static void             ehci_free_sqtd_chain(ehci_softc_t *, ehci_soft_qtd_t *,
 199                                             ehci_soft_qtd_t *);
 200
 201 static usbd_status      ehci_device_request(usbd_xfer_handle xfer);
 202
 203 static usbd_status      ehci_device_setintr(ehci_softc_t *, ehci_soft_qh_t *,
 204                             int ival);
 205
 206 static void             ehci_add_qh(ehci_soft_qh_t *, ehci_soft_qh_t *);
 207 static void             ehci_rem_qh(ehci_softc_t *, ehci_soft_qh_t *,
 208                                     ehci_soft_qh_t *);
 209 static void             ehci_set_qh_qtd(ehci_soft_qh_t *, ehci_soft_qtd_t *);
 210 static void             ehci_sync_hc(ehci_softc_t *);
 211
 212 static void             ehci_close_pipe(usbd_pipe_handle, ehci_soft_qh_t *);
 213 static void             ehci_abort_xfer(usbd_xfer_handle, usbd_status);
 214
 215 #ifdef EHCI_DEBUG
 216 static void             ehci_dump_regs(ehci_softc_t *);
 217 void                    ehci_dump(void);
 218 static ehci_softc_t     *theehci;
 219 static void             ehci_dump_link(ehci_link_t, int);
 220 static void             ehci_dump_sqtds(ehci_soft_qtd_t *);
 221 static void             ehci_dump_sqtd(ehci_soft_qtd_t *);
 222 static void             ehci_dump_qtd(ehci_qtd_t *);
 223 static void             ehci_dump_sqh(ehci_soft_qh_t *);
 224 #ifdef DIAGNOSTIC
 225 static void             ehci_dump_exfer(struct ehci_xfer *);
 226 #endif
 227 #endif
 228
 229 #define EHCI_NULL htole32(EHCI_LINK_TERMINATE)
 230
 231 #define EHCI_INTR_ENDPT 1
 232
 233 #define ehci_add_intr_list(sc, ex) \
 234         LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext);
 235 #define ehci_del_intr_list(ex) \
 236         do { \
 237                 LIST_REMOVE((ex), inext); \
 238                 (ex)->inext.le_prev = NULL; \
 239         } while (0)
 240 #define ehci_active_intr_list(ex) ((ex)->inext.le_prev != NULL)
 241
 242 static struct usbd_bus_methods ehci_bus_methods = {
 243         ehci_open,
 244         ehci_softintr,
 245         ehci_poll,
 246         ehci_allocm,
 247         ehci_freem,
 248         ehci_allocx,
 249         ehci_freex,
 250 };
 251
 252 static struct usbd_pipe_methods ehci_root_ctrl_methods = {
 253         ehci_root_ctrl_transfer,
 254         ehci_root_ctrl_start,
 255         ehci_root_ctrl_abort,
 256         ehci_root_ctrl_close,
 257         ehci_noop,
 258         ehci_root_ctrl_done,
 259 };
 260
 261 static struct usbd_pipe_methods ehci_root_intr_methods = {
 262         ehci_root_intr_transfer,
 263         ehci_root_intr_start,
 264         ehci_root_intr_abort,
 265         ehci_root_intr_close,
 266         ehci_noop,
 267         ehci_root_intr_done,
 268 };
 269
 270 static struct usbd_pipe_methods ehci_device_ctrl_methods = {
 271         ehci_device_ctrl_transfer,
 272         ehci_device_ctrl_start,
 273         ehci_device_ctrl_abort,
 274         ehci_device_ctrl_close,
 275         ehci_noop,
 276         ehci_device_ctrl_done,
 277 };
 278
 279 static struct usbd_pipe_methods ehci_device_intr_methods = {
 280         ehci_device_intr_transfer,
 281         ehci_device_intr_start,
 282         ehci_device_intr_abort,
 283         ehci_device_intr_close,
 284         ehci_device_clear_toggle,
 285         ehci_device_intr_done,
 286 };
 287
 288 static struct usbd_pipe_methods ehci_device_bulk_methods = {
 289         ehci_device_bulk_transfer,
 290         ehci_device_bulk_start,
 291         ehci_device_bulk_abort,
 292         ehci_device_bulk_close,
 293         ehci_device_clear_toggle,
 294         ehci_device_bulk_done,
 295 };
 296
 297 static struct usbd_pipe_methods ehci_device_isoc_methods = {
 298         ehci_device_isoc_transfer,
 299         ehci_device_isoc_start,
 300         ehci_device_isoc_abort,
 301         ehci_device_isoc_close,
 302         ehci_noop,
 303         ehci_device_isoc_done,
 304 };
 305
 306 usbd_status
 307 ehci_init(ehci_softc_t *sc)
 308 {
 309         u_int32_t vers, sparams, cparams, hcr;
 310         u_int i;
 311         usbd_status err;
 312         ehci_soft_qh_t *sqh;
 313         u_int ncomp;
 314         int lev;
 315
 316         DPRINTF(("ehci_init: start\n"));
 317 #ifdef EHCI_DEBUG
 318         theehci = sc;
 319 #endif
 320
 321         sc->sc_offs = EREAD1(sc, EHCI_CAPLENGTH);
 322
 323         vers = EREAD2(sc, EHCI_HCIVERSION);
 324         device_printf(sc->sc_bus.bdev,
 325             "EHCI version %x.%x\n", vers >> 8, vers & 0xff);
 326
 327         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 328         DPRINTF(("ehci_init: sparams=0x%x\n", sparams));
 329         sc->sc_npcomp = EHCI_HCS_N_PCC(sparams);
 330         ncomp = EHCI_HCS_N_CC(sparams);
 331         if (ncomp != sc->sc_ncomp) {
 332                 device_printf(sc->sc_bus.bdev,
 333                     "wrong number of companions (%d != %d)\n",
 334                     ncomp, sc->sc_ncomp);
 335                 if (ncomp < sc->sc_ncomp)
 336                         sc->sc_ncomp = ncomp;
 337         }
 338         if (sc->sc_ncomp > 0) {
 339                 device_printf(sc->sc_bus.bdev,
 340                     "companion controller%s, %d port%s each:",
 341                     sc->sc_ncomp!=1 ? "s" : "",
 342                     EHCI_HCS_N_PCC(sparams),
 343                     EHCI_HCS_N_PCC(sparams)!=1 ? "s" : "");
 344                 for (i = 0; i < sc->sc_ncomp; i++)
 345                         kprintf(" %s", device_get_nameunit(sc->sc_comps[i]->bdev));
 346                 kprintf("\n");
 347         }
 348         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 349         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 350         DPRINTF(("ehci_init: cparams=0x%x\n", cparams));
 351
 352         if (EHCI_HCC_64BIT(cparams)) {
 353                 /* MUST clear segment register if 64 bit capable. */
 354                 EWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 355         }
 356
 357         sc->sc_bus.usbrev = USBREV_2_0;
 358
 359         /* Reset the controller */
 360         DPRINTF(("%s: resetting\n", device_get_nameunit(sc->sc_bus.bdev)));
 361         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 362         usb_delay_ms(&sc->sc_bus, 1);
 363         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 364         for (i = 0; i < 100; i++) {
 365                 usb_delay_ms(&sc->sc_bus, 1);
 366                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 367                 if (!hcr)
 368                         break;
 369         }
 370         if (hcr) {
 371                 device_printf(sc->sc_bus.bdev, "reset timeout\n");
 372                 return (USBD_IOERROR);
 373         }
 374
 375         /* frame list size at default, read back what we got and use that */
 376         switch (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD))) {
 377         case 0: sc->sc_flsize = 1024; break;
 378         case 1: sc->sc_flsize = 512; break;
 379         case 2: sc->sc_flsize = 256; break;
 380         case 3: return (USBD_IOERROR);
 381         }
 382         err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 383                            EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 384         if (err)
 385                 return (err);
 386         DPRINTF(("%s: flsize=%d\n", device_get_nameunit(sc->sc_bus.bdev),sc->sc_flsize));
 387         sc->sc_flist = KERNADDR(&sc->sc_fldma, 0);
 388         EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 389
 390         /* Set up the bus struct. */
 391         sc->sc_bus.methods = &ehci_bus_methods;
 392         sc->sc_bus.pipe_size = sizeof(struct ehci_pipe);
 393
 394         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 395
 396         /*
 397          * Allocate the interrupt dummy QHs. These are arranged to give
 398          * poll intervals that are powers of 2 times 1ms.
 399          */
 400         for (i = 0; i < EHCI_INTRQHS; i++) {
 401                 sqh = ehci_alloc_sqh(sc);
 402                 if (sqh == NULL) {
 403                         err = USBD_NOMEM;
 404                         goto bad1;
 405                 }
 406                 sc->sc_islots[i].sqh = sqh;
 407         }
 408         lev = 0;
 409         for (i = 0; i < EHCI_INTRQHS; i++) {
 410                 if (i == EHCI_IQHIDX(lev + 1, 0))
 411                         lev++;
 412                 sqh = sc->sc_islots[i].sqh;
 413                 if (i == 0) {
 414                         /* The last (1ms) QH terminates. */
 415                         sqh->qh.qh_link = EHCI_NULL;
 416                         sqh->next = NULL;
 417                 } else {
 418                         /* Otherwise the next QH has half the poll interval */
 419                         sqh->next =
 420                             sc->sc_islots[EHCI_IQHIDX(lev - 1, i + 1)].sqh;
 421                         sqh->qh.qh_link = htole32(sqh->next->physaddr |
 422                             EHCI_LINK_QH);
 423                 }
 424                 sqh->qh.qh_endp = htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 425                 sqh->qh.qh_endphub = htole32(EHCI_QH_SET_MULT(1));
 426                 sqh->qh.qh_curqtd = EHCI_NULL;
 427                 sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 428                 sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 429                 sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 430                 sqh->sqtd = NULL;
 431         }
 432         /* Point the frame list at the last level (128ms). */
 433         for (i = 0; i < sc->sc_flsize; i++) {
 434                 sc->sc_flist[i] = htole32(EHCI_LINK_QH |
 435                     sc->sc_islots[EHCI_IQHIDX(EHCI_IPOLLRATES - 1,
 436                     i)].sqh->physaddr);
 437         }
 438
 439         /* Allocate dummy QH that starts the async list. */
 440         sqh = ehci_alloc_sqh(sc);
 441         if (sqh == NULL) {
 442                 err = USBD_NOMEM;
 443                 goto bad1;
 444         }
 445         /* Fill the QH */
 446         sqh->qh.qh_endp =
 447             htole32(EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 448         sqh->qh.qh_link =
 449             htole32(sqh->physaddr | EHCI_LINK_QH);
 450         sqh->qh.qh_curqtd = EHCI_NULL;
 451         sqh->prev = sqh; /*It's a circular list.. */
 452         sqh->next = sqh;
 453         /* Fill the overlay qTD */
 454         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
 455         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
 456         sqh->qh.qh_qtd.qtd_status = htole32(EHCI_QTD_HALTED);
 457         sqh->sqtd = NULL;
 458 #ifdef EHCI_DEBUG
 459         if (ehcidebug) {
 460                 ehci_dump_sqh(sqh);
 461         }
 462 #endif
 463
 464         /* Point to async list */
 465         sc->sc_async_head = sqh;
 466         EOWRITE4(sc, EHCI_ASYNCLISTADDR, sqh->physaddr | EHCI_LINK_QH);
 467
 468         callout_init(&sc->sc_tmo_intrlist);
 469
 470         lockinit(&sc->sc_doorbell_lock, "ehcidb", 0, 0);
 471
 472         /* Enable interrupts */
 473         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 474
 475         /* Turn on controller */
 476         EOWRITE4(sc, EHCI_USBCMD,
 477                  EHCI_CMD_ITC_2 | /* 2 microframes interrupt delay */
 478                  (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 479                  EHCI_CMD_ASE |
 480                  EHCI_CMD_PSE |
 481                  EHCI_CMD_RS);
 482
 483         /* Take over port ownership */
 484         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 485
 486         for (i = 0; i < 100; i++) {
 487                 usb_delay_ms(&sc->sc_bus, 1);
 488                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 489                 if (!hcr)
 490                         break;
 491         }
 492         if (hcr) {
 493                 device_printf(sc->sc_bus.bdev, "run timeout\n");
 494                 return (USBD_IOERROR);
 495         }
 496
 497         return (USBD_NORMAL_COMPLETION);
 498
 499 #if 0
 500  bad2:
 501         ehci_free_sqh(sc, sc->sc_async_head);
 502 #endif
 503  bad1:
 504         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 505         return (err);
 506 }
 507
 508 int
 509 ehci_intr(void *v)
 510 {
 511         ehci_softc_t *sc = v;
 512
 513         if (sc->sc_dying || (sc->sc_flags & EHCI_SCFLG_DONEINIT) == 0)
 514                 return (0);
 515
 516         /* If we get an interrupt while polling, then just ignore it. */
 517         if (sc->sc_bus.use_polling) {
 518                 u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 519
 520                 if (intrs)
 521                         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 522 #ifdef DIAGNOSTIC
 523                 DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n"));
 524 #endif
 525                 return (0);
 526         }
 527
 528         return (ehci_intr1(sc));
 529 }
 530
 531 static int
 532 ehci_intr1(ehci_softc_t *sc)
 533 {
 534         u_int32_t intrs, eintrs;
 535
 536         DPRINTFN(20,("ehci_intr1: enter\n"));
 537
 538         intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 539         if (!intrs)
 540                 return (0);
 541
 542         eintrs = intrs & sc->sc_eintrs;
 543         DPRINTFN(7, ("ehci_intr1: sc=%p intrs=0x%x(0x%x) eintrs=0x%x\n",
 544                      sc, (u_int)intrs, EOREAD4(sc, EHCI_USBSTS),
 545                      (u_int)eintrs));
 546         if (!eintrs)
 547                 return (0);
 548
 549         EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */
 550         sc->sc_bus.intr_context++;
 551         sc->sc_bus.no_intrs++;
 552         if (eintrs & EHCI_STS_IAA) {
 553                 DPRINTF(("ehci_intr1: door bell\n"));
 554                 wakeup(&sc->sc_async_head);
 555                 eintrs &= ~EHCI_STS_IAA;
 556         }
 557         if (eintrs & (EHCI_STS_INT | EHCI_STS_ERRINT)) {
 558                 DPRINTFN(5,("ehci_intr1: %s %s\n",
 559                             eintrs & EHCI_STS_INT ? "INT" : "",
 560                             eintrs & EHCI_STS_ERRINT ? "ERRINT" : ""));
 561                 usb_schedsoftintr(&sc->sc_bus);
 562                 eintrs &= ~(EHCI_STS_INT | EHCI_STS_ERRINT);
 563         }
 564         if (eintrs & EHCI_STS_HSE) {
 565                 device_printf(sc->sc_bus.bdev,
 566                     "unrecoverable error, controller halted\n");
 567                 /* XXX what else */
 568         }
 569         if (eintrs & EHCI_STS_PCD) {
 570                 ehci_pcd(sc, sc->sc_intrxfer);
 571                 eintrs &= ~EHCI_STS_PCD;
 572         }
 573
 574         sc->sc_bus.intr_context--;
 575
 576         if (eintrs != 0) {
 577                 /* Block unprocessed interrupts. */
 578                 sc->sc_eintrs &= ~eintrs;
 579                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 580                 device_printf(sc->sc_bus.bdev,
 581                     "blocking intrs 0x%x\n", eintrs);
 582         }
 583
 584         return (1);
 585 }
 586
 587 void
 588 ehci_pcd(ehci_softc_t *sc, usbd_xfer_handle xfer)
 589 {
 590         usbd_pipe_handle pipe;
 591         u_char *p;
 592         int i, m;
 593
 594         if (xfer == NULL) {
 595                 /* Just ignore the change. */
 596                 return;
 597         }
 598
 599         pipe = xfer->pipe;
 600
 601         p = KERNADDR(&xfer->dmabuf, 0);
 602         m = min(sc->sc_noport, xfer->length * 8 - 1);
 603         memset(p, 0, xfer->length);
 604         for (i = 1; i <= m; i++) {
 605                 /* Pick out CHANGE bits from the status reg. */
 606                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR)
 607                         p[i/8] |= 1 << (i%8);
 608         }
 609         DPRINTF(("ehci_pcd: change=0x%02x\n", *p));
 610         xfer->actlen = xfer->length;
 611         xfer->status = USBD_NORMAL_COMPLETION;
 612
 613         usb_transfer_complete(xfer);
 614 }
 615
 616 void
 617 ehci_softintr(void *v)
 618 {
 619         ehci_softc_t *sc = v;
 620         struct ehci_xfer *ex, *nextex;
 621
 622         DPRINTFN(10,("%s: ehci_softintr (%d)\n", device_get_nameunit(sc->sc_bus.bdev),
 623                      sc->sc_bus.intr_context));
 624
 625         sc->sc_bus.intr_context++;
 626
 627         /*
 628          * The only explanation I can think of for why EHCI is as brain dead
 629          * as UHCI interrupt-wise is that Intel was involved in both.
 630          * An interrupt just tells us that something is done, we have no
 631          * clue what, so we need to scan through all active transfers. :-(
 632          */
 633         for (ex = LIST_FIRST(&sc->sc_intrhead); ex; ex = nextex) {
 634                 nextex = LIST_NEXT(ex, inext);
 635                 ehci_check_intr(sc, ex);
 636         }
 637
 638         /* Schedule a callout to catch any dropped transactions. */
 639         if ((sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) &&
 640             !LIST_EMPTY(&sc->sc_intrhead))
 641                 callout_reset(&sc->sc_tmo_intrlist, hz / 5, ehci_intrlist_timeout,
 642                    sc);
 643
 644 #ifdef USB_USE_SOFTINTR
 645         if (sc->sc_softwake) {
 646                 sc->sc_softwake = 0;
 647                 wakeup(&sc->sc_softwake);
 648         }
 649 #endif /* USB_USE_SOFTINTR */
 650
 651         sc->sc_bus.intr_context--;
 652 }
 653
 654 /* Check for an interrupt. */
 655 void
 656 ehci_check_intr(ehci_softc_t *sc, struct ehci_xfer *ex)
 657 {
 658         ehci_soft_qtd_t *sqtd, *lsqtd;
 659         u_int32_t status;
 660
 661         DPRINTFN(/*15*/2, ("ehci_check_intr: ex=%p\n", ex));
 662
 663         if (ex->sqtdstart == NULL) {
 664                 kprintf("ehci_check_intr: sqtdstart=NULL\n");
 665                 return;
 666         }
 667         lsqtd = ex->sqtdend;
 668 #ifdef DIAGNOSTIC
 669         if (lsqtd == NULL) {
 670                 kprintf("ehci_check_intr: lsqtd==0\n");
 671                 return;
 672         }
 673 #endif
 674         /*
 675          * If the last TD is still active we need to check whether there
 676          * is a an error somewhere in the middle, or whether there was a
 677          * short packet (SPD and not ACTIVE).
 678          */
 679         if (le32toh(lsqtd->qtd.qtd_status) & EHCI_QTD_ACTIVE) {
 680                 DPRINTFN(12, ("ehci_check_intr: active ex=%p\n", ex));
 681                 for (sqtd = ex->sqtdstart; sqtd != lsqtd; sqtd=sqtd->nextqtd) {
 682                         status = le32toh(sqtd->qtd.qtd_status);
 683                         /* If there's an active QTD the xfer isn't done. */
 684                         if (status & EHCI_QTD_ACTIVE)
 685                                 break;
 686                         /* Any kind of error makes the xfer done. */
 687                         if (status & EHCI_QTD_HALTED)
 688                                 goto done;
 689                         /* We want short packets, and it is short: it's done */
 690                         if (EHCI_QTD_GET_BYTES(status) != 0)
 691                                 goto done;
 692                 }
 693                 DPRINTFN(12, ("ehci_check_intr: ex=%p std=%p still active\n",
 694                               ex, ex->sqtdstart));
 695                 return;
 696         }
 697  done:
 698         DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex));
 699         callout_stop(&ex->xfer.timeout_handle);
 700         usb_rem_task(ex->xfer.pipe->device, &ex->abort_task);
 701         ehci_idone(ex);
 702 }
 703
 704 void
 705 ehci_idone(struct ehci_xfer *ex)
 706 {
 707         usbd_xfer_handle xfer = &ex->xfer;
 708 #ifdef USB_DEBUG
 709         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
 710 #endif
 711         ehci_soft_qtd_t *sqtd, *lsqtd;
 712         u_int32_t status = 0, nstatus = 0;
 713         int actlen, cerr;
 714
 715         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p\n", ex));
 716 #ifdef DIAGNOSTIC
 717         {
 718                 crit_enter();
 719                 if (ex->isdone) {
 720                         crit_exit();
 721 #ifdef EHCI_DEBUG
 722                         kprintf("ehci_idone: ex is done!\n   ");
 723                         ehci_dump_exfer(ex);
 724 #else
 725                         kprintf("ehci_idone: ex=%p is done!\n", ex);
 726 #endif
 727                         return;
 728                 }
 729                 ex->isdone = 1;
 730                 crit_exit();
 731         }
 732 #endif
 733
 734         if (xfer->status == USBD_CANCELLED ||
 735             xfer->status == USBD_TIMEOUT) {
 736                 DPRINTF(("ehci_idone: aborted xfer=%p\n", xfer));
 737                 return;
 738         }
 739
 740 #ifdef EHCI_DEBUG
 741         DPRINTFN(/*10*/2, ("ehci_idone: xfer=%p, pipe=%p ready\n", xfer, epipe));
 742         if (ehcidebug > 10)
 743                 ehci_dump_sqtds(ex->sqtdstart);
 744 #endif
 745
 746         /* The transfer is done, compute actual length and status. */
 747         lsqtd = ex->sqtdend;
 748         actlen = 0;
 749         for (sqtd = ex->sqtdstart; sqtd != lsqtd->nextqtd; sqtd=sqtd->nextqtd) {
 750                 nstatus = le32toh(sqtd->qtd.qtd_status);
 751                 if (nstatus & EHCI_QTD_ACTIVE)
 752                         break;
 753
 754                 status = nstatus;
 755                 /* halt is ok if descriptor is last, and complete */
 756                 if (sqtd->qtd.qtd_next == EHCI_NULL &&
 757                     EHCI_QTD_GET_BYTES(status) == 0)
 758                         status &= ~EHCI_QTD_HALTED;
 759                 if (EHCI_QTD_GET_PID(status) != EHCI_QTD_PID_SETUP)
 760                         actlen += sqtd->len - EHCI_QTD_GET_BYTES(status);
 761         }
 762
 763         cerr = EHCI_QTD_GET_CERR(status);
 764         DPRINTFN(/*10*/2, ("ehci_idone: len=%d, actlen=%d, cerr=%d, "
 765             "status=0x%x\n", xfer->length, actlen, cerr, status));
 766         xfer->actlen = actlen;
 767         if ((status & EHCI_QTD_HALTED) != 0) {
 768 #ifdef EHCI_DEBUG
 769                 char sbuf[128];
 770
 771                 bitmask_snprintf((u_int32_t)status,
 772                     "\20\7HALTED\6BUFERR\5BABBLE\4XACTERR"
 773                     "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
 774
 775                 DPRINTFN(2,
 776                          ("ehci_idone: error, addr=%d, endpt=0x%02x, "
 777                           "status 0x%s\n",
 778                           xfer->pipe->device->address,
 779                           xfer->pipe->endpoint->edesc->bEndpointAddress,
 780                           sbuf));
 781                 if (ehcidebug > 2) {
 782                         ehci_dump_sqh(epipe->sqh);
 783                         ehci_dump_sqtds(ex->sqtdstart);
 784                 }
 785 #endif
 786                 if ((status & EHCI_QTD_BABBLE) == 0 && cerr > 0)
 787                         xfer->status = USBD_STALLED;
 788                 else
 789                         xfer->status = USBD_IOERROR; /* more info XXX */
 790         } else {
 791                 xfer->status = USBD_NORMAL_COMPLETION;
 792         }
 793
 794         usb_transfer_complete(xfer);
 795         DPRINTFN(/*12*/2, ("ehci_idone: ex=%p done\n", ex));
 796 }
 797
 798 /*
 799  * Wait here until controller claims to have an interrupt.
 800  * Then call ehci_intr and return.  Use timeout to avoid waiting
 801  * too long.
 802  */
 803 void
 804 ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer)
 805 {
 806         int timo = xfer->timeout;
 807         int usecs;
 808         u_int32_t intrs;
 809
 810         xfer->status = USBD_IN_PROGRESS;
 811         for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) {
 812                 usb_delay_ms(&sc->sc_bus, 1);
 813                 if (sc->sc_dying)
 814                         break;
 815                 intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) &
 816                         sc->sc_eintrs;
 817                 DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs));
 818 #ifdef EHCI_DEBUG
 819                 if (ehcidebug > 15)
 820                         ehci_dump_regs(sc);
 821 #endif
 822                 if (intrs) {
 823                         ehci_intr1(sc);
 824                         if (xfer->status != USBD_IN_PROGRESS)
 825                                 return;
 826                 }
 827         }
 828
 829         /* Timeout */
 830         DPRINTF(("ehci_waitintr: timeout\n"));
 831         xfer->status = USBD_TIMEOUT;
 832         usb_transfer_complete(xfer);
 833         /* XXX should free TD */
 834 }
 835
 836 void
 837 ehci_poll(struct usbd_bus *bus)
 838 {
 839         ehci_softc_t *sc = (ehci_softc_t *)bus;
 840 #ifdef EHCI_DEBUG
 841         static int last;
 842         int new;
 843         new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
 844         if (new != last) {
 845                 DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new));
 846                 last = new;
 847         }
 848 #endif
 849
 850         if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs)
 851                 ehci_intr1(sc);
 852 }
 853
 854 int
 855 ehci_detach(struct ehci_softc *sc, int flags)
 856 {
 857         int rv = 0;
 858
 859         sc->sc_dying = 1;
 860
 861         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 862         EOWRITE4(sc, EHCI_USBCMD, 0);
 863         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 864         callout_stop(&sc->sc_tmo_intrlist);
 865
 866         usb_delay_ms(&sc->sc_bus, 300); /* XXX let stray task complete */
 867
 868         usb_freemem(&sc->sc_bus, &sc->sc_fldma);
 869         /* XXX free other data structures XXX */
 870
 871         return (rv);
 872 }
 873
 874 /*
 875  * Handle suspend/resume.
 876  *
 877  * We need to switch to polling mode here, because this routine is
 878  * called from an interrupt context.  This is all right since we
 879  * are almost suspended anyway.
 880  */
 881 void
 882 ehci_power(int why, void *v)
 883 {
 884         ehci_softc_t *sc = v;
 885         u_int32_t cmd, hcr;
 886         int i;
 887
 888 #ifdef EHCI_DEBUG
 889         DPRINTF(("ehci_power: sc=%p, why=%d\n", sc, why));
 890         if (ehcidebug > 0)
 891                 ehci_dump_regs(sc);
 892 #endif
 893
 894         crit_enter();
 895
 896         switch (why) {
 897         case PWR_SUSPEND:
 898                 sc->sc_bus.use_polling++;
 899
 900                 for (i = 1; i <= sc->sc_noport; i++) {
 901                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 902                         if ((cmd & EHCI_PS_PO) == 0 &&
 903                             (cmd & EHCI_PS_PE) == EHCI_PS_PE)
 904                                 EOWRITE4(sc, EHCI_PORTSC(i),
 905                                     cmd | EHCI_PS_SUSP);
 906                 }
 907
 908                 sc->sc_cmd = EOREAD4(sc, EHCI_USBCMD);
 909
 910                 cmd = sc->sc_cmd & ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
 911                 EOWRITE4(sc, EHCI_USBCMD, cmd);
 912
 913                 for (i = 0; i < 100; i++) {
 914                         hcr = EOREAD4(sc, EHCI_USBSTS) &
 915                             (EHCI_STS_ASS | EHCI_STS_PSS);
 916                         if (hcr == 0)
 917                                 break;
 918
 919                         usb_delay_ms(&sc->sc_bus, 1);
 920                 }
 921                 if (hcr != 0) {
 922                         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 923                 }
 924
 925                 cmd &= ~EHCI_CMD_RS;
 926                 EOWRITE4(sc, EHCI_USBCMD, cmd);
 927
 928                 for (i = 0; i < 100; i++) {
 929                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 930                         if (hcr == EHCI_STS_HCH)
 931                                 break;
 932
 933                         usb_delay_ms(&sc->sc_bus, 1);
 934                 }
 935                 if (hcr != EHCI_STS_HCH) {
 936                         device_printf(sc->sc_bus.bdev, "config timeout\n");
 937                 }
 938
 939                 sc->sc_bus.use_polling--;
 940                 break;
 941
 942         case PWR_RESUME:
 943                 sc->sc_bus.use_polling++;
 944
 945                 /* restore things in case the bios sucks */
 946                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 947                 EOWRITE4(sc, EHCI_PERIODICLISTBASE, DMAADDR(&sc->sc_fldma, 0));
 948                 EOWRITE4(sc, EHCI_ASYNCLISTADDR,
 949                     sc->sc_async_head->physaddr | EHCI_LINK_QH);
 950                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 951
 952                 hcr = 0;
 953                 for (i = 1; i <= sc->sc_noport; i++) {
 954                         cmd = EOREAD4(sc, EHCI_PORTSC(i));
 955                         if ((cmd & EHCI_PS_PO) == 0 &&
 956                             (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP) {
 957                                 EOWRITE4(sc, EHCI_PORTSC(i),
 958                                     cmd | EHCI_PS_FPR);
 959                                 hcr = 1;
 960                         }
 961                 }
 962
 963                 if (hcr) {
 964                         usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
 965
 966                         for (i = 1; i <= sc->sc_noport; i++) {
 967                                 cmd = EOREAD4(sc, EHCI_PORTSC(i));
 968                                 if ((cmd & EHCI_PS_PO) == 0 &&
 969                                     (cmd & EHCI_PS_SUSP) == EHCI_PS_SUSP)
 970                                         EOWRITE4(sc, EHCI_PORTSC(i),
 971                                             cmd & ~EHCI_PS_FPR);
 972                         }
 973                 }
 974
 975                 EOWRITE4(sc, EHCI_USBCMD, sc->sc_cmd);
 976
 977                 for (i = 0; i < 100; i++) {
 978                         hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 979                         if (hcr != EHCI_STS_HCH)
 980                                 break;
 981
 982                         usb_delay_ms(&sc->sc_bus, 1);
 983                 }
 984                 if (hcr == EHCI_STS_HCH) {
 985                         device_printf(sc->sc_bus.bdev, "config timeout\n");
 986                 }
 987
 988                 usb_delay_ms(&sc->sc_bus, USB_RESUME_WAIT);
 989
 990                 sc->sc_bus.use_polling--;
 991                 break;
 992         }
 993         crit_exit();
 994
 995 #ifdef EHCI_DEBUG
 996         DPRINTF(("ehci_power: sc=%p\n", sc));
 997         if (ehcidebug > 0)
 998                 ehci_dump_regs(sc);
 999 #endif
1000 }
1001
1002 /*
1003  * Shut down the controller when the system is going down.
1004  */
1005 void
1006 ehci_shutdown(void *v)
1007 {
1008         ehci_softc_t *sc = v;
1009
1010         DPRINTF(("ehci_shutdown: stopping the HC\n"));
1011         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
1012         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
1013 }
1014
1015 usbd_status
1016 ehci_allocm(struct usbd_bus *bus, usb_dma_t *dma, u_int32_t size)
1017 {
1018         usbd_status err;
1019
1020         err = usb_allocmem(bus, size, 0, dma);
1021 #ifdef EHCI_DEBUG
1022         if (err)
1023                 kprintf("ehci_allocm: usb_allocmem()=%d\n", err);
1024 #endif
1025         return (err);
1026 }
1027
1028 void
1029 ehci_freem(struct usbd_bus *bus, usb_dma_t *dma)
1030 {
1031         usb_freemem(bus, dma);
1032 }
1033
1034 usbd_xfer_handle
1035 ehci_allocx(struct usbd_bus *bus)
1036 {
1037         struct ehci_softc *sc = (struct ehci_softc *)bus;
1038         usbd_xfer_handle xfer;
1039
1040         xfer = STAILQ_FIRST(&sc->sc_free_xfers);
1041         if (xfer != NULL) {
1042                 STAILQ_REMOVE_HEAD(&sc->sc_free_xfers, next);
1043 #ifdef DIAGNOSTIC
1044                 if (xfer->busy_free != XFER_FREE) {
1045                         kprintf("ehci_allocx: xfer=%p not free, 0x%08x\n", xfer,
1046                                xfer->busy_free);
1047                 }
1048 #endif
1049         } else {
1050                 xfer = kmalloc(sizeof(struct ehci_xfer), M_USB, M_INTWAIT);
1051         }
1052         if (xfer != NULL) {
1053                 memset(xfer, 0, sizeof(struct ehci_xfer));
1054                 usb_init_task(&EXFER(xfer)->abort_task, ehci_timeout_task,
1055                     xfer);
1056                 EXFER(xfer)->ehci_xfer_flags = 0;
1057 #ifdef DIAGNOSTIC
1058                 EXFER(xfer)->isdone = 1;
1059                 xfer->busy_free = XFER_BUSY;
1060 #endif
1061         }
1062         return (xfer);
1063 }
1064
1065 void
1066 ehci_freex(struct usbd_bus *bus, usbd_xfer_handle xfer)
1067 {
1068         struct ehci_softc *sc = (struct ehci_softc *)bus;
1069
1070 #ifdef DIAGNOSTIC
1071         if (xfer->busy_free != XFER_BUSY) {
1072                 kprintf("ehci_freex: xfer=%p not busy, 0x%08x\n", xfer,
1073                        xfer->busy_free);
1074                 return;
1075         }
1076         xfer->busy_free = XFER_FREE;
1077         if (!EXFER(xfer)->isdone) {
1078                 kprintf("ehci_freex: !isdone\n");
1079                 return;
1080         }
1081 #endif
1082         STAILQ_INSERT_HEAD(&sc->sc_free_xfers, xfer, next);
1083 }
1084
1085 static void
1086 ehci_device_clear_toggle(usbd_pipe_handle pipe)
1087 {
1088         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1089
1090         DPRINTF(("ehci_device_clear_toggle: epipe=%p status=0x%x\n",
1091                  epipe, epipe->sqh->qh.qh_qtd.qtd_status));
1092 #ifdef USB_DEBUG
1093         if (ehcidebug)
1094                 usbd_dump_pipe(pipe);
1095 #endif
1096         KASSERT((epipe->sqh->qh.qh_qtd.qtd_status &
1097             htole32(EHCI_QTD_ACTIVE)) == 0,
1098             ("ehci_device_clear_toggle: queue active"));
1099         epipe->sqh->qh.qh_qtd.qtd_status &= htole32(~EHCI_QTD_TOGGLE_MASK);
1100 }
1101
1102 static void
1103 ehci_noop(usbd_pipe_handle pipe)
1104 {
1105 }
1106
1107 #ifdef EHCI_DEBUG
1108 void
1109 ehci_dump_regs(ehci_softc_t *sc)
1110 {
1111         int i;
1112         kprintf("cmd=0x%08x, sts=0x%08x, ien=0x%08x\n",
1113                EOREAD4(sc, EHCI_USBCMD),
1114                EOREAD4(sc, EHCI_USBSTS),
1115                EOREAD4(sc, EHCI_USBINTR));
1116         kprintf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
1117                EOREAD4(sc, EHCI_FRINDEX),
1118                EOREAD4(sc, EHCI_CTRLDSSEGMENT),
1119                EOREAD4(sc, EHCI_PERIODICLISTBASE),
1120                EOREAD4(sc, EHCI_ASYNCLISTADDR));
1121         for (i = 1; i <= sc->sc_noport; i++)
1122                 kprintf("port %d status=0x%08x\n", i,
1123                        EOREAD4(sc, EHCI_PORTSC(i)));
1124 }
1125
1126 /*
1127  * Unused function - this is meant to be called from a kernel
1128  * debugger.
1129  */
1130 void
1131 ehci_dump(void)
1132 {
1133         ehci_dump_regs(theehci);
1134 }
1135
1136 void
1137 ehci_dump_link(ehci_link_t link, int type)
1138 {
1139         link = le32toh(link);
1140         kprintf("0x%08x", link);
1141         if (link & EHCI_LINK_TERMINATE)
1142                 kprintf("<T>");
1143         else {
1144                 kprintf("<");
1145                 if (type) {
1146                         switch (EHCI_LINK_TYPE(link)) {
1147                         case EHCI_LINK_ITD: kprintf("ITD"); break;
1148                         case EHCI_LINK_QH: kprintf("QH"); break;
1149                         case EHCI_LINK_SITD: kprintf("SITD"); break;
1150                         case EHCI_LINK_FSTN: kprintf("FSTN"); break;
1151                         }
1152                 }
1153                 kprintf(">");
1154         }
1155 }
1156
1157 void
1158 ehci_dump_sqtds(ehci_soft_qtd_t *sqtd)
1159 {
1160         int i;
1161         u_int32_t stop;
1162
1163         stop = 0;
1164         for (i = 0; sqtd && i < 20 && !stop; sqtd = sqtd->nextqtd, i++) {
1165                 ehci_dump_sqtd(sqtd);
1166                 stop = sqtd->qtd.qtd_next & htole32(EHCI_LINK_TERMINATE);
1167         }
1168         if (sqtd)
1169                 kprintf("dump aborted, too many TDs\n");
1170 }
1171
1172 void
1173 ehci_dump_sqtd(ehci_soft_qtd_t *sqtd)
1174 {
1175         kprintf("QTD(%p) at 0x%08x:\n", sqtd, sqtd->physaddr);
1176         ehci_dump_qtd(&sqtd->qtd);
1177 }
1178
1179 void
1180 ehci_dump_qtd(ehci_qtd_t *qtd)
1181 {
1182         u_int32_t s;
1183         char sbuf[128];
1184
1185         kprintf("  next="); ehci_dump_link(qtd->qtd_next, 0);
1186         kprintf(" altnext="); ehci_dump_link(qtd->qtd_altnext, 0);
1187         kprintf("\n");
1188         s = le32toh(qtd->qtd_status);
1189         bitmask_snprintf(EHCI_QTD_GET_STATUS(s),
1190                          "\20\10ACTIVE\7HALTED\6BUFERR\5BABBLE\4XACTERR"
1191                          "\3MISSED\2SPLIT\1PING", sbuf, sizeof(sbuf));
1192         kprintf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
1193                s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
1194                EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
1195         kprintf("    cerr=%d pid=%d stat=0x%s\n", EHCI_QTD_GET_CERR(s),
1196                EHCI_QTD_GET_PID(s), sbuf);
1197         for (s = 0; s < 5; s++)
1198                 kprintf("  buffer[%d]=0x%08x\n", s, le32toh(qtd->qtd_buffer[s]));
1199 }
1200
1201 void
1202 ehci_dump_sqh(ehci_soft_qh_t *sqh)
1203 {
1204         ehci_qh_t *qh = &sqh->qh;
1205         u_int32_t endp, endphub;
1206
1207         kprintf("QH(%p) at 0x%08x:\n", sqh, sqh->physaddr);
1208         kprintf("  link="); ehci_dump_link(qh->qh_link, 1); kprintf("\n");
1209         endp = le32toh(qh->qh_endp);
1210         kprintf("  endp=0x%08x\n", endp);
1211         kprintf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
1212                EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
1213                EHCI_QH_GET_ENDPT(endp),  EHCI_QH_GET_EPS(endp),
1214                EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
1215         kprintf("    mpl=0x%x ctl=%d nrl=%d\n",
1216                EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
1217                EHCI_QH_GET_NRL(endp));
1218         endphub = le32toh(qh->qh_endphub);
1219         kprintf("  endphub=0x%08x\n", endphub);
1220         kprintf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
1221                EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
1222                EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
1223                EHCI_QH_GET_MULT(endphub));
1224         kprintf("  curqtd="); ehci_dump_link(qh->qh_curqtd, 0); kprintf("\n");
1225         kprintf("Overlay qTD:\n");
1226         ehci_dump_qtd(&qh->qh_qtd);
1227 }
1228
1229 #ifdef DIAGNOSTIC
1230 static void
1231 ehci_dump_exfer(struct ehci_xfer *ex)
1232 {
1233         kprintf("ehci_dump_exfer: ex=%p\n", ex);
1234 }
1235 #endif
1236 #endif
1237
1238 usbd_status
1239 ehci_open(usbd_pipe_handle pipe)
1240 {
1241         usbd_device_handle dev = pipe->device;
1242         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
1243         usb_endpoint_descriptor_t *ed = pipe->endpoint->edesc;
1244         u_int8_t addr = dev->address;
1245         u_int8_t xfertype = ed->bmAttributes & UE_XFERTYPE;
1246         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
1247         ehci_soft_qh_t *sqh;
1248         usbd_status err;
1249         int ival, speed, naks;
1250         int hshubaddr, hshubport;
1251
1252         DPRINTFN(1, ("ehci_open: pipe=%p, addr=%d, endpt=%d (%d)\n",
1253                      pipe, addr, ed->bEndpointAddress, sc->sc_addr));
1254
1255         if (dev->myhsport) {
1256                 hshubaddr = dev->myhsport->parent->address;
1257                 hshubport = dev->myhsport->portno;
1258         } else {
1259                 hshubaddr = 0;
1260                 hshubport = 0;
1261         }
1262
1263         if (sc->sc_dying)
1264                 return (USBD_IOERROR);
1265
1266         if (addr == sc->sc_addr) {
1267                 switch (ed->bEndpointAddress) {
1268                 case USB_CONTROL_ENDPOINT:
1269                         pipe->methods = &ehci_root_ctrl_methods;
1270                         break;
1271                 case UE_DIR_IN | EHCI_INTR_ENDPT:
1272                         pipe->methods = &ehci_root_intr_methods;
1273                         break;
1274                 default:
1275                         return (USBD_INVAL);
1276                 }
1277                 return (USBD_NORMAL_COMPLETION);
1278         }
1279
1280         /* XXX All this stuff is only valid for async. */
1281         switch (dev->speed) {
1282         case USB_SPEED_LOW:  speed = EHCI_QH_SPEED_LOW;  break;
1283         case USB_SPEED_FULL: speed = EHCI_QH_SPEED_FULL; break;
1284         case USB_SPEED_HIGH: speed = EHCI_QH_SPEED_HIGH; break;
1285         default: panic("ehci_open: bad device speed %d", dev->speed);
1286         }
1287         if (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_ISOCHRONOUS) {
1288                 device_printf(sc->sc_bus.bdev,
1289                     "*** WARNING: opening low/full speed device, this "
1290                     "does not work yet.\n");
1291                 DPRINTFN(1,("ehci_open: hshubaddr=%d hshubport=%d\n",
1292                             hshubaddr, hshubport));
1293                 return USBD_INVAL;
1294         }
1295
1296         naks = 8;               /* XXX */
1297         sqh = ehci_alloc_sqh(sc);
1298         if (sqh == NULL)
1299                 goto bad0;
1300         /* qh_link filled when the QH is added */
1301         sqh->qh.qh_endp = htole32(
1302                 EHCI_QH_SET_ADDR(addr) |
1303                 EHCI_QH_SET_ENDPT(UE_GET_ADDR(ed->bEndpointAddress)) |
1304                 EHCI_QH_SET_EPS(speed) |
1305                 (xfertype == UE_CONTROL ? EHCI_QH_DTC : 0) |
1306                 EHCI_QH_SET_MPL(UGETW(ed->wMaxPacketSize)) |
1307                 (speed != EHCI_QH_SPEED_HIGH && xfertype == UE_CONTROL ?
1308                  EHCI_QH_CTL : 0) |
1309                 EHCI_QH_SET_NRL(naks)
1310                 );
1311         sqh->qh.qh_endphub = htole32(
1312                 EHCI_QH_SET_MULT(1) |
1313                 EHCI_QH_SET_HUBA(hshubaddr) |
1314                 EHCI_QH_SET_PORT(hshubport) |
1315                 EHCI_QH_SET_CMASK(0x1c) |
1316                 EHCI_QH_SET_SMASK(xfertype == UE_INTERRUPT ? 0x01 : 0)
1317                 );
1318         sqh->qh.qh_curqtd = EHCI_NULL;
1319         /* Fill the overlay qTD */
1320         sqh->qh.qh_qtd.qtd_next = EHCI_NULL;
1321         sqh->qh.qh_qtd.qtd_altnext = EHCI_NULL;
1322         sqh->qh.qh_qtd.qtd_status =
1323             htole32(EHCI_QTD_SET_TOGGLE(pipe->endpoint->savedtoggle));
1324
1325         epipe->sqh = sqh;
1326
1327         switch (xfertype) {
1328         case UE_CONTROL:
1329                 err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
1330                                    0, &epipe->u.ctl.reqdma);
1331 #ifdef EHCI_DEBUG
1332                 if (err)
1333                         kprintf("ehci_open: usb_allocmem()=%d\n", err);
1334 #endif
1335                 if (err)
1336                         goto bad1;
1337                 pipe->methods = &ehci_device_ctrl_methods;
1338                 crit_enter();
1339                 ehci_add_qh(sqh, sc->sc_async_head);
1340                 crit_exit();
1341                 break;
1342         case UE_BULK:
1343                 pipe->methods = &ehci_device_bulk_methods;
1344                 crit_enter();
1345                 ehci_add_qh(sqh, sc->sc_async_head);
1346                 crit_exit();
1347                 break;
1348         case UE_INTERRUPT:
1349                 pipe->methods = &ehci_device_intr_methods;
1350                 ival = pipe->interval;
1351                 if (ival == USBD_DEFAULT_INTERVAL)
1352                         ival = ed->bInterval;
1353                 return (ehci_device_setintr(sc, sqh, ival));
1354         case UE_ISOCHRONOUS:
1355                 pipe->methods = &ehci_device_isoc_methods;
1356                 return (USBD_INVAL);
1357         default:
1358                 return (USBD_INVAL);
1359         }
1360         return (USBD_NORMAL_COMPLETION);
1361
1362  bad1:
1363         ehci_free_sqh(sc, sqh);
1364  bad0:
1365         return (USBD_NOMEM);
1366 }
1367
1368 /*
1369  * Add an ED to the schedule.  Called while in a critical section.
1370  * If in the async schedule, it will always have a next.
1371  * If in the intr schedule it may not.
1372  */
1373 void
1374 ehci_add_qh(ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1375 {
1376         sqh->next = head->next;
1377         sqh->prev = head;
1378         sqh->qh.qh_link = head->qh.qh_link;
1379         head->next = sqh;
1380         if (sqh->next)
1381                 sqh->next->prev = sqh;
1382         head->qh.qh_link = htole32(sqh->physaddr | EHCI_LINK_QH);
1383
1384 #ifdef EHCI_DEBUG
1385         if (ehcidebug > 5) {
1386                 kprintf("ehci_add_qh:\n");
1387                 ehci_dump_sqh(sqh);
1388         }
1389 #endif
1390 }
1391
1392 /*
1393  * Remove an ED from the schedule.  Called while in a critical section.
1394  * Will always have a 'next' if it's in the async list as it's circular.
1395  */
1396 void
1397 ehci_rem_qh(ehci_softc_t *sc, ehci_soft_qh_t *sqh, ehci_soft_qh_t *head)
1398 {
1399         /* XXX */
1400         sqh->prev->qh.qh_link = sqh->qh.qh_link;
1401         sqh->prev->next = sqh->next;
1402         if (sqh->next)
1403                 sqh->next->prev = sqh->prev;
1404         ehci_sync_hc(sc);
1405 }
1406
1407 void
1408 ehci_set_qh_qtd(ehci_soft_qh_t *sqh, ehci_soft_qtd_t *sqtd)
1409 {
1410         int i;
1411         u_int32_t status;
1412
1413         /* Save toggle bit and ping status. */
1414         status = sqh->qh.qh_qtd.qtd_status &
1415             htole32(EHCI_QTD_TOGGLE_MASK |
1416                     EHCI_QTD_SET_STATUS(EHCI_QTD_PINGSTATE));
1417         /* Set HALTED to make hw leave it alone. */
1418         sqh->qh.qh_qtd.qtd_status =
1419             htole32(EHCI_QTD_SET_STATUS(EHCI_QTD_HALTED));
1420         sqh->qh.qh_curqtd = 0;
1421         sqh->qh.qh_qtd.qtd_next = htole32(sqtd->physaddr);
1422         sqh->qh.qh_qtd.qtd_altnext = 0;
1423         for (i = 0; i < EHCI_QTD_NBUFFERS; i++)
1424                 sqh->qh.qh_qtd.qtd_buffer[i] = 0;
1425         sqh->sqtd = sqtd;
1426         /* Set !HALTED && !ACTIVE to start execution, preserve some fields */
1427         sqh->qh.qh_qtd.qtd_status = status;
1428 }
1429
1430 /*
1431  * Ensure that the HC has released all references to the QH.  We do this
1432  * by asking for a Async Advance Doorbell interrupt and then we wait for
1433  * the interrupt.
1434  * To make this easier we first obtain exclusive use of the doorbell.
1435  */
1436 void
1437 ehci_sync_hc(ehci_softc_t *sc)
1438 {
1439         int error;
1440
1441         if (sc->sc_dying) {
1442                 DPRINTFN(2,("ehci_sync_hc: dying\n"));
1443                 return;
1444         }
1445         DPRINTFN(2,("ehci_sync_hc: enter\n"));
1446         /* get doorbell */
1447         lockmgr(&sc->sc_doorbell_lock, LK_EXCLUSIVE);
1448         crit_enter();
1449         /* ask for doorbell */
1450         EOWRITE4(sc, EHCI_USBCMD, EOREAD4(sc, EHCI_USBCMD) | EHCI_CMD_IAAD);
1451         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1452                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1453         error = tsleep(&sc->sc_async_head, 0, "ehcidi", hz); /* bell wait */
1454         DPRINTFN(1,("ehci_sync_hc: cmd=0x%08x sts=0x%08x\n",
1455                     EOREAD4(sc, EHCI_USBCMD), EOREAD4(sc, EHCI_USBSTS)));
1456         crit_exit();
1457         /* release doorbell */
1458         lockmgr(&sc->sc_doorbell_lock, LK_RELEASE);
1459 #ifdef DIAGNOSTIC
1460         if (error)
1461                 kprintf("ehci_sync_hc: tsleep() = %d\n", error);
1462 #endif
1463         DPRINTFN(2,("ehci_sync_hc: exit\n"));
1464 }
1465
1466 /***********/
1467
1468 /*
1469  * Data structures and routines to emulate the root hub.
1470  */
1471 static usb_device_descriptor_t ehci_devd = {
1472         USB_DEVICE_DESCRIPTOR_SIZE,
1473         UDESC_DEVICE,           /* type */
1474         {0x00, 0x02},           /* USB version */
1475         UDCLASS_HUB,            /* class */
1476         UDSUBCLASS_HUB,         /* subclass */
1477         UDPROTO_HSHUBSTT,       /* protocol */
1478         64,                     /* max packet */
1479         {0},{0},{0x00,0x01},    /* device id */
1480         1,2,0,                  /* string indicies */
1481         1                       /* # of configurations */
1482 };
1483
1484 static usb_device_qualifier_t ehci_odevd = {
1485         USB_DEVICE_DESCRIPTOR_SIZE,
1486         UDESC_DEVICE_QUALIFIER, /* type */
1487         {0x00, 0x02},           /* USB version */
1488         UDCLASS_HUB,            /* class */
1489         UDSUBCLASS_HUB,         /* subclass */
1490         UDPROTO_FSHUB,          /* protocol */
1491         64,                     /* max packet */
1492         1,                      /* # of configurations */
1493         0
1494 };
1495
1496 static usb_config_descriptor_t ehci_confd = {
1497         USB_CONFIG_DESCRIPTOR_SIZE,
1498         UDESC_CONFIG,
1499         {USB_CONFIG_DESCRIPTOR_SIZE +
1500          USB_INTERFACE_DESCRIPTOR_SIZE +
1501          USB_ENDPOINT_DESCRIPTOR_SIZE},
1502         1,
1503         1,
1504         0,
1505         UC_SELF_POWERED,
1506         0                       /* max power */
1507 };
1508
1509 static usb_interface_descriptor_t ehci_ifcd = {
1510         USB_INTERFACE_DESCRIPTOR_SIZE,
1511         UDESC_INTERFACE,
1512         0,
1513         0,
1514         1,
1515         UICLASS_HUB,
1516         UISUBCLASS_HUB,
1517         UIPROTO_HSHUBSTT,
1518         0
1519 };
1520
1521 static usb_endpoint_descriptor_t ehci_endpd = {
1522         USB_ENDPOINT_DESCRIPTOR_SIZE,
1523         UDESC_ENDPOINT,
1524         UE_DIR_IN | EHCI_INTR_ENDPT,
1525         UE_INTERRUPT,
1526         {8, 0},                 /* max packet */
1527         255
1528 };
1529
1530 static usb_hub_descriptor_t ehci_hubd = {
1531         USB_HUB_DESCRIPTOR_SIZE,
1532         UDESC_HUB,
1533         0,
1534         {0,0},
1535         0,
1536         0,
1537         {0},
1538 };
1539
1540 static int
1541 ehci_str(usb_string_descriptor_t *p, int l, char *s)
1542 {
1543         int i;
1544
1545         if (l == 0)
1546                 return (0);
1547         p->bLength = 2 * strlen(s) + 2;
1548         if (l == 1)
1549                 return (1);
1550         p->bDescriptorType = UDESC_STRING;
1551         l -= 2;
1552         for (i = 0; s[i] && l > 1; i++, l -= 2)
1553                 USETW2(p->bString[i], 0, s[i]);
1554         return (2*i+2);
1555 }
1556
1557 /*
1558  * Simulate a hardware hub by handling all the necessary requests.
1559  */
1560 static usbd_status
1561 ehci_root_ctrl_transfer(usbd_xfer_handle xfer)
1562 {
1563         usbd_status err;
1564
1565         /* Insert last in queue. */
1566         err = usb_insert_transfer(xfer);
1567         if (err)
1568                 return (err);
1569
1570         /* Pipe isn't running, start first */
1571         return (ehci_root_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
1572 }
1573
1574 static usbd_status
1575 ehci_root_ctrl_start(usbd_xfer_handle xfer)
1576 {
1577         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
1578         usb_device_request_t *req;
1579         void *buf = NULL;
1580         int port, i;
1581         int len, value, index, l, totlen = 0;
1582         usb_port_status_t ps;
1583         usb_hub_descriptor_t hubd;
1584         usbd_status err;
1585         u_int32_t v;
1586
1587         if (sc->sc_dying)
1588                 return (USBD_IOERROR);
1589
1590 #ifdef DIAGNOSTIC
1591         if (!(xfer->rqflags & URQ_REQUEST))
1592                 /* XXX panic */
1593                 return (USBD_INVAL);
1594 #endif
1595         req = &xfer->request;
1596
1597         DPRINTFN(4,("ehci_root_ctrl_start: type=0x%02x request=%02x\n",
1598                     req->bmRequestType, req->bRequest));
1599
1600         len = UGETW(req->wLength);
1601         value = UGETW(req->wValue);
1602         index = UGETW(req->wIndex);
1603
1604         if (len != 0)
1605                 buf = KERNADDR(&xfer->dmabuf, 0);
1606
1607 #define C(x,y) ((x) | ((y) << 8))
1608         switch(C(req->bRequest, req->bmRequestType)) {
1609         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
1610         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
1611         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
1612                 /*
1613                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
1614                  * for the integrated root hub.
1615                  */
1616                 break;
1617         case C(UR_GET_CONFIG, UT_READ_DEVICE):
1618                 if (len > 0) {
1619                         *(u_int8_t *)buf = sc->sc_conf;
1620                         totlen = 1;
1621                 }
1622                 break;
1623         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
1624                 DPRINTFN(8,("ehci_root_ctrl_start: wValue=0x%04x\n", value));
1625                 switch(value >> 8) {
1626                 case UDESC_DEVICE:
1627                         if ((value & 0xff) != 0) {
1628                                 err = USBD_IOERROR;
1629                                 goto ret;
1630                         }
1631                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1632                         USETW(ehci_devd.idVendor, sc->sc_id_vendor);
1633                         memcpy(buf, &ehci_devd, l);
1634                         break;
1635                 /*
1636                  * We can't really operate at another speed, but the spec says
1637                  * we need this descriptor.
1638                  */
1639                 case UDESC_DEVICE_QUALIFIER:
1640                         if ((value & 0xff) != 0) {
1641                                 err = USBD_IOERROR;
1642                                 goto ret;
1643                         }
1644                         totlen = l = min(len, USB_DEVICE_DESCRIPTOR_SIZE);
1645                         memcpy(buf, &ehci_odevd, l);
1646                         break;
1647                 /*
1648                  * We can't really operate at another speed, but the spec says
1649                  * we need this descriptor.
1650                  */
1651                 case UDESC_OTHER_SPEED_CONFIGURATION:
1652                 case UDESC_CONFIG:
1653                         if ((value & 0xff) != 0) {
1654                                 err = USBD_IOERROR;
1655                                 goto ret;
1656                         }
1657                         totlen = l = min(len, USB_CONFIG_DESCRIPTOR_SIZE);
1658                         memcpy(buf, &ehci_confd, l);
1659                         ((usb_config_descriptor_t *)buf)->bDescriptorType =
1660                                 value >> 8;
1661                         buf = (char *)buf + l;
1662                         len -= l;
1663                         l = min(len, USB_INTERFACE_DESCRIPTOR_SIZE);
1664                         totlen += l;
1665                         memcpy(buf, &ehci_ifcd, l);
1666                         buf = (char *)buf + l;
1667                         len -= l;
1668                         l = min(len, USB_ENDPOINT_DESCRIPTOR_SIZE);
1669                         totlen += l;
1670                         memcpy(buf, &ehci_endpd, l);
1671                         break;
1672                 case UDESC_STRING:
1673                         if (len == 0)
1674                                 break;
1675                         *(u_int8_t *)buf = 0;
1676                         totlen = 1;
1677                         switch (value & 0xff) {
1678                         case 0: /* Language table */
1679                                 totlen = ehci_str(buf, len, "\001");
1680                                 break;
1681                         case 1: /* Vendor */
1682                                 totlen = ehci_str(buf, len, sc->sc_vendor);
1683                                 break;
1684                         case 2: /* Product */
1685                                 totlen = ehci_str(buf, len, "EHCI root hub");
1686                                 break;
1687                         }
1688                         break;
1689                 default:
1690                         err = USBD_IOERROR;
1691                         goto ret;
1692                 }
1693                 break;
1694         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
1695                 if (len > 0) {
1696                         *(u_int8_t *)buf = 0;
1697                         totlen = 1;
1698                 }
1699                 break;
1700         case C(UR_GET_STATUS, UT_READ_DEVICE):
1701                 if (len > 1) {
1702                         USETW(((usb_status_t *)buf)->wStatus,UDS_SELF_POWERED);
1703                         totlen = 2;
1704                 }
1705                 break;
1706         case C(UR_GET_STATUS, UT_READ_INTERFACE):
1707         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
1708                 if (len > 1) {
1709                         USETW(((usb_status_t *)buf)->wStatus, 0);
1710                         totlen = 2;
1711                 }
1712                 break;
1713         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
1714                 if (value >= USB_MAX_DEVICES) {
1715                         err = USBD_IOERROR;
1716                         goto ret;
1717                 }
1718                 sc->sc_addr = value;
1719                 break;
1720         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
1721                 if (value != 0 && value != 1) {
1722                         err = USBD_IOERROR;
1723                         goto ret;
1724                 }
1725                 sc->sc_conf = value;
1726                 break;
1727         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
1728                 break;
1729         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
1730         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
1731         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
1732                 err = USBD_IOERROR;
1733                 goto ret;
1734         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
1735                 break;
1736         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
1737                 break;
1738         /* Hub requests */
1739         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
1740                 break;
1741         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
1742                 DPRINTFN(8, ("ehci_root_ctrl_start: UR_CLEAR_PORT_FEATURE "
1743                              "port=%d feature=%d\n",
1744                              index, value));
1745                 if (index < 1 || index > sc->sc_noport) {
1746                         err = USBD_IOERROR;
1747                         goto ret;
1748                 }
1749                 port = EHCI_PORTSC(index);
1750                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1751                 switch(value) {
1752                 case UHF_PORT_ENABLE:
1753                         EOWRITE4(sc, port, v &~ EHCI_PS_PE);
1754                         break;
1755                 case UHF_PORT_SUSPEND:
1756                         EOWRITE4(sc, port, v &~ EHCI_PS_SUSP);
1757                         break;
1758                 case UHF_PORT_POWER:
1759                         EOWRITE4(sc, port, v &~ EHCI_PS_PP);
1760                         break;
1761                 case UHF_PORT_TEST:
1762                         DPRINTFN(2,("ehci_root_ctrl_start: clear port test "
1763                                     "%d\n", index));
1764                         break;
1765                 case UHF_PORT_INDICATOR:
1766                         DPRINTFN(2,("ehci_root_ctrl_start: clear port ind "
1767                                     "%d\n", index));
1768                         EOWRITE4(sc, port, v &~ EHCI_PS_PIC);
1769                         break;
1770                 case UHF_C_PORT_CONNECTION:
1771                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
1772                         break;
1773                 case UHF_C_PORT_ENABLE:
1774                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
1775                         break;
1776                 case UHF_C_PORT_SUSPEND:
1777                         /* how? */
1778                         break;
1779                 case UHF_C_PORT_OVER_CURRENT:
1780                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
1781                         break;
1782                 case UHF_C_PORT_RESET:
1783                         sc->sc_isreset = 0;
1784                         break;
1785                 default:
1786                         err = USBD_IOERROR;
1787                         goto ret;
1788                 }
1789                 break;
1790         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
1791                 if ((value & 0xff) != 0) {
1792                         err = USBD_IOERROR;
1793                         goto ret;
1794                 }
1795                 hubd = ehci_hubd;
1796                 hubd.bNbrPorts = sc->sc_noport;
1797                 v = EOREAD4(sc, EHCI_HCSPARAMS);
1798                 USETW(hubd.wHubCharacteristics,
1799                     EHCI_HCS_PPC(v) ? UHD_PWR_INDIVIDUAL : UHD_PWR_NO_SWITCH |
1800                     EHCI_HCS_P_INDICATOR(EREAD4(sc, EHCI_HCSPARAMS))
1801                         ? UHD_PORT_IND : 0);
1802                 hubd.bPwrOn2PwrGood = 200; /* XXX can't find out? */
1803                 for (i = 0, l = sc->sc_noport; l > 0; i++, l -= 8, v >>= 8)
1804                         hubd.DeviceRemovable[i++] = 0; /* XXX can't find out? */
1805                 hubd.bDescLength = USB_HUB_DESCRIPTOR_SIZE + i;
1806                 l = min(len, hubd.bDescLength);
1807                 totlen = l;
1808                 memcpy(buf, &hubd, l);
1809                 break;
1810         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
1811                 if (len != 4) {
1812                         err = USBD_IOERROR;
1813                         goto ret;
1814                 }
1815                 memset(buf, 0, len); /* ? XXX */
1816                 totlen = len;
1817                 break;
1818         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
1819                 DPRINTFN(8,("ehci_root_ctrl_start: get port status i=%d\n",
1820                             index));
1821                 if (index < 1 || index > sc->sc_noport) {
1822                         err = USBD_IOERROR;
1823                         goto ret;
1824                 }
1825                 if (len != 4) {
1826                         err = USBD_IOERROR;
1827                         goto ret;
1828                 }
1829                 v = EOREAD4(sc, EHCI_PORTSC(index));
1830                 DPRINTFN(8,("ehci_root_ctrl_start: port status=0x%04x\n", v));
1831                 i = UPS_HIGH_SPEED;
1832                 if (v & EHCI_PS_CS)     i |= UPS_CURRENT_CONNECT_STATUS;
1833                 if (v & EHCI_PS_PE)     i |= UPS_PORT_ENABLED;
1834                 if (v & EHCI_PS_SUSP)   i |= UPS_SUSPEND;
1835                 if (v & EHCI_PS_OCA)    i |= UPS_OVERCURRENT_INDICATOR;
1836                 if (v & EHCI_PS_PR)     i |= UPS_RESET;
1837                 if (v & EHCI_PS_PP)     i |= UPS_PORT_POWER;
1838                 USETW(ps.wPortStatus, i);
1839                 i = 0;
1840                 if (v & EHCI_PS_CSC)    i |= UPS_C_CONNECT_STATUS;
1841                 if (v & EHCI_PS_PEC)    i |= UPS_C_PORT_ENABLED;
1842                 if (v & EHCI_PS_OCC)    i |= UPS_C_OVERCURRENT_INDICATOR;
1843                 if (sc->sc_isreset)     i |= UPS_C_PORT_RESET;
1844                 USETW(ps.wPortChange, i);
1845                 l = min(len, sizeof ps);
1846                 memcpy(buf, &ps, l);
1847                 totlen = l;
1848                 break;
1849         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
1850                 err = USBD_IOERROR;
1851                 goto ret;
1852         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
1853                 break;
1854         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
1855                 if (index < 1 || index > sc->sc_noport) {
1856                         err = USBD_IOERROR;
1857                         goto ret;
1858                 }
1859                 port = EHCI_PORTSC(index);
1860                 v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1861                 switch(value) {
1862                 case UHF_PORT_ENABLE:
1863                         EOWRITE4(sc, port, v | EHCI_PS_PE);
1864                         break;
1865                 case UHF_PORT_SUSPEND:
1866                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
1867                         break;
1868                 case UHF_PORT_RESET:
1869                         DPRINTFN(5,("ehci_root_ctrl_start: reset port %d\n",
1870                                     index));
1871                         if (EHCI_PS_IS_LOWSPEED(v)) {
1872                                 /* Low speed device, give up ownership. */
1873                                 ehci_disown(sc, index, 1);
1874                                 break;
1875                         }
1876                         /* Start reset sequence. */
1877                         v &= ~ (EHCI_PS_PE | EHCI_PS_PR);
1878                         EOWRITE4(sc, port, v | EHCI_PS_PR);
1879                         /* Wait for reset to complete. */
1880                         usb_delay_ms(&sc->sc_bus, USB_PORT_ROOT_RESET_DELAY);
1881                         if (sc->sc_dying) {
1882                                 err = USBD_IOERROR;
1883                                 goto ret;
1884                         }
1885                         /* Terminate reset sequence. */
1886                         EOWRITE4(sc, port, v);
1887                         /* Wait for HC to complete reset. */
1888                         usb_delay_ms(&sc->sc_bus, EHCI_PORT_RESET_COMPLETE);
1889                         if (sc->sc_dying) {
1890                                 err = USBD_IOERROR;
1891                                 goto ret;
1892                         }
1893                         v = EOREAD4(sc, port);
1894                         DPRINTF(("ehci after reset, status=0x%08x\n", v));
1895                         if (v & EHCI_PS_PR) {
1896                                 device_printf(sc->sc_bus.bdev,
1897                                     "port reset timeout\n");
1898                                 return (USBD_TIMEOUT);
1899                         }
1900                         if (!(v & EHCI_PS_PE)) {
1901                                 /* Not a high speed device, give up ownership.*/
1902                                 ehci_disown(sc, index, 0);
1903                                 break;
1904                         }
1905                         sc->sc_isreset = 1;
1906                         DPRINTF(("ehci port %d reset, status = 0x%08x\n",
1907                                  index, v));
1908                         break;
1909                 case UHF_PORT_POWER:
1910                         DPRINTFN(2,("ehci_root_ctrl_start: set port power "
1911                                     "%d\n", index));
1912                         EOWRITE4(sc, port, v | EHCI_PS_PP);
1913                         break;
1914                 case UHF_PORT_TEST:
1915                         DPRINTFN(2,("ehci_root_ctrl_start: set port test "
1916                                     "%d\n", index));
1917                         break;
1918                 case UHF_PORT_INDICATOR:
1919                         DPRINTFN(2,("ehci_root_ctrl_start: set port ind "
1920                                     "%d\n", index));
1921                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
1922                         break;
1923                 default:
1924                         err = USBD_IOERROR;
1925                         goto ret;
1926                 }
1927                 break;
1928         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
1929         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
1930         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
1931         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
1932                 break;
1933         default:
1934                 err = USBD_IOERROR;
1935                 goto ret;
1936         }
1937         xfer->actlen = totlen;
1938         err = USBD_NORMAL_COMPLETION;
1939  ret:
1940         xfer->status = err;
1941         crit_enter();
1942         usb_transfer_complete(xfer);
1943         crit_exit();
1944         return (USBD_IN_PROGRESS);
1945 }
1946
1947 void
1948 ehci_disown(ehci_softc_t *sc, int index, int lowspeed)
1949 {
1950         int port;
1951         u_int32_t v;
1952
1953         DPRINTF(("ehci_disown: index=%d lowspeed=%d\n", index, lowspeed));
1954 #ifdef DIAGNOSTIC
1955         if (sc->sc_npcomp != 0) {
1956                 int i = (index-1) / sc->sc_npcomp;
1957                 if (i >= sc->sc_ncomp)
1958                         device_printf(sc->sc_bus.bdev, "strange port\n");
1959                 else
1960                         device_printf(sc->sc_bus.bdev,
1961                             "handing over %s speed device on port %d to %s\n",
1962                             lowspeed ? "low" : "full",
1963                             index, device_get_nameunit(sc->sc_comps[i]->bdev));
1964         } else {
1965                 device_printf(sc->sc_bus.bdev, "npcomp == 0\n");
1966         }
1967 #endif
1968         port = EHCI_PORTSC(index);
1969         v = EOREAD4(sc, port) &~ EHCI_PS_CLEAR;
1970         EOWRITE4(sc, port, v | EHCI_PS_PO);
1971 }
1972
1973 /* Abort a root control request. */
1974 static void
1975 ehci_root_ctrl_abort(usbd_xfer_handle xfer)
1976 {
1977         /* Nothing to do, all transfers are synchronous. */
1978 }
1979
1980 /* Close the root pipe. */
1981 static void
1982 ehci_root_ctrl_close(usbd_pipe_handle pipe)
1983 {
1984         DPRINTF(("ehci_root_ctrl_close\n"));
1985         /* Nothing to do. */
1986 }
1987
1988 void
1989 ehci_root_intr_done(usbd_xfer_handle xfer)
1990 {
1991 }
1992
1993 static usbd_status
1994 ehci_root_intr_transfer(usbd_xfer_handle xfer)
1995 {
1996         usbd_status err;
1997
1998         /* Insert last in queue. */
1999         err = usb_insert_transfer(xfer);
2000         if (err)
2001                 return (err);
2002
2003         /* Pipe isn't running, start first */
2004         return (ehci_root_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
2005 }
2006
2007 static usbd_status
2008 ehci_root_intr_start(usbd_xfer_handle xfer)
2009 {
2010         usbd_pipe_handle pipe = xfer->pipe;
2011         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2012
2013         if (sc->sc_dying)
2014                 return (USBD_IOERROR);
2015
2016         sc->sc_intrxfer = xfer;
2017
2018         return (USBD_IN_PROGRESS);
2019 }
2020
2021 /* Abort a root interrupt request. */
2022 static void
2023 ehci_root_intr_abort(usbd_xfer_handle xfer)
2024 {
2025         if (xfer->pipe->intrxfer == xfer) {
2026                 DPRINTF(("ehci_root_intr_abort: remove\n"));
2027                 xfer->pipe->intrxfer = NULL;
2028         }
2029         xfer->status = USBD_CANCELLED;
2030         crit_enter();
2031         usb_transfer_complete(xfer);
2032         crit_exit();
2033 }
2034
2035 /* Close the root pipe. */
2036 static void
2037 ehci_root_intr_close(usbd_pipe_handle pipe)
2038 {
2039         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2040
2041         DPRINTF(("ehci_root_intr_close\n"));
2042
2043         sc->sc_intrxfer = NULL;
2044 }
2045
2046 void
2047 ehci_root_ctrl_done(usbd_xfer_handle xfer)
2048 {
2049 }
2050
2051 /************************/
2052
2053 ehci_soft_qh_t *
2054 ehci_alloc_sqh(ehci_softc_t *sc)
2055 {
2056         ehci_soft_qh_t *sqh;
2057         usbd_status err;
2058         int i, offs;
2059         usb_dma_t dma;
2060
2061         if (sc->sc_freeqhs == NULL) {
2062                 DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
2063                 err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
2064                           EHCI_PAGE_SIZE, &dma);
2065 #ifdef EHCI_DEBUG
2066                 if (err)
2067                         kprintf("ehci_alloc_sqh: usb_allocmem()=%d\n", err);
2068 #endif
2069                 if (err)
2070                         return (NULL);
2071                 for(i = 0; i < EHCI_SQH_CHUNK; i++) {
2072                         offs = i * EHCI_SQH_SIZE;
2073                         sqh = KERNADDR(&dma, offs);
2074                         sqh->physaddr = DMAADDR(&dma, offs);
2075                         sqh->next = sc->sc_freeqhs;
2076                         sc->sc_freeqhs = sqh;
2077                 }
2078         }
2079         sqh = sc->sc_freeqhs;
2080         sc->sc_freeqhs = sqh->next;
2081         memset(&sqh->qh, 0, sizeof(ehci_qh_t));
2082         sqh->next = NULL;
2083         sqh->prev = NULL;
2084         return (sqh);
2085 }
2086
2087 void
2088 ehci_free_sqh(ehci_softc_t *sc, ehci_soft_qh_t *sqh)
2089 {
2090         sqh->next = sc->sc_freeqhs;
2091         sc->sc_freeqhs = sqh;
2092 }
2093
2094 ehci_soft_qtd_t *
2095 ehci_alloc_sqtd(ehci_softc_t *sc)
2096 {
2097         ehci_soft_qtd_t *sqtd;
2098         usbd_status err;
2099         int i, offs;
2100         usb_dma_t dma;
2101
2102         if (sc->sc_freeqtds == NULL) {
2103                 DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
2104                 err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
2105                           EHCI_PAGE_SIZE, &dma);
2106 #ifdef EHCI_DEBUG
2107                 if (err)
2108                         kprintf("ehci_alloc_sqtd: usb_allocmem()=%d\n", err);
2109 #endif
2110                 if (err)
2111                         return (NULL);
2112                 crit_enter();
2113                 for(i = 0; i < EHCI_SQTD_CHUNK; i++) {
2114                         offs = i * EHCI_SQTD_SIZE;
2115                         sqtd = KERNADDR(&dma, offs);
2116                         sqtd->physaddr = DMAADDR(&dma, offs);
2117                         sqtd->nextqtd = sc->sc_freeqtds;
2118                         sc->sc_freeqtds = sqtd;
2119                 }
2120                 crit_exit();
2121         }
2122
2123         crit_enter();
2124         sqtd = sc->sc_freeqtds;
2125         sc->sc_freeqtds = sqtd->nextqtd;
2126         memset(&sqtd->qtd, 0, sizeof(ehci_qtd_t));
2127         sqtd->nextqtd = NULL;
2128         sqtd->xfer = NULL;
2129         crit_exit();
2130
2131         return (sqtd);
2132 }
2133
2134 void
2135 ehci_free_sqtd(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd)
2136 {
2137         crit_enter();
2138         sqtd->nextqtd = sc->sc_freeqtds;
2139         sc->sc_freeqtds = sqtd;
2140         crit_exit();
2141 }
2142
2143 usbd_status
2144 ehci_alloc_sqtd_chain(struct ehci_pipe *epipe, ehci_softc_t *sc,
2145                      int alen, int rd, usbd_xfer_handle xfer,
2146                      ehci_soft_qtd_t **sp, ehci_soft_qtd_t **ep)
2147 {
2148         ehci_soft_qtd_t *next, *cur;
2149         ehci_physaddr_t dataphys, dataphyspage, dataphyslastpage, nextphys;
2150         u_int32_t qtdstatus;
2151         int len, curlen, mps, offset;
2152         int i, iscontrol;
2153         usb_dma_t *dma = &xfer->dmabuf;
2154
2155         DPRINTFN(alen<4*4096,("ehci_alloc_sqtd_chain: start len=%d\n", alen));
2156
2157         offset = 0;
2158         len = alen;
2159         iscontrol = (epipe->pipe.endpoint->edesc->bmAttributes & UE_XFERTYPE) ==
2160             UE_CONTROL;
2161         dataphys = DMAADDR(dma, 0);
2162         dataphyslastpage = EHCI_PAGE(DMAADDR(dma, len - 1));
2163         qtdstatus = EHCI_QTD_ACTIVE |
2164             EHCI_QTD_SET_PID(rd ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT) |
2165             EHCI_QTD_SET_CERR(3)
2166             /* IOC set below */
2167             /* BYTES set below */
2168             ;
2169         mps = UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize);
2170         /*
2171          * The control transfer data stage always starts with a toggle of 1.
2172          * For other transfers we let the hardware track the toggle state.
2173          */
2174         if (iscontrol)
2175                 qtdstatus |= EHCI_QTD_SET_TOGGLE(1);
2176
2177         cur = ehci_alloc_sqtd(sc);
2178         *sp = cur;
2179         if (cur == NULL)
2180                 goto nomem;
2181         for (;;) {
2182                 dataphyspage = EHCI_PAGE(dataphys);
2183                 /* XXX This is pretty broken: Because we do not allocate
2184                  * a contiguous buffer (contiguous in physical pages) we
2185                  * can only transfer one page in one go.
2186                  * So check whether the start and end of the buffer are on
2187                  * the same page.
2188                  */
2189                 if (dataphyspage == dataphyslastpage) {
2190                         curlen = len;
2191                 }
2192                 else {
2193                         /* See comment above (XXX) */
2194                         curlen = EHCI_PAGE_SIZE -
2195                                  EHCI_PAGE_MASK(dataphys);
2196                         /* the length must be a multiple of the max size */
2197                         curlen -= curlen % mps;
2198                         DPRINTFN(1,("ehci_alloc_sqtd_chain: multiple QTDs, "
2199                                     "curlen=%d\n", curlen));
2200                         KASSERT(curlen != 0, ("ehci_alloc_std: curlen == 0"));
2201                 }
2202                 DPRINTFN(4,("ehci_alloc_sqtd_chain: dataphys=0x%08x "
2203                             "dataphyslastpage=0x%08x len=%d curlen=%d\n",
2204                             dataphys, dataphyslastpage,
2205                             len, curlen));
2206                 len -= curlen;
2207
2208                 if (len != 0) {
2209                         next = ehci_alloc_sqtd(sc);
2210                         if (next == NULL)
2211                                 goto nomem;
2212                         nextphys = htole32(next->physaddr);
2213                 } else {
2214                         next = NULL;
2215                         nextphys = EHCI_NULL;
2216                 }
2217
2218                 for (i = 0; i * EHCI_PAGE_SIZE < curlen; i++) {
2219                         ehci_physaddr_t a = dataphys + i * EHCI_PAGE_SIZE;
2220                         if (i != 0) /* use offset only in first buffer */
2221                                 a = EHCI_PAGE(a);
2222                         cur->qtd.qtd_buffer[i] = htole32(a);
2223                         cur->qtd.qtd_buffer_hi[i] = 0;
2224 #ifdef DIAGNOSTIC
2225                         if (i >= EHCI_QTD_NBUFFERS) {
2226                                 kprintf("ehci_alloc_sqtd_chain: i=%d\n", i);
2227                                 goto nomem;
2228                         }
2229 #endif
2230                 }
2231                 cur->nextqtd = next;
2232                 cur->qtd.qtd_next = cur->qtd.qtd_altnext = nextphys;
2233                 cur->qtd.qtd_status =
2234                     htole32(qtdstatus | EHCI_QTD_SET_BYTES(curlen));
2235                 cur->xfer = xfer;
2236                 cur->len = curlen;
2237                 DPRINTFN(10,("ehci_alloc_sqtd_chain: cbp=0x%08x end=0x%08x\n",
2238                             dataphys, dataphys + curlen));
2239                 if (iscontrol) {
2240                         /*
2241                          * adjust the toggle based on the number of packets
2242                          * in this qtd
2243                          */
2244                         if (((curlen + mps - 1) / mps) & 1)
2245                                 qtdstatus ^= EHCI_QTD_TOGGLE_MASK;
2246                 }
2247                 if (len == 0)
2248                         break;
2249                 DPRINTFN(10,("ehci_alloc_sqtd_chain: extend chain\n"));
2250                 offset += curlen;
2251                 dataphys = DMAADDR(dma, offset);
2252                 cur = next;
2253         }
2254         cur->qtd.qtd_status |= htole32(EHCI_QTD_IOC);
2255         *ep = cur;
2256
2257         DPRINTFN(10,("ehci_alloc_sqtd_chain: return sqtd=%p sqtdend=%p\n",
2258                      *sp, *ep));
2259
2260         return (USBD_NORMAL_COMPLETION);
2261
2262  nomem:
2263         /* XXX free chain */
2264         DPRINTFN(-1,("ehci_alloc_sqtd_chain: no memory\n"));
2265         return (USBD_NOMEM);
2266 }
2267
2268 static void
2269 ehci_free_sqtd_chain(ehci_softc_t *sc, ehci_soft_qtd_t *sqtd,
2270                     ehci_soft_qtd_t *sqtdend)
2271 {
2272         ehci_soft_qtd_t *p;
2273         int i;
2274
2275         DPRINTFN(10,("ehci_free_sqtd_chain: sqtd=%p sqtdend=%p\n",
2276                      sqtd, sqtdend));
2277
2278         for (i = 0; sqtd != sqtdend; sqtd = p, i++) {
2279                 p = sqtd->nextqtd;
2280                 ehci_free_sqtd(sc, sqtd);
2281         }
2282 }
2283
2284 /****************/
2285
2286 /*
2287  * Close a reqular pipe.
2288  * Assumes that there are no pending transactions.
2289  */
2290 void
2291 ehci_close_pipe(usbd_pipe_handle pipe, ehci_soft_qh_t *head)
2292 {
2293         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
2294         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2295         ehci_soft_qh_t *sqh = epipe->sqh;
2296
2297         crit_enter();
2298         ehci_rem_qh(sc, sqh, head);
2299         crit_exit();
2300         pipe->endpoint->savedtoggle =
2301             EHCI_QTD_GET_TOGGLE(le32toh(sqh->qh.qh_qtd.qtd_status));
2302         ehci_free_sqh(sc, epipe->sqh);
2303 }
2304
2305 /*
2306  * Abort a device request.
2307  * If this routine is called from a critical section it guarantees that the
2308  * request will be removed from the hardware scheduling and that the callback
2309  * for it will be called with USBD_CANCELLED status.
2310  * It's impossible to guarantee that the requested transfer will not
2311  * have happened since the hardware runs concurrently.
2312  * If the transaction has already happened we rely on the ordinary
2313  * interrupt processing to process it.
2314  * XXX This is most probably wrong.
2315  */
2316 void
2317 ehci_abort_xfer(usbd_xfer_handle xfer, usbd_status status)
2318 {
2319 #define exfer EXFER(xfer)
2320         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2321         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2322         ehci_soft_qh_t *sqh = epipe->sqh;
2323         ehci_soft_qtd_t *sqtd, *snext, **psqtd;
2324         ehci_physaddr_t cur, us, next;
2325         int hit;
2326         /* int count = 0; */
2327         ehci_soft_qh_t *psqh;
2328
2329         DPRINTF(("ehci_abort_xfer: xfer=%p pipe=%p\n", xfer, epipe));
2330
2331         if (sc->sc_dying) {
2332                 /* If we're dying, just do the software part. */
2333                 crit_enter();
2334                 xfer->status = status;  /* make software ignore it */
2335                 callout_stop(&xfer->timeout_handle);
2336                 usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2337                 usb_transfer_complete(xfer);
2338                 crit_exit();
2339                 return;
2340         }
2341
2342         if (xfer->device->bus->intr_context /* || !curproc REMOVED DFly */)
2343                 panic("ehci_abort_xfer: not in process context");
2344
2345         /*
2346          * If an abort is already in progress then just wait for it to
2347          * complete and return.
2348          */
2349         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING) {
2350                 DPRINTFN(2, ("ehci_abort_xfer: already aborting\n"));
2351                 /* No need to wait if we're aborting from a timeout. */
2352                 if (status == USBD_TIMEOUT)
2353                         return;
2354                 /* Override the status which might be USBD_TIMEOUT. */
2355                 xfer->status = status;
2356                 DPRINTFN(2, ("ehci_abort_xfer: waiting for abort to finish\n"));
2357                 exfer->ehci_xfer_flags |= EHCI_XFER_ABORTWAIT;
2358                 while (exfer->ehci_xfer_flags & EHCI_XFER_ABORTING)
2359                         tsleep(&exfer->ehci_xfer_flags, 0, "ehciaw", 0);
2360                 return;
2361         }
2362
2363         /*
2364          * Step 1: Make interrupt routine and timeouts ignore xfer.
2365          */
2366         crit_enter();
2367         exfer->ehci_xfer_flags |= EHCI_XFER_ABORTING;
2368         xfer->status = status;  /* make software ignore it */
2369         callout_stop(&xfer->timeout_handle);
2370         usb_rem_task(epipe->pipe.device, &exfer->abort_task);
2371         crit_exit();
2372
2373         /*
2374          * Step 2: Wait until we know hardware has finished any possible
2375          * use of the xfer. We do this by removing the entire
2376          * queue from the async schedule and waiting for the doorbell.
2377          * Nothing else should be touching the queue now.
2378          */
2379         psqh = sqh->prev;
2380         ehci_rem_qh(sc, sqh, psqh);
2381
2382         /*
2383          * Step 3:  make sure the soft interrupt routine
2384          * has run. This should remove any completed items off the queue.
2385          * The hardware has no reference to completed items (TDs).
2386          * It's safe to remove them at any time.
2387          */
2388         crit_enter();
2389 #ifdef USB_USE_SOFTINTR
2390         sc->sc_softwake = 1;
2391 #endif /* USB_USE_SOFTINTR */
2392         usb_schedsoftintr(&sc->sc_bus);
2393 #ifdef USB_USE_SOFTINTR
2394         tsleep(&sc->sc_softwake, 0, "ehciab", 0);
2395 #endif /* USB_USE_SOFTINTR */
2396
2397         /*
2398          * Step 4: Remove any vestiges of the xfer from the hardware.
2399          * The complication here is that the hardware may have executed
2400          * into or even beyond the xfer we're trying to abort.
2401          * So as we're scanning the TDs of this xfer we check if
2402          * the hardware points to any of them.
2403          *
2404          * first we need to see if there are any transfers
2405          * on this queue before the xfer we are aborting.. we need
2406          * to update any pointers that point to us to point past
2407          * the aborting xfer.  (If there is something past us).
2408          * Hardware and software.
2409          */
2410         cur = EHCI_LINK_ADDR(le32toh(sqh->qh.qh_curqtd));
2411         hit = 0;
2412
2413         /* If they initially point here. */
2414         us = exfer->sqtdstart->physaddr;
2415
2416         /* We will change them to point here */
2417         snext = exfer->sqtdend->nextqtd;
2418         next = snext ? htole32(snext->physaddr) : EHCI_NULL;
2419
2420         /*
2421          * Now loop through any qTDs before us and keep track of the pointer
2422          * that points to us for the end.
2423          */
2424         psqtd = &sqh->sqtd;
2425         sqtd = sqh->sqtd;
2426         while (sqtd && sqtd != exfer->sqtdstart) {
2427                 hit |= (cur == sqtd->physaddr);
2428                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_next)) == us)
2429                         sqtd->qtd.qtd_next = next;
2430                 if (EHCI_LINK_ADDR(le32toh(sqtd->qtd.qtd_altnext)) == us)
2431                         sqtd->qtd.qtd_altnext = next;
2432                 psqtd = &sqtd->nextqtd;
2433                 sqtd = sqtd->nextqtd;
2434         }
2435                 /* make the software pointer bypass us too */
2436         *psqtd = exfer->sqtdend->nextqtd;
2437
2438         /*
2439          * If we already saw the active one then we are pretty much done.
2440          * We've done all the relinking we need to do.
2441          */
2442         if (!hit) {
2443
2444                 /*
2445                  * Now reinitialise the QH to point to the next qTD
2446                  * (if there is one). We only need to do this if
2447                  * it was previously pointing to us.
2448                  */
2449                 sqtd = exfer->sqtdstart;
2450                 for (sqtd = exfer->sqtdstart; ; sqtd = sqtd->nextqtd) {
2451                         if (cur == sqtd->physaddr) {
2452                                 hit++;
2453                         }
2454                         if (sqtd == exfer->sqtdend)
2455                                 break;
2456                 }
2457                 sqtd = sqtd->nextqtd;
2458                 /*
2459                  * Only need to alter the QH if it was pointing at a qTD
2460                  * that we are removing.
2461                  */
2462                 if (hit) {
2463                         if (snext) {
2464                                 ehci_set_qh_qtd(sqh, snext);
2465                         } else {
2466
2467                                 sqh->qh.qh_curqtd = 0; /* unlink qTDs */
2468                                 sqh->qh.qh_qtd.qtd_status &=
2469                                     htole32(EHCI_QTD_TOGGLE_MASK);
2470                                 sqh->qh.qh_qtd.qtd_next =
2471                                     sqh->qh.qh_qtd.qtd_altnext
2472                                         = EHCI_NULL;
2473                                 DPRINTFN(1,("ehci_abort_xfer: no hit\n"));
2474                         }
2475                 }
2476         }
2477         ehci_add_qh(sqh, psqh);
2478         /*
2479          * Step 5: Execute callback.
2480          */
2481 #ifdef DIAGNOSTIC
2482         exfer->isdone = 1;
2483 #endif
2484         /* Do the wakeup first to avoid touching the xfer after the callback. */
2485         exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTING;
2486         if (exfer->ehci_xfer_flags & EHCI_XFER_ABORTWAIT) {
2487                 exfer->ehci_xfer_flags &= ~EHCI_XFER_ABORTWAIT;
2488                 wakeup(&exfer->ehci_xfer_flags);
2489         }
2490         usb_transfer_complete(xfer);
2491
2492         /* kprintf("%s: %d TDs aborted\n", __func__, count); */
2493         crit_exit();
2494 #undef exfer
2495 }
2496
2497 void
2498 ehci_timeout(void *addr)
2499 {
2500         struct ehci_xfer *exfer = addr;
2501         struct ehci_pipe *epipe = (struct ehci_pipe *)exfer->xfer.pipe;
2502         ehci_softc_t *sc = (ehci_softc_t *)epipe->pipe.device->bus;
2503
2504         DPRINTF(("ehci_timeout: exfer=%p\n", exfer));
2505 #ifdef USB_DEBUG
2506         if (ehcidebug > 1)
2507                 usbd_dump_pipe(exfer->xfer.pipe);
2508 #endif
2509
2510         if (sc->sc_dying) {
2511                 ehci_abort_xfer(&exfer->xfer, USBD_TIMEOUT);
2512                 return;
2513         }
2514
2515         /* Execute the abort in a process context. */
2516         usb_add_task(exfer->xfer.pipe->device, &exfer->abort_task,
2517                      USB_TASKQ_HC);
2518 }
2519
2520 void
2521 ehci_timeout_task(void *addr)
2522 {
2523         usbd_xfer_handle xfer = addr;
2524
2525         DPRINTF(("ehci_timeout_task: xfer=%p\n", xfer));
2526         crit_enter();
2527         ehci_abort_xfer(xfer, USBD_TIMEOUT);
2528         crit_exit();
2529 }
2530
2531 /*
2532  * Some EHCI chips from VIA / ATI seem to trigger interrupts before writing
2533  * back the qTD status, or miss signalling occasionally under heavy load.
2534  * If the host machine is too fast, we can miss transaction completion - when
2535  * we scan the active list the transaction still seems to be active. This
2536  * generally exhibits itself as a umass stall that never recovers.
2537  *
2538  * We work around this behaviour by setting up this callback after any softintr
2539  * that completes with transactions still pending, giving us another chance to
2540  * check for completion after the writeback has taken place.
2541  */
2542 void
2543 ehci_intrlist_timeout(void *arg)
2544 {
2545         ehci_softc_t *sc = arg;
2546
2547         DPRINTFN(3, ("ehci_intrlist_timeout\n"));
2548         usb_schedsoftintr(&sc->sc_bus);
2549 }
2550
2551 /************************/
2552
2553 static usbd_status
2554 ehci_device_ctrl_transfer(usbd_xfer_handle xfer)
2555 {
2556         usbd_status err;
2557
2558         /* Insert last in queue. */
2559         err = usb_insert_transfer(xfer);
2560         if (err)
2561                 return (err);
2562
2563         /* Pipe isn't running, start first */
2564         return (ehci_device_ctrl_start(STAILQ_FIRST(&xfer->pipe->queue)));
2565 }
2566
2567 static usbd_status
2568 ehci_device_ctrl_start(usbd_xfer_handle xfer)
2569 {
2570         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2571         usbd_status err;
2572
2573         if (sc->sc_dying)
2574                 return (USBD_IOERROR);
2575
2576 #ifdef DIAGNOSTIC
2577         if (!(xfer->rqflags & URQ_REQUEST)) {
2578                 /* XXX panic */
2579                 kprintf("ehci_device_ctrl_transfer: not a request\n");
2580                 return (USBD_INVAL);
2581         }
2582 #endif
2583
2584         err = ehci_device_request(xfer);
2585         if (err)
2586                 return (err);
2587
2588         if (sc->sc_bus.use_polling)
2589                 ehci_waitintr(sc, xfer);
2590         return (USBD_IN_PROGRESS);
2591 }
2592
2593 void
2594 ehci_device_ctrl_done(usbd_xfer_handle xfer)
2595 {
2596         struct ehci_xfer *ex = EXFER(xfer);
2597         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2598         /*struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;*/
2599
2600         DPRINTFN(10,("ehci_ctrl_done: xfer=%p\n", xfer));
2601
2602 #ifdef DIAGNOSTIC
2603         if (!(xfer->rqflags & URQ_REQUEST)) {
2604                 panic("ehci_ctrl_done: not a request");
2605         }
2606 #endif
2607
2608         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2609                 ehci_del_intr_list(ex); /* remove from active list */
2610                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
2611         }
2612
2613         DPRINTFN(5, ("ehci_ctrl_done: length=%d\n", xfer->actlen));
2614 }
2615
2616 /* Abort a device control request. */
2617 static void
2618 ehci_device_ctrl_abort(usbd_xfer_handle xfer)
2619 {
2620         DPRINTF(("ehci_device_ctrl_abort: xfer=%p\n", xfer));
2621         ehci_abort_xfer(xfer, USBD_CANCELLED);
2622 }
2623
2624 /* Close a device control pipe. */
2625 static void
2626 ehci_device_ctrl_close(usbd_pipe_handle pipe)
2627 {
2628         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2629         /*struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;*/
2630
2631         DPRINTF(("ehci_device_ctrl_close: pipe=%p\n", pipe));
2632         ehci_close_pipe(pipe, sc->sc_async_head);
2633 }
2634
2635 usbd_status
2636 ehci_device_request(usbd_xfer_handle xfer)
2637 {
2638 #define exfer EXFER(xfer)
2639         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2640         usb_device_request_t *req = &xfer->request;
2641         usbd_device_handle dev = epipe->pipe.device;
2642         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2643         int addr = dev->address;
2644         ehci_soft_qtd_t *setup, *stat, *next;
2645         ehci_soft_qh_t *sqh;
2646         int isread;
2647         int len;
2648         usbd_status err;
2649
2650         isread = req->bmRequestType & UT_READ;
2651         len = UGETW(req->wLength);
2652
2653         DPRINTFN(3,("ehci_device_request: type=0x%02x, request=0x%02x, "
2654                     "wValue=0x%04x, wIndex=0x%04x len=%d, addr=%d, endpt=%d\n",
2655                     req->bmRequestType, req->bRequest, UGETW(req->wValue),
2656                     UGETW(req->wIndex), len, addr,
2657                     epipe->pipe.endpoint->edesc->bEndpointAddress));
2658
2659         setup = ehci_alloc_sqtd(sc);
2660         if (setup == NULL) {
2661                 err = USBD_NOMEM;
2662                 goto bad1;
2663         }
2664         stat = ehci_alloc_sqtd(sc);
2665         if (stat == NULL) {
2666                 err = USBD_NOMEM;
2667                 goto bad2;
2668         }
2669
2670         sqh = epipe->sqh;
2671         epipe->u.ctl.length = len;
2672
2673         /* Update device address and length since they may have changed
2674            during the setup of the control pipe in usbd_new_device(). */
2675         /* XXX This only needs to be done once, but it's too early in open. */
2676         /* XXXX Should not touch ED here! */
2677         sqh->qh.qh_endp =
2678             (sqh->qh.qh_endp & htole32(~(EHCI_QH_ADDRMASK | EHCI_QH_MPLMASK))) |
2679             htole32(
2680              EHCI_QH_SET_ADDR(addr) |
2681              EHCI_QH_SET_MPL(UGETW(epipe->pipe.endpoint->edesc->wMaxPacketSize))
2682             );
2683
2684         /* Set up data transaction */
2685         if (len != 0) {
2686                 ehci_soft_qtd_t *end;
2687
2688                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
2689                           &next, &end);
2690                 if (err)
2691                         goto bad3;
2692                 end->qtd.qtd_status &= htole32(~EHCI_QTD_IOC);
2693                 end->nextqtd = stat;
2694                 end->qtd.qtd_next =
2695                 end->qtd.qtd_altnext = htole32(stat->physaddr);
2696         } else {
2697                 next = stat;
2698         }
2699
2700         memcpy(KERNADDR(&epipe->u.ctl.reqdma, 0), req, sizeof *req);
2701
2702         /* Clear toggle */
2703         setup->qtd.qtd_status = htole32(
2704             EHCI_QTD_ACTIVE |
2705             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
2706             EHCI_QTD_SET_CERR(3) |
2707             EHCI_QTD_SET_TOGGLE(0) |
2708             EHCI_QTD_SET_BYTES(sizeof *req)
2709             );
2710         setup->qtd.qtd_buffer[0] = htole32(DMAADDR(&epipe->u.ctl.reqdma, 0));
2711         setup->qtd.qtd_buffer_hi[0] = 0;
2712         setup->nextqtd = next;
2713         setup->qtd.qtd_next = setup->qtd.qtd_altnext = htole32(next->physaddr);
2714         setup->xfer = xfer;
2715         setup->len = sizeof *req;
2716
2717         stat->qtd.qtd_status = htole32(
2718             EHCI_QTD_ACTIVE |
2719             EHCI_QTD_SET_PID(isread ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN) |
2720             EHCI_QTD_SET_CERR(3) |
2721             EHCI_QTD_SET_TOGGLE(1) |
2722             EHCI_QTD_IOC
2723             );
2724         stat->qtd.qtd_buffer[0] = 0; /* XXX not needed? */
2725         stat->qtd.qtd_buffer_hi[0] = 0; /* XXX not needed? */
2726         stat->nextqtd = NULL;
2727         stat->qtd.qtd_next = stat->qtd.qtd_altnext = EHCI_NULL;
2728         stat->xfer = xfer;
2729         stat->len = 0;
2730
2731 #ifdef EHCI_DEBUG
2732         if (ehcidebug > 5) {
2733                 DPRINTF(("ehci_device_request:\n"));
2734                 ehci_dump_sqh(sqh);
2735                 ehci_dump_sqtds(setup);
2736         }
2737 #endif
2738
2739         exfer->sqtdstart = setup;
2740         exfer->sqtdend = stat;
2741 #ifdef DIAGNOSTIC
2742         if (!exfer->isdone) {
2743                 kprintf("ehci_device_request: not done, exfer=%p\n", exfer);
2744         }
2745         exfer->isdone = 0;
2746 #endif
2747
2748         /* Insert qTD in QH list. */
2749         crit_enter();
2750         ehci_set_qh_qtd(sqh, setup);
2751         if (xfer->timeout && !sc->sc_bus.use_polling) {
2752                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2753                             ehci_timeout, xfer);
2754         }
2755         ehci_add_intr_list(sc, exfer);
2756         xfer->status = USBD_IN_PROGRESS;
2757         crit_exit();
2758
2759 #ifdef EHCI_DEBUG
2760         if (ehcidebug > 10) {
2761                 DPRINTF(("ehci_device_request: status=%x\n",
2762                          EOREAD4(sc, EHCI_USBSTS)));
2763                 DELAY(10000);
2764                 ehci_dump_regs(sc);
2765                 ehci_dump_sqh(sc->sc_async_head);
2766                 ehci_dump_sqh(sqh);
2767                 ehci_dump_sqtds(setup);
2768         }
2769 #endif
2770
2771         return (USBD_NORMAL_COMPLETION);
2772
2773  bad3:
2774         ehci_free_sqtd(sc, stat);
2775  bad2:
2776         ehci_free_sqtd(sc, setup);
2777  bad1:
2778         DPRINTFN(-1,("ehci_device_request: no memory\n"));
2779         xfer->status = err;
2780         usb_transfer_complete(xfer);
2781         return (err);
2782 #undef exfer
2783 }
2784
2785 /************************/
2786
2787 static usbd_status
2788 ehci_device_bulk_transfer(usbd_xfer_handle xfer)
2789 {
2790         usbd_status err;
2791
2792         /* Insert last in queue. */
2793         err = usb_insert_transfer(xfer);
2794         if (err)
2795                 return (err);
2796
2797         /* Pipe isn't running, start first */
2798         return (ehci_device_bulk_start(STAILQ_FIRST(&xfer->pipe->queue)));
2799 }
2800
2801 usbd_status
2802 ehci_device_bulk_start(usbd_xfer_handle xfer)
2803 {
2804 #define exfer EXFER(xfer)
2805         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2806         usbd_device_handle dev = epipe->pipe.device;
2807         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2808         ehci_soft_qtd_t *data, *dataend;
2809         ehci_soft_qh_t *sqh;
2810         usbd_status err;
2811         int len, isread, endpt;
2812
2813         DPRINTFN(2, ("ehci_device_bulk_start: xfer=%p len=%d flags=%d\n",
2814                      xfer, xfer->length, xfer->flags));
2815
2816         if (sc->sc_dying)
2817                 return (USBD_IOERROR);
2818
2819 #ifdef DIAGNOSTIC
2820         if (xfer->rqflags & URQ_REQUEST)
2821                 panic("ehci_device_bulk_start: a request");
2822 #endif
2823
2824         len = xfer->length;
2825         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
2826         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
2827         sqh = epipe->sqh;
2828
2829         epipe->u.bulk.length = len;
2830
2831         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
2832                                    &dataend);
2833         if (err) {
2834                 DPRINTFN(-1,("ehci_device_bulk_start: no memory\n"));
2835                 xfer->status = err;
2836                 usb_transfer_complete(xfer);
2837                 return (err);
2838         }
2839
2840 #ifdef EHCI_DEBUG
2841         if (ehcidebug > 5) {
2842                 DPRINTF(("ehci_device_bulk_start: data(1)\n"));
2843                 ehci_dump_sqh(sqh);
2844                 ehci_dump_sqtds(data);
2845         }
2846 #endif
2847
2848         /* Set up interrupt info. */
2849         exfer->sqtdstart = data;
2850         exfer->sqtdend = dataend;
2851 #ifdef DIAGNOSTIC
2852         if (!exfer->isdone) {
2853                 kprintf("ehci_device_bulk_start: not done, ex=%p\n", exfer);
2854         }
2855         exfer->isdone = 0;
2856 #endif
2857
2858         crit_enter();
2859         ehci_set_qh_qtd(sqh, data);
2860         if (xfer->timeout && !sc->sc_bus.use_polling) {
2861                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
2862                             ehci_timeout, xfer);
2863         }
2864         ehci_add_intr_list(sc, exfer);
2865         xfer->status = USBD_IN_PROGRESS;
2866         crit_exit();
2867
2868 #ifdef EHCI_DEBUG
2869         if (ehcidebug > 10) {
2870                 DPRINTF(("ehci_device_bulk_start: data(2)\n"));
2871                 DELAY(10000);
2872                 DPRINTF(("ehci_device_bulk_start: data(3)\n"));
2873                 ehci_dump_regs(sc);
2874 #if 0
2875                 kprintf("async_head:\n");
2876                 ehci_dump_sqh(sc->sc_async_head);
2877 #endif
2878                 kprintf("sqh:\n");
2879                 ehci_dump_sqh(sqh);
2880                 ehci_dump_sqtds(data);
2881         }
2882 #endif
2883
2884         if (sc->sc_bus.use_polling)
2885                 ehci_waitintr(sc, xfer);
2886
2887         return (USBD_IN_PROGRESS);
2888 #undef exfer
2889 }
2890
2891 static void
2892 ehci_device_bulk_abort(usbd_xfer_handle xfer)
2893 {
2894         DPRINTF(("ehci_device_bulk_abort: xfer=%p\n", xfer));
2895         ehci_abort_xfer(xfer, USBD_CANCELLED);
2896 }
2897
2898 /*
2899  * Close a device bulk pipe.
2900  */
2901 static void
2902 ehci_device_bulk_close(usbd_pipe_handle pipe)
2903 {
2904         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
2905
2906         DPRINTF(("ehci_device_bulk_close: pipe=%p\n", pipe));
2907         ehci_close_pipe(pipe, sc->sc_async_head);
2908 }
2909
2910 void
2911 ehci_device_bulk_done(usbd_xfer_handle xfer)
2912 {
2913         struct ehci_xfer *ex = EXFER(xfer);
2914         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
2915         /*struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;*/
2916
2917         DPRINTFN(10,("ehci_bulk_done: xfer=%p, actlen=%d\n",
2918                      xfer, xfer->actlen));
2919
2920         if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
2921                 ehci_del_intr_list(ex); /* remove from active list */
2922                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
2923         }
2924
2925         DPRINTFN(5, ("ehci_bulk_done: length=%d\n", xfer->actlen));
2926 }
2927
2928 /************************/
2929
2930 static usbd_status
2931 ehci_device_setintr(ehci_softc_t *sc, ehci_soft_qh_t *sqh, int ival)
2932 {
2933         struct ehci_soft_islot *isp;
2934         int islot, lev;
2935
2936         /* Find a poll rate that is large enough. */
2937         for (lev = EHCI_IPOLLRATES - 1; lev > 0; lev--)
2938                 if (EHCI_ILEV_IVAL(lev) <= ival)
2939                         break;
2940
2941         /* Pick an interrupt slot at the right level. */
2942         /* XXX could do better than picking at random. */
2943         islot = EHCI_IQHIDX(lev, karc4random());
2944
2945         sqh->islot = islot;
2946         isp = &sc->sc_islots[islot];
2947         ehci_add_qh(sqh, isp->sqh);
2948
2949         return (USBD_NORMAL_COMPLETION);
2950 }
2951
2952 static usbd_status
2953 ehci_device_intr_transfer(usbd_xfer_handle xfer)
2954 {
2955         usbd_status err;
2956
2957         /* Insert last in queue. */
2958         err = usb_insert_transfer(xfer);
2959         if (err)
2960                 return (err);
2961
2962         /*
2963          * Pipe isn't running (otherwise err would be USBD_INPROG),
2964          * so start it first.
2965          */
2966         return (ehci_device_intr_start(STAILQ_FIRST(&xfer->pipe->queue)));
2967 }
2968
2969 static usbd_status
2970 ehci_device_intr_start(usbd_xfer_handle xfer)
2971 {
2972 #define exfer EXFER(xfer)
2973         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
2974         usbd_device_handle dev = xfer->pipe->device;
2975         ehci_softc_t *sc = (ehci_softc_t *)dev->bus;
2976         ehci_soft_qtd_t *data, *dataend;
2977         ehci_soft_qh_t *sqh;
2978         usbd_status err;
2979         int len, isread, endpt;
2980
2981         DPRINTFN(2, ("ehci_device_intr_start: xfer=%p len=%d flags=%d\n",
2982             xfer, xfer->length, xfer->flags));
2983
2984         if (sc->sc_dying)
2985                 return (USBD_IOERROR);
2986
2987 #ifdef DIAGNOSTIC
2988         if (xfer->rqflags & URQ_REQUEST)
2989                 panic("ehci_device_intr_start: a request");
2990 #endif
2991
2992         len = xfer->length;
2993         endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
2994         isread = UE_GET_DIR(endpt) == UE_DIR_IN;
2995         sqh = epipe->sqh;
2996
2997         epipe->u.intr.length = len;
2998
2999         err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer, &data,
3000             &dataend);
3001         if (err) {
3002                 DPRINTFN(-1, ("ehci_device_intr_start: no memory\n"));
3003                 xfer->status = err;
3004                 usb_transfer_complete(xfer);
3005                 return (err);
3006         }
3007
3008 #ifdef EHCI_DEBUG
3009         if (ehcidebug > 5) {
3010                 DPRINTF(("ehci_device_intr_start: data(1)\n"));
3011                 ehci_dump_sqh(sqh);
3012                 ehci_dump_sqtds(data);
3013         }
3014 #endif
3015
3016         /* Set up interrupt info. */
3017         exfer->sqtdstart = data;
3018         exfer->sqtdend = dataend;
3019 #ifdef DIAGNOSTIC
3020         if (!exfer->isdone) {
3021                 kprintf("ehci_device_intr_start: not done, ex=%p\n", exfer);
3022         }
3023         exfer->isdone = 0;
3024 #endif
3025
3026         crit_enter();
3027         ehci_set_qh_qtd(sqh, data);
3028         if (xfer->timeout && !sc->sc_bus.use_polling) {
3029                 callout_reset(&xfer->timeout_handle, MS_TO_TICKS(xfer->timeout),
3030                     ehci_timeout, xfer);
3031         }
3032         ehci_add_intr_list(sc, exfer);
3033         xfer->status = USBD_IN_PROGRESS;
3034         crit_exit();
3035
3036 #ifdef EHCI_DEBUG
3037         if (ehcidebug > 10) {
3038                 DPRINTF(("ehci_device_intr_start: data(2)\n"));
3039                 DELAY(10000);
3040                 DPRINTF(("ehci_device_intr_start: data(3)\n"));
3041                 ehci_dump_regs(sc);
3042                 kprintf("sqh:\n");
3043                 ehci_dump_sqh(sqh);
3044                 ehci_dump_sqtds(data);
3045         }
3046 #endif
3047
3048         if (sc->sc_bus.use_polling)
3049                 ehci_waitintr(sc, xfer);
3050
3051         return (USBD_IN_PROGRESS);
3052 #undef exfer
3053 }
3054
3055 static void
3056 ehci_device_intr_abort(usbd_xfer_handle xfer)
3057 {
3058         DPRINTFN(1, ("ehci_device_intr_abort: xfer=%p\n", xfer));
3059         if (xfer->pipe->intrxfer == xfer) {
3060                 DPRINTFN(1, ("ehci_device_intr_abort: remove\n"));
3061                 xfer->pipe->intrxfer = NULL;
3062         }
3063         ehci_abort_xfer(xfer, USBD_CANCELLED);
3064 }
3065
3066 static void
3067 ehci_device_intr_close(usbd_pipe_handle pipe)
3068 {
3069         ehci_softc_t *sc = (ehci_softc_t *)pipe->device->bus;
3070         struct ehci_pipe *epipe = (struct ehci_pipe *)pipe;
3071         struct ehci_soft_islot *isp;
3072
3073         isp = &sc->sc_islots[epipe->sqh->islot];
3074         ehci_close_pipe(pipe, isp->sqh);
3075 }
3076
3077 static void
3078 ehci_device_intr_done(usbd_xfer_handle xfer)
3079 {
3080 #define exfer EXFER(xfer)
3081         struct ehci_xfer *ex = EXFER(xfer);
3082         ehci_softc_t *sc = (ehci_softc_t *)xfer->pipe->device->bus;
3083         struct ehci_pipe *epipe = (struct ehci_pipe *)xfer->pipe;
3084         ehci_soft_qtd_t *data, *dataend;
3085         ehci_soft_qh_t *sqh;
3086         usbd_status err;
3087         int len, isread, endpt;
3088
3089         DPRINTFN(10, ("ehci_device_intr_done: xfer=%p, actlen=%d\n",
3090             xfer, xfer->actlen));
3091
3092         if (xfer->pipe->repeat) {
3093                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3094
3095                 len = epipe->u.intr.length;
3096                 xfer->length = len;
3097                 endpt = epipe->pipe.endpoint->edesc->bEndpointAddress;
3098                 isread = UE_GET_DIR(endpt) == UE_DIR_IN;
3099                 sqh = epipe->sqh;
3100
3101                 err = ehci_alloc_sqtd_chain(epipe, sc, len, isread, xfer,
3102                     &data, &dataend);
3103                 if (err) {
3104                         DPRINTFN(-1, ("ehci_device_intr_done: no memory\n"));
3105                         xfer->status = err;
3106                         return;
3107                 }
3108
3109                 /* Set up interrupt info. */
3110                 exfer->sqtdstart = data;
3111                 exfer->sqtdend = dataend;
3112 #ifdef DIAGNOSTIC
3113                 if (!exfer->isdone) {
3114                         kprintf("ehci_device_intr_done: not done, ex=%p\n",
3115                             exfer);
3116                 }
3117                 exfer->isdone = 0;
3118 #endif
3119
3120                 crit_enter();
3121                 ehci_set_qh_qtd(sqh, data);
3122                 if (xfer->timeout && !sc->sc_bus.use_polling) {
3123                         callout_reset(&xfer->timeout_handle,
3124                             MS_TO_TICKS(xfer->timeout), ehci_timeout, xfer);
3125                 }
3126                 crit_exit();
3127
3128                 xfer->status = USBD_IN_PROGRESS;
3129         } else if (xfer->status != USBD_NOMEM && ehci_active_intr_list(ex)) {
3130                 ehci_del_intr_list(ex); /* remove from active list */
3131                 ehci_free_sqtd_chain(sc, ex->sqtdstart, NULL);
3132         }
3133 #undef exfer
3134 }
3135
3136 /************************/
3137
3138 static usbd_status
3139 ehci_device_isoc_transfer(usbd_xfer_handle xfer)
3140 {
3141         return USBD_IOERROR;
3142 }
3143
3144 static usbd_status
3145 ehci_device_isoc_start(usbd_xfer_handle xfer)
3146 {
3147         return USBD_IOERROR;
3148 }
3149
3150 static void
3151 ehci_device_isoc_abort(usbd_xfer_handle xfer)
3152 {
3153 }
3154
3155 static void
3156 ehci_device_isoc_close(usbd_pipe_handle pipe)
3157 {
3158 }
3159
3160 static void
3161 ehci_device_isoc_done(usbd_xfer_handle xfer)
3162 {
3163 }