sys/bus/u4b/controller/ehci.c

   1 /*-
   2  * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
   3  * Copyright (c) 2004 The NetBSD Foundation, Inc. All rights reserved.
   4  * Copyright (c) 2004 Lennart Augustsson. All rights reserved.
   5  * Copyright (c) 2004 Charles M. Hannum. All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * USB Enhanced Host Controller Driver, a.k.a. USB 2.0 controller.
  31  *
  32  * The EHCI 0.96 spec can be found at
  33  * http://developer.intel.com/technology/usb/download/ehci-r096.pdf
  34  * The EHCI 1.0 spec can be found at
  35  * http://developer.intel.com/technology/usb/download/ehci-r10.pdf
  36  * and the USB 2.0 spec at
  37  * http://www.usb.org/developers/docs/usb_20.zip
  38  *
  39  */
  40
  41 /*
  42  * TODO:
  43  * 1) command failures are not recovered correctly
  44  */
  45
  46 #include <sys/stdint.h>
  47 #include <sys/param.h>
  48 #include <sys/queue.h>
  49 #include <sys/types.h>
  50 #include <sys/systm.h>
  51 #include <sys/kernel.h>
  52 #include <sys/bus.h>
  53 #include <sys/module.h>
  54 #include <sys/lock.h>
  55 #include <sys/condvar.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/unistd.h>
  58 #include <sys/callout.h>
  59 #include <sys/malloc.h>
  60 #include <sys/priv.h>
  61
  62 #include <bus/u4b/usb.h>
  63 #include <bus/u4b/usbdi.h>
  64
  65 #define USB_DEBUG_VAR ehcidebug
  66
  67 #include <bus/u4b/usb_core.h>
  68 #include <bus/u4b/usb_debug.h>
  69 #include <bus/u4b/usb_busdma.h>
  70 #include <bus/u4b/usb_process.h>
  71 #include <bus/u4b/usb_transfer.h>
  72 #include <bus/u4b/usb_device.h>
  73 #include <bus/u4b/usb_hub.h>
  74 #include <bus/u4b/usb_util.h>
  75
  76 #include <bus/u4b/usb_controller.h>
  77 #include <bus/u4b/usb_bus.h>
  78 #include <bus/u4b/controller/ehci.h>
  79 #include <bus/u4b/controller/ehcireg.h>
  80
  81 #define EHCI_BUS2SC(bus) \
  82    ((ehci_softc_t *)(((uint8_t *)(bus)) - \
  83     ((uint8_t *)&(((ehci_softc_t *)0)->sc_bus))))
  84
  85 #ifdef USB_DEBUG
  86 static int ehcidebug = 0;
  87 static int ehcinohighspeed = 0;
  88 static int ehciiaadbug = 0;
  89 static int ehcilostintrbug = 0;
  90
  91 static SYSCTL_NODE(_hw_usb, OID_AUTO, ehci, CTLFLAG_RW, 0, "USB ehci");
  92 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, debug, CTLFLAG_RW,
  93     &ehcidebug, 0, "Debug level");
  94 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, no_hs, CTLFLAG_RW,
  95     &ehcinohighspeed, 0, "Disable High Speed USB");
  96 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, iaadbug, CTLFLAG_RW,
  97     &ehciiaadbug, 0, "Enable doorbell bug workaround");
  98 SYSCTL_INT(_hw_usb_ehci, OID_AUTO, lostintrbug, CTLFLAG_RW,
  99     &ehcilostintrbug, 0, "Enable lost interrupt bug workaround");
 100
 101 TUNABLE_INT("hw.usb.ehci.debug", &ehcidebug);
 102 TUNABLE_INT("hw.usb.ehci.no_hs", &ehcinohighspeed);
 103 TUNABLE_INT("hw.usb.ehci.iaadbug", &ehciiaadbug);
 104 TUNABLE_INT("hw.usb.ehci.lostintrbug", &ehcilostintrbug);
 105
 106 static void ehci_dump_regs(ehci_softc_t *sc);
 107 static void ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *sqh);
 108
 109 #endif
 110
 111 #define EHCI_INTR_ENDPT 1
 112
 113 static const struct usb_bus_methods ehci_bus_methods;
 114 static const struct usb_pipe_methods ehci_device_bulk_methods;
 115 static const struct usb_pipe_methods ehci_device_ctrl_methods;
 116 static const struct usb_pipe_methods ehci_device_intr_methods;
 117 static const struct usb_pipe_methods ehci_device_isoc_fs_methods;
 118 static const struct usb_pipe_methods ehci_device_isoc_hs_methods;
 119
 120 static void ehci_do_poll(struct usb_bus *);
 121 static void ehci_device_done(struct usb_xfer *, usb_error_t);
 122 static uint8_t ehci_check_transfer(struct usb_xfer *xfer, int finish);
 123 static void ehci_timeout(void *);
 124 static void ehci_poll_timeout(void *);
 125
 126 static void ehci_root_intr(ehci_softc_t *sc);
 127
 128 struct ehci_std_temp {
 129         ehci_softc_t *sc;
 130         struct usb_page_cache *pc;
 131         ehci_qtd_t *td;
 132         ehci_qtd_t *td_next;
 133         uint32_t average;
 134         uint32_t qtd_status;
 135         uint32_t len;
 136         uint16_t max_frame_size;
 137         uint8_t shortpkt;
 138         uint8_t auto_data_toggle;
 139         uint8_t setup_alt_next;
 140         uint8_t last_frame;
 141 };
 142
 143 void
 144 ehci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
 145 {
 146         ehci_softc_t *sc = EHCI_BUS2SC(bus);
 147         uint32_t i;
 148
 149         cb(bus, &sc->sc_hw.pframes_pc, &sc->sc_hw.pframes_pg,
 150             sizeof(uint32_t) * EHCI_FRAMELIST_COUNT, EHCI_FRAMELIST_ALIGN);
 151
 152         cb(bus, &sc->sc_hw.terminate_pc, &sc->sc_hw.terminate_pg,
 153             sizeof(struct ehci_qh_sub), EHCI_QH_ALIGN);
 154
 155         cb(bus, &sc->sc_hw.async_start_pc, &sc->sc_hw.async_start_pg,
 156             sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 157
 158         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 159                 cb(bus, sc->sc_hw.intr_start_pc + i,
 160                     sc->sc_hw.intr_start_pg + i,
 161                     sizeof(ehci_qh_t), EHCI_QH_ALIGN);
 162         }
 163
 164         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 165                 cb(bus, sc->sc_hw.isoc_hs_start_pc + i,
 166                     sc->sc_hw.isoc_hs_start_pg + i,
 167                     sizeof(ehci_itd_t), EHCI_ITD_ALIGN);
 168         }
 169
 170         for (i = 0; i != EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 171                 cb(bus, sc->sc_hw.isoc_fs_start_pc + i,
 172                     sc->sc_hw.isoc_fs_start_pg + i,
 173                     sizeof(ehci_sitd_t), EHCI_SITD_ALIGN);
 174         }
 175 }
 176
 177 usb_error_t
 178 ehci_reset(ehci_softc_t *sc)
 179 {
 180         uint32_t hcr;
 181         int i;
 182
 183         EOWRITE4(sc, EHCI_USBCMD, EHCI_CMD_HCRESET);
 184         for (i = 0; i < 100; i++) {
 185                 usb_pause_mtx(NULL, hz / 128);
 186                 hcr = EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_HCRESET;
 187                 if (!hcr) {
 188                         if (sc->sc_flags & (EHCI_SCFLG_SETMODE | EHCI_SCFLG_BIGEMMIO)) {
 189                                 /*
 190                                  * Force USBMODE as requested.  Controllers
 191                                  * may have multiple operating modes.
 192                                  */
 193                                 uint32_t usbmode = EOREAD4(sc, EHCI_USBMODE);
 194                                 if (sc->sc_flags & EHCI_SCFLG_SETMODE) {
 195                                         usbmode = (usbmode &~ EHCI_UM_CM) | EHCI_UM_CM_HOST;
 196                                         device_printf(sc->sc_bus.bdev,
 197                                             "set host controller mode\n");
 198                                 }
 199                                 if (sc->sc_flags & EHCI_SCFLG_BIGEMMIO) {
 200                                         usbmode = (usbmode &~ EHCI_UM_ES) | EHCI_UM_ES_BE;
 201                                         device_printf(sc->sc_bus.bdev,
 202                                             "set big-endian mode\n");
 203                                 }
 204                                 EOWRITE4(sc,  EHCI_USBMODE, usbmode);
 205                         }
 206                         return (0);
 207                 }
 208         }
 209         device_printf(sc->sc_bus.bdev, "Reset timeout\n");
 210         return (USB_ERR_IOERROR);
 211 }
 212
 213 static usb_error_t
 214 ehci_hcreset(ehci_softc_t *sc)
 215 {
 216         uint32_t hcr;
 217         int i;
 218
 219         EOWRITE4(sc, EHCI_USBCMD, 0);   /* Halt controller */
 220         for (i = 0; i < 100; i++) {
 221                 usb_pause_mtx(NULL, hz / 128);
 222                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 223                 if (hcr)
 224                         break;
 225         }
 226         if (!hcr)
 227                 /*
 228                  * Fall through and try reset anyway even though
 229                  * Table 2-9 in the EHCI spec says this will result
 230                  * in undefined behavior.
 231                  */
 232                 device_printf(sc->sc_bus.bdev, "stop timeout\n");
 233
 234         return (ehci_reset(sc));
 235 }
 236
 237 static int
 238 ehci_init_sub(struct ehci_softc *sc)
 239 {
 240         struct usb_page_search buf_res;
 241         uint32_t cparams;
 242         uint32_t hcr;
 243         uint8_t i;
 244
 245         cparams = EREAD4(sc, EHCI_HCCPARAMS);
 246
 247         DPRINTF("cparams=0x%x\n", cparams);
 248
 249         if (EHCI_HCC_64BIT(cparams)) {
 250                 DPRINTF("HCC uses 64-bit structures\n");
 251
 252                 /* MUST clear segment register if 64 bit capable */
 253                 EOWRITE4(sc, EHCI_CTRLDSSEGMENT, 0);
 254         }
 255
 256         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 257         EOWRITE4(sc, EHCI_PERIODICLISTBASE, buf_res.physaddr);
 258
 259         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 260         EOWRITE4(sc, EHCI_ASYNCLISTADDR, buf_res.physaddr | EHCI_LINK_QH);
 261
 262         /* enable interrupts */
 263         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
 264
 265         /* turn on controller */
 266         EOWRITE4(sc, EHCI_USBCMD,
 267             EHCI_CMD_ITC_1 |            /* 1 microframes interrupt delay */
 268             (EOREAD4(sc, EHCI_USBCMD) & EHCI_CMD_FLS_M) |
 269             EHCI_CMD_ASPME | EHCI_CMD_ASPMC |
 270             EHCI_CMD_ASE |
 271             EHCI_CMD_PSE |
 272             EHCI_CMD_RS);
 273
 274         /* Take over port ownership */
 275         EOWRITE4(sc, EHCI_CONFIGFLAG, EHCI_CONF_CF);
 276
 277         for (i = 0; i < 100; i++) {
 278                 usb_pause_mtx(NULL, hz / 128);
 279                 hcr = EOREAD4(sc, EHCI_USBSTS) & EHCI_STS_HCH;
 280                 if (!hcr) {
 281                         break;
 282                 }
 283         }
 284         if (hcr) {
 285                 device_printf(sc->sc_bus.bdev, "Run timeout\n");
 286                 return (USB_ERR_IOERROR);
 287         }
 288         return (USB_ERR_NORMAL_COMPLETION);
 289 }
 290
 291 usb_error_t
 292 ehci_init(ehci_softc_t *sc)
 293 {
 294         struct usb_page_search buf_res;
 295         uint32_t version;
 296         uint32_t sparams;
 297         uint16_t i;
 298         uint16_t x;
 299         uint16_t y;
 300         uint16_t bit;
 301         usb_error_t err = 0;
 302
 303         DPRINTF("start\n");
 304
 305         usb_callout_init_mtx(&sc->sc_tmo_pcd, &sc->sc_bus.bus_lock, 0);
 306         usb_callout_init_mtx(&sc->sc_tmo_poll, &sc->sc_bus.bus_lock, 0);
 307
 308         sc->sc_offs = EHCI_CAPLENGTH(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
 309
 310 #ifdef USB_DEBUG
 311         if (ehciiaadbug)
 312                 sc->sc_flags |= EHCI_SCFLG_IAADBUG;
 313         if (ehcilostintrbug)
 314                 sc->sc_flags |= EHCI_SCFLG_LOSTINTRBUG;
 315         if (ehcidebug > 2) {
 316                 ehci_dump_regs(sc);
 317         }
 318 #endif
 319
 320         version = EHCI_HCIVERSION(EREAD4(sc, EHCI_CAPLEN_HCIVERSION));
 321         device_printf(sc->sc_bus.bdev, "EHCI version %x.%x\n",
 322             version >> 8, version & 0xff);
 323
 324         sparams = EREAD4(sc, EHCI_HCSPARAMS);
 325         DPRINTF("sparams=0x%x\n", sparams);
 326
 327         sc->sc_noport = EHCI_HCS_N_PORTS(sparams);
 328         sc->sc_bus.usbrev = USB_REV_2_0;
 329
 330         if (!(sc->sc_flags & EHCI_SCFLG_DONTRESET)) {
 331                 /* Reset the controller */
 332                 DPRINTF("%s: resetting\n",
 333                     device_get_nameunit(sc->sc_bus.bdev));
 334
 335                 err = ehci_hcreset(sc);
 336                 if (err) {
 337                         device_printf(sc->sc_bus.bdev, "reset timeout\n");
 338                         return (err);
 339                 }
 340         }
 341
 342         /*
 343          * use current frame-list-size selection 0: 1024*4 bytes 1:  512*4
 344          * bytes 2:  256*4 bytes 3:      unknown
 345          */
 346         if (EHCI_CMD_FLS(EOREAD4(sc, EHCI_USBCMD)) == 3) {
 347                 device_printf(sc->sc_bus.bdev, "invalid frame-list-size\n");
 348                 return (USB_ERR_IOERROR);
 349         }
 350         /* set up the bus struct */
 351         sc->sc_bus.methods = &ehci_bus_methods;
 352
 353         sc->sc_eintrs = EHCI_NORMAL_INTRS;
 354
 355         if (1) {
 356                 struct ehci_qh_sub *qh;
 357
 358                 usbd_get_page(&sc->sc_hw.terminate_pc, 0, &buf_res);
 359
 360                 qh = buf_res.buffer;
 361
 362                 sc->sc_terminate_self = htohc32(sc, buf_res.physaddr);
 363
 364                 /* init terminate TD */
 365                 qh->qtd_next =
 366                     htohc32(sc, EHCI_LINK_TERMINATE);
 367                 qh->qtd_altnext =
 368                     htohc32(sc, EHCI_LINK_TERMINATE);
 369                 qh->qtd_status =
 370                     htohc32(sc, EHCI_QTD_HALTED);
 371         }
 372
 373         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 374                 ehci_qh_t *qh;
 375
 376                 usbd_get_page(sc->sc_hw.intr_start_pc + i, 0, &buf_res);
 377
 378                 qh = buf_res.buffer;
 379
 380                 /* initialize page cache pointer */
 381
 382                 qh->page_cache = sc->sc_hw.intr_start_pc + i;
 383
 384                 /* store a pointer to queue head */
 385
 386                 sc->sc_intr_p_last[i] = qh;
 387
 388                 qh->qh_self =
 389                     htohc32(sc, buf_res.physaddr) |
 390                     htohc32(sc, EHCI_LINK_QH);
 391
 392                 qh->qh_endp =
 393                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH));
 394                 qh->qh_endphub =
 395                     htohc32(sc, EHCI_QH_SET_MULT(1));
 396                 qh->qh_curqtd = 0;
 397
 398                 qh->qh_qtd.qtd_next =
 399                     htohc32(sc, EHCI_LINK_TERMINATE);
 400                 qh->qh_qtd.qtd_altnext =
 401                     htohc32(sc, EHCI_LINK_TERMINATE);
 402                 qh->qh_qtd.qtd_status =
 403                     htohc32(sc, EHCI_QTD_HALTED);
 404         }
 405
 406         /*
 407          * the QHs are arranged to give poll intervals that are
 408          * powers of 2 times 1ms
 409          *
 410          * e.g. 64-127, 32-63, 16-31, 8-15, 4-7, 2-3, 1-1, 0
 411          *        1ms    2ms    4ms    8ms  16ms 32ms 64ms 128m
 412          */
 413         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
 414         while (bit) {
 415                 x = bit;
 416                 while (x & bit) {
 417                         ehci_qh_t *qh_x;
 418                         ehci_qh_t *qh_y;
 419
 420                         y = (x ^ bit) | (bit / 2);
 421
 422                         qh_x = sc->sc_intr_p_last[x];
 423                         qh_y = sc->sc_intr_p_last[y];
 424
 425                         /*
 426                          * the next QH has half the poll interval
 427                          */
 428                         qh_x->qh_link = qh_y->qh_self;
 429
 430                         x++;
 431                 }
 432                 bit >>= 1;
 433         }
 434
 435         if (1) {
 436                 ehci_qh_t *qh;
 437
 438                 qh = sc->sc_intr_p_last[0];
 439
 440                 /* the last (1ms) QH terminates */
 441                 qh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE);
 442         }
 443         for (i = 0; i < EHCI_VIRTUAL_FRAMELIST_COUNT; i++) {
 444                 ehci_sitd_t *sitd;
 445                 ehci_itd_t *itd;
 446
 447                 usbd_get_page(sc->sc_hw.isoc_fs_start_pc + i, 0, &buf_res);
 448
 449                 sitd = buf_res.buffer;
 450
 451                 /* initialize page cache pointer */
 452
 453                 sitd->page_cache = sc->sc_hw.isoc_fs_start_pc + i;
 454
 455                 /* store a pointer to the transfer descriptor */
 456
 457                 sc->sc_isoc_fs_p_last[i] = sitd;
 458
 459                 /* initialize full speed isochronous */
 460
 461                 sitd->sitd_self =
 462                     htohc32(sc, buf_res.physaddr) |
 463                     htohc32(sc, EHCI_LINK_SITD);
 464
 465                 sitd->sitd_back =
 466                     htohc32(sc, EHCI_LINK_TERMINATE);
 467
 468                 /*
 469                  * Point to 1ms interrupt queue heads 1:1
 470                  */
 471                 sitd->sitd_next =
 472                     sc->sc_intr_p_last[i | (EHCI_VIRTUAL_FRAMELIST_COUNT / 2)]->qh_self;
 473
 474
 475                 usbd_get_page(sc->sc_hw.isoc_hs_start_pc + i, 0, &buf_res);
 476
 477                 itd = buf_res.buffer;
 478
 479                 /* initialize page cache pointer */
 480
 481                 itd->page_cache = sc->sc_hw.isoc_hs_start_pc + i;
 482
 483                 /* store a pointer to the transfer descriptor */
 484
 485                 sc->sc_isoc_hs_p_last[i] = itd;
 486
 487                 /* initialize high speed isochronous */
 488
 489                 itd->itd_self =
 490                     htohc32(sc, buf_res.physaddr) |
 491                     htohc32(sc, EHCI_LINK_ITD);
 492
 493                 /*
 494                  * 1:1 HS -> FS
 495                  */
 496                 itd->itd_next =
 497                     sitd->sitd_self;
 498         }
 499
 500         usbd_get_page(&sc->sc_hw.pframes_pc, 0, &buf_res);
 501
 502         if (1) {
 503                 uint32_t *pframes;
 504
 505                 pframes = buf_res.buffer;
 506
 507                 /*
 508                  * execution order:
 509                  * pframes -> high speed isochronous ->
 510                  *    full speed isochronous -> interrupt QH's
 511                  *
 512                  * Each pframe chains through 10 queue heads.
 513                  *    HS FS IQ1ms IQ2ms IQ4ms ... IQ128ms -> term
 514                  */
 515                 for (i = 0; i < EHCI_FRAMELIST_COUNT; i++) {
 516                         pframes[i] = sc->sc_isoc_hs_p_last
 517                             [i & (EHCI_VIRTUAL_FRAMELIST_COUNT - 1)]->itd_self;
 518                 }
 519         }
 520         usbd_get_page(&sc->sc_hw.async_start_pc, 0, &buf_res);
 521
 522         /*
 523          * Asynchronous schedule requires a QH the points to itself initially.
 524          * (it is not terminated with EHCI_LINK_TERMINATE).
 525          */
 526         if (1) {
 527
 528                 ehci_qh_t *qh;
 529
 530                 qh = buf_res.buffer;
 531
 532                 /* initialize page cache pointer */
 533
 534                 qh->page_cache = &sc->sc_hw.async_start_pc;
 535
 536                 /* store a pointer to the queue head */
 537
 538                 sc->sc_async_p_last = qh;
 539
 540                 /* init dummy QH that starts the async list */
 541
 542                 qh->qh_self =
 543                     htohc32(sc, buf_res.physaddr) |
 544                     htohc32(sc, EHCI_LINK_QH);
 545
 546                 /* fill the QH */
 547                 qh->qh_endp =
 548                     htohc32(sc, EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH) | EHCI_QH_HRECL);
 549                 qh->qh_endphub = htohc32(sc, EHCI_QH_SET_MULT(1));
 550                 qh->qh_link = qh->qh_self;
 551                 qh->qh_curqtd = 0;
 552
 553                 /* fill the overlay qTD */
 554                 qh->qh_qtd.qtd_next = htohc32(sc, EHCI_LINK_TERMINATE);
 555                 qh->qh_qtd.qtd_altnext = htohc32(sc, EHCI_LINK_TERMINATE);
 556                 qh->qh_qtd.qtd_status = htohc32(sc, EHCI_QTD_HALTED);
 557         }
 558         /* flush all cache into memory */
 559
 560         usb_bus_mem_flush_all(&sc->sc_bus, &ehci_iterate_hw_softc);
 561
 562 #ifdef USB_DEBUG
 563         if (ehcidebug) {
 564                 ehci_dump_sqh(sc, sc->sc_async_p_last);
 565         }
 566 #endif
 567
 568         /* finial setup */
 569         err = ehci_init_sub(sc);
 570
 571         if (!err) {
 572                 /* catch any lost interrupts */
 573                 ehci_do_poll(&sc->sc_bus);
 574         }
 575         return (err);
 576 }
 577
 578 /*
 579  * shut down the controller when the system is going down
 580  */
 581 void
 582 ehci_detach(ehci_softc_t *sc)
 583 {
 584         USB_BUS_LOCK(&sc->sc_bus);
 585
 586         usb_callout_stop(&sc->sc_tmo_pcd);
 587         usb_callout_stop(&sc->sc_tmo_poll);
 588
 589         EOWRITE4(sc, EHCI_USBINTR, 0);
 590         USB_BUS_UNLOCK(&sc->sc_bus);
 591
 592         if (ehci_hcreset(sc)) {
 593                 DPRINTF("reset failed!\n");
 594         }
 595
 596         /* XXX let stray task complete */
 597         usb_pause_mtx(NULL, hz / 20);
 598
 599         usb_callout_drain(&sc->sc_tmo_pcd);
 600         usb_callout_drain(&sc->sc_tmo_poll);
 601 }
 602
 603 static void
 604 ehci_suspend(ehci_softc_t *sc)
 605 {
 606         DPRINTF("stopping the HC\n");
 607
 608         /* reset HC */
 609         ehci_hcreset(sc);
 610 }
 611
 612 static void
 613 ehci_resume(ehci_softc_t *sc)
 614 {
 615         /* reset HC */
 616         ehci_hcreset(sc);
 617
 618         /* setup HC */
 619         ehci_init_sub(sc);
 620
 621         /* catch any lost interrupts */
 622         ehci_do_poll(&sc->sc_bus);
 623 }
 624
 625 #ifdef USB_DEBUG
 626 static void
 627 ehci_dump_regs(ehci_softc_t *sc)
 628 {
 629         uint32_t i;
 630
 631         i = EOREAD4(sc, EHCI_USBCMD);
 632         kprintf("cmd=0x%08x\n", i);
 633
 634         if (i & EHCI_CMD_ITC_1)
 635                 kprintf(" EHCI_CMD_ITC_1\n");
 636         if (i & EHCI_CMD_ITC_2)
 637                 kprintf(" EHCI_CMD_ITC_2\n");
 638         if (i & EHCI_CMD_ITC_4)
 639                 kprintf(" EHCI_CMD_ITC_4\n");
 640         if (i & EHCI_CMD_ITC_8)
 641                 kprintf(" EHCI_CMD_ITC_8\n");
 642         if (i & EHCI_CMD_ITC_16)
 643                 kprintf(" EHCI_CMD_ITC_16\n");
 644         if (i & EHCI_CMD_ITC_32)
 645                 kprintf(" EHCI_CMD_ITC_32\n");
 646         if (i & EHCI_CMD_ITC_64)
 647                 kprintf(" EHCI_CMD_ITC_64\n");
 648         if (i & EHCI_CMD_ASPME)
 649                 kprintf(" EHCI_CMD_ASPME\n");
 650         if (i & EHCI_CMD_ASPMC)
 651                 kprintf(" EHCI_CMD_ASPMC\n");
 652         if (i & EHCI_CMD_LHCR)
 653                 kprintf(" EHCI_CMD_LHCR\n");
 654         if (i & EHCI_CMD_IAAD)
 655                 kprintf(" EHCI_CMD_IAAD\n");
 656         if (i & EHCI_CMD_ASE)
 657                 kprintf(" EHCI_CMD_ASE\n");
 658         if (i & EHCI_CMD_PSE)
 659                 kprintf(" EHCI_CMD_PSE\n");
 660         if (i & EHCI_CMD_FLS_M)
 661                 kprintf(" EHCI_CMD_FLS_M\n");
 662         if (i & EHCI_CMD_HCRESET)
 663                 kprintf(" EHCI_CMD_HCRESET\n");
 664         if (i & EHCI_CMD_RS)
 665                 kprintf(" EHCI_CMD_RS\n");
 666
 667         i = EOREAD4(sc, EHCI_USBSTS);
 668
 669         kprintf("sts=0x%08x\n", i);
 670
 671         if (i & EHCI_STS_ASS)
 672                 kprintf(" EHCI_STS_ASS\n");
 673         if (i & EHCI_STS_PSS)
 674                 kprintf(" EHCI_STS_PSS\n");
 675         if (i & EHCI_STS_REC)
 676                 kprintf(" EHCI_STS_REC\n");
 677         if (i & EHCI_STS_HCH)
 678                 kprintf(" EHCI_STS_HCH\n");
 679         if (i & EHCI_STS_IAA)
 680                 kprintf(" EHCI_STS_IAA\n");
 681         if (i & EHCI_STS_HSE)
 682                 kprintf(" EHCI_STS_HSE\n");
 683         if (i & EHCI_STS_FLR)
 684                 kprintf(" EHCI_STS_FLR\n");
 685         if (i & EHCI_STS_PCD)
 686                 kprintf(" EHCI_STS_PCD\n");
 687         if (i & EHCI_STS_ERRINT)
 688                 kprintf(" EHCI_STS_ERRINT\n");
 689         if (i & EHCI_STS_INT)
 690                 kprintf(" EHCI_STS_INT\n");
 691
 692         kprintf("ien=0x%08x\n",
 693             EOREAD4(sc, EHCI_USBINTR));
 694         kprintf("frindex=0x%08x ctrdsegm=0x%08x periodic=0x%08x async=0x%08x\n",
 695             EOREAD4(sc, EHCI_FRINDEX),
 696             EOREAD4(sc, EHCI_CTRLDSSEGMENT),
 697             EOREAD4(sc, EHCI_PERIODICLISTBASE),
 698             EOREAD4(sc, EHCI_ASYNCLISTADDR));
 699         for (i = 1; i <= sc->sc_noport; i++) {
 700                 kprintf("port %d status=0x%08x\n", i,
 701                     EOREAD4(sc, EHCI_PORTSC(i)));
 702         }
 703 }
 704
 705 static void
 706 ehci_dump_link(ehci_softc_t *sc, uint32_t link, int type)
 707 {
 708         link = hc32toh(sc, link);
 709         kprintf("0x%08x", link);
 710         if (link & EHCI_LINK_TERMINATE)
 711                 kprintf("<T>");
 712         else {
 713                 kprintf("<");
 714                 if (type) {
 715                         switch (EHCI_LINK_TYPE(link)) {
 716                         case EHCI_LINK_ITD:
 717                                 kprintf("ITD");
 718                                 break;
 719                         case EHCI_LINK_QH:
 720                                 kprintf("QH");
 721                                 break;
 722                         case EHCI_LINK_SITD:
 723                                 kprintf("SITD");
 724                                 break;
 725                         case EHCI_LINK_FSTN:
 726                                 kprintf("FSTN");
 727                                 break;
 728                         }
 729                 }
 730                 kprintf(">");
 731         }
 732 }
 733
 734 static void
 735 ehci_dump_qtd(ehci_softc_t *sc, ehci_qtd_t *qtd)
 736 {
 737         uint32_t s;
 738
 739         kprintf("  next=");
 740         ehci_dump_link(sc, qtd->qtd_next, 0);
 741         kprintf(" altnext=");
 742         ehci_dump_link(sc, qtd->qtd_altnext, 0);
 743         kprintf("\n");
 744         s = hc32toh(sc, qtd->qtd_status);
 745         kprintf("  status=0x%08x: toggle=%d bytes=0x%x ioc=%d c_page=0x%x\n",
 746             s, EHCI_QTD_GET_TOGGLE(s), EHCI_QTD_GET_BYTES(s),
 747             EHCI_QTD_GET_IOC(s), EHCI_QTD_GET_C_PAGE(s));
 748         kprintf("    cerr=%d pid=%d stat=%s%s%s%s%s%s%s%s\n",
 749             EHCI_QTD_GET_CERR(s), EHCI_QTD_GET_PID(s),
 750             (s & EHCI_QTD_ACTIVE) ? "ACTIVE" : "NOT_ACTIVE",
 751             (s & EHCI_QTD_HALTED) ? "-HALTED" : "",
 752             (s & EHCI_QTD_BUFERR) ? "-BUFERR" : "",
 753             (s & EHCI_QTD_BABBLE) ? "-BABBLE" : "",
 754             (s & EHCI_QTD_XACTERR) ? "-XACTERR" : "",
 755             (s & EHCI_QTD_MISSEDMICRO) ? "-MISSED" : "",
 756             (s & EHCI_QTD_SPLITXSTATE) ? "-SPLIT" : "",
 757             (s & EHCI_QTD_PINGSTATE) ? "-PING" : "");
 758
 759         for (s = 0; s < 5; s++) {
 760                 kprintf("  buffer[%d]=0x%08x\n", s,
 761                     hc32toh(sc, qtd->qtd_buffer[s]));
 762         }
 763         for (s = 0; s < 5; s++) {
 764                 kprintf("  buffer_hi[%d]=0x%08x\n", s,
 765                     hc32toh(sc, qtd->qtd_buffer_hi[s]));
 766         }
 767 }
 768
 769 static uint8_t
 770 ehci_dump_sqtd(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 771 {
 772         uint8_t temp;
 773
 774         usb_pc_cpu_invalidate(sqtd->page_cache);
 775         kprintf("QTD(%p) at 0x%08x:\n", sqtd, hc32toh(sc, sqtd->qtd_self));
 776         ehci_dump_qtd(sc, sqtd);
 777         temp = (sqtd->qtd_next & htohc32(sc, EHCI_LINK_TERMINATE)) ? 1 : 0;
 778         return (temp);
 779 }
 780
 781 static void
 782 ehci_dump_sqtds(ehci_softc_t *sc, ehci_qtd_t *sqtd)
 783 {
 784         uint16_t i;
 785         uint8_t stop;
 786
 787         stop = 0;
 788         for (i = 0; sqtd && (i < 20) && !stop; sqtd = sqtd->obj_next, i++) {
 789                 stop = ehci_dump_sqtd(sc, sqtd);
 790         }
 791         if (sqtd) {
 792                 kprintf("dump aborted, too many TDs\n");
 793         }
 794 }
 795
 796 static void
 797 ehci_dump_sqh(ehci_softc_t *sc, ehci_qh_t *qh)
 798 {
 799         uint32_t endp;
 800         uint32_t endphub;
 801
 802         usb_pc_cpu_invalidate(qh->page_cache);
 803         kprintf("QH(%p) at 0x%08x:\n", qh, hc32toh(sc, qh->qh_self) & ~0x1F);
 804         kprintf("  link=");
 805         ehci_dump_link(sc, qh->qh_link, 1);
 806         kprintf("\n");
 807         endp = hc32toh(sc, qh->qh_endp);
 808         kprintf("  endp=0x%08x\n", endp);
 809         kprintf("    addr=0x%02x inact=%d endpt=%d eps=%d dtc=%d hrecl=%d\n",
 810             EHCI_QH_GET_ADDR(endp), EHCI_QH_GET_INACT(endp),
 811             EHCI_QH_GET_ENDPT(endp), EHCI_QH_GET_EPS(endp),
 812             EHCI_QH_GET_DTC(endp), EHCI_QH_GET_HRECL(endp));
 813         kprintf("    mpl=0x%x ctl=%d nrl=%d\n",
 814             EHCI_QH_GET_MPL(endp), EHCI_QH_GET_CTL(endp),
 815             EHCI_QH_GET_NRL(endp));
 816         endphub = hc32toh(sc, qh->qh_endphub);
 817         kprintf("  endphub=0x%08x\n", endphub);
 818         kprintf("    smask=0x%02x cmask=0x%02x huba=0x%02x port=%d mult=%d\n",
 819             EHCI_QH_GET_SMASK(endphub), EHCI_QH_GET_CMASK(endphub),
 820             EHCI_QH_GET_HUBA(endphub), EHCI_QH_GET_PORT(endphub),
 821             EHCI_QH_GET_MULT(endphub));
 822         kprintf("  curqtd=");
 823         ehci_dump_link(sc, qh->qh_curqtd, 0);
 824         kprintf("\n");
 825         kprintf("Overlay qTD:\n");
 826         ehci_dump_qtd(sc, (void *)&qh->qh_qtd);
 827 }
 828
 829 static void
 830 ehci_dump_sitd(ehci_softc_t *sc, ehci_sitd_t *sitd)
 831 {
 832         usb_pc_cpu_invalidate(sitd->page_cache);
 833         kprintf("SITD(%p) at 0x%08x\n", sitd, hc32toh(sc, sitd->sitd_self) & ~0x1F);
 834         kprintf(" next=0x%08x\n", hc32toh(sc, sitd->sitd_next));
 835         kprintf(" portaddr=0x%08x dir=%s addr=%d endpt=0x%x port=0x%x huba=0x%x\n",
 836             hc32toh(sc, sitd->sitd_portaddr),
 837             (sitd->sitd_portaddr & htohc32(sc, EHCI_SITD_SET_DIR_IN))
 838             ? "in" : "out",
 839             EHCI_SITD_GET_ADDR(hc32toh(sc, sitd->sitd_portaddr)),
 840             EHCI_SITD_GET_ENDPT(hc32toh(sc, sitd->sitd_portaddr)),
 841             EHCI_SITD_GET_PORT(hc32toh(sc, sitd->sitd_portaddr)),
 842             EHCI_SITD_GET_HUBA(hc32toh(sc, sitd->sitd_portaddr)));
 843         kprintf(" mask=0x%08x\n", hc32toh(sc, sitd->sitd_mask));
 844         kprintf(" status=0x%08x <%s> len=0x%x\n", hc32toh(sc, sitd->sitd_status),
 845             (sitd->sitd_status & htohc32(sc, EHCI_SITD_ACTIVE)) ? "ACTIVE" : "",
 846             EHCI_SITD_GET_LEN(hc32toh(sc, sitd->sitd_status)));
 847         kprintf(" back=0x%08x, bp=0x%08x,0x%08x,0x%08x,0x%08x\n",
 848             hc32toh(sc, sitd->sitd_back),
 849             hc32toh(sc, sitd->sitd_bp[0]),
 850             hc32toh(sc, sitd->sitd_bp[1]),
 851             hc32toh(sc, sitd->sitd_bp_hi[0]),
 852             hc32toh(sc, sitd->sitd_bp_hi[1]));
 853 }
 854
 855 static void
 856 ehci_dump_itd(ehci_softc_t *sc, ehci_itd_t *itd)
 857 {
 858         usb_pc_cpu_invalidate(itd->page_cache);
 859         kprintf("ITD(%p) at 0x%08x\n", itd, hc32toh(sc, itd->itd_self) & ~0x1F);
 860         kprintf(" next=0x%08x\n", hc32toh(sc, itd->itd_next));
 861         kprintf(" status[0]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[0]),
 862             (itd->itd_status[0] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 863         kprintf(" status[1]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[1]),
 864             (itd->itd_status[1] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 865         kprintf(" status[2]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[2]),
 866             (itd->itd_status[2] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 867         kprintf(" status[3]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[3]),
 868             (itd->itd_status[3] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 869         kprintf(" status[4]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[4]),
 870             (itd->itd_status[4] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 871         kprintf(" status[5]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[5]),
 872             (itd->itd_status[5] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 873         kprintf(" status[6]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[6]),
 874             (itd->itd_status[6] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 875         kprintf(" status[7]=0x%08x; <%s>\n", hc32toh(sc, itd->itd_status[7]),
 876             (itd->itd_status[7] & htohc32(sc, EHCI_ITD_ACTIVE)) ? "ACTIVE" : "");
 877         kprintf(" bp[0]=0x%08x\n", hc32toh(sc, itd->itd_bp[0]));
 878         kprintf("  addr=0x%02x; endpt=0x%01x\n",
 879             EHCI_ITD_GET_ADDR(hc32toh(sc, itd->itd_bp[0])),
 880             EHCI_ITD_GET_ENDPT(hc32toh(sc, itd->itd_bp[0])));
 881         kprintf(" bp[1]=0x%08x\n", hc32toh(sc, itd->itd_bp[1]));
 882         kprintf(" dir=%s; mpl=0x%02x\n",
 883             (hc32toh(sc, itd->itd_bp[1]) & EHCI_ITD_SET_DIR_IN) ? "in" : "out",
 884             EHCI_ITD_GET_MPL(hc32toh(sc, itd->itd_bp[1])));
 885         kprintf(" bp[2..6]=0x%08x,0x%08x,0x%08x,0x%08x,0x%08x\n",
 886             hc32toh(sc, itd->itd_bp[2]),
 887             hc32toh(sc, itd->itd_bp[3]),
 888             hc32toh(sc, itd->itd_bp[4]),
 889             hc32toh(sc, itd->itd_bp[5]),
 890             hc32toh(sc, itd->itd_bp[6]));
 891         kprintf(" bp_hi=0x%08x,0x%08x,0x%08x,0x%08x,\n"
 892             "       0x%08x,0x%08x,0x%08x\n",
 893             hc32toh(sc, itd->itd_bp_hi[0]),
 894             hc32toh(sc, itd->itd_bp_hi[1]),
 895             hc32toh(sc, itd->itd_bp_hi[2]),
 896             hc32toh(sc, itd->itd_bp_hi[3]),
 897             hc32toh(sc, itd->itd_bp_hi[4]),
 898             hc32toh(sc, itd->itd_bp_hi[5]),
 899             hc32toh(sc, itd->itd_bp_hi[6]));
 900 }
 901
 902 static void
 903 ehci_dump_isoc(ehci_softc_t *sc)
 904 {
 905         ehci_itd_t *itd;
 906         ehci_sitd_t *sitd;
 907         uint16_t max = 1000;
 908         uint16_t pos;
 909
 910         pos = (EOREAD4(sc, EHCI_FRINDEX) / 8) &
 911             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
 912
 913         kprintf("%s: isochronous dump from frame 0x%03x:\n",
 914             __func__, pos);
 915
 916         itd = sc->sc_isoc_hs_p_last[pos];
 917         sitd = sc->sc_isoc_fs_p_last[pos];
 918
 919         while (itd && max && max--) {
 920                 ehci_dump_itd(sc, itd);
 921                 itd = itd->prev;
 922         }
 923
 924         while (sitd && max && max--) {
 925                 ehci_dump_sitd(sc, sitd);
 926                 sitd = sitd->prev;
 927         }
 928 }
 929
 930 #endif
 931
 932 static void
 933 ehci_ring_doorbell(ehci_softc_t *sc, struct usb_xfer *xfer)
 934 {
 935         uint32_t temp;
 936
 937         /*
 938          * Request a doorbell interrupt.  We can't safely disturb previously
 939          * used QH's until controller has synchronized the list changes we've
 940          * made.
 941          *
 942          * Deal with controller races if a doorbell has already been
 943          * requested, we have to wait 2 doorbells in this case to be sure.
 944          *
 945          * (interlocked against races by the bus lock)
 946          */
 947         temp = EOREAD4(sc, EHCI_USBCMD);
 948         if (sc->doorbell_again == 0) {
 949                 if ((temp & EHCI_CMD_IAAD) == 0)
 950                         EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
 951                 if (xfer)
 952                         xfer->doorbell_match = sc->doorbell_match + 1;
 953                 sc->doorbell_again = 1;
 954         } else {
 955                 if ((temp & EHCI_CMD_IAAD) == 0)
 956                         EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
 957                 if (xfer)
 958                         xfer->doorbell_match = sc->doorbell_match + 2;
 959                 sc->doorbell_again = 2;
 960         }
 961 }
 962
 963 static void
 964 ehci_wait_doorbell(ehci_softc_t *sc, int match)
 965 {
 966         while ((int)(match - sc->doorbell_match) > 0)
 967                 lksleep(sc, &sc->sc_bus.bus_lock, 0, "ehcidoor", hz);
 968 }
 969
 970 static void
 971 ehci_transfer_intr_enqueue(struct usb_xfer *xfer)
 972 {
 973         /* check for early completion */
 974         if (ehci_check_transfer(xfer, 1)) {
 975                 return;
 976         }
 977         /* put transfer on interrupt queue */
 978         usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
 979
 980         /* start timeout, if any */
 981         if (xfer->timeout != 0) {
 982                 usbd_transfer_timeout_ms(xfer, &ehci_timeout, xfer->timeout);
 983         }
 984 }
 985
 986 #define EHCI_APPEND_FS_TD(std,last) (last) = _ehci_append_fs_td(std,last)
 987 static ehci_sitd_t *
 988 _ehci_append_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
 989 {
 990         DPRINTFN(11, "%p to %p\n", std, last);
 991
 992         /* (sc->sc_bus.mtx) must be locked */
 993
 994         std->next = last->next;
 995         std->sitd_next = last->sitd_next;
 996
 997         std->prev = last;
 998
 999         usb_pc_cpu_flush(std->page_cache);
1000
1001         /*
1002          * the last->next->prev is never followed: std->next->prev = std;
1003          */
1004         last->next = std;
1005         last->sitd_next = std->sitd_self;
1006
1007         usb_pc_cpu_flush(last->page_cache);
1008
1009         return (std);
1010 }
1011
1012 #define EHCI_APPEND_HS_TD(std,last) (last) = _ehci_append_hs_td(std,last)
1013 static ehci_itd_t *
1014 _ehci_append_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1015 {
1016         DPRINTFN(11, "%p to %p\n", std, last);
1017
1018         /* (sc->sc_bus.mtx) must be locked */
1019
1020         std->next = last->next;
1021         std->itd_next = last->itd_next;
1022
1023         std->prev = last;
1024
1025         usb_pc_cpu_flush(std->page_cache);
1026
1027         /*
1028          * the last->next->prev is never followed: std->next->prev = std;
1029          */
1030         last->next = std;
1031         last->itd_next = std->itd_self;
1032
1033         usb_pc_cpu_flush(last->page_cache);
1034
1035         return (std);
1036 }
1037
1038 #define EHCI_APPEND_QH(sqh,last) (last) = _ehci_append_qh(sqh,last)
1039 static ehci_qh_t *
1040 _ehci_append_qh(ehci_qh_t *sqh, ehci_qh_t *last)
1041 {
1042         DPRINTFN(11, "%p to %p\n", sqh, last);
1043
1044         if (sqh->prev != NULL) {
1045                 /* should not happen */
1046                 DPRINTFN(0, "QH already linked!\n");
1047                 return (last);
1048         }
1049         /* (sc->sc_bus.mtx) must be locked */
1050
1051         sqh->next = last->next;
1052         sqh->qh_link = last->qh_link;
1053
1054         sqh->prev = last;
1055
1056         usb_pc_cpu_flush(sqh->page_cache);
1057
1058         /*
1059          * The last->next->prev is never followed: sqh->next->prev = sqh;
1060          *
1061          * This should activate the command.
1062          */
1063
1064         last->next = sqh;
1065         last->qh_link = sqh->qh_self;
1066
1067         usb_pc_cpu_flush(last->page_cache);
1068
1069         return (sqh);
1070 }
1071
1072 static void
1073 ehci_stop_xfer(struct usb_xfer *xfer)
1074 {
1075 #if 0
1076         if (xfer->flags_int.onhwqueue == 0)
1077                 return;
1078         return;
1079         if (ehci_check_transfer(xfer, 0))
1080                 return;
1081         kprintf("R");
1082         retry = 10;
1083         while (retry && ehci_check_transfer(xfer, 0) == 0) {
1084                 kprintf("R");
1085                 DELAY(10000);
1086                 --retry;
1087         }
1088         if (retry == 0)
1089                 kprintf("[can't stop xfer]\n");
1090 #endif
1091 }
1092
1093 #define EHCI_REMOVE_FS_TD(std,last) (last) = _ehci_remove_fs_td(std,last)
1094 static ehci_sitd_t *
1095 _ehci_remove_fs_td(ehci_sitd_t *std, ehci_sitd_t *last)
1096 {
1097         DPRINTFN(11, "%p from %p\n", std, last);
1098
1099         /* (sc->sc_bus.mtx) must be locked */
1100
1101         std->prev->next = std->next;
1102         std->prev->sitd_next = std->sitd_next;
1103
1104         usb_pc_cpu_flush(std->prev->page_cache);
1105
1106         if (std->next) {
1107                 std->next->prev = std->prev;
1108                 usb_pc_cpu_flush(std->next->page_cache);
1109         }
1110         return ((last == std) ? std->prev : last);
1111 }
1112
1113 #define EHCI_REMOVE_HS_TD(std,last) (last) = _ehci_remove_hs_td(std,last)
1114 static ehci_itd_t *
1115 _ehci_remove_hs_td(ehci_itd_t *std, ehci_itd_t *last)
1116 {
1117         DPRINTFN(11, "%p from %p\n", std, last);
1118
1119         /* (sc->sc_bus.mtx) must be locked */
1120
1121         std->prev->next = std->next;
1122         std->prev->itd_next = std->itd_next;
1123
1124         usb_pc_cpu_flush(std->prev->page_cache);
1125
1126         if (std->next) {
1127                 std->next->prev = std->prev;
1128                 usb_pc_cpu_flush(std->next->page_cache);
1129         }
1130         return ((last == std) ? std->prev : last);
1131 }
1132
1133 #define EHCI_REMOVE_QH(sc,sqh,last) (last) = _ehci_remove_qh(sc,sqh,last)
1134 static ehci_qh_t *
1135 _ehci_remove_qh(ehci_softc_t *sc, ehci_qh_t *sqh, ehci_qh_t *last)
1136 {
1137         DPRINTFN(11, "%p from %p\n", sqh, last);
1138
1139         /* (sc->sc_bus.mtx) must be locked */
1140
1141         /* only remove if not removed from a queue */
1142         if (sqh->prev) {
1143                 sqh->prev->next = sqh->next;
1144                 if (sqh->next)
1145                         sqh->next->prev = sqh->prev;
1146                 last = ((last == sqh) ? sqh->prev : last);
1147
1148                 /*
1149                  * This is a horrible hack because basically the entire
1150                  * U4B QH handling is broken.  To do this correctly the
1151                  * QH needs be unlinked but then not touched until the
1152                  * next rollup doorbell interrupt.  That is, the controller
1153                  * could be racing us accessing the QH and qh_link, so
1154                  * the links have to remain valid enough to not cause the
1155                  * controller to blow up.
1156                  *
1157                  * When we mess with the links it is possible for the
1158                  * controller to end up on the deleted chain(s) and miss
1159                  * new requests entered after the deletion.  Then it thinks
1160                  * it is finished and stops scanning.  Thus we must issue a
1161                  * doorbell to force the controller to re-scan and pick
1162                  * up anything it missed if it got off-track.
1163                  *
1164                  * I wanted to try to make this more reliable by setting
1165                  * qh_link to htohc32(sc, EHCI_LINK_TERMINATE), but it
1166                  * doesn't work.  The controller barfs badly and isn't
1167                  * able to probe all the usb devices.  I'm not sure
1168                  * why it doesn't work but... whatever.
1169                  */
1170                 sqh->prev->qh_link = sqh->qh_link;
1171                 usb_pc_cpu_flush(sqh->prev->page_cache);
1172                 /* sqh->qh_link = htohc32(sc, EHCI_LINK_TERMINATE); */
1173 #if 0
1174                 if ((sc->sc_flags & EHCI_SCFLG_IAADBUG) == 0) {
1175                         uint32_t temp = EOREAD4(sc, EHCI_USBCMD);
1176                         EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
1177                 }
1178 #endif
1179                 sqh->prev = NULL;
1180                 usb_pc_cpu_flush(sqh->page_cache);
1181         }
1182         return (last);
1183 }
1184
1185 static void
1186 ehci_data_toggle_update(struct usb_xfer *xfer, uint16_t actlen, uint16_t xlen)
1187 {
1188         uint16_t rem;
1189         uint8_t dt;
1190
1191         /* count number of full packets */
1192         dt = (actlen / xfer->max_packet_size) & 1;
1193
1194         /* compute remainder */
1195         rem = actlen % xfer->max_packet_size;
1196
1197         if (rem > 0)
1198                 dt ^= 1;        /* short packet at the end */
1199         else if (actlen != xlen)
1200                 dt ^= 1;        /* zero length packet at the end */
1201         else if (xlen == 0)
1202                 dt ^= 1;        /* zero length transfer */
1203
1204         xfer->endpoint->toggle_next ^= dt;
1205 }
1206
1207 static usb_error_t
1208 ehci_non_isoc_done_sub(struct usb_xfer *xfer)
1209 {
1210         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1211         ehci_qtd_t *td;
1212         ehci_qtd_t *td_alt_next;
1213         uint32_t status;
1214         uint16_t len;
1215
1216         td = xfer->td_transfer_cache;
1217         td_alt_next = td->alt_next;
1218
1219         if (xfer->aframes != xfer->nframes) {
1220                 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
1221         }
1222         while (1) {
1223                 usb_pc_cpu_invalidate(td->page_cache);
1224                 status = hc32toh(sc, td->qtd_status);
1225
1226                 len = EHCI_QTD_GET_BYTES(status);
1227
1228                 /*
1229                  * Verify the status length and
1230                  * add the length to "frlengths[]":
1231                  */
1232                 if (len > td->len) {
1233                         /* should not happen */
1234                         DPRINTF("Invalid status length, "
1235                             "0x%04x/0x%04x bytes\n", len, td->len);
1236                         status |= EHCI_QTD_HALTED;
1237                 } else if (xfer->aframes != xfer->nframes) {
1238                         xfer->frlengths[xfer->aframes] += td->len - len;
1239                         /* manually update data toggle */
1240                         ehci_data_toggle_update(xfer, td->len - len, td->len);
1241                 }
1242
1243                 /* Check for last transfer */
1244                 if (((void *)td) == xfer->td_transfer_last) {
1245                         td = NULL;
1246                         break;
1247                 }
1248                 /* Check for transfer error */
1249                 if (status & EHCI_QTD_HALTED) {
1250                         /* the transfer is finished */
1251                         td = NULL;
1252                         break;
1253                 }
1254                 /* Check for short transfer */
1255                 if (len > 0) {
1256                         if (xfer->flags_int.short_frames_ok) {
1257                                 /* follow alt next */
1258                                 td = td->alt_next;
1259                         } else {
1260                                 /* the transfer is finished */
1261                                 td = NULL;
1262                         }
1263                         break;
1264                 }
1265                 td = td->obj_next;
1266
1267                 if (td->alt_next != td_alt_next) {
1268                         /* this USB frame is complete */
1269                         break;
1270                 }
1271         }
1272
1273         /* update transfer cache */
1274
1275         xfer->td_transfer_cache = td;
1276
1277 #ifdef USB_DEBUG
1278         if (status & EHCI_QTD_STATERRS) {
1279                 DPRINTFN(11, "error, addr=%d, endpt=0x%02x, frame=0x%02x"
1280                     "status=%s%s%s%s%s%s%s%s\n",
1281                     xfer->address, xfer->endpointno, xfer->aframes,
1282                     (status & EHCI_QTD_ACTIVE) ? "[ACTIVE]" : "[NOT_ACTIVE]",
1283                     (status & EHCI_QTD_HALTED) ? "[HALTED]" : "",
1284                     (status & EHCI_QTD_BUFERR) ? "[BUFERR]" : "",
1285                     (status & EHCI_QTD_BABBLE) ? "[BABBLE]" : "",
1286                     (status & EHCI_QTD_XACTERR) ? "[XACTERR]" : "",
1287                     (status & EHCI_QTD_MISSEDMICRO) ? "[MISSED]" : "",
1288                     (status & EHCI_QTD_SPLITXSTATE) ? "[SPLIT]" : "",
1289                     (status & EHCI_QTD_PINGSTATE) ? "[PING]" : "");
1290         }
1291 #endif
1292
1293         return ((status & EHCI_QTD_HALTED) ?
1294             USB_ERR_STALLED : USB_ERR_NORMAL_COMPLETION);
1295 }
1296
1297 static void
1298 ehci_non_isoc_done(struct usb_xfer *xfer)
1299 {
1300         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1301         ehci_qh_t *qh;
1302         uint32_t status;
1303         usb_error_t err = 0;
1304
1305         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
1306             xfer, xfer->endpoint);
1307
1308 #ifdef USB_DEBUG
1309         if (ehcidebug > 10) {
1310                 ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1311
1312                 ehci_dump_sqtds(sc, xfer->td_transfer_first);
1313         }
1314 #endif
1315
1316         /* extract data toggle directly from the QH's overlay area */
1317
1318         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1319
1320         usb_pc_cpu_invalidate(qh->page_cache);
1321
1322         status = hc32toh(sc, qh->qh_qtd.qtd_status);
1323
1324         /* reset scanner */
1325
1326         xfer->td_transfer_cache = xfer->td_transfer_first;
1327
1328         if (xfer->flags_int.control_xfr) {
1329
1330                 if (xfer->flags_int.control_hdr) {
1331
1332                         err = ehci_non_isoc_done_sub(xfer);
1333                 }
1334                 xfer->aframes = 1;
1335
1336                 if (xfer->td_transfer_cache == NULL) {
1337                         goto done;
1338                 }
1339         }
1340         while (xfer->aframes != xfer->nframes) {
1341
1342                 err = ehci_non_isoc_done_sub(xfer);
1343                 xfer->aframes++;
1344
1345                 if (xfer->td_transfer_cache == NULL) {
1346                         goto done;
1347                 }
1348         }
1349
1350         if (xfer->flags_int.control_xfr &&
1351             !xfer->flags_int.control_act) {
1352
1353                 err = ehci_non_isoc_done_sub(xfer);
1354         }
1355 done:
1356         ehci_device_done(xfer, err);
1357 }
1358
1359 /*------------------------------------------------------------------------*
1360  *      ehci_check_transfer
1361  *
1362  * Return values:
1363  *    0: USB transfer is not finished
1364  * Else: USB transfer is finished
1365  *------------------------------------------------------------------------*/
1366 static uint8_t
1367 ehci_check_transfer(struct usb_xfer *xfer, int finish)
1368 {
1369         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
1370         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
1371
1372         uint32_t status;
1373
1374         DPRINTFN(13, "xfer=%p checking transfer\n", xfer);
1375
1376         if (methods == &ehci_device_isoc_fs_methods) {
1377                 ehci_sitd_t *td;
1378
1379                 /* isochronous full speed transfer */
1380
1381                 td = xfer->td_transfer_last;
1382                 usb_pc_cpu_invalidate(td->page_cache);
1383                 status = hc32toh(sc, td->sitd_status);
1384
1385                 /* also check if first is complete */
1386
1387                 td = xfer->td_transfer_first;
1388                 usb_pc_cpu_invalidate(td->page_cache);
1389                 status |= hc32toh(sc, td->sitd_status);
1390
1391                 if (!(status & EHCI_SITD_ACTIVE)) {
1392                         ehci_device_done(xfer, USB_ERR_NORMAL_COMPLETION);
1393                         goto transferred;
1394                 }
1395         } else if (methods == &ehci_device_isoc_hs_methods) {
1396                 ehci_itd_t *td;
1397
1398                 /* isochronous high speed transfer */
1399
1400                 /* check last transfer */
1401                 td = xfer->td_transfer_last;
1402                 usb_pc_cpu_invalidate(td->page_cache);
1403                 status = td->itd_status[0];
1404                 status |= td->itd_status[1];
1405                 status |= td->itd_status[2];
1406                 status |= td->itd_status[3];
1407                 status |= td->itd_status[4];
1408                 status |= td->itd_status[5];
1409                 status |= td->itd_status[6];
1410                 status |= td->itd_status[7];
1411
1412                 /* also check first transfer */
1413                 td = xfer->td_transfer_first;
1414                 usb_pc_cpu_invalidate(td->page_cache);
1415                 status |= td->itd_status[0];
1416                 status |= td->itd_status[1];
1417                 status |= td->itd_status[2];
1418                 status |= td->itd_status[3];
1419                 status |= td->itd_status[4];
1420                 status |= td->itd_status[5];
1421                 status |= td->itd_status[6];
1422                 status |= td->itd_status[7];
1423
1424                 /* if no transactions are active we continue */
1425                 if (!(status & htohc32(sc, EHCI_ITD_ACTIVE))) {
1426                         if (finish) {
1427                                 ehci_device_done(xfer,
1428                                                  USB_ERR_NORMAL_COMPLETION);
1429                         }
1430                         goto transferred;
1431                 }
1432         } else {
1433                 ehci_qtd_t *td;
1434                 ehci_qh_t *qh;
1435
1436                 /* non-isochronous transfer */
1437
1438                 /*
1439                  * check whether there is an error somewhere in the middle,
1440                  * or whether there was a short packet (SPD and not ACTIVE)
1441                  */
1442                 td = xfer->td_transfer_cache;
1443
1444                 qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
1445
1446                 usb_pc_cpu_invalidate(qh->page_cache);
1447
1448                 status = hc32toh(sc, qh->qh_qtd.qtd_status);
1449                 if (status & EHCI_QTD_ACTIVE) {
1450                         /* transfer is pending */
1451                         goto done;
1452                 }
1453
1454                 while (1) {
1455                         usb_pc_cpu_invalidate(td->page_cache);
1456                         status = hc32toh(sc, td->qtd_status);
1457
1458                         /*
1459                          * Check if there is an active TD which
1460                          * indicates that the transfer isn't done.
1461                          */
1462                         if (status & EHCI_QTD_ACTIVE) {
1463                                 /* update cache */
1464                                 xfer->td_transfer_cache = td;
1465                                 goto done;
1466                         }
1467                         /*
1468                          * last transfer descriptor makes the transfer done
1469                          */
1470                         if (((void *)td) == xfer->td_transfer_last) {
1471                                 break;
1472                         }
1473                         /*
1474                          * any kind of error makes the transfer done
1475                          */
1476                         if (status & EHCI_QTD_HALTED) {
1477                                 break;
1478                         }
1479                         /*
1480                          * if there is no alternate next transfer, a short
1481                          * packet also makes the transfer done
1482                          */
1483                         if (EHCI_QTD_GET_BYTES(status)) {
1484                                 if (xfer->flags_int.short_frames_ok) {
1485                                         /* follow alt next */
1486                                         if (td->alt_next) {
1487                                                 td = td->alt_next;
1488                                                 continue;
1489                                         }
1490                                 }
1491                                 /* transfer is done */
1492                                 break;
1493                         }
1494                         td = td->obj_next;
1495                 }
1496                 if (finish)
1497                         ehci_non_isoc_done(xfer);
1498                 goto transferred;
1499         }
1500
1501 done:
1502         DPRINTFN(13, "xfer=%p is still active\n", xfer);
1503         return (0);
1504
1505 transferred:
1506         return (1);
1507 }
1508
1509 static void
1510 ehci_pcd_enable(ehci_softc_t *sc)
1511 {
1512         USB_BUS_LOCK_ASSERT(&sc->sc_bus);
1513
1514         sc->sc_eintrs |= EHCI_STS_PCD;
1515         EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1516
1517         /* acknowledge any PCD interrupt */
1518         EOWRITE4(sc, EHCI_USBSTS, EHCI_STS_PCD);
1519
1520         ehci_root_intr(sc);
1521 }
1522
1523 static void
1524 ehci_interrupt_poll(ehci_softc_t *sc)
1525 {
1526         struct usb_xfer *xfer;
1527
1528 repeat:
1529         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
1530                 /*
1531                  * check if transfer is transferred
1532                  */
1533                 if (ehci_check_transfer(xfer, 1)) {
1534                         /* queue has been modified */
1535                         goto repeat;
1536                 }
1537         }
1538 }
1539
1540 /*
1541  * Some EHCI chips from VIA / ATI seem to trigger interrupts before
1542  * writing back the qTD status, or miss signalling occasionally under
1543  * heavy load.  If the host machine is too fast, we can miss
1544  * transaction completion - when we scan the active list the
1545  * transaction still seems to be active. This generally exhibits
1546  * itself as a umass stall that never recovers.
1547  *
1548  * We work around this behaviour by setting up this callback after any
1549  * softintr that completes with transactions still pending, giving us
1550  * another chance to check for completion after the writeback has
1551  * taken place.
1552  */
1553 static void
1554 ehci_poll_timeout(void *arg)
1555 {
1556         ehci_softc_t *sc = arg;
1557
1558         DPRINTFN(3, "\n");
1559         ehci_interrupt_poll(sc);
1560 }
1561
1562 /*------------------------------------------------------------------------*
1563  *      ehci_interrupt - EHCI interrupt handler
1564  *
1565  * NOTE: Do not access "sc->sc_bus.bdev" inside the interrupt handler,
1566  * hence the interrupt handler will be setup before "sc->sc_bus.bdev"
1567  * is present !
1568  *------------------------------------------------------------------------*/
1569 void
1570 ehci_interrupt(ehci_softc_t *sc)
1571 {
1572         uint32_t status;
1573
1574         USB_BUS_LOCK(&sc->sc_bus);
1575
1576         DPRINTFN(16, "real interrupt\n");
1577
1578 #ifdef USB_DEBUG
1579         if (ehcidebug > 15) {
1580                 ehci_dump_regs(sc);
1581         }
1582 #endif
1583
1584         status = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS));
1585 #if 0
1586         /*
1587          * XXX hack workaround bug where interrupt status says
1588          * nothing is pending but xfer completions might be present.
1589          */
1590         if (status == 0) {
1591                 /* the interrupt was not for us */
1592                 goto done;
1593         }
1594         if (!(status & sc->sc_eintrs)) {
1595                 goto done;
1596         }
1597 #endif
1598         /*
1599          * Acknowlege interrupts except IAA
1600          */
1601         EOWRITE4(sc, EHCI_USBSTS, status);
1602
1603         status &= sc->sc_eintrs;
1604
1605         if (status & EHCI_STS_HSE) {
1606                 kprintf("%s: unrecoverable error, "
1607                     "controller halted\n", __func__);
1608 #ifdef USB_DEBUG
1609                 ehci_dump_regs(sc);
1610                 ehci_dump_isoc(sc);
1611 #endif
1612         }
1613         if (status & EHCI_STS_PCD) {
1614                 /*
1615                  * Disable PCD interrupt for now, because it will be
1616                  * on until the port has been reset.
1617                  */
1618                 sc->sc_eintrs &= ~EHCI_STS_PCD;
1619                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1620
1621                 ehci_root_intr(sc);
1622
1623                 /* do not allow RHSC interrupts > 1 per second */
1624                 usb_callout_reset(&sc->sc_tmo_pcd, hz,
1625                     (void *)&ehci_pcd_enable, sc);
1626         }
1627
1628         /*
1629          * Handle IAA doorbells, acknowledge IAA after clearing CMD_IAAD.
1630          *
1631          * The frontend may have tried to ring the doorbell after it had
1632          * already been rung, in which case it has to wait for two doorbells.
1633          * We track this with the doorbell_again counter.  This counter never
1634          * goes above 2.
1635          *
1636          * NOTE: ehci_ring_doorbell() bumps doorbell_again so we need to
1637          *       decrement it twice to net-out at a single decrement.
1638          */
1639         if (status & EHCI_STS_IAA) {
1640                 ++sc->doorbell_match;
1641                 wakeup(sc);
1642                 if (sc->doorbell_again && --sc->doorbell_again > 0) {
1643                         --sc->doorbell_again;
1644                         ehci_ring_doorbell(sc, NULL);
1645                 }
1646         }
1647
1648         /*
1649          * Block unprocessed interrupts
1650          */
1651         status &= ~(EHCI_STS_INT | EHCI_STS_ERRINT |
1652                     EHCI_STS_PCD | EHCI_STS_IAA);
1653         if (status != 0) {
1654                 sc->sc_eintrs &= ~status;
1655                 EOWRITE4(sc, EHCI_USBINTR, sc->sc_eintrs);
1656                 kprintf("%s: blocking interrupts 0x%x\n", __func__, status);
1657         }
1658
1659         /* poll all the USB transfers */
1660         ehci_interrupt_poll(sc);
1661
1662         if (sc->sc_flags & EHCI_SCFLG_LOSTINTRBUG) {
1663                 usb_callout_reset(&sc->sc_tmo_poll, hz / 128,
1664                     (void *)&ehci_poll_timeout, sc);
1665         }
1666
1667 #if 0
1668 done:
1669 #endif
1670         USB_BUS_UNLOCK(&sc->sc_bus);
1671 }
1672
1673 /*
1674  * called when a request does not complete
1675  */
1676 static void
1677 ehci_timeout(void *arg)
1678 {
1679         struct usb_xfer *xfer = arg;
1680
1681         DPRINTF("xfer=%p\n", xfer);
1682
1683         USB_BUS_LOCK_ASSERT(xfer->xroot->bus);
1684
1685         /* transfer is transferred */
1686         ehci_device_done(xfer, USB_ERR_TIMEOUT);
1687 }
1688
1689 static void
1690 ehci_do_poll(struct usb_bus *bus)
1691 {
1692         ehci_softc_t *sc = EHCI_BUS2SC(bus);
1693
1694         USB_BUS_LOCK(&sc->sc_bus);
1695         ehci_interrupt_poll(sc);
1696         USB_BUS_UNLOCK(&sc->sc_bus);
1697 }
1698
1699 static void
1700 ehci_setup_standard_chain_sub(struct ehci_std_temp *temp)
1701 {
1702         struct usb_page_search buf_res;
1703         ehci_qtd_t *td;
1704         ehci_qtd_t *td_next;
1705         ehci_qtd_t *td_alt_next;
1706         uint32_t buf_offset;
1707         uint32_t average;
1708         uint32_t len_old;
1709         uint32_t terminate;
1710         uint32_t qtd_altnext;
1711         uint8_t shortpkt_old;
1712         uint8_t precompute;
1713
1714         terminate = temp->sc->sc_terminate_self;
1715         qtd_altnext = temp->sc->sc_terminate_self;
1716         td_alt_next = NULL;
1717         buf_offset = 0;
1718         shortpkt_old = temp->shortpkt;
1719         len_old = temp->len;
1720         precompute = 1;
1721
1722 restart:
1723
1724         td = temp->td;                  /* starts out NULL */
1725         td_next = temp->td_next;        /* starts with first td */
1726
1727         while (1) {
1728
1729                 if (temp->len == 0) {
1730
1731                         if (temp->shortpkt) {
1732                                 break;
1733                         }
1734                         /* send a Zero Length Packet, ZLP, last */
1735
1736                         temp->shortpkt = 1;
1737                         average = 0;
1738
1739                 } else {
1740
1741                         average = temp->average;
1742
1743                         if (average > temp->len) {
1744                                 if (temp->len % temp->max_frame_size) {
1745                                         temp->shortpkt = 1;
1746                                 }
1747                                 average = temp->len;
1748                         }
1749                 }
1750
1751                 if (td_next == NULL) {
1752                         panic("%s: out of EHCI transfer descriptors!", __func__);
1753                 }
1754                 /* get next TD */
1755
1756                 td = td_next;
1757                 td_next = td->obj_next;
1758
1759                 /* check if we are pre-computing */
1760
1761                 if (precompute) {
1762
1763                         /* update remaining length */
1764
1765                         temp->len -= average;
1766
1767                         continue;
1768                 }
1769                 /* fill out current TD */
1770
1771                 td->qtd_status =
1772                     temp->qtd_status |
1773                     htohc32(temp->sc, EHCI_QTD_IOC |
1774                         EHCI_QTD_SET_BYTES(average));
1775
1776                 if (average == 0) {
1777
1778                         if (temp->auto_data_toggle == 0) {
1779
1780                                 /* update data toggle, ZLP case */
1781
1782                                 temp->qtd_status ^=
1783                                     htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1784                         }
1785                         td->len = 0;
1786
1787                         td->qtd_buffer[0] = 0;
1788                         td->qtd_buffer_hi[0] = 0;
1789
1790                         td->qtd_buffer[1] = 0;
1791                         td->qtd_buffer_hi[1] = 0;
1792
1793                 } else {
1794
1795                         uint8_t x;
1796
1797                         if (temp->auto_data_toggle == 0) {
1798
1799                                 /* update data toggle */
1800
1801                                 if (((average + temp->max_frame_size - 1) /
1802                                     temp->max_frame_size) & 1) {
1803                                         temp->qtd_status ^=
1804                                             htohc32(temp->sc, EHCI_QTD_TOGGLE_MASK);
1805                                 }
1806                         }
1807                         td->len = average;
1808
1809                         /* update remaining length */
1810
1811                         temp->len -= average;
1812
1813                         /* fill out buffer pointers */
1814
1815                         usbd_get_page(temp->pc, buf_offset, &buf_res);
1816                         td->qtd_buffer[0] =
1817                             htohc32(temp->sc, buf_res.physaddr);
1818                         td->qtd_buffer_hi[0] = 0;
1819
1820                         x = 1;
1821
1822                         while (average > EHCI_PAGE_SIZE) {
1823                                 average -= EHCI_PAGE_SIZE;
1824                                 buf_offset += EHCI_PAGE_SIZE;
1825                                 usbd_get_page(temp->pc, buf_offset, &buf_res);
1826                                 td->qtd_buffer[x] =
1827                                     htohc32(temp->sc,
1828                                     buf_res.physaddr & (~0xFFF));
1829                                 td->qtd_buffer_hi[x] = 0;
1830                                 x++;
1831                         }
1832
1833                         /*
1834                          * NOTE: The "average" variable is never zero after
1835                          * exiting the loop above !
1836                          *
1837                          * NOTE: We have to subtract one from the offset to
1838                          * ensure that we are computing the physical address
1839                          * of a valid page !
1840                          */
1841                         buf_offset += average;
1842                         usbd_get_page(temp->pc, buf_offset - 1, &buf_res);
1843                         td->qtd_buffer[x] =
1844                             htohc32(temp->sc,
1845                             buf_res.physaddr & (~0xFFF));
1846                         td->qtd_buffer_hi[x] = 0;
1847                 }
1848
1849                 if (td_next) {
1850                         /* link the current TD with the next one */
1851                         td->qtd_next = td_next->qtd_self;
1852                 }
1853                 td->qtd_altnext = qtd_altnext;
1854                 td->alt_next = td_alt_next;
1855
1856                 usb_pc_cpu_flush(td->page_cache);
1857         }
1858
1859         if (precompute) {
1860                 precompute = 0;
1861
1862                 /* setup alt next pointer, if any */
1863                 if (temp->last_frame) {
1864                         td_alt_next = NULL;
1865                         qtd_altnext = terminate;
1866                 } else {
1867                         /* we use this field internally */
1868                         td_alt_next = td_next;
1869                         if (temp->setup_alt_next) {
1870                                 qtd_altnext = td_next->qtd_self;
1871                         } else {
1872                                 qtd_altnext = terminate;
1873                         }
1874                 }
1875
1876                 /* restore */
1877                 temp->shortpkt = shortpkt_old;
1878                 temp->len = len_old;
1879                 goto restart;
1880         }
1881         temp->td = td;
1882         temp->td_next = td_next;
1883 }
1884
1885 static void
1886 ehci_setup_standard_chain(struct usb_xfer *xfer, ehci_qh_t **qh_last)
1887 {
1888         struct ehci_std_temp temp;
1889         const struct usb_pipe_methods *methods;
1890         ehci_qh_t *qh;
1891         ehci_qtd_t *td;
1892         uint32_t qh_endp;
1893         uint32_t qh_endphub;
1894         uint32_t x;
1895
1896         DPRINTFN(9, "addr=%d endpt=%d sumlen=%d speed=%d\n",
1897             xfer->address, UE_GET_ADDR(xfer->endpointno),
1898             xfer->sumlen, usbd_get_speed(xfer->xroot->udev));
1899
1900         temp.average = xfer->max_hc_frame_size;
1901         temp.max_frame_size = xfer->max_frame_size;
1902         temp.sc = EHCI_BUS2SC(xfer->xroot->bus);
1903
1904         if (xfer->flags_int.doorbell_wait)
1905                 ehci_wait_doorbell(temp.sc, xfer->doorbell_match);
1906         xfer->flags_int.doorbell_wait = 1;
1907
1908         /* toggle the DMA set we are using */
1909         xfer->flags_int.curr_dma_set ^= 1;
1910
1911         /* get next DMA set */
1912         td = xfer->td_start[xfer->flags_int.curr_dma_set];
1913
1914         xfer->td_transfer_first = td;
1915         xfer->td_transfer_cache = td;
1916
1917         temp.td = NULL;
1918         temp.td_next = td;
1919         temp.qtd_status = 0;
1920         temp.last_frame = 0;
1921         temp.setup_alt_next = xfer->flags_int.short_frames_ok;
1922
1923         if (xfer->flags_int.control_xfr) {
1924                 if (xfer->endpoint->toggle_next) {
1925                         /* DATA1 is next */
1926                         temp.qtd_status |=
1927                             htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
1928                 }
1929                 temp.auto_data_toggle = 0;
1930         } else {
1931                 temp.auto_data_toggle = 1;
1932         }
1933
1934         if ((xfer->xroot->udev->parent_hs_hub != NULL) ||
1935             (xfer->xroot->udev->address != 0)) {
1936                 /* max 3 retries */
1937                 temp.qtd_status |=
1938                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1939         }
1940         /* check if we should prepend a setup message */
1941
1942         if (xfer->flags_int.control_xfr) {
1943                 if (xfer->flags_int.control_hdr) {
1944
1945                         xfer->endpoint->toggle_next = 0;
1946
1947                         temp.qtd_status &=
1948                             htohc32(temp.sc, EHCI_QTD_SET_CERR(3));
1949                         temp.qtd_status |= htohc32(temp.sc,
1950                             EHCI_QTD_ACTIVE |
1951                             EHCI_QTD_SET_PID(EHCI_QTD_PID_SETUP) |
1952                             EHCI_QTD_SET_TOGGLE(0));
1953
1954                         temp.len = xfer->frlengths[0];
1955                         temp.pc = xfer->frbuffers + 0;
1956                         temp.shortpkt = temp.len ? 1 : 0;
1957                         /* check for last frame */
1958                         if (xfer->nframes == 1) {
1959                                 /* no STATUS stage yet, SETUP is last */
1960                                 if (xfer->flags_int.control_act) {
1961                                         temp.last_frame = 1;
1962                                         temp.setup_alt_next = 0;
1963                                 }
1964                         }
1965                         ehci_setup_standard_chain_sub(&temp);
1966                 }
1967                 x = 1;
1968         } else {
1969                 x = 0;
1970         }
1971
1972         while (x != xfer->nframes) {
1973
1974                 /* DATA0 / DATA1 message */
1975
1976                 temp.len = xfer->frlengths[x];
1977                 temp.pc = xfer->frbuffers + x;
1978
1979                 x++;
1980
1981                 if (x == xfer->nframes) {
1982                         if (xfer->flags_int.control_xfr) {
1983                                 /* no STATUS stage yet, DATA is last */
1984                                 if (xfer->flags_int.control_act) {
1985                                         temp.last_frame = 1;
1986                                         temp.setup_alt_next = 0;
1987                                 }
1988                         } else {
1989                                 temp.last_frame = 1;
1990                                 temp.setup_alt_next = 0;
1991                         }
1992                 }
1993                 /* keep previous data toggle and error count */
1994
1995                 temp.qtd_status &=
1996                     htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
1997                     EHCI_QTD_SET_TOGGLE(1));
1998
1999                 if (temp.len == 0) {
2000
2001                         /* make sure that we send an USB packet */
2002
2003                         temp.shortpkt = 0;
2004
2005                 } else {
2006
2007                         /* regular data transfer */
2008
2009                         temp.shortpkt = (xfer->flags.force_short_xfer) ? 0 : 1;
2010                 }
2011
2012                 /* set endpoint direction */
2013
2014                 temp.qtd_status |=
2015                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) ?
2016                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
2017                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN)) :
2018                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
2019                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT));
2020
2021                 ehci_setup_standard_chain_sub(&temp);
2022         }
2023
2024         /* check if we should append a status stage */
2025
2026         if (xfer->flags_int.control_xfr &&
2027             !xfer->flags_int.control_act) {
2028
2029                 /*
2030                  * Send a DATA1 message and invert the current endpoint
2031                  * direction.
2032                  */
2033
2034                 temp.qtd_status &= htohc32(temp.sc, EHCI_QTD_SET_CERR(3) |
2035                     EHCI_QTD_SET_TOGGLE(1));
2036                 temp.qtd_status |=
2037                     (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) ?
2038                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
2039                     EHCI_QTD_SET_PID(EHCI_QTD_PID_IN) |
2040                     EHCI_QTD_SET_TOGGLE(1)) :
2041                     htohc32(temp.sc, EHCI_QTD_ACTIVE |
2042                     EHCI_QTD_SET_PID(EHCI_QTD_PID_OUT) |
2043                     EHCI_QTD_SET_TOGGLE(1));
2044
2045                 temp.len = 0;
2046                 temp.pc = NULL;
2047                 temp.shortpkt = 0;
2048                 temp.last_frame = 1;
2049                 temp.setup_alt_next = 0;
2050
2051                 ehci_setup_standard_chain_sub(&temp);
2052         }
2053         td = temp.td;
2054
2055         /* the last TD terminates the transfer: */
2056         td->qtd_next = htohc32(temp.sc, EHCI_LINK_TERMINATE);
2057         td->qtd_altnext = htohc32(temp.sc, EHCI_LINK_TERMINATE);
2058
2059         usb_pc_cpu_flush(td->page_cache);
2060
2061         /* must have at least one frame! */
2062
2063         xfer->td_transfer_last = td;
2064
2065 #ifdef USB_DEBUG
2066         if (ehcidebug > 8) {
2067                 DPRINTF("nexttog=%d; data before transfer:\n",
2068                     xfer->endpoint->toggle_next);
2069                 ehci_dump_sqtds(temp.sc,
2070                     xfer->td_transfer_first);
2071         }
2072 #endif
2073
2074         methods = xfer->endpoint->methods;
2075
2076         qh = xfer->qh_start[xfer->flags_int.curr_dma_set];
2077
2078         /* the "qh_link" field is filled when the QH is added */
2079
2080         qh_endp =
2081             (EHCI_QH_SET_ADDR(xfer->address) |
2082             EHCI_QH_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2083             EHCI_QH_SET_MPL(xfer->max_packet_size));
2084
2085         if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_HIGH) {
2086                 qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_HIGH);
2087                 if (methods != &ehci_device_intr_methods)
2088                         qh_endp |= EHCI_QH_SET_NRL(8);
2089         } else {
2090
2091                 if (usbd_get_speed(xfer->xroot->udev) == USB_SPEED_FULL) {
2092                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_FULL);
2093                 } else {
2094                         qh_endp |= EHCI_QH_SET_EPS(EHCI_QH_SPEED_LOW);
2095                 }
2096
2097                 if (methods == &ehci_device_ctrl_methods) {
2098                         qh_endp |= EHCI_QH_CTL;
2099                 }
2100                 if (methods != &ehci_device_intr_methods) {
2101                         /* Only try one time per microframe! */
2102                         qh_endp |= EHCI_QH_SET_NRL(1);
2103                 }
2104         }
2105
2106         if (temp.auto_data_toggle == 0) {
2107                 /* software computes the data toggle */
2108                 qh_endp |= EHCI_QH_DTC;
2109         }
2110
2111         qh->qh_endp = htohc32(temp.sc, qh_endp);
2112
2113         qh_endphub =
2114             (EHCI_QH_SET_MULT(xfer->max_packet_count & 3) |
2115             EHCI_QH_SET_CMASK(xfer->endpoint->usb_cmask) |
2116             EHCI_QH_SET_SMASK(xfer->endpoint->usb_smask) |
2117             EHCI_QH_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2118             EHCI_QH_SET_PORT(xfer->xroot->udev->hs_port_no));
2119
2120         qh->qh_endphub = htohc32(temp.sc, qh_endphub);
2121         qh->qh_curqtd = 0;
2122
2123         /* fill the overlay qTD */
2124
2125         if (temp.auto_data_toggle && xfer->endpoint->toggle_next) {
2126                 /* DATA1 is next */
2127                 qh->qh_qtd.qtd_status = htohc32(temp.sc, EHCI_QTD_SET_TOGGLE(1));
2128         } else {
2129                 qh->qh_qtd.qtd_status = 0;
2130         }
2131
2132         td = xfer->td_transfer_first;
2133
2134         qh->qh_qtd.qtd_next = td->qtd_self;
2135         qh->qh_qtd.qtd_altnext =
2136             htohc32(temp.sc, EHCI_LINK_TERMINATE);
2137
2138         usb_pc_cpu_flush(qh->page_cache);
2139
2140         if (xfer->xroot->udev->flags.self_suspended == 0) {
2141                 KKASSERT(xfer->flags_int.onhwqueue == 0);
2142                 xfer->flags_int.onhwqueue = 1;
2143                 EHCI_APPEND_QH(qh, *qh_last);
2144         }
2145         ehci_ring_doorbell(temp.sc, NULL);
2146 }
2147
2148 static void
2149 ehci_root_intr(ehci_softc_t *sc)
2150 {
2151         uint16_t i;
2152         uint16_t m;
2153
2154         USB_BUS_LOCK_ASSERT(&sc->sc_bus);
2155
2156         /* clear any old interrupt data */
2157         memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2158
2159         /* set bits */
2160         m = (sc->sc_noport + 1);
2161         if (m > (8 * sizeof(sc->sc_hub_idata))) {
2162                 m = (8 * sizeof(sc->sc_hub_idata));
2163         }
2164         for (i = 1; i < m; i++) {
2165                 /* pick out CHANGE bits from the status register */
2166                 if (EOREAD4(sc, EHCI_PORTSC(i)) & EHCI_PS_CLEAR) {
2167                         sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2168                         DPRINTF("port %d changed\n", i);
2169                 }
2170         }
2171         uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2172             sizeof(sc->sc_hub_idata));
2173 }
2174
2175 static void
2176 ehci_isoc_fs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2177 {
2178         uint32_t nframes = xfer->nframes;
2179         uint32_t status;
2180         uint32_t *plen = xfer->frlengths;
2181         uint16_t len = 0;
2182         ehci_sitd_t *td = xfer->td_transfer_first;
2183         ehci_sitd_t **pp_last = &sc->sc_isoc_fs_p_last[xfer->qh_pos];
2184
2185         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2186             xfer, xfer->endpoint);
2187
2188         while (xfer->flags_int.onhwqueue && nframes--) {
2189                 if (td == NULL) {
2190                         panic("%s:%d: out of TD's\n",
2191                             __func__, __LINE__);
2192                 }
2193                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2194                         pp_last = &sc->sc_isoc_fs_p_last[0];
2195                 }
2196 #ifdef USB_DEBUG
2197                 if (ehcidebug > 15) {
2198                         DPRINTF("isoc FS-TD\n");
2199                         ehci_dump_sitd(sc, td);
2200                 }
2201 #endif
2202                 usb_pc_cpu_invalidate(td->page_cache);
2203                 status = hc32toh(sc, td->sitd_status);
2204
2205                 len = EHCI_SITD_GET_LEN(status);
2206
2207                 DPRINTFN(2, "status=0x%08x, rem=%u\n", status, len);
2208
2209                 if (*plen >= len) {
2210                         len = *plen - len;
2211                 } else {
2212                         len = 0;
2213                 }
2214
2215                 *plen = len;
2216
2217                 /* remove FS-TD from schedule */
2218                 EHCI_REMOVE_FS_TD(td, *pp_last);
2219
2220                 pp_last++;
2221                 plen++;
2222                 td = td->obj_next;
2223         }
2224         xfer->flags_int.onhwqueue = 0;
2225
2226         xfer->aframes = xfer->nframes;
2227 }
2228
2229 static void
2230 ehci_isoc_hs_done(ehci_softc_t *sc, struct usb_xfer *xfer)
2231 {
2232         uint32_t nframes = xfer->nframes;
2233         uint32_t status;
2234         uint32_t *plen = xfer->frlengths;
2235         uint16_t len = 0;
2236         uint8_t td_no = 0;
2237         ehci_itd_t *td = xfer->td_transfer_first;
2238         ehci_itd_t **pp_last = &sc->sc_isoc_hs_p_last[xfer->qh_pos];
2239
2240         DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
2241             xfer, xfer->endpoint);
2242
2243         while (nframes && xfer->flags_int.onhwqueue) {
2244                 if (td == NULL) {
2245                         panic("%s:%d: out of TD's\n",
2246                             __func__, __LINE__);
2247                 }
2248                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2249                         pp_last = &sc->sc_isoc_hs_p_last[0];
2250                 }
2251 #ifdef USB_DEBUG
2252                 if (ehcidebug > 15) {
2253                         DPRINTF("isoc HS-TD\n");
2254                         ehci_dump_itd(sc, td);
2255                 }
2256 #endif
2257
2258                 usb_pc_cpu_invalidate(td->page_cache);
2259                 status = hc32toh(sc, td->itd_status[td_no]);
2260
2261                 len = EHCI_ITD_GET_LEN(status);
2262
2263                 DPRINTFN(2, "status=0x%08x, len=%u\n", status, len);
2264
2265                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
2266
2267                         if (*plen >= len) {
2268                                 /*
2269                                  * The length is valid. NOTE: The
2270                                  * complete length is written back
2271                                  * into the status field, and not the
2272                                  * remainder like with other transfer
2273                                  * descriptor types.
2274                                  */
2275                         } else {
2276                                 /* Invalid length - truncate */
2277                                 len = 0;
2278                         }
2279
2280                         *plen = len;
2281                         plen++;
2282                         nframes--;
2283                 }
2284
2285                 td_no++;
2286
2287                 if ((td_no == 8) || (nframes == 0)) {
2288                         /* remove HS-TD from schedule */
2289                         EHCI_REMOVE_HS_TD(td, *pp_last);
2290                         pp_last++;
2291
2292                         td_no = 0;
2293                         td = td->obj_next;
2294                 }
2295         }
2296         xfer->aframes = xfer->nframes;
2297         xfer->flags_int.onhwqueue = 0;
2298 }
2299
2300 /* NOTE: "done" can be run two times in a row,
2301  * from close and from interrupt
2302  */
2303 static void
2304 ehci_device_done(struct usb_xfer *xfer, usb_error_t error)
2305 {
2306         const struct usb_pipe_methods *methods = xfer->endpoint->methods;
2307         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2308
2309         USB_BUS_LOCK_ASSERT(&sc->sc_bus);
2310
2311         DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2312             xfer, xfer->endpoint, error);
2313
2314         /*
2315          * Don't throw away a timed-out transfer without safely removing
2316          * it from the controller's notice!
2317          */
2318         if (error != USB_ERR_NORMAL_COMPLETION)
2319                 ehci_stop_xfer(xfer);
2320
2321         if (xfer->flags_int.onhwqueue) {
2322             if ((methods == &ehci_device_bulk_methods) ||
2323                 (methods == &ehci_device_ctrl_methods)) {
2324 #ifdef USB_DEBUG
2325                     if (ehcidebug > 8) {
2326                             DPRINTF("nexttog=%d; data after transfer:\n",
2327                                 xfer->endpoint->toggle_next);
2328                             ehci_dump_sqtds(sc,
2329                                 xfer->td_transfer_first);
2330                     }
2331 #endif
2332
2333                     EHCI_REMOVE_QH(sc, xfer->qh_start[xfer->flags_int.curr_dma_set],
2334                         sc->sc_async_p_last);
2335             }
2336             if (methods == &ehci_device_intr_methods) {
2337                     EHCI_REMOVE_QH(sc, xfer->qh_start[xfer->flags_int.curr_dma_set],
2338                         sc->sc_intr_p_last[xfer->qh_pos]);
2339             }
2340             xfer->flags_int.onhwqueue = 0;
2341             ehci_ring_doorbell(sc, xfer);
2342         }
2343
2344         /*
2345          * Only finish isochronous transfers once which will update
2346          * "xfer->frlengths".
2347          */
2348         if (xfer->td_transfer_first &&
2349             xfer->td_transfer_last) {
2350                 if (methods == &ehci_device_isoc_fs_methods) {
2351                         ehci_isoc_fs_done(sc, xfer);
2352                 }
2353                 if (methods == &ehci_device_isoc_hs_methods) {
2354                         ehci_isoc_hs_done(sc, xfer);
2355                 }
2356                 xfer->td_transfer_first = NULL;
2357                 xfer->td_transfer_last = NULL;
2358         }
2359         /* dequeue transfer and start next transfer */
2360         usbd_transfer_done(xfer, error);
2361 }
2362
2363 /*------------------------------------------------------------------------*
2364  * ehci bulk support
2365  *------------------------------------------------------------------------*/
2366 static void
2367 ehci_device_bulk_open(struct usb_xfer *xfer)
2368 {
2369         return;
2370 }
2371
2372 static void
2373 ehci_device_bulk_close(struct usb_xfer *xfer)
2374 {
2375         ehci_device_done(xfer, USB_ERR_CANCELLED);
2376 }
2377
2378 static void
2379 ehci_device_bulk_enter(struct usb_xfer *xfer)
2380 {
2381         return;
2382 }
2383
2384 static void
2385 ehci_device_bulk_start(struct usb_xfer *xfer)
2386 {
2387         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2388 #if 0
2389         uint32_t temp;
2390 #endif
2391
2392         /* setup TD's and QH */
2393         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2394
2395         /* put transfer on interrupt queue */
2396         ehci_transfer_intr_enqueue(xfer);
2397
2398 #if 0
2399         /*
2400          * XXX Certain nVidia chipsets choke when using the IAAD
2401          * feature too frequently.
2402          *
2403          * XXX This code has been moved to another place which handles
2404          *     more cases.  EHCI appears to need the doorbell to be hit
2405          *     in the other queueing cases or it can wind up not
2406          *     processing the queue.
2407          */
2408         if (sc->sc_flags & EHCI_SCFLG_IAADBUG)
2409                 return;
2410
2411         /* XXX Performance quirk: Some Host Controllers have a too low
2412          * interrupt rate. Issue an IAAD to stimulate the Host
2413          * Controller after queueing the BULK transfer.
2414          */
2415         temp = EOREAD4(sc, EHCI_USBCMD);
2416         if (!(temp & EHCI_CMD_IAAD))
2417                 EOWRITE4(sc, EHCI_USBCMD, temp | EHCI_CMD_IAAD);
2418 #endif
2419 }
2420
2421 static const struct usb_pipe_methods ehci_device_bulk_methods =
2422 {
2423         .open = ehci_device_bulk_open,
2424         .close = ehci_device_bulk_close,
2425         .enter = ehci_device_bulk_enter,
2426         .start = ehci_device_bulk_start,
2427 };
2428
2429 /*------------------------------------------------------------------------*
2430  * ehci control support
2431  *------------------------------------------------------------------------*/
2432 static void
2433 ehci_device_ctrl_open(struct usb_xfer *xfer)
2434 {
2435         return;
2436 }
2437
2438 static void
2439 ehci_device_ctrl_close(struct usb_xfer *xfer)
2440 {
2441         ehci_device_done(xfer, USB_ERR_CANCELLED);
2442 }
2443
2444 static void
2445 ehci_device_ctrl_enter(struct usb_xfer *xfer)
2446 {
2447         return;
2448 }
2449
2450 static void
2451 ehci_device_ctrl_start(struct usb_xfer *xfer)
2452 {
2453         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2454
2455         /* setup TD's and QH */
2456         ehci_setup_standard_chain(xfer, &sc->sc_async_p_last);
2457
2458         /* put transfer on interrupt queue */
2459         ehci_transfer_intr_enqueue(xfer);
2460 }
2461
2462 static const struct usb_pipe_methods ehci_device_ctrl_methods =
2463 {
2464         .open = ehci_device_ctrl_open,
2465         .close = ehci_device_ctrl_close,
2466         .enter = ehci_device_ctrl_enter,
2467         .start = ehci_device_ctrl_start,
2468 };
2469
2470 /*------------------------------------------------------------------------*
2471  * ehci interrupt support
2472  *------------------------------------------------------------------------*/
2473 static void
2474 ehci_device_intr_open(struct usb_xfer *xfer)
2475 {
2476         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2477         uint16_t best;
2478         uint16_t bit;
2479         uint16_t x;
2480
2481         usb_hs_bandwidth_alloc(xfer);
2482
2483         /*
2484          * Find the best QH position corresponding to the given interval:
2485          */
2486
2487         best = 0;
2488         bit = EHCI_VIRTUAL_FRAMELIST_COUNT / 2;
2489         while (bit) {
2490                 if (xfer->interval >= bit) {
2491                         x = bit;
2492                         best = bit;
2493                         while (x & bit) {
2494                                 if (sc->sc_intr_stat[x] <
2495                                     sc->sc_intr_stat[best]) {
2496                                         best = x;
2497                                 }
2498                                 x++;
2499                         }
2500                         break;
2501                 }
2502                 bit >>= 1;
2503         }
2504
2505         sc->sc_intr_stat[best]++;
2506         xfer->qh_pos = best;
2507
2508         DPRINTFN(3, "best=%d interval=%d\n",
2509             best, xfer->interval);
2510 }
2511
2512 static void
2513 ehci_device_intr_close(struct usb_xfer *xfer)
2514 {
2515         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2516
2517         sc->sc_intr_stat[xfer->qh_pos]--;
2518
2519         ehci_device_done(xfer, USB_ERR_CANCELLED);
2520
2521         /* bandwidth must be freed after device done */
2522         usb_hs_bandwidth_free(xfer);
2523 }
2524
2525 static void
2526 ehci_device_intr_enter(struct usb_xfer *xfer)
2527 {
2528         return;
2529 }
2530
2531 static void
2532 ehci_device_intr_start(struct usb_xfer *xfer)
2533 {
2534         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2535
2536         /* setup TD's and QH */
2537         ehci_setup_standard_chain(xfer, &sc->sc_intr_p_last[xfer->qh_pos]);
2538
2539         /* put transfer on interrupt queue */
2540         ehci_transfer_intr_enqueue(xfer);
2541 }
2542
2543 static const struct usb_pipe_methods ehci_device_intr_methods =
2544 {
2545         .open = ehci_device_intr_open,
2546         .close = ehci_device_intr_close,
2547         .enter = ehci_device_intr_enter,
2548         .start = ehci_device_intr_start,
2549 };
2550
2551 /*------------------------------------------------------------------------*
2552  * ehci full speed isochronous support
2553  *------------------------------------------------------------------------*/
2554 static void
2555 ehci_device_isoc_fs_open(struct usb_xfer *xfer)
2556 {
2557         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2558         ehci_sitd_t *td;
2559         uint32_t sitd_portaddr;
2560         uint8_t ds;
2561
2562         sitd_portaddr =
2563             EHCI_SITD_SET_ADDR(xfer->address) |
2564             EHCI_SITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)) |
2565             EHCI_SITD_SET_HUBA(xfer->xroot->udev->hs_hub_addr) |
2566             EHCI_SITD_SET_PORT(xfer->xroot->udev->hs_port_no);
2567
2568         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2569                 sitd_portaddr |= EHCI_SITD_SET_DIR_IN;
2570         }
2571         sitd_portaddr = htohc32(sc, sitd_portaddr);
2572
2573         /* initialize all TD's */
2574
2575         for (ds = 0; ds != 2; ds++) {
2576
2577                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2578
2579                         td->sitd_portaddr = sitd_portaddr;
2580
2581                         /*
2582                          * TODO: make some kind of automatic
2583                          * SMASK/CMASK selection based on micro-frame
2584                          * usage
2585                          *
2586                          * micro-frame usage (8 microframes per 1ms)
2587                          */
2588                         td->sitd_back = htohc32(sc, EHCI_LINK_TERMINATE);
2589
2590                         usb_pc_cpu_flush(td->page_cache);
2591                 }
2592         }
2593 }
2594
2595 static void
2596 ehci_device_isoc_fs_close(struct usb_xfer *xfer)
2597 {
2598         ehci_device_done(xfer, USB_ERR_CANCELLED);
2599 }
2600
2601 static void
2602 ehci_device_isoc_fs_enter(struct usb_xfer *xfer)
2603 {
2604         struct usb_page_search buf_res;
2605         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2606         ehci_sitd_t *td;
2607         ehci_sitd_t *td_last = NULL;
2608         ehci_sitd_t **pp_last;
2609         uint32_t *plen;
2610         uint32_t buf_offset;
2611         uint32_t nframes;
2612         uint32_t temp;
2613         uint32_t sitd_mask;
2614         uint16_t tlen;
2615         uint8_t sa;
2616         uint8_t sb;
2617
2618 #ifdef USB_DEBUG
2619         uint8_t once = 1;
2620
2621 #endif
2622
2623         DPRINTFN(6, "xfer=%p next=%d nframes=%d\n",
2624             xfer, xfer->endpoint->isoc_next, xfer->nframes);
2625
2626         /* get the current frame index */
2627
2628         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2629
2630         /*
2631          * check if the frame index is within the window where the frames
2632          * will be inserted
2633          */
2634         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2635             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2636
2637         if ((xfer->endpoint->is_synced == 0) ||
2638             (buf_offset < xfer->nframes)) {
2639                 /*
2640                  * If there is data underflow or the pipe queue is empty we
2641                  * schedule the transfer a few frames ahead of the current
2642                  * frame position. Else two isochronous transfers might
2643                  * overlap.
2644                  */
2645                 xfer->endpoint->isoc_next = (nframes + 3) &
2646                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2647                 xfer->endpoint->is_synced = 1;
2648                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2649         }
2650         /*
2651          * compute how many milliseconds the insertion is ahead of the
2652          * current frame position:
2653          */
2654         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2655             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2656
2657         /*
2658          * pre-compute when the isochronous transfer will be finished:
2659          */
2660         xfer->isoc_time_complete =
2661                 usb_isoc_time_expand(&sc->sc_bus, nframes) +
2662                 buf_offset + xfer->nframes;
2663
2664         /* get the real number of frames */
2665
2666         nframes = xfer->nframes;
2667
2668         buf_offset = 0;
2669
2670         plen = xfer->frlengths;
2671
2672         /* toggle the DMA set we are using */
2673         xfer->flags_int.curr_dma_set ^= 1;
2674
2675         /* get next DMA set */
2676         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2677         xfer->td_transfer_first = td;
2678
2679         pp_last = &sc->sc_isoc_fs_p_last[xfer->endpoint->isoc_next];
2680
2681         /* store starting position */
2682
2683         xfer->qh_pos = xfer->endpoint->isoc_next;
2684
2685         while (nframes--) {
2686                 if (td == NULL) {
2687                         panic("%s:%d: out of TD's\n",
2688                             __func__, __LINE__);
2689                 }
2690                 if (pp_last >= &sc->sc_isoc_fs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2691                         pp_last = &sc->sc_isoc_fs_p_last[0];
2692                 }
2693                 /* reuse sitd_portaddr and sitd_back from last transfer */
2694
2695                 if (*plen > xfer->max_frame_size) {
2696 #ifdef USB_DEBUG
2697                         if (once) {
2698                                 once = 0;
2699                                 kprintf("%s: frame length(%d) exceeds %d "
2700                                     "bytes (frame truncated)\n",
2701                                     __func__, *plen,
2702                                     xfer->max_frame_size);
2703                         }
2704 #endif
2705                         *plen = xfer->max_frame_size;
2706                 }
2707
2708                 /* allocate a slot */
2709
2710                 sa = usbd_fs_isoc_schedule_alloc_slot(xfer,
2711                     xfer->isoc_time_complete - nframes - 1);
2712
2713                 if(sa == 255) {
2714                         /*
2715                          * Schedule is FULL, set length to zero:
2716                          */
2717
2718                         *plen = 0;
2719                         sa = USB_FS_ISOC_UFRAME_MAX - 1;
2720                 }
2721                 if (*plen) {
2722                         /*
2723                          * only call "usbd_get_page()" when we have a
2724                          * non-zero length
2725                          */
2726                         usbd_get_page(xfer->frbuffers, buf_offset, &buf_res);
2727                         td->sitd_bp[0] = htohc32(sc, buf_res.physaddr);
2728                         buf_offset += *plen;
2729                         /*
2730                          * NOTE: We need to subtract one from the offset so
2731                          * that we are on a valid page!
2732                          */
2733                         usbd_get_page(xfer->frbuffers, buf_offset - 1,
2734                             &buf_res);
2735                         temp = buf_res.physaddr & ~0xFFF;
2736                 } else {
2737                         td->sitd_bp[0] = 0;
2738                         temp = 0;
2739                 }
2740
2741                 if (UE_GET_DIR(xfer->endpointno) == UE_DIR_OUT) {
2742                         tlen = *plen;
2743                         if (tlen <= 188) {
2744                                 temp |= 1;      /* T-count = 1, TP = ALL */
2745                                 tlen = 1;
2746                         } else {
2747                                 tlen += 187;
2748                                 tlen /= 188;
2749                                 temp |= tlen;   /* T-count = [1..6] */
2750                                 temp |= 8;      /* TP = Begin */
2751                         }
2752
2753                         tlen += sa;
2754
2755                         if (tlen >= 8) {
2756                                 sb = 0;
2757                         } else {
2758                                 sb = (1 << tlen);
2759                         }
2760
2761                         sa = (1 << sa);
2762                         sa = (sb - sa) & 0x3F;
2763                         sb = 0;
2764                 } else {
2765                         sb = (-(4 << sa)) & 0xFE;
2766                         sa = (1 << sa) & 0x3F;
2767                 }
2768
2769                 sitd_mask = (EHCI_SITD_SET_SMASK(sa) |
2770                     EHCI_SITD_SET_CMASK(sb));
2771
2772                 td->sitd_bp[1] = htohc32(sc, temp);
2773
2774                 td->sitd_mask = htohc32(sc, sitd_mask);
2775
2776                 if (nframes == 0) {
2777                         td->sitd_status = htohc32(sc,
2778                             EHCI_SITD_IOC |
2779                             EHCI_SITD_ACTIVE |
2780                             EHCI_SITD_SET_LEN(*plen));
2781                 } else {
2782                         td->sitd_status = htohc32(sc,
2783                             EHCI_SITD_ACTIVE |
2784                             EHCI_SITD_SET_LEN(*plen));
2785                 }
2786                 usb_pc_cpu_flush(td->page_cache);
2787
2788 #ifdef USB_DEBUG
2789                 if (ehcidebug > 15) {
2790                         DPRINTF("FS-TD %d\n", nframes);
2791                         ehci_dump_sitd(sc, td);
2792                 }
2793 #endif
2794                 /* insert TD into schedule */
2795                 xfer->flags_int.onhwqueue = 1;
2796                 EHCI_APPEND_FS_TD(td, *pp_last);
2797                 pp_last++;
2798
2799                 plen++;
2800                 td_last = td;
2801                 td = td->obj_next;
2802         }
2803
2804         xfer->td_transfer_last = td_last;
2805
2806         /* update isoc_next */
2807         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_fs_p_last[0]) &
2808             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2809
2810         /*
2811          * We don't allow cancelling of the SPLIT transaction USB FULL
2812          * speed transfer, because it disturbs the bandwidth
2813          * computation algorithm.
2814          */
2815         xfer->flags_int.can_cancel_immed = 0;
2816         ehci_ring_doorbell(sc, NULL);
2817 }
2818
2819 static void
2820 ehci_device_isoc_fs_start(struct usb_xfer *xfer)
2821 {
2822         /*
2823          * We don't allow cancelling of the SPLIT transaction USB FULL
2824          * speed transfer, because it disturbs the bandwidth
2825          * computation algorithm.
2826          */
2827         xfer->flags_int.can_cancel_immed = 0;
2828
2829         /* set a default timeout */
2830         if (xfer->timeout == 0)
2831                 xfer->timeout = 1000; /* ms */
2832
2833         /* put transfer on interrupt queue */
2834         ehci_transfer_intr_enqueue(xfer);
2835 }
2836
2837 static const struct usb_pipe_methods ehci_device_isoc_fs_methods =
2838 {
2839         .open = ehci_device_isoc_fs_open,
2840         .close = ehci_device_isoc_fs_close,
2841         .enter = ehci_device_isoc_fs_enter,
2842         .start = ehci_device_isoc_fs_start,
2843 };
2844
2845 /*------------------------------------------------------------------------*
2846  * ehci high speed isochronous support
2847  *------------------------------------------------------------------------*/
2848 static void
2849 ehci_device_isoc_hs_open(struct usb_xfer *xfer)
2850 {
2851         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2852         ehci_itd_t *td;
2853         uint32_t temp;
2854         uint8_t ds;
2855
2856         usb_hs_bandwidth_alloc(xfer);
2857
2858         /* initialize all TD's */
2859
2860         for (ds = 0; ds != 2; ds++) {
2861
2862                 for (td = xfer->td_start[ds]; td; td = td->obj_next) {
2863
2864                         /* set TD inactive */
2865                         td->itd_status[0] = 0;
2866                         td->itd_status[1] = 0;
2867                         td->itd_status[2] = 0;
2868                         td->itd_status[3] = 0;
2869                         td->itd_status[4] = 0;
2870                         td->itd_status[5] = 0;
2871                         td->itd_status[6] = 0;
2872                         td->itd_status[7] = 0;
2873
2874                         /* set endpoint and address */
2875                         td->itd_bp[0] = htohc32(sc,
2876                             EHCI_ITD_SET_ADDR(xfer->address) |
2877                             EHCI_ITD_SET_ENDPT(UE_GET_ADDR(xfer->endpointno)));
2878
2879                         temp =
2880                             EHCI_ITD_SET_MPL(xfer->max_packet_size & 0x7FF);
2881
2882                         /* set direction */
2883                         if (UE_GET_DIR(xfer->endpointno) == UE_DIR_IN) {
2884                                 temp |= EHCI_ITD_SET_DIR_IN;
2885                         }
2886                         /* set maximum packet size */
2887                         td->itd_bp[1] = htohc32(sc, temp);
2888
2889                         /* set transfer multiplier */
2890                         td->itd_bp[2] = htohc32(sc, xfer->max_packet_count & 3);
2891
2892                         usb_pc_cpu_flush(td->page_cache);
2893                 }
2894         }
2895 }
2896
2897 static void
2898 ehci_device_isoc_hs_close(struct usb_xfer *xfer)
2899 {
2900         ehci_device_done(xfer, USB_ERR_CANCELLED);
2901
2902         /* bandwidth must be freed after device done */
2903         usb_hs_bandwidth_free(xfer);
2904 }
2905
2906 static void
2907 ehci_device_isoc_hs_enter(struct usb_xfer *xfer)
2908 {
2909         struct usb_page_search buf_res;
2910         ehci_softc_t *sc = EHCI_BUS2SC(xfer->xroot->bus);
2911         ehci_itd_t *td;
2912         ehci_itd_t *td_last = NULL;
2913         ehci_itd_t **pp_last;
2914         bus_size_t page_addr;
2915         uint32_t *plen;
2916         uint32_t status;
2917         uint32_t buf_offset;
2918         uint32_t nframes;
2919         uint32_t itd_offset[8 + 1];
2920         uint8_t x;
2921         uint8_t td_no;
2922         uint8_t page_no;
2923         uint8_t shift = usbd_xfer_get_fps_shift(xfer);
2924
2925 #ifdef USB_DEBUG
2926         uint8_t once = 1;
2927
2928 #endif
2929
2930         DPRINTFN(6, "xfer=%p next=%d nframes=%d shift=%d\n",
2931             xfer, xfer->endpoint->isoc_next, xfer->nframes, (int)shift);
2932
2933         /* get the current frame index */
2934
2935         nframes = EOREAD4(sc, EHCI_FRINDEX) / 8;
2936
2937         /*
2938          * check if the frame index is within the window where the frames
2939          * will be inserted
2940          */
2941         buf_offset = (nframes - xfer->endpoint->isoc_next) &
2942             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2943
2944         if ((xfer->endpoint->is_synced == 0) ||
2945             (buf_offset < (((xfer->nframes << shift) + 7) / 8))) {
2946                 /*
2947                  * If there is data underflow or the pipe queue is empty we
2948                  * schedule the transfer a few frames ahead of the current
2949                  * frame position. Else two isochronous transfers might
2950                  * overlap.
2951                  */
2952                 xfer->endpoint->isoc_next = (nframes + 3) &
2953                     (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2954                 xfer->endpoint->is_synced = 1;
2955                 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
2956         }
2957         /*
2958          * compute how many milliseconds the insertion is ahead of the
2959          * current frame position:
2960          */
2961         buf_offset = (xfer->endpoint->isoc_next - nframes) &
2962             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
2963
2964         /*
2965          * pre-compute when the isochronous transfer will be finished:
2966          */
2967         xfer->isoc_time_complete =
2968             usb_isoc_time_expand(&sc->sc_bus, nframes) + buf_offset +
2969             (((xfer->nframes << shift) + 7) / 8);
2970
2971         /* get the real number of frames */
2972
2973         nframes = xfer->nframes;
2974
2975         buf_offset = 0;
2976         td_no = 0;
2977
2978         plen = xfer->frlengths;
2979
2980         /* toggle the DMA set we are using */
2981         xfer->flags_int.curr_dma_set ^= 1;
2982
2983         /* get next DMA set */
2984         td = xfer->td_start[xfer->flags_int.curr_dma_set];
2985         xfer->td_transfer_first = td;
2986
2987         pp_last = &sc->sc_isoc_hs_p_last[xfer->endpoint->isoc_next];
2988
2989         /* store starting position */
2990
2991         xfer->qh_pos = xfer->endpoint->isoc_next;
2992
2993         while (nframes) {
2994                 if (td == NULL) {
2995                         panic("%s:%d: out of TD's\n",
2996                             __func__, __LINE__);
2997                 }
2998                 if (pp_last >= &sc->sc_isoc_hs_p_last[EHCI_VIRTUAL_FRAMELIST_COUNT]) {
2999                         pp_last = &sc->sc_isoc_hs_p_last[0];
3000                 }
3001                 /* range check */
3002                 if (*plen > xfer->max_frame_size) {
3003 #ifdef USB_DEBUG
3004                         if (once) {
3005                                 once = 0;
3006                                 kprintf("%s: frame length(%d) exceeds %d bytes "
3007                                     "(frame truncated)\n",
3008                                     __func__, *plen, xfer->max_frame_size);
3009                         }
3010 #endif
3011                         *plen = xfer->max_frame_size;
3012                 }
3013
3014                 if (xfer->endpoint->usb_smask & (1 << td_no)) {
3015                         status = (EHCI_ITD_SET_LEN(*plen) |
3016                             EHCI_ITD_ACTIVE |
3017                             EHCI_ITD_SET_PG(0));
3018                         td->itd_status[td_no] = htohc32(sc, status);
3019                         itd_offset[td_no] = buf_offset;
3020                         buf_offset += *plen;
3021                         plen++;
3022                         nframes --;
3023                 } else {
3024                         td->itd_status[td_no] = 0;      /* not active */
3025                         itd_offset[td_no] = buf_offset;
3026                 }
3027
3028                 td_no++;
3029
3030                 if ((td_no == 8) || (nframes == 0)) {
3031
3032                         /* the rest of the transfers are not active, if any */
3033                         for (x = td_no; x != 8; x++) {
3034                                 td->itd_status[x] = 0;  /* not active */
3035                         }
3036
3037                         /* check if there is any data to be transferred */
3038                         if (itd_offset[0] != buf_offset) {
3039                                 page_no = 0;
3040                                 itd_offset[td_no] = buf_offset;
3041
3042                                 /* get first page offset */
3043                                 usbd_get_page(xfer->frbuffers, itd_offset[0], &buf_res);
3044                                 /* get page address */
3045                                 page_addr = buf_res.physaddr & ~0xFFF;
3046                                 /* update page address */
3047                                 td->itd_bp[0] &= htohc32(sc, 0xFFF);
3048                                 td->itd_bp[0] |= htohc32(sc, page_addr);
3049
3050                                 for (x = 0; x != td_no; x++) {
3051                                         /* set page number and page offset */
3052                                         status = (EHCI_ITD_SET_PG(page_no) |
3053                                             (buf_res.physaddr & 0xFFF));
3054                                         td->itd_status[x] |= htohc32(sc, status);
3055
3056                                         /* get next page offset */
3057                                         if (itd_offset[x + 1] == buf_offset) {
3058                                                 /*
3059                                                  * We subtract one so that
3060                                                  * we don't go off the last
3061                                                  * page!
3062                                                  */
3063                                                 usbd_get_page(xfer->frbuffers, buf_offset - 1, &buf_res);
3064                                         } else {
3065                                                 usbd_get_page(xfer->frbuffers, itd_offset[x + 1], &buf_res);
3066                                         }
3067
3068                                         /* check if we need a new page */
3069                                         if ((buf_res.physaddr ^ page_addr) & ~0xFFF) {
3070                                                 /* new page needed */
3071                                                 page_addr = buf_res.physaddr & ~0xFFF;
3072                                                 if (page_no == 6) {
3073                                                         panic("%s: too many pages\n", __func__);
3074                                                 }
3075                                                 page_no++;
3076                                                 /* update page address */
3077                                                 td->itd_bp[page_no] &= htohc32(sc, 0xFFF);
3078                                                 td->itd_bp[page_no] |= htohc32(sc, page_addr);
3079                                         }
3080                                 }
3081                         }
3082                         /* set IOC bit if we are complete */
3083                         if (nframes == 0) {
3084                                 td->itd_status[td_no - 1] |= htohc32(sc, EHCI_ITD_IOC);
3085                         }
3086                         usb_pc_cpu_flush(td->page_cache);
3087 #ifdef USB_DEBUG
3088                         if (ehcidebug > 15) {
3089                                 DPRINTF("HS-TD %d\n", nframes);
3090                                 ehci_dump_itd(sc, td);
3091                         }
3092 #endif
3093                         /* insert TD into schedule */
3094                         xfer->flags_int.onhwqueue = 1;
3095                         EHCI_APPEND_HS_TD(td, *pp_last);
3096                         pp_last++;
3097
3098                         td_no = 0;
3099                         td_last = td;
3100                         td = td->obj_next;
3101                 }
3102         }
3103
3104         xfer->td_transfer_last = td_last;
3105
3106         /* update isoc_next */
3107         xfer->endpoint->isoc_next = (pp_last - &sc->sc_isoc_hs_p_last[0]) &
3108             (EHCI_VIRTUAL_FRAMELIST_COUNT - 1);
3109         ehci_ring_doorbell(sc, NULL);
3110 }
3111
3112 static void
3113 ehci_device_isoc_hs_start(struct usb_xfer *xfer)
3114 {
3115         /* put transfer on interrupt queue */
3116         ehci_transfer_intr_enqueue(xfer);
3117 }
3118
3119 static const struct usb_pipe_methods ehci_device_isoc_hs_methods =
3120 {
3121         .open = ehci_device_isoc_hs_open,
3122         .close = ehci_device_isoc_hs_close,
3123         .enter = ehci_device_isoc_hs_enter,
3124         .start = ehci_device_isoc_hs_start,
3125 };
3126
3127 /*------------------------------------------------------------------------*
3128  * ehci root control support
3129  *------------------------------------------------------------------------*
3130  * Simulate a hardware hub by handling all the necessary requests.
3131  *------------------------------------------------------------------------*/
3132
3133 static const
3134 struct usb_device_descriptor ehci_devd =
3135 {
3136         sizeof(struct usb_device_descriptor),
3137         UDESC_DEVICE,                   /* type */
3138         {0x00, 0x02},                   /* USB version */
3139         UDCLASS_HUB,                    /* class */
3140         UDSUBCLASS_HUB,                 /* subclass */
3141         UDPROTO_HSHUBSTT,               /* protocol */
3142         64,                             /* max packet */
3143         {0}, {0}, {0x00, 0x01},         /* device id */
3144         1, 2, 0,                        /* string indicies */
3145         1                               /* # of configurations */
3146 };
3147
3148 static const
3149 struct usb_device_qualifier ehci_odevd =
3150 {
3151         sizeof(struct usb_device_qualifier),
3152         UDESC_DEVICE_QUALIFIER,         /* type */
3153         {0x00, 0x02},                   /* USB version */
3154         UDCLASS_HUB,                    /* class */
3155         UDSUBCLASS_HUB,                 /* subclass */
3156         UDPROTO_FSHUB,                  /* protocol */
3157         0,                              /* max packet */
3158         0,                              /* # of configurations */
3159         0
3160 };
3161
3162 static const struct ehci_config_desc ehci_confd = {
3163         .confd = {
3164                 .bLength = sizeof(struct usb_config_descriptor),
3165                 .bDescriptorType = UDESC_CONFIG,
3166                 .wTotalLength[0] = sizeof(ehci_confd),
3167                 .bNumInterface = 1,
3168                 .bConfigurationValue = 1,
3169                 .iConfiguration = 0,
3170                 .bmAttributes = UC_SELF_POWERED,
3171                 .bMaxPower = 0          /* max power */
3172         },
3173         .ifcd = {
3174                 .bLength = sizeof(struct usb_interface_descriptor),
3175                 .bDescriptorType = UDESC_INTERFACE,
3176                 .bNumEndpoints = 1,
3177                 .bInterfaceClass = UICLASS_HUB,
3178                 .bInterfaceSubClass = UISUBCLASS_HUB,
3179                 .bInterfaceProtocol = 0,
3180         },
3181         .endpd = {
3182                 .bLength = sizeof(struct usb_endpoint_descriptor),
3183                 .bDescriptorType = UDESC_ENDPOINT,
3184                 .bEndpointAddress = UE_DIR_IN | EHCI_INTR_ENDPT,
3185                 .bmAttributes = UE_INTERRUPT,
3186                 .wMaxPacketSize[0] = 8, /* max packet (63 ports) */
3187                 .bInterval = 255,
3188         },
3189 };
3190
3191 static const
3192 struct usb_hub_descriptor ehci_hubd =
3193 {
3194         .bDescLength = 0,               /* dynamic length */
3195         .bDescriptorType = UDESC_HUB,
3196 };
3197
3198 static void
3199 ehci_disown(ehci_softc_t *sc, uint16_t index, uint8_t lowspeed)
3200 {
3201         uint32_t port;
3202         uint32_t v;
3203
3204         DPRINTF("index=%d lowspeed=%d\n", index, lowspeed);
3205
3206         port = EHCI_PORTSC(index);
3207         v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3208         EOWRITE4(sc, port, v | EHCI_PS_PO);
3209 }
3210
3211 static usb_error_t
3212 ehci_roothub_exec(struct usb_device *udev,
3213     struct usb_device_request *req, const void **pptr, uint16_t *plength)
3214 {
3215         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3216         const char *str_ptr;
3217         const void *ptr;
3218         uint32_t port;
3219         uint32_t v;
3220         uint16_t len;
3221         uint16_t i;
3222         uint16_t value;
3223         uint16_t index;
3224         usb_error_t err;
3225
3226         USB_BUS_LOCK_ASSERT(&sc->sc_bus);
3227
3228         /* buffer reset */
3229         ptr = (const void *)&sc->sc_hub_desc;
3230         len = 0;
3231         err = 0;
3232
3233         value = UGETW(req->wValue);
3234         index = UGETW(req->wIndex);
3235
3236         DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3237             "wValue=0x%04x wIndex=0x%04x\n",
3238             req->bmRequestType, req->bRequest,
3239             UGETW(req->wLength), value, index);
3240
3241 #define C(x,y) ((x) | ((y) << 8))
3242         switch (C(req->bRequest, req->bmRequestType)) {
3243         case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3244         case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3245         case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3246                 /*
3247                  * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3248                  * for the integrated root hub.
3249                  */
3250                 break;
3251         case C(UR_GET_CONFIG, UT_READ_DEVICE):
3252                 len = 1;
3253                 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3254                 break;
3255         case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3256                 switch (value >> 8) {
3257                 case UDESC_DEVICE:
3258                         if ((value & 0xff) != 0) {
3259                                 err = USB_ERR_IOERROR;
3260                                 goto done;
3261                         }
3262                         len = sizeof(ehci_devd);
3263                         ptr = (const void *)&ehci_devd;
3264                         break;
3265                         /*
3266                          * We can't really operate at another speed,
3267                          * but the specification says we need this
3268                          * descriptor:
3269                          */
3270                 case UDESC_DEVICE_QUALIFIER:
3271                         if ((value & 0xff) != 0) {
3272                                 err = USB_ERR_IOERROR;
3273                                 goto done;
3274                         }
3275                         len = sizeof(ehci_odevd);
3276                         ptr = (const void *)&ehci_odevd;
3277                         break;
3278
3279                 case UDESC_CONFIG:
3280                         if ((value & 0xff) != 0) {
3281                                 err = USB_ERR_IOERROR;
3282                                 goto done;
3283                         }
3284                         len = sizeof(ehci_confd);
3285                         ptr = (const void *)&ehci_confd;
3286                         break;
3287
3288                 case UDESC_STRING:
3289                         switch (value & 0xff) {
3290                         case 0: /* Language table */
3291                                 str_ptr = "\001";
3292                                 break;
3293
3294                         case 1: /* Vendor */
3295                                 str_ptr = sc->sc_vendor;
3296                                 break;
3297
3298                         case 2: /* Product */
3299                                 str_ptr = "EHCI root HUB";
3300                                 break;
3301
3302                         default:
3303                                 str_ptr = "";
3304                                 break;
3305                         }
3306
3307                         len = usb_make_str_desc(
3308                             sc->sc_hub_desc.temp,
3309                             sizeof(sc->sc_hub_desc.temp),
3310                             str_ptr);
3311                         break;
3312                 default:
3313                         err = USB_ERR_IOERROR;
3314                         goto done;
3315                 }
3316                 break;
3317         case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3318                 len = 1;
3319                 sc->sc_hub_desc.temp[0] = 0;
3320                 break;
3321         case C(UR_GET_STATUS, UT_READ_DEVICE):
3322                 len = 2;
3323                 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3324                 break;
3325         case C(UR_GET_STATUS, UT_READ_INTERFACE):
3326         case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3327                 len = 2;
3328                 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3329                 break;
3330         case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3331                 if (value >= EHCI_MAX_DEVICES) {
3332                         err = USB_ERR_IOERROR;
3333                         goto done;
3334                 }
3335                 sc->sc_addr = value;
3336                 break;
3337         case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3338                 if ((value != 0) && (value != 1)) {
3339                         err = USB_ERR_IOERROR;
3340                         goto done;
3341                 }
3342                 sc->sc_conf = value;
3343                 break;
3344         case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3345                 break;
3346         case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3347         case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3348         case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3349                 err = USB_ERR_IOERROR;
3350                 goto done;
3351         case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3352                 break;
3353         case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3354                 break;
3355                 /* Hub requests */
3356         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3357                 break;
3358         case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3359                 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3360
3361                 if ((index < 1) ||
3362                     (index > sc->sc_noport)) {
3363                         err = USB_ERR_IOERROR;
3364                         goto done;
3365                 }
3366                 port = EHCI_PORTSC(index);
3367                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3368                 switch (value) {
3369                 case UHF_PORT_ENABLE:
3370                         EOWRITE4(sc, port, v & ~EHCI_PS_PE);
3371                         break;
3372                 case UHF_PORT_SUSPEND:
3373                         if ((v & EHCI_PS_SUSP) && (!(v & EHCI_PS_FPR))) {
3374
3375                                 /*
3376                                  * waking up a High Speed device is rather
3377                                  * complicated if
3378                                  */
3379                                 EOWRITE4(sc, port, v | EHCI_PS_FPR);
3380                         }
3381                         /* wait 20ms for resume sequence to complete */
3382                         usb_pause_mtx(&sc->sc_bus.bus_lock, hz / 50);
3383
3384                         EOWRITE4(sc, port, v & ~(EHCI_PS_SUSP |
3385                             EHCI_PS_FPR | (3 << 10) /* High Speed */ ));
3386
3387                         /* 4ms settle time */
3388                         usb_pause_mtx(&sc->sc_bus.bus_lock, hz / 250);
3389                         break;
3390                 case UHF_PORT_POWER:
3391                         EOWRITE4(sc, port, v & ~EHCI_PS_PP);
3392                         break;
3393                 case UHF_PORT_TEST:
3394                         DPRINTFN(3, "clear port test "
3395                             "%d\n", index);
3396                         break;
3397                 case UHF_PORT_INDICATOR:
3398                         DPRINTFN(3, "clear port ind "
3399                             "%d\n", index);
3400                         EOWRITE4(sc, port, v & ~EHCI_PS_PIC);
3401                         break;
3402                 case UHF_C_PORT_CONNECTION:
3403                         EOWRITE4(sc, port, v | EHCI_PS_CSC);
3404                         break;
3405                 case UHF_C_PORT_ENABLE:
3406                         EOWRITE4(sc, port, v | EHCI_PS_PEC);
3407                         break;
3408                 case UHF_C_PORT_SUSPEND:
3409                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3410                         break;
3411                 case UHF_C_PORT_OVER_CURRENT:
3412                         EOWRITE4(sc, port, v | EHCI_PS_OCC);
3413                         break;
3414                 case UHF_C_PORT_RESET:
3415                         sc->sc_isreset = 0;
3416                         break;
3417                 default:
3418                         err = USB_ERR_IOERROR;
3419                         goto done;
3420                 }
3421                 break;
3422         case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3423                 if ((value & 0xff) != 0) {
3424                         err = USB_ERR_IOERROR;
3425                         goto done;
3426                 }
3427                 v = EREAD4(sc, EHCI_HCSPARAMS);
3428
3429                 sc->sc_hub_desc.hubd = ehci_hubd;
3430                 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3431
3432                 if (EHCI_HCS_PPC(v))
3433                         i = UHD_PWR_INDIVIDUAL;
3434                 else
3435                         i = UHD_PWR_NO_SWITCH;
3436
3437                 if (EHCI_HCS_P_INDICATOR(v))
3438                         i |= UHD_PORT_IND;
3439
3440                 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3441                 /* XXX can't find out? */
3442                 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 200;
3443                 /* XXX don't know if ports are removable or not */
3444                 sc->sc_hub_desc.hubd.bDescLength =
3445                     8 + ((sc->sc_noport + 7) / 8);
3446                 len = sc->sc_hub_desc.hubd.bDescLength;
3447                 break;
3448         case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3449                 len = 16;
3450                 memset(sc->sc_hub_desc.temp, 0, 16);
3451                 break;
3452         case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3453                 DPRINTFN(9, "get port status i=%d\n",
3454                     index);
3455                 if ((index < 1) ||
3456                     (index > sc->sc_noport)) {
3457                         err = USB_ERR_IOERROR;
3458                         goto done;
3459                 }
3460                 v = EOREAD4(sc, EHCI_PORTSC(index));
3461                 DPRINTFN(9, "port status=0x%04x\n", v);
3462                 if (sc->sc_flags & (EHCI_SCFLG_FORCESPEED | EHCI_SCFLG_TT)) {
3463                         if ((v & 0xc000000) == 0x8000000)
3464                                 i = UPS_HIGH_SPEED;
3465                         else if ((v & 0xc000000) == 0x4000000)
3466                                 i = UPS_LOW_SPEED;
3467                         else
3468                                 i = 0;
3469                 } else {
3470                         i = UPS_HIGH_SPEED;
3471                 }
3472                 if (v & EHCI_PS_CS)
3473                         i |= UPS_CURRENT_CONNECT_STATUS;
3474                 if (v & EHCI_PS_PE)
3475                         i |= UPS_PORT_ENABLED;
3476                 if ((v & EHCI_PS_SUSP) && !(v & EHCI_PS_FPR))
3477                         i |= UPS_SUSPEND;
3478                 if (v & EHCI_PS_OCA)
3479                         i |= UPS_OVERCURRENT_INDICATOR;
3480                 if (v & EHCI_PS_PR)
3481                         i |= UPS_RESET;
3482                 if (v & EHCI_PS_PP)
3483                         i |= UPS_PORT_POWER;
3484                 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3485                 i = 0;
3486                 if (v & EHCI_PS_CSC)
3487                         i |= UPS_C_CONNECT_STATUS;
3488                 if (v & EHCI_PS_PEC)
3489                         i |= UPS_C_PORT_ENABLED;
3490                 if (v & EHCI_PS_OCC)
3491                         i |= UPS_C_OVERCURRENT_INDICATOR;
3492                 if (v & EHCI_PS_FPR)
3493                         i |= UPS_C_SUSPEND;
3494                 if (sc->sc_isreset)
3495                         i |= UPS_C_PORT_RESET;
3496                 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3497                 len = sizeof(sc->sc_hub_desc.ps);
3498                 break;
3499         case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3500                 err = USB_ERR_IOERROR;
3501                 goto done;
3502         case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3503                 break;
3504         case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3505                 if ((index < 1) ||
3506                     (index > sc->sc_noport)) {
3507                         err = USB_ERR_IOERROR;
3508                         goto done;
3509                 }
3510                 port = EHCI_PORTSC(index);
3511                 v = EOREAD4(sc, port) & ~EHCI_PS_CLEAR;
3512                 switch (value) {
3513                 case UHF_PORT_ENABLE:
3514                         EOWRITE4(sc, port, v | EHCI_PS_PE);
3515                         break;
3516                 case UHF_PORT_SUSPEND:
3517                         EOWRITE4(sc, port, v | EHCI_PS_SUSP);
3518                         break;
3519                 case UHF_PORT_RESET:
3520                         DPRINTFN(6, "reset port %d\n", index);
3521 #ifdef USB_DEBUG
3522                         if (ehcinohighspeed) {
3523                                 /*
3524                                  * Connect USB device to companion
3525                                  * controller.
3526                                  */
3527                                 ehci_disown(sc, index, 1);
3528                                 break;
3529                         }
3530 #endif
3531                         if (EHCI_PS_IS_LOWSPEED(v) &&
3532                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3533                                 /* Low speed device, give up ownership. */
3534                                 ehci_disown(sc, index, 1);
3535                                 break;
3536                         }
3537                         /* Start reset sequence. */
3538                         v &= ~(EHCI_PS_PE | EHCI_PS_PR);
3539                         EOWRITE4(sc, port, v | EHCI_PS_PR);
3540
3541                         /* Wait for reset to complete. */
3542                         usb_pause_mtx(&sc->sc_bus.bus_lock,
3543                             USB_MS_TO_TICKS(usb_port_root_reset_delay));
3544
3545                         /* Terminate reset sequence. */
3546                         if (!(sc->sc_flags & EHCI_SCFLG_NORESTERM))
3547                                 EOWRITE4(sc, port, v);
3548
3549                         /* Wait for HC to complete reset. */
3550                         usb_pause_mtx(&sc->sc_bus.bus_lock,
3551                             USB_MS_TO_TICKS(EHCI_PORT_RESET_COMPLETE));
3552
3553                         v = EOREAD4(sc, port);
3554                         DPRINTF("ehci after reset, status=0x%08x\n", v);
3555                         if (v & EHCI_PS_PR) {
3556                                 device_printf(sc->sc_bus.bdev,
3557                                     "port reset timeout\n");
3558                                 err = USB_ERR_TIMEOUT;
3559                                 goto done;
3560                         }
3561                         if (!(v & EHCI_PS_PE) &&
3562                             (sc->sc_flags & EHCI_SCFLG_TT) == 0) {
3563                                 /* Not a high speed device, give up ownership.*/
3564                                 ehci_disown(sc, index, 0);
3565                                 break;
3566                         }
3567                         sc->sc_isreset = 1;
3568                         DPRINTF("ehci port %d reset, status = 0x%08x\n",
3569                             index, v);
3570                         break;
3571
3572                 case UHF_PORT_POWER:
3573                         DPRINTFN(3, "set port power %d\n", index);
3574                         EOWRITE4(sc, port, v | EHCI_PS_PP);
3575                         break;
3576
3577                 case UHF_PORT_TEST:
3578                         DPRINTFN(3, "set port test %d\n", index);
3579                         break;
3580
3581                 case UHF_PORT_INDICATOR:
3582                         DPRINTFN(3, "set port ind %d\n", index);
3583                         EOWRITE4(sc, port, v | EHCI_PS_PIC);
3584                         break;
3585
3586                 default:
3587                         err = USB_ERR_IOERROR;
3588                         goto done;
3589                 }
3590                 break;
3591         case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3592         case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3593         case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3594         case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3595                 break;
3596         default:
3597                 err = USB_ERR_IOERROR;
3598                 goto done;
3599         }
3600 done:
3601         *plength = len;
3602         *pptr = ptr;
3603         return (err);
3604 }
3605
3606 static void
3607 ehci_xfer_setup(struct usb_setup_params *parm)
3608 {
3609         struct usb_page_search page_info;
3610         struct usb_page_cache *pc;
3611         ehci_softc_t *sc;
3612         struct usb_xfer *xfer;
3613         void *last_obj;
3614         uint32_t nqtd;
3615         uint32_t nqh;
3616         uint32_t nsitd;
3617         uint32_t nitd;
3618         uint32_t n;
3619
3620         sc = EHCI_BUS2SC(parm->udev->bus);
3621         xfer = parm->curr_xfer;
3622
3623         nqtd = 0;
3624         nqh = 0;
3625         nsitd = 0;
3626         nitd = 0;
3627
3628         /*
3629          * compute maximum number of some structures
3630          */
3631         if (parm->methods == &ehci_device_ctrl_methods) {
3632
3633                 /*
3634                  * The proof for the "nqtd" formula is illustrated like
3635                  * this:
3636                  *
3637                  * +------------------------------------+
3638                  * |                                    |
3639                  * |         |remainder ->              |
3640                  * |   +-----+---+                      |
3641                  * |   | xxx | x | frm 0                |
3642                  * |   +-----+---++                     |
3643                  * |   | xxx | xx | frm 1               |
3644                  * |   +-----+----+                     |
3645                  * |            ...                     |
3646                  * +------------------------------------+
3647                  *
3648                  * "xxx" means a completely full USB transfer descriptor
3649                  *
3650                  * "x" and "xx" means a short USB packet
3651                  *
3652                  * For the remainder of an USB transfer modulo
3653                  * "max_data_length" we need two USB transfer descriptors.
3654                  * One to transfer the remaining data and one to finalise
3655                  * with a zero length packet in case the "force_short_xfer"
3656                  * flag is set. We only need two USB transfer descriptors in
3657                  * the case where the transfer length of the first one is a
3658                  * factor of "max_frame_size". The rest of the needed USB
3659                  * transfer descriptors is given by the buffer size divided
3660                  * by the maximum data payload.
3661                  */
3662                 parm->hc_max_packet_size = 0x400;
3663                 parm->hc_max_packet_count = 1;
3664                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3665                 xfer->flags_int.bdma_enable = 1;
3666
3667                 usbd_transfer_setup_sub(parm);
3668
3669                 nqh = 1;
3670                 nqtd = ((2 * xfer->nframes) + 1 /* STATUS */
3671                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3672         } else if (parm->methods == &ehci_device_bulk_methods) {
3673
3674                 parm->hc_max_packet_size = 0x400;
3675                 parm->hc_max_packet_count = 1;
3676                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3677                 xfer->flags_int.bdma_enable = 1;
3678
3679                 usbd_transfer_setup_sub(parm);
3680
3681                 nqh = 1;
3682                 nqtd = ((2 * xfer->nframes)
3683                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3684
3685         } else if (parm->methods == &ehci_device_intr_methods) {
3686
3687                 if (parm->speed == USB_SPEED_HIGH) {
3688                         parm->hc_max_packet_size = 0x400;
3689                         parm->hc_max_packet_count = 3;
3690                 } else if (parm->speed == USB_SPEED_FULL) {
3691                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME;
3692                         parm->hc_max_packet_count = 1;
3693                 } else {
3694                         parm->hc_max_packet_size = USB_FS_BYTES_PER_HS_UFRAME / 8;
3695                         parm->hc_max_packet_count = 1;
3696                 }
3697
3698                 parm->hc_max_frame_size = EHCI_QTD_PAYLOAD_MAX;
3699                 xfer->flags_int.bdma_enable = 1;
3700
3701                 usbd_transfer_setup_sub(parm);
3702
3703                 nqh = 1;
3704                 nqtd = ((2 * xfer->nframes)
3705                     + (xfer->max_data_length / xfer->max_hc_frame_size));
3706
3707         } else if (parm->methods == &ehci_device_isoc_fs_methods) {
3708
3709                 parm->hc_max_packet_size = 0x3FF;
3710                 parm->hc_max_packet_count = 1;
3711                 parm->hc_max_frame_size = 0x3FF;
3712                 xfer->flags_int.bdma_enable = 1;
3713
3714                 usbd_transfer_setup_sub(parm);
3715
3716                 nsitd = xfer->nframes;
3717
3718         } else if (parm->methods == &ehci_device_isoc_hs_methods) {
3719
3720                 parm->hc_max_packet_size = 0x400;
3721                 parm->hc_max_packet_count = 3;
3722                 parm->hc_max_frame_size = 0xC00;
3723                 xfer->flags_int.bdma_enable = 1;
3724
3725                 usbd_transfer_setup_sub(parm);
3726
3727                 nitd = ((xfer->nframes + 7) / 8) <<
3728                     usbd_xfer_get_fps_shift(xfer);
3729
3730         } else {
3731
3732                 parm->hc_max_packet_size = 0x400;
3733                 parm->hc_max_packet_count = 1;
3734                 parm->hc_max_frame_size = 0x400;
3735
3736                 usbd_transfer_setup_sub(parm);
3737         }
3738
3739 alloc_dma_set:
3740
3741         if (parm->err) {
3742                 return;
3743         }
3744         /*
3745          * Allocate queue heads and transfer descriptors
3746          */
3747         last_obj = NULL;
3748
3749         if (usbd_transfer_setup_sub_malloc(
3750             parm, &pc, sizeof(ehci_itd_t),
3751             EHCI_ITD_ALIGN, nitd)) {
3752                 parm->err = USB_ERR_NOMEM;
3753                 return;
3754         }
3755         if (parm->buf) {
3756                 for (n = 0; n != nitd; n++) {
3757                         ehci_itd_t *td;
3758
3759                         usbd_get_page(pc + n, 0, &page_info);
3760
3761                         td = page_info.buffer;
3762
3763                         /* init TD */
3764                         td->itd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_ITD);
3765                         td->obj_next = last_obj;
3766                         td->page_cache = pc + n;
3767
3768                         last_obj = td;
3769
3770                         usb_pc_cpu_flush(pc + n);
3771                 }
3772         }
3773         if (usbd_transfer_setup_sub_malloc(
3774             parm, &pc, sizeof(ehci_sitd_t),
3775             EHCI_SITD_ALIGN, nsitd)) {
3776                 parm->err = USB_ERR_NOMEM;
3777                 return;
3778         }
3779         if (parm->buf) {
3780                 for (n = 0; n != nsitd; n++) {
3781                         ehci_sitd_t *td;
3782
3783                         usbd_get_page(pc + n, 0, &page_info);
3784
3785                         td = page_info.buffer;
3786
3787                         /* init TD */
3788                         td->sitd_self = htohc32(sc, page_info.physaddr | EHCI_LINK_SITD);
3789                         td->obj_next = last_obj;
3790                         td->page_cache = pc + n;
3791
3792                         last_obj = td;
3793
3794                         usb_pc_cpu_flush(pc + n);
3795                 }
3796         }
3797         if (usbd_transfer_setup_sub_malloc(
3798             parm, &pc, sizeof(ehci_qtd_t),
3799             EHCI_QTD_ALIGN, nqtd)) {
3800                 parm->err = USB_ERR_NOMEM;
3801                 return;
3802         }
3803         if (parm->buf) {
3804                 for (n = 0; n != nqtd; n++) {
3805                         ehci_qtd_t *qtd;
3806
3807                         usbd_get_page(pc + n, 0, &page_info);
3808
3809                         qtd = page_info.buffer;
3810
3811                         /* init TD */
3812                         qtd->qtd_self = htohc32(sc, page_info.physaddr);
3813                         qtd->obj_next = last_obj;
3814                         qtd->page_cache = pc + n;
3815
3816                         last_obj = qtd;
3817
3818                         usb_pc_cpu_flush(pc + n);
3819                 }
3820         }
3821         xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3822
3823         last_obj = NULL;
3824
3825         if (usbd_transfer_setup_sub_malloc(
3826             parm, &pc, sizeof(ehci_qh_t),
3827             EHCI_QH_ALIGN, nqh)) {
3828                 parm->err = USB_ERR_NOMEM;
3829                 return;
3830         }
3831         if (parm->buf) {
3832                 for (n = 0; n != nqh; n++) {
3833                         ehci_qh_t *qh;
3834
3835                         usbd_get_page(pc + n, 0, &page_info);
3836
3837                         qh = page_info.buffer;
3838
3839                         /* init QH */
3840                         qh->qh_self = htohc32(sc, page_info.physaddr | EHCI_LINK_QH);
3841                         qh->obj_next = last_obj;
3842                         qh->page_cache = pc + n;
3843
3844                         last_obj = qh;
3845
3846                         usb_pc_cpu_flush(pc + n);
3847                 }
3848         }
3849         xfer->qh_start[xfer->flags_int.curr_dma_set] = last_obj;
3850
3851         if (!xfer->flags_int.curr_dma_set) {
3852                 xfer->flags_int.curr_dma_set = 1;
3853                 goto alloc_dma_set;
3854         }
3855 }
3856
3857 static void
3858 ehci_xfer_unsetup(struct usb_xfer *xfer)
3859 {
3860         return;
3861 }
3862
3863 static void
3864 ehci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3865     struct usb_endpoint *ep)
3866 {
3867         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3868
3869         DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d (%d)\n",
3870             ep, udev->address,
3871             edesc->bEndpointAddress, udev->flags.usb_mode,
3872             sc->sc_addr);
3873
3874         if (udev->device_index != sc->sc_addr) {
3875
3876                 if ((udev->speed != USB_SPEED_HIGH) &&
3877                     ((udev->hs_hub_addr == 0) ||
3878                     (udev->hs_port_no == 0) ||
3879                     (udev->parent_hs_hub == NULL) ||
3880                     (udev->parent_hs_hub->hub == NULL))) {
3881                         /* We need a transaction translator */
3882                         goto done;
3883                 }
3884                 switch (edesc->bmAttributes & UE_XFERTYPE) {
3885                 case UE_CONTROL:
3886                         ep->methods = &ehci_device_ctrl_methods;
3887                         break;
3888                 case UE_INTERRUPT:
3889                         ep->methods = &ehci_device_intr_methods;
3890                         break;
3891                 case UE_ISOCHRONOUS:
3892                         if (udev->speed == USB_SPEED_HIGH) {
3893                                 ep->methods = &ehci_device_isoc_hs_methods;
3894                         } else if (udev->speed == USB_SPEED_FULL) {
3895                                 ep->methods = &ehci_device_isoc_fs_methods;
3896                         }
3897                         break;
3898                 case UE_BULK:
3899                         ep->methods = &ehci_device_bulk_methods;
3900                         break;
3901                 default:
3902                         /* do nothing */
3903                         break;
3904                 }
3905         }
3906 done:
3907         return;
3908 }
3909
3910 static void
3911 ehci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3912 {
3913         /*
3914          * Wait until the hardware has finished any possible use of
3915          * the transfer descriptor(s) and QH
3916          */
3917         *pus = (188);                   /* microseconds */
3918 }
3919
3920 static void
3921 ehci_device_resume(struct usb_device *udev)
3922 {
3923         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3924         struct usb_xfer *xfer;
3925         const struct usb_pipe_methods *methods;
3926
3927         DPRINTF("\n");
3928
3929         USB_BUS_LOCK(udev->bus);
3930
3931         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3932
3933                 if (xfer->xroot->udev == udev) {
3934
3935                         methods = xfer->endpoint->methods;
3936                         KKASSERT(xfer->flags_int.onhwqueue == 0);
3937                         if ((methods == &ehci_device_bulk_methods) ||
3938                             (methods == &ehci_device_ctrl_methods)) {
3939                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3940                                     sc->sc_async_p_last);
3941                         }
3942                         if (methods == &ehci_device_intr_methods) {
3943                                 EHCI_APPEND_QH(xfer->qh_start[xfer->flags_int.curr_dma_set],
3944                                     sc->sc_intr_p_last[xfer->qh_pos]);
3945                         }
3946                         xfer->flags_int.onhwqueue = 1;
3947                 }
3948         }
3949         ehci_ring_doorbell(sc, NULL);
3950
3951         USB_BUS_UNLOCK(udev->bus);
3952
3953         return;
3954 }
3955
3956 static void
3957 ehci_device_suspend(struct usb_device *udev)
3958 {
3959         ehci_softc_t *sc = EHCI_BUS2SC(udev->bus);
3960         struct usb_xfer *xfer;
3961         const struct usb_pipe_methods *methods;
3962
3963         DPRINTF("\n");
3964
3965         USB_BUS_LOCK(udev->bus);
3966
3967         TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3968
3969                 if (xfer->xroot->udev == udev) {
3970                         methods = xfer->endpoint->methods;
3971
3972                         if ((methods == &ehci_device_bulk_methods) ||
3973                             (methods == &ehci_device_ctrl_methods)) {
3974                                 EHCI_REMOVE_QH(sc, xfer->qh_start[xfer->flags_int.curr_dma_set],
3975                                     sc->sc_async_p_last);
3976                         }
3977                         if (methods == &ehci_device_intr_methods) {
3978                                 EHCI_REMOVE_QH(sc, xfer->qh_start[xfer->flags_int.curr_dma_set],
3979                                     sc->sc_intr_p_last[xfer->qh_pos]);
3980                         }
3981                         xfer->flags_int.onhwqueue = 0;
3982                 }
3983         }
3984         ehci_ring_doorbell(sc, NULL);
3985
3986         USB_BUS_UNLOCK(udev->bus);
3987 }
3988
3989 static void
3990 ehci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
3991 {
3992         struct ehci_softc *sc = EHCI_BUS2SC(bus);
3993
3994         switch (state) {
3995         case USB_HW_POWER_SUSPEND:
3996         case USB_HW_POWER_SHUTDOWN:
3997                 ehci_suspend(sc);
3998                 break;
3999         case USB_HW_POWER_RESUME:
4000                 ehci_resume(sc);
4001                 break;
4002         default:
4003                 break;
4004         }
4005 }
4006
4007 static void
4008 ehci_set_hw_power(struct usb_bus *bus)
4009 {
4010         ehci_softc_t *sc = EHCI_BUS2SC(bus);
4011         uint32_t temp;
4012         uint32_t flags;
4013
4014         DPRINTF("\n");
4015
4016         USB_BUS_LOCK(bus);
4017
4018         flags = bus->hw_power_state;
4019
4020         temp = EOREAD4(sc, EHCI_USBCMD);
4021
4022         temp &= ~(EHCI_CMD_ASE | EHCI_CMD_PSE);
4023
4024         if (flags & (USB_HW_POWER_CONTROL |
4025             USB_HW_POWER_BULK)) {
4026                 DPRINTF("Async is active\n");
4027                 temp |= EHCI_CMD_ASE;
4028         }
4029         if (flags & (USB_HW_POWER_INTERRUPT |
4030             USB_HW_POWER_ISOC)) {
4031                 DPRINTF("Periodic is active\n");
4032                 temp |= EHCI_CMD_PSE;
4033         }
4034         EOWRITE4(sc, EHCI_USBCMD, temp);
4035
4036         USB_BUS_UNLOCK(bus);
4037
4038         return;
4039 }
4040
4041 static const struct usb_bus_methods ehci_bus_methods =
4042 {
4043         .endpoint_init = ehci_ep_init,
4044         .xfer_setup = ehci_xfer_setup,
4045         .xfer_unsetup = ehci_xfer_unsetup,
4046         .get_dma_delay = ehci_get_dma_delay,
4047         .device_resume = ehci_device_resume,
4048         .device_suspend = ehci_device_suspend,
4049         .set_hw_power = ehci_set_hw_power,
4050         .set_hw_power_sleep = ehci_set_hw_power_sleep,
4051         .roothub_exec = ehci_roothub_exec,
4052         .xfer_poll = ehci_do_poll,
4053 };