From 258223a30e6f99a6a4cdae5b969947ce79286b78 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Thu, 4 Jun 2009 11:21:05 -0700 Subject: [PATCH] AHCI Driver port from OpenBSD to DragonFly - Initial commit This is the initial work synchronization commit for the AHCI driver port. About 85% of the logic is now working. This also includes initial wiring of interrupt bits and status tests for hot-plug support. --- sys/dev/disk/ahci/Makefile | 10 + sys/dev/disk/ahci/TODO | 16 + sys/dev/disk/ahci/ahci.c | 1824 ++++++++++++++++++++++++++++ sys/dev/disk/ahci/ahci.h | 466 +++++++ sys/dev/disk/ahci/ahci_attach.c | 482 ++++++++ sys/dev/disk/ahci/ahci_cam.c | 998 +++++++++++++++ sys/dev/disk/ahci/ahci_dragonfly.c | 141 +++ sys/dev/disk/ahci/ahci_dragonfly.h | 72 ++ sys/dev/disk/ahci/atascsi.h | 310 +++++ 9 files changed, 4319 insertions(+) create mode 100644 sys/dev/disk/ahci/Makefile create mode 100644 sys/dev/disk/ahci/TODO create mode 100644 sys/dev/disk/ahci/ahci.c create mode 100644 sys/dev/disk/ahci/ahci.h create mode 100644 sys/dev/disk/ahci/ahci_attach.c create mode 100644 sys/dev/disk/ahci/ahci_cam.c create mode 100644 sys/dev/disk/ahci/ahci_dragonfly.c create mode 100644 sys/dev/disk/ahci/ahci_dragonfly.h create mode 100644 sys/dev/disk/ahci/atascsi.h diff --git a/sys/dev/disk/ahci/Makefile b/sys/dev/disk/ahci/Makefile new file mode 100644 index 0000000000..401f6904c8 --- /dev/null +++ b/sys/dev/disk/ahci/Makefile @@ -0,0 +1,10 @@ +# Native AHCI driver, ported from OpenBSD +# +KMOD= ahci + +.PATH: ${.CURDIR} + +SRCS= ahci_dragonfly.c ahci_attach.c ahci_cam.c ahci.c \ + bus_if.h device_if.h pci_if.h opt_scsi.h opt_cam.h + +.include diff --git a/sys/dev/disk/ahci/TODO b/sys/dev/disk/ahci/TODO new file mode 100644 index 0000000000..210c2f535b --- /dev/null +++ b/sys/dev/disk/ahci/TODO @@ -0,0 +1,16 @@ + +DELAY's will tsleep, so interrupts might run. fix poll loop to detect +completion via other interrupts. + +Locking serialize_enter/exit. Lots of recursion. Needs help. Use +lockmgr()? + +Hot plug/unplug SCSI bus rescan sequencing. + +Port multiplier support. + +ATAPI support. + +Simulate various mode pages (serial number access and so forth). + +ESATA issues with MyBook (port fails after ident even at 1.5 GBits). diff --git a/sys/dev/disk/ahci/ahci.c b/sys/dev/disk/ahci/ahci.c new file mode 100644 index 0000000000..0e909f31b3 --- /dev/null +++ b/sys/dev/disk/ahci/ahci.c @@ -0,0 +1,1824 @@ +/* + * Copyright (c) 2006 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * + * Copyright (c) 2009 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $ + */ + +#include "ahci.h" + +int ahci_port_start(struct ahci_port *, int); +int ahci_port_stop(struct ahci_port *, int); +int ahci_port_clo(struct ahci_port *); +int ahci_port_softreset(struct ahci_port *); +int ahci_port_portreset(struct ahci_port *); + +int ahci_load_prdt(struct ahci_ccb *); +void ahci_unload_prdt(struct ahci_ccb *); +static void ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, + int nsegs, int error); +int ahci_poll(struct ahci_ccb *, int, void (*)(void *)); +void ahci_start(struct ahci_ccb *); + +static void ahci_ata_cmd_timeout_unserialized(void *arg); +static void ahci_ata_cmd_timeout(void *arg); + +void ahci_issue_pending_ncq_commands(struct ahci_port *); +void ahci_issue_pending_commands(struct ahci_port *, int); + +u_int32_t ahci_port_intr(struct ahci_port *, u_int32_t); + +struct ahci_ccb *ahci_get_ccb(struct ahci_port *); +void ahci_put_ccb(struct ahci_ccb *); + +struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *); +void ahci_put_err_ccb(struct ahci_ccb *); + +int ahci_port_read_ncq_error(struct ahci_port *, int *); + +struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, bus_dma_tag_t tag); +void ahci_dmamem_free(struct ahci_softc *, struct ahci_dmamem *); +static void ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error); + +void ahci_empty_done(struct ahci_ccb *ccb); +void ahci_ata_cmd_done(struct ahci_ccb *ccb); + +/* Wait for all bits in _b to be cleared */ +#define ahci_pwait_clr(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), 0) + +/* Wait for all bits in _b to be set */ +#define ahci_pwait_set(_ap, _r, _b) ahci_pwait_eq((_ap), (_r), (_b), (_b)) + +int +ahci_init(struct ahci_softc *sc) +{ + u_int32_t cap, pi; + + DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b", + ahci_read(sc, AHCI_REG_GHC), AHCI_FMT_GHC); + + /* save BIOS initialised parameters, enable staggered spin up */ + cap = ahci_read(sc, AHCI_REG_CAP); + cap &= AHCI_REG_CAP_SMPS; + cap |= AHCI_REG_CAP_SSS; + pi = ahci_read(sc, AHCI_REG_PI); + + if (AHCI_REG_GHC_AE & ahci_read(sc, AHCI_REG_GHC)) { + /* reset the controller */ + ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR); + if (ahci_wait_ne(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR, + AHCI_REG_GHC_HR) != 0) { + device_printf(sc->sc_dev, + "unable to reset controller\n"); + return (1); + } + } + + /* enable ahci (global interrupts disabled) */ + ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE); + + /* restore parameters */ + ahci_write(sc, AHCI_REG_CAP, cap); + ahci_write(sc, AHCI_REG_PI, pi); + + return (0); +} + +int +ahci_port_alloc(struct ahci_softc *sc, u_int port) +{ + struct ahci_port *ap; + struct ahci_ccb *ccb; + u_int64_t dva; + u_int32_t cmd; + struct ahci_cmd_hdr *hdr; + struct ahci_cmd_table *table; + int rc = ENOMEM; + int error; + int i; + + ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO); + if (ap == NULL) { + device_printf(sc->sc_dev, + "unable to allocate memory for port %d\n", + port); + goto reterr; + } + + ksnprintf(ap->ap_name, sizeof(ap->ap_name), "%s%d.%d", + device_get_name(sc->sc_dev), + device_get_unit(sc->sc_dev), + port); + sc->sc_ports[port] = ap; + + if (bus_space_subregion(sc->sc_iot, sc->sc_ioh, + AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) { + device_printf(sc->sc_dev, + "unable to create register window for port %d\n", + port); + goto freeport; + } + + ap->ap_sc = sc; + ap->ap_num = port; + TAILQ_INIT(&ap->ap_ccb_free); + TAILQ_INIT(&ap->ap_ccb_pending); + lockinit(&ap->ap_ccb_lock, "ahcipo", 0, 0); + + /* Disable port interrupts */ + ahci_pwrite(ap, AHCI_PREG_IE, 0); + + /* Sec 10.1.2 - deinitialise port if it is already running */ + cmd = ahci_pread(ap, AHCI_PREG_CMD); + if ((cmd & (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR | + AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) || + (ahci_pread(ap, AHCI_PREG_SCTL) & AHCI_PREG_SCTL_DET)) { + int r; + + r = ahci_port_stop(ap, 1); + if (r) { + device_printf(sc->sc_dev, + "unable to disable %s, ignoring port %d\n", + ((r == 2) ? "CR" : "FR"), port); + rc = ENXIO; + goto freeport; + } + + /* Write DET to zero */ + ahci_pwrite(ap, AHCI_PREG_SCTL, 0); + } + + /* Allocate RFIS */ + ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sc->sc_tag_rfis); + if (ap->ap_dmamem_rfis == NULL) { + kprintf("NORFIS\n"); + goto nomem; + } + + /* Setup RFIS base address */ + ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis); + dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis); + ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32)); + ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva); + + /* Enable FIS reception and activate port. */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD; + ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE); + + /* Check whether port activated. Skip it if not. */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + if ((cmd & AHCI_PREG_CMD_FRE) == 0) { + kprintf("NOT-ACTIVATED\n"); + rc = ENXIO; + goto freeport; + } + + /* Allocate a CCB for each command slot */ + ap->ap_ccbs = kmalloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF, + M_WAITOK | M_ZERO); + if (ap->ap_ccbs == NULL) { + device_printf(sc->sc_dev, + "unable to allocate command list for port %d\n", + port); + goto freeport; + } + + /* Command List Structures and Command Tables */ + ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, sc->sc_tag_cmdh); + ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, sc->sc_tag_cmdt); + if (ap->ap_dmamem_cmd_table == NULL || + ap->ap_dmamem_cmd_list == NULL) { +nomem: + device_printf(sc->sc_dev, + "unable to allocate DMA memory for port %d\n", + port); + goto freeport; + } + + /* Setup command list base address */ + dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list); + ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32)); + ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva); + + /* Split CCB allocation into CCBs and assign to command header/table */ + hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list); + table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table); + for (i = 0; i < sc->sc_ncmds; i++) { + ccb = &ap->ap_ccbs[i]; + + error = bus_dmamap_create(sc->sc_tag_data, BUS_DMA_ALLOCNOW, + &ccb->ccb_dmamap); + if (error) { + device_printf(sc->sc_dev, + "unable to create dmamap for port %d " + "ccb %d\n", port, i); + goto freeport; + } + + callout_init(&ccb->ccb_timeout); + ccb->ccb_slot = i; + ccb->ccb_port = ap; + ccb->ccb_cmd_hdr = &hdr[i]; + ccb->ccb_cmd_table = &table[i]; + dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) + + ccb->ccb_slot * sizeof(struct ahci_cmd_table); + ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32)); + ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva); + + ccb->ccb_xa.fis = + (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; + ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd; + ccb->ccb_xa.tag = i; + + ccb->ccb_xa.ata_put_xfer = ahci_ata_put_xfer; + + ccb->ccb_xa.state = ATA_S_COMPLETE; + ahci_put_ccb(ccb); + } + + /* Wait for ICC change to complete */ + ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC); + + /* Reset port */ + rc = ahci_port_portreset(ap); + switch (rc) { + case ENODEV: + switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { + case AHCI_PREG_SSTS_DET_DEV_NE: + device_printf(sc->sc_dev, + "device not communicating on port %d\n", + port); + break; + case AHCI_PREG_SSTS_DET_PHYOFFLINE: + device_printf(sc->sc_dev, + "PHY offline on port %d\n", + port); + break; + default: + device_printf(sc->sc_dev, + "no device detected on port %d\n", + port); + break; + } + break; + + case EBUSY: + device_printf(sc->sc_dev, + "device on port %d didn't come ready, " + "TFD: 0x%b\n", + port, + ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); + + /* Try a soft reset to clear busy */ + rc = ahci_port_softreset(ap); + if (rc) { + device_printf(sc->sc_dev, + "unable to communicate " + "with device on port %d\n", + port); + goto freeport; + } + break; + + default: + break; + } + + /* + * Enable command transfers on the port if a device was detected. + * Otherwise leave them disabled but leave the port structure + * intact so we get hot-plug interrupts. + */ + if (rc == 0) { + DPRINTF(AHCI_D_VERBOSE, "%s: detected device on port %d\n", + device_get_name(sc->sc_dev), port); + if (ahci_port_start(ap, 0)) { + device_printf(sc->sc_dev, + "failed to start command DMA on port %d, " + "disabling\n", port); + rc = ENXIO; /* couldn't start port */ + } + } + + /* + * A missing or busy device is not fatal for the purposes of + * port allocation. We still want to detect hot-plug + * state changes. + */ + if (rc == ENODEV || rc == EBUSY) { + rc = 0; + } + + /* Flush interrupts for port */ + ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); + ahci_write(sc, AHCI_REG_IS, 1 << port); + + /* Enable port interrupts */ + ahci_pwrite(ap, AHCI_PREG_IE, + AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | + AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | + AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE | + AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE | +#ifdef AHCI_COALESCE + ((sc->sc_ccc_ports & (1 << port)) ? + 0 : (AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE)) +#else + AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE +#endif + ); + +freeport: + if (rc != 0) + ahci_port_free(sc, port); +reterr: + return (rc); +} + +void +ahci_port_free(struct ahci_softc *sc, u_int port) +{ + struct ahci_port *ap = sc->sc_ports[port]; + struct ahci_ccb *ccb; + + /* Ensure port is disabled and its interrupts are flushed */ + if (ap->ap_sc) { + ahci_pwrite(ap, AHCI_PREG_CMD, 0); + ahci_pwrite(ap, AHCI_PREG_IE, 0); + ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); + ahci_write(sc, AHCI_REG_IS, 1 << port); + } + + if (ap->ap_ccbs) { + while ((ccb = ahci_get_ccb(ap)) != NULL) { + if (ccb->ccb_dmamap) { + bus_dmamap_destroy(sc->sc_tag_data, + ccb->ccb_dmamap); + ccb->ccb_dmamap = NULL; + } + } + kfree(ap->ap_ccbs, M_DEVBUF); + ap->ap_ccbs = NULL; + } + + if (ap->ap_dmamem_cmd_list) { + ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list); + ap->ap_dmamem_cmd_list = NULL; + } + if (ap->ap_dmamem_rfis) { + ahci_dmamem_free(sc, ap->ap_dmamem_rfis); + ap->ap_dmamem_rfis = NULL; + } + if (ap->ap_dmamem_cmd_table) { + ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table); + ap->ap_dmamem_cmd_table = NULL; + } + + /* bus_space(9) says we dont free the subregions handle */ + + kfree(ap, M_DEVBUF); + sc->sc_ports[port] = NULL; +} + +int +ahci_port_start(struct ahci_port *ap, int fre_only) +{ + u_int32_t r; + + /* Turn on FRE (and ST) */ + r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + r |= AHCI_PREG_CMD_FRE; + if (!fre_only) + r |= AHCI_PREG_CMD_ST; + ahci_pwrite(ap, AHCI_PREG_CMD, r); + +#ifdef AHCI_COALESCE + /* (Re-)enable coalescing on the port. */ + if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { + ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num); + ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, + ap->ap_sc->sc_ccc_ports_cur); + } +#endif + + if (!(ap->ap_sc->sc_flags & AHCI_F_IGN_FR)) { + /* Wait for FR to come on */ + if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) + return (2); + } + + /* Wait for CR to come on */ + if (!fre_only && ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) + return (1); + + return (0); +} + +int +ahci_port_stop(struct ahci_port *ap, int stop_fis_rx) +{ + u_int32_t r; + +#ifdef AHCI_COALESCE + /* Disable coalescing on the port while it is stopped. */ + if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) { + ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num); + ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS, + ap->ap_sc->sc_ccc_ports_cur); + } +#endif + + /* Turn off ST (and FRE) */ + r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + r &= ~AHCI_PREG_CMD_ST; + if (stop_fis_rx) + r &= ~AHCI_PREG_CMD_FRE; + ahci_pwrite(ap, AHCI_PREG_CMD, r); + + /* Wait for CR to go off */ + if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) + return (1); + + /* Wait for FR to go off */ + if (stop_fis_rx && ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) + return (2); + + return (0); +} + +/* AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ} */ +int +ahci_port_clo(struct ahci_port *ap) +{ + struct ahci_softc *sc = ap->ap_sc; + u_int32_t cmd; + + /* Only attempt CLO if supported by controller */ + if ((ahci_read(sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) == 0) + return (1); + + /* Issue CLO */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; +#ifdef DIAGNOSTIC + if (cmd & AHCI_PREG_CMD_ST) { + kprintf("%s: CLO requested while port running\n", + PORTNAME(ap)); + } +#endif + ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO); + + /* Wait for completion */ + if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) { + kprintf("%s: CLO did not complete\n", PORTNAME(ap)); + return (1); + } + + return (0); +} + +/* AHCI soft reset, Section 10.4.1 */ +int +ahci_port_softreset(struct ahci_port *ap) +{ + struct ahci_ccb *ccb = NULL; + struct ahci_cmd_hdr *cmd_slot; + u_int8_t *fis; + int rc = EIO; + u_int32_t cmd; + + DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap)); + + crit_enter(); + + /* Save previous command register state */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + + /* Idle port */ + if (ahci_port_stop(ap, 0)) { + kprintf("%s: failed to stop port, cannot softreset\n", + PORTNAME(ap)); + goto err; + } + + /* Request CLO if device appears hung */ + if (ahci_pread(ap, AHCI_PREG_TFD) & + (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { + ahci_port_clo(ap); + } + + /* Clear port errors to permit TFD transfer */ + ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); + + /* Restart port */ + if (ahci_port_start(ap, 0)) { + kprintf("%s: failed to start port, cannot softreset\n", + PORTNAME(ap)); + goto err; + } + + /* Check whether CLO worked */ + if (ahci_pwait_clr(ap, AHCI_PREG_TFD, + AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { + kprintf("%s: CLO %s, need port reset\n", + PORTNAME(ap), + (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) + ? "failed" : "unsupported"); + rc = EBUSY; + goto err; + } + + /* Prep first D2H command with SRST feature & clear busy/reset flags */ + ccb = ahci_get_err_ccb(ap); + cmd_slot = ccb->ccb_cmd_hdr; + bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table)); + + fis = ccb->ccb_cmd_table->cfis; + fis[0] = 0x27; /* Host to device */ + fis[15] = 0x04; /* SRST DEVCTL */ + + cmd_slot->prdtl = 0; + cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */ + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */ + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); /* Write */ + + ccb->ccb_xa.state = ATA_S_PENDING; + if (ahci_poll(ccb, hz, NULL) != 0) + goto err; + + /* Prep second D2H command to read status and complete reset sequence */ + fis[0] = 0x27; /* Host to device */ + fis[15] = 0; + + cmd_slot->prdtl = 0; + cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); + + ccb->ccb_xa.state = ATA_S_PENDING; + if (ahci_poll(ccb, hz, NULL) != 0) + goto err; + + if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | + AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) { + kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n", + PORTNAME(ap), + ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); + rc = EBUSY; + goto err; + } + + rc = 0; +err: + if (ccb != NULL) { + /* Abort our command, if it failed, by stopping command DMA. */ + if (rc != 0 && (ap->ap_active & (1 << ccb->ccb_slot))) { + kprintf("%s: stopping the port, softreset slot " + "%d was still active.\n", + PORTNAME(ap), + ccb->ccb_slot); + ahci_port_stop(ap, 0); + } + ccb->ccb_xa.state = ATA_S_ERROR; + ahci_put_err_ccb(ccb); + } + + /* Restore saved CMD register state */ + ahci_pwrite(ap, AHCI_PREG_CMD, cmd); + + crit_exit(); + + return (rc); +} + +/* AHCI port reset, Section 10.4.2 */ +int +ahci_port_portreset(struct ahci_port *ap) +{ + u_int32_t cmd, r; + int rc; + + DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap)); + + /* Save previous command register state */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + + /* Clear ST, ignoring failure */ + ahci_port_stop(ap, 0); + + /* Perform device detection */ + ap->ap_ata.ap_type = ATA_PORT_T_NONE; + ahci_pwrite(ap, AHCI_PREG_SCTL, 0); + DELAY(10000); + r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT; + + if (AhciForceGen1 & (1 << ap->ap_num)) { + kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap)); + r |= AHCI_PREG_SCTL_SPD_GEN1; + } else { + r |= AHCI_PREG_SCTL_SPD_ANY; + } + ahci_pwrite(ap, AHCI_PREG_SCTL, r); + DELAY(10000); /* wait at least 1ms for COMRESET to be sent */ + r &= ~AHCI_PREG_SCTL_DET_INIT; + r |= AHCI_PREG_SCTL_DET_NONE; + ahci_pwrite(ap, AHCI_PREG_SCTL, r); + DELAY(10000); + + /* Wait for device to be detected and communications established */ + if (ahci_pwait_eq(ap, AHCI_PREG_SSTS, AHCI_PREG_SSTS_DET, + AHCI_PREG_SSTS_DET_DEV)) { + rc = ENODEV; + goto err; + } + + /* Clear SERR (incl X bit), so TFD can update */ + ahci_pwrite(ap, AHCI_PREG_SERR, ahci_pread(ap, AHCI_PREG_SERR)); + + /* Wait for device to become ready */ + /* XXX maybe more than the default wait is appropriate here? */ + if (ahci_pwait_clr(ap, AHCI_PREG_TFD, AHCI_PREG_TFD_STS_BSY | + AHCI_PREG_TFD_STS_DRQ | AHCI_PREG_TFD_STS_ERR)) { + rc = EBUSY; + kprintf("%s: Device will not come ready 0x%b\n", + PORTNAME(ap), + ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS); + goto err; + } + + /* + * Figure out if we are a ATAPI or DISK device + */ + u_int32_t sig; + sig = ahci_pread(ap, AHCI_PREG_SIG); + if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) { + ap->ap_ata.ap_type = ATA_PORT_T_ATAPI; + } else { + ap->ap_ata.ap_type = ATA_PORT_T_DISK; + } + rc = 0; +err: + /* Restore preserved port state */ + ahci_pwrite(ap, AHCI_PREG_CMD, cmd); + + return (rc); +} + +int +ahci_load_prdt(struct ahci_ccb *ccb) +{ + struct ahci_port *ap = ccb->ccb_port; + struct ahci_softc *sc = ap->ap_sc; + struct ata_xfer *xa = &ccb->ccb_xa; + struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt; + bus_dmamap_t dmap = ccb->ccb_dmamap; + struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr; + int error; + + if (xa->datalen == 0) { + ccb->ccb_cmd_hdr->prdtl = 0; + return (0); + } + + error = bus_dmamap_load(sc->sc_tag_data, dmap, + xa->data, xa->datalen, + ahci_load_prdt_callback, + &prdt, + ((xa->flags & ATA_F_NOWAIT) ? + BUS_DMA_NOWAIT : BUS_DMA_WAITOK)); + if (error != 0) { + kprintf("%s: error %d loading dmamap\n", PORTNAME(ap), error); + return (1); + } + if (xa->flags & ATA_F_PIO) + prdt->flags |= htole32(AHCI_PRDT_FLAG_INTR); + + cmd_slot->prdtl = htole16(prdt - ccb->ccb_cmd_table->prdt + 1); + + bus_dmamap_sync(sc->sc_tag_data, dmap, + (xa->flags & ATA_F_READ) ? + BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); + + return (0); + +#ifdef DIAGNOSTIC +diagerr: + bus_dmamap_unload(sc->sc_tag_data, dmap); + return (1); +#endif +} + +/* + * Callback from BUSDMA system to load the segment list. The passed segment + * list is a temporary structure. + */ +static +void +ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, int nsegs, + int error) +{ + struct ahci_prdt *prd = *(void **)info; + u_int64_t addr; + + KKASSERT(nsegs <= AHCI_MAX_PRDT); + + while (nsegs) { + addr = segs->ds_addr; + prd->dba_hi = htole32((u_int32_t)(addr >> 32)); + prd->dba_lo = htole32((u_int32_t)addr); +#ifdef DIAGNOSTIC + KKASSERT((addr & 1) == 0); + KKASSERT((segs->ds_len & 1) == 0); +#endif + prd->flags = htole32(segs->ds_len - 1); + --nsegs; + if (nsegs) + ++prd; + ++segs; + } + *(void **)info = prd; /* return last valid segment */ +} + +void +ahci_unload_prdt(struct ahci_ccb *ccb) +{ + struct ahci_port *ap = ccb->ccb_port; + struct ahci_softc *sc = ap->ap_sc; + struct ata_xfer *xa = &ccb->ccb_xa; + bus_dmamap_t dmap = ccb->ccb_dmamap; + + if (xa->datalen != 0) { + bus_dmamap_sync(sc->sc_tag_data, dmap, + (xa->flags & ATA_F_READ) ? + BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); + + bus_dmamap_unload(sc->sc_tag_data, dmap); + + if (ccb->ccb_xa.flags & ATA_F_NCQ) + xa->resid = 0; + else + xa->resid = xa->datalen - + le32toh(ccb->ccb_cmd_hdr->prdbc); + } +} + +int +ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) +{ + struct ahci_port *ap = ccb->ccb_port; + + crit_enter(); + ahci_start(ccb); + do { + if (ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS) & + (1 << ccb->ccb_slot)) { + crit_exit(); + return (0); + } + DELAY(1000000 / hz); + } while (--timeout > 0); + kprintf("timeout ccb state %d\n", ccb->ccb_xa.state); + + if (timeout_fn != NULL) + timeout_fn(ccb); + crit_exit(); + + return (1); +} + +void +ahci_start(struct ahci_ccb *ccb) +{ + struct ahci_port *ap = ccb->ccb_port; + struct ahci_softc *sc = ap->ap_sc; + + KKASSERT(ccb->ccb_xa.state == ATA_S_PENDING); + + /* Zero transferred byte count before transfer */ + ccb->ccb_cmd_hdr->prdbc = 0; + + /* Sync command list entry and corresponding command table entry */ + bus_dmamap_sync(sc->sc_tag_cmdh, + AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), + BUS_DMASYNC_PREWRITE); + bus_dmamap_sync(sc->sc_tag_cmdt, + AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), + BUS_DMASYNC_PREWRITE); + + /* Prepare RFIS area for write by controller */ + bus_dmamap_sync(sc->sc_tag_rfis, + AHCI_DMA_MAP(ap->ap_dmamem_rfis), + BUS_DMASYNC_PREREAD); + + if (ccb->ccb_xa.flags & ATA_F_NCQ) { + /* Issue NCQ commands only when there are no outstanding + * standard commands. */ + if (ap->ap_active != 0 || !TAILQ_EMPTY(&ap->ap_ccb_pending)) + TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); + else { + KKASSERT(ap->ap_active_cnt == 0); + ap->ap_sactive |= (1 << ccb->ccb_slot); + ccb->ccb_xa.state = ATA_S_ONCHIP; + ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot); + ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); + } + } else { + /* Wait for all NCQ commands to finish before issuing standard + * command. */ + if (ap->ap_sactive != 0 || ap->ap_active_cnt == 2) + TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry); + else if (ap->ap_active_cnt < 2) { + ap->ap_active |= 1 << ccb->ccb_slot; + ccb->ccb_xa.state = ATA_S_ONCHIP; + ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot); + ap->ap_active_cnt++; + } + } +} + +void +ahci_issue_pending_ncq_commands(struct ahci_port *ap) +{ + struct ahci_ccb *nextccb; + u_int32_t sact_change = 0; + + KKASSERT(ap->ap_active_cnt == 0); + + nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); + if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ)) + return; + + /* Start all the NCQ commands at the head of the pending list. */ + do { + TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); + sact_change |= 1 << nextccb->ccb_slot; + nextccb->ccb_xa.state = ATA_S_ONCHIP; + nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); + } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)); + + ap->ap_sactive |= sact_change; + ahci_pwrite(ap, AHCI_PREG_SACT, sact_change); + ahci_pwrite(ap, AHCI_PREG_CI, sact_change); + + return; +} + +void +ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) +{ + struct ahci_ccb *nextccb; + + nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); + if (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ)) { + KKASSERT(last_was_ncq == 0); /* otherwise it should have + * been started already. */ + + /* Issue NCQ commands only when there are no outstanding + * standard commands. */ + ap->ap_active_cnt--; + if (ap->ap_active == 0) + ahci_issue_pending_ncq_commands(ap); + else + KKASSERT(ap->ap_active_cnt == 1); + } else if (nextccb) { + if (ap->ap_sactive != 0 || last_was_ncq) + KKASSERT(ap->ap_active_cnt == 0); + + /* Wait for all NCQ commands to finish before issuing standard + * command. */ + if (ap->ap_sactive != 0) + return; + + /* Keep up to 2 standard commands on-chip at a time. */ + do { + TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry); + ap->ap_active |= 1 << nextccb->ccb_slot; + nextccb->ccb_xa.state = ATA_S_ONCHIP; + ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot); + if (last_was_ncq) + ap->ap_active_cnt++; + if (ap->ap_active_cnt == 2) + break; + KKASSERT(ap->ap_active_cnt == 1); + nextccb = TAILQ_FIRST(&ap->ap_ccb_pending); + } while (nextccb && !(nextccb->ccb_xa.flags & ATA_F_NCQ)); + } else if (!last_was_ncq) { + KKASSERT(ap->ap_active_cnt == 1 || ap->ap_active_cnt == 2); + + /* Standard command finished, none waiting to start. */ + ap->ap_active_cnt--; + } else { + KKASSERT(ap->ap_active_cnt == 0); + + /* NCQ command finished. */ + } +} + +void +ahci_intr(void *arg) +{ + struct ahci_softc *sc = arg; + u_int32_t is, ack = 0; + int port; + + /* Read global interrupt status */ + is = ahci_read(sc, AHCI_REG_IS); + if (is == 0 || is == 0xffffffff) + return; + ack = is; + +#ifdef AHCI_COALESCE + /* Check coalescing interrupt first */ + if (is & sc->sc_ccc_mask) { + DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n", + DEVNAME(sc)); + is &= ~sc->sc_ccc_mask; + is |= sc->sc_ccc_ports_cur; + } +#endif + + /* Process interrupts for each port */ + while (is) { + port = ffs(is) - 1; + if (sc->sc_ports[port]) { + ahci_port_intr(sc->sc_ports[port], + AHCI_PREG_CI_ALL_SLOTS); + } + is &= ~(1 << port); + } + + /* Finally, acknowledge global interrupt */ + ahci_write(sc, AHCI_REG_IS, ack); +} + +u_int32_t +ahci_port_intr(struct ahci_port *ap, u_int32_t ci_mask) +{ + struct ahci_softc *sc = ap->ap_sc; + u_int32_t is, ci_saved, ci_masked, processed = 0; + int slot, need_restart = 0; + struct ahci_ccb *ccb = NULL; + volatile u_int32_t *active; +#ifdef DIAGNOSTIC + u_int32_t tmp; +#endif + + is = ahci_pread(ap, AHCI_PREG_IS); + + /* Ack port interrupt only if checking all command slots. */ + if (ci_mask == AHCI_PREG_CI_ALL_SLOTS) + ahci_pwrite(ap, AHCI_PREG_IS, is); + + if (is) + DPRINTF(AHCI_D_INTR, "%s: interrupt: %b\n", PORTNAME(ap), + is, AHCI_PFMT_IS); + + if (ap->ap_sactive) { + /* Active NCQ commands - use SActive instead of CI */ + KKASSERT(ap->ap_active == 0); + KKASSERT(ap->ap_active_cnt == 0); + ci_saved = ahci_pread(ap, AHCI_PREG_SACT); + active = &ap->ap_sactive; + } else { + /* Save CI */ + ci_saved = ahci_pread(ap, AHCI_PREG_CI); + active = &ap->ap_active; + } + + /* Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. */ + if (is & AHCI_PREG_IS_TFES) { + u_int32_t tfd, serr; + int err_slot; + + tfd = ahci_pread(ap, AHCI_PREG_TFD); + serr = ahci_pread(ap, AHCI_PREG_SERR); + + if (ap->ap_sactive == 0) { + /* Errored slot is easy to determine from CMD. */ + err_slot = AHCI_PREG_CMD_CCS(ahci_pread(ap, + AHCI_PREG_CMD)); + ccb = &ap->ap_ccbs[err_slot]; + + /* Preserve received taskfile data from the RFIS. */ + memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis, + sizeof(struct ata_fis_d2h)); + } else + err_slot = -1; /* Must extract error from log page */ + + DPRINTF(AHCI_D_VERBOSE, "%s: errored slot %d, TFD: %b, SERR:" + " %b, DIAG: %b\n", PORTNAME(ap), err_slot, tfd, + AHCI_PFMT_TFD_STS, AHCI_PREG_SERR_ERR(serr), + AHCI_PFMT_SERR_ERR, AHCI_PREG_SERR_DIAG(serr), + AHCI_PFMT_SERR_DIAG); + + /* Turn off ST to clear CI and SACT. */ + ahci_port_stop(ap, 0); + need_restart = 1; + + /* Clear SERR to enable capturing new errors. */ + ahci_pwrite(ap, AHCI_PREG_SERR, serr); + + /* Acknowledge the interrupts we can recover from. */ + ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_TFES | + AHCI_PREG_IS_IFS); + is = ahci_pread(ap, AHCI_PREG_IS); + + /* If device hasn't cleared its busy status, try to idle it. */ + if (tfd & (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) { + kprintf("%s: attempting to idle device\n", + PORTNAME(ap)); + if (ahci_port_softreset(ap)) { + kprintf("%s: failed to soft reset device\n", + PORTNAME(ap)); + if (ahci_port_portreset(ap)) { + kprintf("%s: failed to port reset " + "device, give up on it\n", + PORTNAME(ap)); + goto fatal; + } + } + + /* Had to reset device, can't gather extended info. */ + } else if (ap->ap_sactive) { + /* Recover the NCQ error from log page 10h. */ + ahci_port_read_ncq_error(ap, &err_slot); + if (err_slot < 0) + goto failall; + + DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n", + PORTNAME(ap), err_slot); + + ccb = &ap->ap_ccbs[err_slot]; + } else { + /* Didn't reset, could gather extended info from log. */ + } + + /* + * If we couldn't determine the errored slot, reset the port + * and fail all the active slots. + */ + if (err_slot == -1) { + if (ahci_port_softreset(ap) != 0 && + ahci_port_portreset(ap) != 0) { + kprintf("%s: couldn't reset after NCQ error, " + "disabling device.\n", + PORTNAME(ap)); + goto fatal; + } + kprintf("%s: couldn't recover NCQ error, failing " + "all outstanding commands.\n", + PORTNAME(ap)); + goto failall; + } + + /* Clear the failed command in saved CI so completion runs. */ + ci_saved &= ~(1 << err_slot); + + /* Note the error in the ata_xfer. */ + KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); + ccb->ccb_xa.state = ATA_S_ERROR; + +#ifdef DIAGNOSTIC + /* There may only be one outstanding standard command now. */ + if (ap->ap_sactive == 0) { + tmp = ci_saved; + if (tmp) { + slot = ffs(tmp) - 1; + tmp &= ~(1 << slot); + KKASSERT(tmp == 0); + } + } +#endif + } + + /* + * Port change (hot-plug). + * + * A PCS interrupt will occur on hot-plug once communication is + * established. + * + * A PRCS interrupt will occur on hot-unplug (and possibly also + * on hot-plug). + * + * We can then check the CPS (Cold Presence State) bit to determine + * if a device is plugged in or not and do the right thing. + */ + if (is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)) { + ahci_pwrite(ap, AHCI_PREG_SERR, + (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X) << 16); + switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) { + case AHCI_PREG_SSTS_DET_DEV: + if (ap->ap_ata.ap_type == ATA_PORT_T_NONE) { + kprintf("%s: HOTPLUG - Device added\n", + PORTNAME(ap)); + if (ahci_port_portreset(ap) == 0) + ahci_cam_changed(ap); + } + break; + default: + if (ap->ap_ata.ap_type != ATA_PORT_T_NONE) { + kprintf("%s: HOTPLUG - Device removed\n", + PORTNAME(ap)); + ahci_port_portreset(ap); + ahci_cam_changed(ap); + } + break; + } + } + + /* Check for remaining errors - they are fatal. */ + if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | + AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { + kprintf("%s: unrecoverable errors (IS: %b), disabling port.\n", + PORTNAME(ap), is, AHCI_PFMT_IS); + + /* XXX try recovery first */ + goto fatal; + } + + /* Fail all outstanding commands if we know the port won't recover. */ + if (ap->ap_state == AP_S_FATAL_ERROR) { +fatal: + ap->ap_state = AP_S_FATAL_ERROR; +failall: + + /* Ensure port is shut down. */ + ahci_port_stop(ap, 1); + + /* Error all the active slots. */ + ci_masked = ci_saved & *active; + while (ci_masked) { + slot = ffs(ci_masked) - 1; + ccb = &ap->ap_ccbs[slot]; + ci_masked &= ~(1 << slot); + ccb->ccb_xa.state = ATA_S_ERROR; + } + + /* Run completion for all active slots. */ + ci_saved &= ~*active; + + /* + * Don't restart the port if our problems were deemed fatal. + * + * Also acknowlege all fatal interrupt sources to prevent + * a livelock. + */ + if (ap->ap_state == AP_S_FATAL_ERROR) { + need_restart = 0; + ahci_pwrite(ap, AHCI_PREG_IS, + AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | + AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | + AHCI_PREG_IS_UFS); + } + } + + /* + * CCB completion is detected by noticing its slot's bit in CI has + * changed to zero some time after we activated it. + * If we are polling, we may only be interested in particular slot(s). + */ + ci_masked = ~ci_saved & *active & ci_mask; + while (ci_masked) { + slot = ffs(ci_masked) - 1; + ccb = &ap->ap_ccbs[slot]; + ci_masked &= ~(1 << slot); + + DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n", + PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ? + " (error)" : ""); + + bus_dmamap_sync(sc->sc_tag_cmdh, + AHCI_DMA_MAP(ap->ap_dmamem_cmd_list), + BUS_DMASYNC_POSTWRITE); + + bus_dmamap_sync(sc->sc_tag_cmdt, + AHCI_DMA_MAP(ap->ap_dmamem_cmd_table), + BUS_DMASYNC_POSTWRITE); + + bus_dmamap_sync(sc->sc_tag_rfis, + AHCI_DMA_MAP(ap->ap_dmamem_rfis), + BUS_DMASYNC_POSTREAD); + + *active &= ~(1 << ccb->ccb_slot); + ccb->ccb_done(ccb); + + processed |= 1 << ccb->ccb_slot; + } + + if (need_restart) { + /* Restart command DMA on the port */ + ahci_port_start(ap, 0); + + /* Re-enable outstanding commands on port. */ + if (ci_saved) { +#ifdef DIAGNOSTIC + tmp = ci_saved; + while (tmp) { + slot = ffs(tmp) - 1; + tmp &= ~(1 << slot); + ccb = &ap->ap_ccbs[slot]; + KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP); + KKASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) == + (!!ap->ap_sactive)); + } +#endif + DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr " + "re-enabling%s slots %08x\n", PORTNAME(ap), + ap->ap_sactive ? " NCQ" : "", ci_saved); + + if (ap->ap_sactive) + ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved); + ahci_pwrite(ap, AHCI_PREG_CI, ci_saved); + } + } + + return (processed); +} + +struct ahci_ccb * +ahci_get_ccb(struct ahci_port *ap) +{ + struct ahci_ccb *ccb; + + lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); + ccb = TAILQ_FIRST(&ap->ap_ccb_free); + if (ccb != NULL) { + KKASSERT(ccb->ccb_xa.state == ATA_S_PUT); + TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry); + ccb->ccb_xa.state = ATA_S_SETUP; + } + lockmgr(&ap->ap_ccb_lock, LK_RELEASE); + + return (ccb); +} + +void +ahci_put_ccb(struct ahci_ccb *ccb) +{ + struct ahci_port *ap = ccb->ccb_port; + +#ifdef DIAGNOSTIC + if (ccb->ccb_xa.state != ATA_S_COMPLETE && + ccb->ccb_xa.state != ATA_S_TIMEOUT && + ccb->ccb_xa.state != ATA_S_ERROR) { + kprintf("%s: invalid ata_xfer state %02x in ahci_put_ccb, " + "slot %d\n", + PORTNAME(ccb->ccb_port), ccb->ccb_xa.state, + ccb->ccb_slot); + } +#endif + + ccb->ccb_xa.state = ATA_S_PUT; + lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE); + TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry); + lockmgr(&ap->ap_ccb_lock, LK_RELEASE); +} + +struct ahci_ccb * +ahci_get_err_ccb(struct ahci_port *ap) +{ + struct ahci_ccb *err_ccb; + u_int32_t sact; + + /* No commands may be active on the chip. */ + sact = ahci_pread(ap, AHCI_PREG_SACT); + if (sact != 0) + kprintf("ahci_get_err_ccb but SACT %08x != 0?\n", sact); + KKASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); + +#ifdef DIAGNOSTIC + KKASSERT(ap->ap_err_busy == 0); + ap->ap_err_busy = 1; +#endif + /* Save outstanding command state. */ + ap->ap_err_saved_active = ap->ap_active; + ap->ap_err_saved_active_cnt = ap->ap_active_cnt; + ap->ap_err_saved_sactive = ap->ap_sactive; + + /* + * Pretend we have no commands outstanding, so that completions won't + * run prematurely. + */ + ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0; + + /* + * Grab a CCB to use for error recovery. This should never fail, as + * we ask atascsi to reserve one for us at init time. + */ + err_ccb = ahci_get_ccb(ap); + KKASSERT(err_ccb != NULL); + err_ccb->ccb_xa.flags = 0; + err_ccb->ccb_done = ahci_empty_done; + + return err_ccb; +} + +void +ahci_put_err_ccb(struct ahci_ccb *ccb) +{ + struct ahci_port *ap = ccb->ccb_port; + u_int32_t sact; + +#ifdef DIAGNOSTIC + KKASSERT(ap->ap_err_busy); +#endif + /* No commands may be active on the chip */ + sact = ahci_pread(ap, AHCI_PREG_SACT); + if (sact != 0) { + kprintf("ahci_port_err_ccb_restore but SACT %08x != 0?\n", + sact); + } + KKASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0); + + /* Done with the CCB */ + ahci_put_ccb(ccb); + + /* Restore outstanding command state */ + ap->ap_sactive = ap->ap_err_saved_sactive; + ap->ap_active_cnt = ap->ap_err_saved_active_cnt; + ap->ap_active = ap->ap_err_saved_active; + +#ifdef DIAGNOSTIC + ap->ap_err_busy = 0; +#endif +} + +int +ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp) +{ + struct ahci_ccb *ccb; + struct ahci_cmd_hdr *cmd_slot; + u_int32_t cmd; + struct ata_fis_h2d *fis; + int rc = EIO; + + DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap)); + + /* Save command register state. */ + cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC; + + /* Port should have been idled already. Start it. */ + KKASSERT((cmd & AHCI_PREG_CMD_CR) == 0); + ahci_port_start(ap, 0); + + /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */ + ccb = ahci_get_err_ccb(ap); + ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL; + ccb->ccb_xa.data = ap->ap_err_scratch; + ccb->ccb_xa.datalen = 512; + cmd_slot = ccb->ccb_cmd_hdr; + bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table)); + + fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis; + fis->type = ATA_FIS_TYPE_H2D; + fis->flags = ATA_H2D_FLAGS_CMD; + fis->command = ATA_C_READ_LOG_EXT; + fis->lba_low = 0x10; /* queued error log page (10h) */ + fis->sector_count = 1; /* number of sectors (1) */ + fis->sector_count_exp = 0; + fis->lba_mid = 0; /* starting offset */ + fis->lba_mid_exp = 0; + fis->device = 0; + + cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */ + + if (ahci_load_prdt(ccb) != 0) { + rc = ENOMEM; /* XXX caller must abort all commands */ + goto err; + } + + ccb->ccb_xa.state = ATA_S_PENDING; + if (ahci_poll(ccb, hz, NULL) != 0) + goto err; + + rc = 0; +err: + /* Abort our command, if it failed, by stopping command DMA. */ + if (rc != 0 && (ap->ap_active & (1 << ccb->ccb_slot))) { + kprintf("%s: log page read failed, slot %d was still active.\n", + PORTNAME(ap), ccb->ccb_slot); + ahci_port_stop(ap, 0); + } + + /* Done with the error CCB now. */ + ahci_unload_prdt(ccb); + ahci_put_err_ccb(ccb); + + /* Extract failed register set and tags from the scratch space. */ + if (rc == 0) { + struct ata_log_page_10h *log; + int err_slot; + + log = (struct ata_log_page_10h *)ap->ap_err_scratch; + if (log->err_regs.type & ATA_LOG_10H_TYPE_NOTQUEUED) { + /* Not queued bit was set - wasn't an NCQ error? */ + kprintf("%s: read NCQ error page, but not an NCQ " + "error?\n", + PORTNAME(ap)); + rc = ESRCH; + } else { + /* Copy back the log record as a D2H register FIS. */ + *err_slotp = err_slot = log->err_regs.type & + ATA_LOG_10H_TYPE_TAG_MASK; + + ccb = &ap->ap_ccbs[err_slot]; + memcpy(&ccb->ccb_xa.rfis, &log->err_regs, + sizeof(struct ata_fis_d2h)); + ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H; + ccb->ccb_xa.rfis.flags = 0; + } + } + + /* Restore saved CMD register state */ + ahci_pwrite(ap, AHCI_PREG_CMD, cmd); + + return (rc); +} + +/* + * Allocate memory for various structures DMAd by hardware. The maximum + * number of segments for these tags is 1 so the DMA memory will have a + * single physical base address. + */ +struct ahci_dmamem * +ahci_dmamem_alloc(struct ahci_softc *sc, bus_dma_tag_t tag) +{ + struct ahci_dmamem *adm; + int error; + + adm = kmalloc(sizeof(*adm), M_DEVBUF, M_INTWAIT | M_ZERO); + + error = bus_dmamem_alloc(tag, (void **)&adm->adm_kva, + BUS_DMA_ZERO, &adm->adm_map); + if (error == 0) { + adm->adm_tag = tag; + error = bus_dmamap_load(tag, adm->adm_map, + adm->adm_kva, + bus_dma_tag_getmaxsize(tag), + ahci_dmamem_saveseg, &adm->adm_busaddr, + 0); + } + if (error) { + if (adm->adm_map) { + bus_dmamap_destroy(tag, adm->adm_map); + adm->adm_map = NULL; + adm->adm_tag = NULL; + adm->adm_kva = NULL; + } + kfree(adm, M_DEVBUF); + adm = NULL; + } + return (adm); +} + +static +void +ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error) +{ + KKASSERT(error == 0); + KKASSERT(nsegs == 1); + *(bus_addr_t *)info = segs->ds_addr; +} + + +void +ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm) +{ + if (adm->adm_map) { + bus_dmamap_unload(adm->adm_tag, adm->adm_map); + bus_dmamap_destroy(adm->adm_tag, adm->adm_map); + adm->adm_map = NULL; + adm->adm_tag = NULL; + adm->adm_kva = NULL; + } + kfree(adm, M_DEVBUF); +} + +u_int32_t +ahci_read(struct ahci_softc *sc, bus_size_t r) +{ + bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, + BUS_SPACE_BARRIER_READ); + return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r)); +} + +void +ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v) +{ + bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v); + bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4, + BUS_SPACE_BARRIER_WRITE); +} + +int +ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask, + u_int32_t target) +{ + int i; + + for (i = 0; i < 1000; i++) { + if ((ahci_read(sc, r) & mask) != target) + return (0); + DELAY(1000); + } + + return (1); +} + +u_int32_t +ahci_pread(struct ahci_port *ap, bus_size_t r) +{ + bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, + BUS_SPACE_BARRIER_READ); + return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r)); +} + +void +ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v) +{ + bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v); + bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4, + BUS_SPACE_BARRIER_WRITE); +} + +int +ahci_pwait_eq(struct ahci_port *ap, bus_size_t r, u_int32_t mask, + u_int32_t target) +{ + int i; + + for (i = 0; i < 1000; i++) { + if ((ahci_pread(ap, r) & mask) == target) + return (0); + DELAY(1000); + } + + return (1); +} + +struct ata_xfer * +ahci_ata_get_xfer(struct ahci_port *ap) +{ + /*struct ahci_softc *sc = ap->ap_sc;*/ + struct ahci_ccb *ccb; + + ccb = ahci_get_ccb(ap); + if (ccb == NULL) { + DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n", + PORTNAME(ap)); + return (NULL); + } + + DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n", + PORTNAME(ap), ccb->ccb_slot); + + ccb->ccb_xa.fis->type = ATA_FIS_TYPE_H2D; + + return (&ccb->ccb_xa); +} + +void +ahci_ata_put_xfer(struct ata_xfer *xa) +{ + struct ahci_ccb *ccb = (struct ahci_ccb *)xa; + + DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot); + + ahci_put_ccb(ccb); +} + +int +ahci_ata_cmd(struct ata_xfer *xa) +{ + struct ahci_ccb *ccb = (struct ahci_ccb *)xa; + struct ahci_cmd_hdr *cmd_slot; + + KKASSERT(xa->state == ATA_S_SETUP); + + if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) + goto failcmd; + + ccb->ccb_done = ahci_ata_cmd_done; + + cmd_slot = ccb->ccb_cmd_hdr; + cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */ + + if (xa->flags & ATA_F_WRITE) + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W); + + if (xa->flags & ATA_F_PACKET) + cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A); + + if (ahci_load_prdt(ccb) != 0) + goto failcmd; + + xa->state = ATA_S_PENDING; + + if (xa->flags & ATA_F_POLL) { + ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout); + return (ATA_COMPLETE); + } + + crit_enter(); + xa->flags |= ATA_F_TIMEOUT_RUNNING; + callout_reset(&ccb->ccb_timeout, xa->timeout, + ahci_ata_cmd_timeout_unserialized, ccb); + ahci_start(ccb); + crit_exit(); + return (ATA_QUEUED); + +failcmd: + crit_enter(); + xa->state = ATA_S_ERROR; + xa->complete(xa); + crit_exit(); + return (ATA_ERROR); +} + +void +ahci_ata_cmd_done(struct ahci_ccb *ccb) +{ + struct ata_xfer *xa = &ccb->ccb_xa; + + if (xa->flags & ATA_F_TIMEOUT_RUNNING) { + xa->flags &= ~ATA_F_TIMEOUT_RUNNING; + callout_stop(&ccb->ccb_timeout); + } + + if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) + ahci_issue_pending_commands(ccb->ccb_port, + xa->flags & ATA_F_NCQ); + + ahci_unload_prdt(ccb); + + if (xa->state == ATA_S_ONCHIP) + xa->state = ATA_S_COMPLETE; +#ifdef DIAGNOSTIC + else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT) + kprintf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, " + "slot %d\n", + PORTNAME(ccb->ccb_port), xa->state, ccb->ccb_slot); +#endif + if (xa->state != ATA_S_TIMEOUT) + xa->complete(xa); +} + +static void +ahci_ata_cmd_timeout_unserialized(void *arg) +{ + struct ahci_ccb *ccb = arg; + struct ahci_port *ap = ccb->ccb_port; + + lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_ata_cmd_timeout(arg); + lwkt_serialize_exit(&ap->ap_sc->sc_serializer); +} + +static void +ahci_ata_cmd_timeout(void *arg) +{ + struct ahci_ccb *ccb = arg; + struct ata_xfer *xa = &ccb->ccb_xa; + struct ahci_port *ap = ccb->ccb_port; + volatile u_int32_t *active; + int ccb_was_started, ncq_cmd; + + crit_enter(); + kprintf("CMD TIMEOUT port-cmd-reg 0x%b\n" + "\tactive=%08x sactive=%08x\n" + "\t sact=%08x ci=%08x\n", + ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD, + ap->ap_active, ap->ap_sactive, + ahci_pread(ap, AHCI_PREG_SACT), + ahci_pread(ap, AHCI_PREG_CI)); + + KKASSERT(xa->flags & ATA_F_TIMEOUT_RUNNING); + xa->flags &= ~ATA_F_TIMEOUT_RUNNING; + ncq_cmd = (xa->flags & ATA_F_NCQ); + active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; + + if (ccb->ccb_xa.state == ATA_S_PENDING) { + DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out " + "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot); + TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry); + ccb_was_started = 0; + } else if (ccb->ccb_xa.state == ATA_S_ONCHIP && ahci_port_intr(ap, + 1 << ccb->ccb_slot)) { + DPRINTF(AHCI_D_TIMEOUT, "%s: final poll of port completed " + "command in slot %d\n", PORTNAME(ap), ccb->ccb_slot); + goto ret; + } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) { + DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already " + "handled%s\n", PORTNAME(ap), ccb->ccb_slot, + (*active & (1 << ccb->ccb_slot)) ? + " but slot is still active?" : "."); + goto ret; + } else if ((ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : AHCI_PREG_CI) & + (1 << ccb->ccb_slot)) == 0 && + (*active & (1 << ccb->ccb_slot))) { + DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d completed but " + "IRQ handler didn't detect it. Why?\n", PORTNAME(ap), + ccb->ccb_slot); + *active &= ~(1 << ccb->ccb_slot); + ccb->ccb_done(ccb); + goto ret; + } else { + kprintf("X5\n"); + ccb_was_started = 1; + } + + /* Complete the slot with a timeout error. */ + ccb->ccb_xa.state = ATA_S_TIMEOUT; + *active &= ~(1 << ccb->ccb_slot); + DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap)); + ccb->ccb_done(ccb); /* This won't issue pending commands or run the + atascsi completion. */ + + /* Reset port to abort running command. */ + if (ccb_was_started) { + DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command " + "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" + : "", ccb->ccb_slot, *active); + if (ahci_port_softreset(ap) != 0 && ahci_port_portreset(ap) + != 0) { + kprintf("%s: failed to reset port during timeout " + "handling, disabling it\n", + PORTNAME(ap)); + ap->ap_state = AP_S_FATAL_ERROR; + } + + /* Restart any other commands that were aborted by the reset. */ + if (*active) { + DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots " + "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "", + *active); + if (ncq_cmd) + ahci_pwrite(ap, AHCI_PREG_SACT, *active); + ahci_pwrite(ap, AHCI_PREG_CI, *active); + } + } + + /* Issue any pending commands now. */ + DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap)); + if (ccb_was_started) + ahci_issue_pending_commands(ap, ncq_cmd); + else if (ap->ap_active == 0) + ahci_issue_pending_ncq_commands(ap); + + /* Complete the timed out ata_xfer I/O (may generate new I/O). */ + DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap)); + xa->complete(xa); + + DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap)); +ret: + crit_exit(); +} + +void +ahci_empty_done(struct ahci_ccb *ccb) +{ + ccb->ccb_xa.state = ATA_S_COMPLETE; +} diff --git a/sys/dev/disk/ahci/ahci.h b/sys/dev/disk/ahci/ahci.h new file mode 100644 index 0000000000..df31f7677a --- /dev/null +++ b/sys/dev/disk/ahci/ahci.h @@ -0,0 +1,466 @@ +/* + * Copyright (c) 2006 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $ + */ + +#if defined(__DragonFly__) +#include "ahci_dragonfly.h" +#else +#error "build for OS unknown" +#endif +#include "atascsi.h" + +/* change to AHCI_DEBUG for dmesg spam */ +#define NO_AHCI_DEBUG + +#ifdef AHCI_DEBUG +#define DPRINTF(m, f...) do { if ((ahcidebug & (m)) == (m)) kprintf(f); } \ + while (0) +#define AHCI_D_TIMEOUT 0x00 +#define AHCI_D_VERBOSE 0x01 +#define AHCI_D_INTR 0x02 +#define AHCI_D_XFER 0x08 +int ahcidebug = AHCI_D_VERBOSE; +#else +#define DPRINTF(m, f...) +#endif + +#define AHCI_PCI_BAR 0x24 +#define AHCI_PCI_ATI_SB600_MAGIC 0x40 +#define AHCI_PCI_ATI_SB600_LOCKED 0x01 +#define AHCI_PCI_INTERFACE 0x01 + +#define AHCI_REG_CAP 0x000 /* HBA Capabilities */ +#define AHCI_REG_CAP_NP(_r) (((_r) & 0x1f)+1) /* Number of Ports */ +#define AHCI_REG_CAP_SXS (1<<5) /* External SATA */ +#define AHCI_REG_CAP_EMS (1<<6) /* Enclosure Mgmt */ +#define AHCI_REG_CAP_CCCS (1<<7) /* Cmd Coalescing */ +#define AHCI_REG_CAP_NCS(_r) ((((_r) & 0x1f00)>>8)+1) /* NCmds*/ +#define AHCI_REG_CAP_PSC (1<<13) /* Partial State Capable */ +#define AHCI_REG_CAP_SSC (1<<14) /* Slumber State Capable */ +#define AHCI_REG_CAP_PMD (1<<15) /* PIO Multiple DRQ Block */ +#define AHCI_REG_CAP_FBSS (1<<16) /* FIS-Based Switching */ +#define AHCI_REG_CAP_SPM (1<<17) /* Port Multiplier */ +#define AHCI_REG_CAP_SAM (1<<18) /* AHCI Only mode */ +#define AHCI_REG_CAP_SNZO (1<<19) /* Non Zero DMA Offsets */ +#define AHCI_REG_CAP_ISS (0xf<<20) /* Interface Speed Support */ +#define AHCI_REG_CAP_ISS_G1 (0x1<<20) /* Gen 1 (1.5 Gbps) */ +#define AHCI_REG_CAP_ISS_G1_2 (0x2<<20) /* Gen 1 and 2 (3 Gbps) */ +#define AHCI_REG_CAP_SCLO (1<<24) /* Cmd List Override */ +#define AHCI_REG_CAP_SAL (1<<25) /* Activity LED */ +#define AHCI_REG_CAP_SALP (1<<26) /* Aggressive Link Pwr Mgmt */ +#define AHCI_REG_CAP_SSS (1<<27) /* Staggered Spinup */ +#define AHCI_REG_CAP_SMPS (1<<28) /* Mech Presence Switch */ +#define AHCI_REG_CAP_SSNTF (1<<29) /* SNotification Register */ +#define AHCI_REG_CAP_SNCQ (1<<30) /* Native Cmd Queuing */ +#define AHCI_REG_CAP_S64A (1<<31) /* 64bit Addressing */ +#define AHCI_FMT_CAP "\020" "\040S64A" "\037NCQ" "\036SSNTF" \ + "\035SMPS" "\034SSS" "\033SALP" "\032SAL" \ + "\031SCLO" "\024SNZO" "\023SAM" "\022SPM" \ + "\021FBSS" "\020PMD" "\017SSC" "\016PSC" \ + "\010CCCS" "\007EMS" "\006SXS" + +#define AHCI_REG_GHC 0x004 /* Global HBA Control */ +#define AHCI_REG_GHC_HR (1<<0) /* HBA Reset */ +#define AHCI_REG_GHC_IE (1<<1) /* Interrupt Enable */ +#define AHCI_REG_GHC_MRSM (1<<2) /* MSI Revert to Single Msg */ +#define AHCI_REG_GHC_AE (1<<31) /* AHCI Enable */ +#define AHCI_FMT_GHC "\020" "\040AE" "\003MRSM" "\002IE" "\001HR" + +#define AHCI_REG_IS 0x008 /* Interrupt Status */ +#define AHCI_REG_PI 0x00c /* Ports Implemented */ + +#define AHCI_REG_VS 0x010 /* AHCI Version */ +#define AHCI_REG_VS_0_95 0x00000905 /* 0.95 */ +#define AHCI_REG_VS_1_0 0x00010000 /* 1.0 */ +#define AHCI_REG_VS_1_1 0x00010100 /* 1.1 */ +#define AHCI_REG_VS_1_2 0x00010200 /* 1.2 */ + +#define AHCI_REG_CCC_CTL 0x014 /* Coalescing Control */ +#define AHCI_REG_CCC_CTL_INT(_r) (((_r) & 0xf8) >> 3) /* CCC INT slot */ + +#define AHCI_REG_CCC_PORTS 0x018 /* Coalescing Ports */ +#define AHCI_REG_EM_LOC 0x01c /* Enclosure Mgmt Location */ +#define AHCI_REG_EM_CTL 0x020 /* Enclosure Mgmt Control */ + +#define AHCI_PORT_REGION(_p) (0x100 + ((_p) * 0x80)) +#define AHCI_PORT_SIZE 0x80 + +#define AHCI_PREG_CLB 0x00 /* Cmd List Base Addr */ +#define AHCI_PREG_CLBU 0x04 /* Cmd List Base Hi Addr */ +#define AHCI_PREG_FB 0x08 /* FIS Base Addr */ +#define AHCI_PREG_FBU 0x0c /* FIS Base Hi Addr */ + +#define AHCI_PREG_IS 0x10 /* Interrupt Status */ +#define AHCI_PREG_IS_DHRS (1<<0) /* Device to Host FIS */ +#define AHCI_PREG_IS_PSS (1<<1) /* PIO Setup FIS */ +#define AHCI_PREG_IS_DSS (1<<2) /* DMA Setup FIS */ +#define AHCI_PREG_IS_SDBS (1<<3) /* Set Device Bits FIS */ +#define AHCI_PREG_IS_UFS (1<<4) /* Unknown FIS */ +#define AHCI_PREG_IS_DPS (1<<5) /* Descriptor Processed */ +#define AHCI_PREG_IS_PCS (1<<6) /* Port Change */ +#define AHCI_PREG_IS_DMPS (1<<7) /* Device Mechanical Presence */ +#define AHCI_PREG_IS_PRCS (1<<22) /* PhyRdy Change */ +#define AHCI_PREG_IS_IPMS (1<<23) /* Incorrect Port Multiplier */ +#define AHCI_PREG_IS_OFS (1<<24) /* Overflow */ +#define AHCI_PREG_IS_INFS (1<<26) /* Interface Non-fatal Error */ +#define AHCI_PREG_IS_IFS (1<<27) /* Interface Fatal Error */ +#define AHCI_PREG_IS_HBDS (1<<28) /* Host Bus Data Error */ +#define AHCI_PREG_IS_HBFS (1<<29) /* Host Bus Fatal Error */ +#define AHCI_PREG_IS_TFES (1<<30) /* Task File Error */ +#define AHCI_PREG_IS_CPDS (1<<31) /* Cold Presence Detect */ +#define AHCI_PFMT_IS "\20" "\040CPDS" "\037TFES" "\036HBFS" \ + "\035HBDS" "\034IFS" "\033INFS" "\031OFS" \ + "\030IPMS" "\027PRCS" "\010DMPS" "\006DPS" \ + "\007PCS" "\005UFS" "\004SDBS" "\003DSS" \ + "\002PSS" "\001DHRS" + +#define AHCI_PREG_IE 0x14 /* Interrupt Enable */ +#define AHCI_PREG_IE_DHRE (1<<0) /* Device to Host FIS */ +#define AHCI_PREG_IE_PSE (1<<1) /* PIO Setup FIS */ +#define AHCI_PREG_IE_DSE (1<<2) /* DMA Setup FIS */ +#define AHCI_PREG_IE_SDBE (1<<3) /* Set Device Bits FIS */ +#define AHCI_PREG_IE_UFE (1<<4) /* Unknown FIS */ +#define AHCI_PREG_IE_DPE (1<<5) /* Descriptor Processed */ +#define AHCI_PREG_IE_PCE (1<<6) /* Port Change */ +#define AHCI_PREG_IE_DMPE (1<<7) /* Device Mechanical Presence */ +#define AHCI_PREG_IE_PRCE (1<<22) /* PhyRdy Change */ +#define AHCI_PREG_IE_IPME (1<<23) /* Incorrect Port Multiplier */ +#define AHCI_PREG_IE_OFE (1<<24) /* Overflow */ +#define AHCI_PREG_IE_INFE (1<<26) /* Interface Non-fatal Error */ +#define AHCI_PREG_IE_IFE (1<<27) /* Interface Fatal Error */ +#define AHCI_PREG_IE_HBDE (1<<28) /* Host Bus Data Error */ +#define AHCI_PREG_IE_HBFE (1<<29) /* Host Bus Fatal Error */ +#define AHCI_PREG_IE_TFEE (1<<30) /* Task File Error */ +#define AHCI_PREG_IE_CPDE (1<<31) /* Cold Presence Detect */ +#define AHCI_PFMT_IE "\20" "\040CPDE" "\037TFEE" "\036HBFE" \ + "\035HBDE" "\034IFE" "\033INFE" "\031OFE" \ + "\030IPME" "\027PRCE" "\010DMPE" "\007PCE" \ + "\006DPE" "\005UFE" "\004SDBE" "\003DSE" \ + "\002PSE" "\001DHRE" + +#define AHCI_PREG_CMD 0x18 /* Command and Status */ +#define AHCI_PREG_CMD_ST (1<<0) /* Start */ +#define AHCI_PREG_CMD_SUD (1<<1) /* Spin Up Device */ +#define AHCI_PREG_CMD_POD (1<<2) /* Power On Device */ +#define AHCI_PREG_CMD_CLO (1<<3) /* Command List Override */ +#define AHCI_PREG_CMD_FRE (1<<4) /* FIS Receive Enable */ +#define AHCI_PREG_CMD_CCS(_r) (((_r) >> 8) & 0x1f) /* Curr CmdSlot# */ +#define AHCI_PREG_CMD_MPSS (1<<13) /* Mech Presence State */ +#define AHCI_PREG_CMD_FR (1<<14) /* FIS Receive Running */ +#define AHCI_PREG_CMD_CR (1<<15) /* Command List Running */ +#define AHCI_PREG_CMD_CPS (1<<16) /* Cold Presence State */ +#define AHCI_PREG_CMD_PMA (1<<17) /* Port Multiplier Attached */ +#define AHCI_PREG_CMD_HPCP (1<<18) /* Hot Plug Capable */ +#define AHCI_PREG_CMD_MPSP (1<<19) /* Mech Presence Switch */ +#define AHCI_PREG_CMD_CPD (1<<20) /* Cold Presence Detection */ +#define AHCI_PREG_CMD_ESP (1<<21) /* External SATA Port */ +#define AHCI_PREG_CMD_ATAPI (1<<24) /* Device is ATAPI */ +#define AHCI_PREG_CMD_DLAE (1<<25) /* Drv LED on ATAPI Enable */ +#define AHCI_PREG_CMD_ALPE (1<<26) /* Aggro Pwr Mgmt Enable */ +#define AHCI_PREG_CMD_ASP (1<<27) /* Aggro Slumber/Partial */ +#define AHCI_PREG_CMD_ICC 0xf0000000 /* Interface Comm Ctrl */ +#define AHCI_PREG_CMD_ICC_SLUMBER 0x60000000 +#define AHCI_PREG_CMD_ICC_PARTIAL 0x20000000 +#define AHCI_PREG_CMD_ICC_ACTIVE 0x10000000 +#define AHCI_PREG_CMD_ICC_IDLE 0x00000000 +#define AHCI_PFMT_CMD "\020" "\034ASP" "\033ALPE" "\032DLAE" \ + "\031ATAPI" "\026ESP" "\025CPD" "\024MPSP" \ + "\023HPCP" "\022PMA" "\021CPS" "\020CR" \ + "\017FR" "\016MPSS" "\005FRE" "\004CLO" \ + "\003POD" "\002SUD" "\001ST" + +#define AHCI_PREG_TFD 0x20 /* Task File Data*/ +#define AHCI_PREG_TFD_STS 0xff +#define AHCI_PREG_TFD_STS_ERR (1<<0) +#define AHCI_PREG_TFD_STS_DRQ (1<<3) +#define AHCI_PREG_TFD_STS_BSY (1<<7) +#define AHCI_PREG_TFD_ERR 0xff00 + +#define AHCI_PFMT_TFD_STS "\20" "\010BSY" "\004DRQ" "\001ERR" +#define AHCI_PREG_SIG 0x24 /* Signature */ + +#define AHCI_PREG_SSTS 0x28 /* SATA Status */ +#define AHCI_PREG_SSTS_DET 0xf /* Device Detection */ +#define AHCI_PREG_SSTS_DET_NONE 0x0 +#define AHCI_PREG_SSTS_DET_DEV_NE 0x1 +#define AHCI_PREG_SSTS_DET_DEV 0x3 +#define AHCI_PREG_SSTS_DET_PHYOFFLINE 0x4 +#define AHCI_PREG_SSTS_SPD 0xf0 /* Current Interface Speed */ +#define AHCI_PREG_SSTS_SPD_NONE 0x00 +#define AHCI_PREG_SSTS_SPD_GEN1 0x10 +#define AHCI_PREG_SSTS_SPD_GEN2 0x20 +#define AHCI_PREG_SSTS_IPM 0xf00 /* Interface Power Management */ +#define AHCI_PREG_SSTS_IPM_NONE 0x000 +#define AHCI_PREG_SSTS_IPM_ACTIVE 0x100 +#define AHCI_PREG_SSTS_IPM_PARTIAL 0x200 +#define AHCI_PREG_SSTS_IPM_SLUMBER 0x600 + +#define AHCI_PREG_SCTL 0x2c /* SATA Control */ +#define AHCI_PREG_SCTL_DET 0xf /* Device Detection */ +#define AHCI_PREG_SCTL_DET_NONE 0x0 +#define AHCI_PREG_SCTL_DET_INIT 0x1 +#define AHCI_PREG_SCTL_DET_DISABLE 0x4 +#define AHCI_PREG_SCTL_SPD 0xf0 /* Speed Allowed */ +#define AHCI_PREG_SCTL_SPD_ANY 0x00 +#define AHCI_PREG_SCTL_SPD_GEN1 0x10 +#define AHCI_PREG_SCTL_SPD_GEN2 0x20 +#define AHCI_PREG_SCTL_IPM 0xf00 /* Interface Power Management */ +#define AHCI_PREG_SCTL_IPM_NONE 0x000 +#define AHCI_PREG_SCTL_IPM_NOPARTIAL 0x100 +#define AHCI_PREG_SCTL_IPM_NOSLUMBER 0x200 +#define AHCI_PREG_SCTL_IPM_DISABLED 0x300 + +#define AHCI_PREG_SERR 0x30 /* SATA Error */ +#define AHCI_PREG_SERR_ERR(_r) ((_r) & 0xffff) +#define AHCI_PREG_SERR_ERR_I (1<<0) /* Recovered Data Integrity */ +#define AHCI_PREG_SERR_ERR_M (1<<1) /* Recovered Communications */ +#define AHCI_PREG_SERR_ERR_T (1<<8) /* Transient Data Integrity */ +#define AHCI_PREG_SERR_ERR_C (1<<9) /* Persistent Comm/Data */ +#define AHCI_PREG_SERR_ERR_P (1<<10) /* Protocol */ +#define AHCI_PREG_SERR_ERR_E (1<<11) /* Internal */ +#define AHCI_PFMT_SERR_ERR "\020" "\014E" "\013P" "\012C" "\011T" "\002M" \ + "\001I" +#define AHCI_PREG_SERR_DIAG(_r) (((_r) >> 16) & 0xffff) +#define AHCI_PREG_SERR_DIAG_N (1<<0) /* PhyRdy Change */ +#define AHCI_PREG_SERR_DIAG_I (1<<1) /* Phy Internal Error */ +#define AHCI_PREG_SERR_DIAG_W (1<<2) /* Comm Wake */ +#define AHCI_PREG_SERR_DIAG_B (1<<3) /* 10B to 8B Decode Error */ +#define AHCI_PREG_SERR_DIAG_D (1<<4) /* Disparity Error */ +#define AHCI_PREG_SERR_DIAG_C (1<<5) /* CRC Error */ +#define AHCI_PREG_SERR_DIAG_H (1<<6) /* Handshake Error */ +#define AHCI_PREG_SERR_DIAG_S (1<<7) /* Link Sequence Error */ +#define AHCI_PREG_SERR_DIAG_T (1<<8) /* Transport State Trans Err */ +#define AHCI_PREG_SERR_DIAG_F (1<<9) /* Unknown FIS Type */ +#define AHCI_PREG_SERR_DIAG_X (1<<10) /* Exchanged */ +#define AHCI_PFMT_SERR_DIAG "\020" "\013X" "\012F" "\011T" "\010S" "\007H" \ + "\006C" "\005D" "\004B" "\003W" "\002I" \ + "\001N" + +#define AHCI_PREG_SACT 0x34 /* SATA Active */ +#define AHCI_PREG_CI 0x38 /* Command Issue */ +#define AHCI_PREG_CI_ALL_SLOTS 0xffffffff +#define AHCI_PREG_SNTF 0x3c /* SNotification */ + +struct ahci_cmd_hdr { + u_int16_t flags; +#define AHCI_CMD_LIST_FLAG_CFL 0x001f /* Command FIS Length */ +#define AHCI_CMD_LIST_FLAG_A (1<<5) /* ATAPI */ +#define AHCI_CMD_LIST_FLAG_W (1<<6) /* Write */ +#define AHCI_CMD_LIST_FLAG_P (1<<7) /* Prefetchable */ +#define AHCI_CMD_LIST_FLAG_R (1<<8) /* Reset */ +#define AHCI_CMD_LIST_FLAG_B (1<<9) /* BIST */ +#define AHCI_CMD_LIST_FLAG_C (1<<10) /* Clear Busy upon R_OK */ +#define AHCI_CMD_LIST_FLAG_PMP 0xf000 /* Port Multiplier Port */ + u_int16_t prdtl; /* sgl len */ + + u_int32_t prdbc; /* transferred byte count */ + + u_int32_t ctba_lo; + u_int32_t ctba_hi; + + u_int32_t reserved[4]; +} __packed; + +struct ahci_rfis { + u_int8_t dsfis[28]; + u_int8_t reserved1[4]; + u_int8_t psfis[24]; + u_int8_t reserved2[8]; + u_int8_t rfis[24]; + u_int8_t reserved3[4]; + u_int8_t sdbfis[4]; + u_int8_t ufis[64]; + u_int8_t reserved4[96]; +} __packed; + +struct ahci_prdt { + u_int32_t dba_lo; + u_int32_t dba_hi; + u_int32_t reserved; + u_int32_t flags; +#define AHCI_PRDT_FLAG_INTR (1<<31) /* interrupt on completion */ +} __packed; + +/* + * The base command table structure is 128 bytes. Each prdt is 16 bytes. + * We need to accomodate MAXPHYS (128K) which is at least 32 entries, + * plus one for page slop. + * + * Making the ahci_cmd_table 1024 bytes (a reasonable power of 2) + * thus requires MAX_PRDT to be set to 56. + */ +#define AHCI_MAX_PRDT 56 + +#if MAXPHYS / PAGE_SIZE + 1 > AHCI_MAX_PRDT +#error "AHCI_MAX_PRDT is not big enough" +#endif + +struct ahci_cmd_table { + u_int8_t cfis[64]; /* Command FIS */ + u_int8_t acmd[16]; /* ATAPI Command */ + u_int8_t reserved[48]; + + struct ahci_prdt prdt[AHCI_MAX_PRDT]; +} __packed; + +#define AHCI_MAX_PORTS 32 + +struct ahci_dmamem { + bus_dma_tag_t adm_tag; + bus_dmamap_t adm_map; + bus_dma_segment_t adm_seg; + bus_addr_t adm_busaddr; + caddr_t adm_kva; +}; +#define AHCI_DMA_MAP(_adm) ((_adm)->adm_map) +#define AHCI_DMA_DVA(_adm) ((_adm)->adm_busaddr) +#define AHCI_DMA_KVA(_adm) ((void *)(_adm)->adm_kva) + +struct ahci_softc; +struct ahci_port; +struct ahci_device; + +struct ahci_ccb { + /* ATA xfer associated with this CCB. Must be 1st struct member. */ + struct ata_xfer ccb_xa; + struct callout ccb_timeout; + + int ccb_slot; + struct ahci_port *ccb_port; + + bus_dmamap_t ccb_dmamap; + struct ahci_cmd_hdr *ccb_cmd_hdr; + struct ahci_cmd_table *ccb_cmd_table; + + void (*ccb_done)(struct ahci_ccb *); + + TAILQ_ENTRY(ahci_ccb) ccb_entry; +}; + +struct ahci_port { + struct ahci_softc *ap_sc; + bus_space_handle_t ap_ioh; + + int ap_num; + int ap_flags; +#define AP_F_BUS_REGISTERED 0x0001 +#define AP_F_CAM_ATTACHED 0x0002 + struct cam_sim *ap_sim; + struct cam_path *ap_path; + + struct ahci_rfis *ap_rfis; + struct ahci_dmamem *ap_dmamem_rfis; + + struct ahci_dmamem *ap_dmamem_cmd_list; + struct ahci_dmamem *ap_dmamem_cmd_table; + + volatile u_int32_t ap_active; + volatile u_int32_t ap_active_cnt; + volatile u_int32_t ap_sactive; + struct ahci_ccb *ap_ccbs; + + TAILQ_HEAD(, ahci_ccb) ap_ccb_free; + TAILQ_HEAD(, ahci_ccb) ap_ccb_pending; + struct lock ap_ccb_lock; + + struct ata_port ap_ata; + + u_int32_t ap_state; +#define AP_S_NORMAL 0 +#define AP_S_FATAL_ERROR 1 + + /* For error recovery. */ +#ifdef DIAGNOSTIC + int ap_err_busy; +#endif + u_int32_t ap_err_saved_sactive; + u_int32_t ap_err_saved_active; + u_int32_t ap_err_saved_active_cnt; + + u_int8_t ap_err_scratch[512]; + + char ap_name[16]; +}; + +#define PORTNAME(_ap) ((_ap)->ap_name) + +struct ahci_softc { + device_t sc_dev; + const struct ahci_device *sc_ad; /* special casing */ + struct lwkt_serialize sc_serializer; + + struct resource *sc_irq; /* bus resources */ + struct resource *sc_regs; /* bus resources */ + bus_space_tag_t sc_iot; /* split from sc_regs */ + bus_space_handle_t sc_ioh; /* split from sc_regs */ + + int sc_rid_irq; /* saved bus RIDs */ + int sc_rid_regs; + u_int32_t sc_cap; /* capabilities */ + + void *sc_irq_handle; /* installed irq vector */ + + bus_dma_tag_t sc_tag_rfis; /* bus DMA tags */ + bus_dma_tag_t sc_tag_cmdh; + bus_dma_tag_t sc_tag_cmdt; + bus_dma_tag_t sc_tag_data; + + int sc_flags; +#define AHCI_F_NO_NCQ (1<<0) +#define AHCI_F_IGN_FR (1<<1) + + u_int sc_ncmds; + + struct ahci_port *sc_ports[AHCI_MAX_PORTS]; + +#ifdef AHCI_COALESCE + u_int32_t sc_ccc_mask; + u_int32_t sc_ccc_ports; + u_int32_t sc_ccc_ports_cur; +#endif +}; +#define DEVNAME(_s) ((_s)->sc_dev.dv_xname) + +struct ahci_device { + pci_vendor_id_t ad_vendor; + pci_product_id_t ad_product; + int (*ad_attach)(device_t dev); + int (*ad_detach)(device_t dev); + char *name; +}; + +const struct ahci_device *ahci_lookup_device(device_t dev); +int ahci_init(struct ahci_softc *); +int ahci_port_alloc(struct ahci_softc *, u_int); +void ahci_port_free(struct ahci_softc *, u_int); +u_int32_t ahci_read(struct ahci_softc *, bus_size_t); +void ahci_write(struct ahci_softc *, bus_size_t, u_int32_t); +int ahci_wait_ne(struct ahci_softc *, bus_size_t, u_int32_t, u_int32_t); +u_int32_t ahci_pread(struct ahci_port *, bus_size_t); +void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); +int ahci_pwait_eq(struct ahci_port *, bus_size_t, u_int32_t, u_int32_t); +void ahci_intr(void *); + +int ahci_cam_attach(struct ahci_port *ap); +void ahci_cam_changed(struct ahci_port *ap); +void ahci_cam_detach(struct ahci_port *ap); + +struct ata_xfer *ahci_ata_get_xfer(struct ahci_port *ap); +void ahci_ata_put_xfer(struct ata_xfer *xa); +int ahci_ata_cmd(struct ata_xfer *xa); + +extern u_int32_t AhciForceGen1; diff --git a/sys/dev/disk/ahci/ahci_attach.c b/sys/dev/disk/ahci/ahci_attach.c new file mode 100644 index 0000000000..4bc1aba98f --- /dev/null +++ b/sys/dev/disk/ahci/ahci_attach.c @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2006 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * + * Copyright (c) 2009 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $ + */ + +#include "ahci.h" + +static int ahci_vt8251_attach(device_t); +static int ahci_ati_sb600_attach(device_t); +static int ahci_nvidia_mcp_attach(device_t); +static int ahci_pci_attach(device_t); +static int ahci_pci_detach(device_t); + +static const struct ahci_device ahci_devices[] = { + { PCI_VENDOR_VIATECH, PCI_PRODUCT_VIATECH_VT8251_SATA, + ahci_vt8251_attach, ahci_pci_detach, "ViaTech-VT8251-SATA" }, + { PCI_VENDOR_ATI, PCI_PRODUCT_ATI_SB600_SATA, + ahci_ati_sb600_attach, ahci_pci_detach, "ATI-SB600-SATA" }, + { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP65_AHCI_2, + ahci_nvidia_mcp_attach, ahci_pci_detach, "NVidia-MCP65-SATA" }, + { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP67_AHCI_1, + ahci_nvidia_mcp_attach, ahci_pci_detach, "NVidia-MCP67-SATA" }, + { PCI_VENDOR_NVIDIA, PCI_PRODUCT_NVIDIA_MCP77_AHCI_5, + ahci_nvidia_mcp_attach, ahci_pci_detach, "NVidia-MCP77-SATA" }, + { 0, 0, + ahci_pci_attach, ahci_pci_detach, "AHCI-PCI-SATA" } +}; + +u_int32_t AhciForceGen1 = 0; /* XXX add sysctl/kenv support */ + +/* + * Match during probe and attach. The device does not yet have a softc. + */ +const struct ahci_device * +ahci_lookup_device(device_t dev) +{ + const struct ahci_device *ad; + u_int16_t vendor = pci_get_vendor(dev); + u_int16_t product = pci_get_device(dev); + u_int8_t class = pci_get_class(dev); + u_int8_t subclass = pci_get_subclass(dev); + u_int8_t progif = pci_read_config(dev, PCIR_PROGIF, 1); + + for (ad = &ahci_devices[0]; ad->ad_vendor; ++ad) { + if (ad->ad_vendor == vendor && ad->ad_product == product) + return (ad); + } + + /* + * Last ad is the default match if the PCI device matches SATA. + */ + if (class == PCIC_STORAGE && subclass == PCIS_STORAGE_SATA && + progif == PCIP_STORAGE_SATA_AHCI_1_0) { + kprintf("match generic sata\n"); + return (ad); + } + + return (NULL); +} + +/* + * Attach functions. They all eventually fall through to ahci_pci_attach(). + */ +static int +ahci_vt8251_attach(device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + + sc->sc_flags |= AHCI_F_NO_NCQ; + return (ahci_pci_attach(dev)); +} + +static int +ahci_ati_sb600_attach(device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + pcireg_t magic; + u_int8_t subclass = pci_get_subclass(dev); + u_int8_t revid; + + if (subclass == PCIS_STORAGE_IDE) { + revid = pci_read_config(dev, PCIR_REVID, 1); + magic = pci_read_config(dev, AHCI_PCI_ATI_SB600_MAGIC, 4); + pci_write_config(dev, AHCI_PCI_ATI_SB600_MAGIC, + magic | AHCI_PCI_ATI_SB600_LOCKED, 4); + pci_write_config(dev, PCIR_REVID, + (PCIC_STORAGE << 24) | + (PCIS_STORAGE_SATA << 16) | + (PCIP_STORAGE_SATA_AHCI_1_0 << 8) | + revid, 4); + pci_write_config(dev, AHCI_PCI_ATI_SB600_MAGIC, magic, 4); + } + + sc->sc_flags |= AHCI_F_IGN_FR; + return (ahci_pci_attach(dev)); +} + +static int +ahci_nvidia_mcp_attach(device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + + sc->sc_flags |= AHCI_F_IGN_FR; + return (ahci_pci_attach(dev)); +} + +static int +ahci_pci_attach(device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + const char *gen; + u_int32_t cap, pi, reg; + bus_addr_t addr; + int i; + int error; + const char *revision; + + /* + * Map the AHCI controller's IRQ and BAR(5) (hardware registers) + */ + sc->sc_dev = dev; + sc->sc_rid_irq = AHCI_IRQ_RID; + sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_rid_irq, + RF_SHAREABLE | RF_ACTIVE); + lwkt_serialize_init(&sc->sc_serializer); + lwkt_serialize_enter(&sc->sc_serializer); + if (sc->sc_irq == NULL) { + device_printf(dev, "unable to map interrupt\n"); + lwkt_serialize_exit(&sc->sc_serializer); + ahci_pci_detach(dev); + return (ENXIO); + } + + /* + * When mapping the register window store the tag and handle + * separately so we can use the tag with per-port bus handle + * sub-spaces. + */ + sc->sc_rid_regs = PCIR_BAR(5); + sc->sc_regs = bus_alloc_resource_any(dev, SYS_RES_MEMORY, + &sc->sc_rid_regs, RF_ACTIVE); + if (sc->sc_regs == NULL) { + device_printf(dev, "unable to map registers\n"); + lwkt_serialize_exit(&sc->sc_serializer); + ahci_pci_detach(dev); + return (ENXIO); + } + sc->sc_iot = rman_get_bustag(sc->sc_regs); + sc->sc_ioh = rman_get_bushandle(sc->sc_regs); + + /* + * Initialize the chipset and then set the interrupt vector up + */ + error = ahci_init(sc); + if (error) { + lwkt_serialize_exit(&sc->sc_serializer); + ahci_pci_detach(dev); + return (ENXIO); + } + + /* + * Get the AHCI capabilities and max number of concurrent + * command tags and set up the DMA tags. + */ + cap = ahci_read(sc, AHCI_REG_CAP); + if (sc->sc_flags & AHCI_F_NO_NCQ) + cap &= ~AHCI_REG_CAP_SNCQ; + sc->sc_cap = cap; + sc->sc_ncmds = AHCI_REG_CAP_NCS(cap); + + addr = (cap & AHCI_REG_CAP_S64A) ? + BUS_SPACE_MAXADDR : BUS_SPACE_MAXADDR_32BIT; + + /* + * DMA tags for allocation of DMA memory buffers, lists, and so + * forth. These are typically per-port. + */ + error = 0; + error += bus_dma_tag_create( + NULL, /* parent tag */ + 256, /* alignment */ + PAGE_SIZE, /* boundary */ + addr, /* loaddr? */ + BUS_SPACE_MAXADDR, /* hiaddr */ + NULL, /* filter */ + NULL, /* filterarg */ + sizeof(struct ahci_rfis), /* [max]size */ + 1, /* maxsegs */ + sizeof(struct ahci_rfis), /* maxsegsz */ + 0, /* flags */ + &sc->sc_tag_rfis); /* return tag */ + + error += bus_dma_tag_create( + NULL, /* parent tag */ + 32, /* alignment */ + 4096 * 1024, /* boundary */ + addr, /* loaddr? */ + BUS_SPACE_MAXADDR, /* hiaddr */ + NULL, /* filter */ + NULL, /* filterarg */ + sc->sc_ncmds * sizeof(struct ahci_cmd_hdr), + 1, /* maxsegs */ + sc->sc_ncmds * sizeof(struct ahci_cmd_hdr), + 0, /* flags */ + &sc->sc_tag_cmdh); /* return tag */ + + /* + * NOTE: ahci_cmd_table is sized to a power of 2 + */ + error += bus_dma_tag_create( + NULL, /* parent tag */ + sizeof(struct ahci_cmd_table), /* alignment */ + 4096 * 1024, /* boundary */ + addr, /* loaddr? */ + BUS_SPACE_MAXADDR, /* hiaddr */ + NULL, /* filter */ + NULL, /* filterarg */ + sc->sc_ncmds * sizeof(struct ahci_cmd_table), + 1, /* maxsegs */ + sc->sc_ncmds * sizeof(struct ahci_cmd_table), + 0, /* flags */ + &sc->sc_tag_cmdt); /* return tag */ + + /* + * The data tag is used for later dmamaps and not immediately + * allocated. + */ + error += bus_dma_tag_create( + NULL, /* parent tag */ + 4, /* alignment */ + 0, /* boundary */ + addr, /* loaddr? */ + BUS_SPACE_MAXADDR, /* hiaddr */ + NULL, /* filter */ + NULL, /* filterarg */ + 4096 * 1024, /* maxiosize */ + AHCI_MAX_PRDT, /* maxsegs */ + 65536, /* maxsegsz */ + 0, /* flags */ + &sc->sc_tag_data); /* return tag */ + + if (error) { + device_printf(dev, "unable to create dma tags\n"); + lwkt_serialize_exit(&sc->sc_serializer); + ahci_pci_detach(dev); + return (ENXIO); + } + + switch (cap & AHCI_REG_CAP_ISS) { + case AHCI_REG_CAP_ISS_G1: + gen = "1 (1.5Gbps)"; + break; + case AHCI_REG_CAP_ISS_G1_2: + gen = "1 (1.5Gbps) and 2 (3Gbps)"; + break; + default: + gen = "unknown"; + break; + } + + /* check the revision */ + reg = ahci_read(sc, AHCI_REG_VS); + switch (reg) { + case AHCI_REG_VS_0_95: + revision = "AHCI 0.95"; + break; + case AHCI_REG_VS_1_0: + revision = "AHCI 1.0"; + break; + case AHCI_REG_VS_1_1: + revision = "AHCI 1.1"; + break; + case AHCI_REG_VS_1_2: + revision = "AHCI 1.2"; + break; + default: + device_printf(sc->sc_dev, + "Warning: Unknown AHCI revision 0x%08x\n", reg); + revision = "AHCI "; + break; + } + + device_printf(dev, + "%s capabilities 0x%b, %d ports, %d tags/port, gen %s\n", + revision, + cap, AHCI_FMT_CAP, + AHCI_REG_CAP_NP(cap), sc->sc_ncmds, gen); + + pi = ahci_read(sc, AHCI_REG_PI); + DPRINTF(AHCI_D_VERBOSE, "%s: ports implemented: 0x%08x\n", + DEVNAME(sc), pi); + +#ifdef AHCI_COALESCE + /* Naive coalescing support - enable for all ports. */ + if (cap & AHCI_REG_CAP_CCCS) { + u_int16_t ccc_timeout = 20; + u_int8_t ccc_numcomplete = 12; + u_int32_t ccc_ctl; + + /* disable coalescing during reconfiguration. */ + ccc_ctl = ahci_read(sc, AHCI_REG_CCC_CTL); + ccc_ctl &= ~0x00000001; + ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); + + sc->sc_ccc_mask = 1 << AHCI_REG_CCC_CTL_INT(ccc_ctl); + if (pi & sc->sc_ccc_mask) { + /* A conflict with the implemented port list? */ + printf("%s: coalescing interrupt/implemented port list " + "conflict, PI: %08x, ccc_mask: %08x\n", + DEVNAME(sc), pi, sc->sc_ccc_mask); + sc->sc_ccc_mask = 0; + goto noccc; + } + + /* ahci_port_start will enable each port when it starts. */ + sc->sc_ccc_ports = pi; + sc->sc_ccc_ports_cur = 0; + + /* program thresholds and enable overall coalescing. */ + ccc_ctl &= ~0xffffff00; + ccc_ctl |= (ccc_timeout << 16) | (ccc_numcomplete << 8); + ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl); + ahci_write(sc, AHCI_REG_CCC_PORTS, 0); + ahci_write(sc, AHCI_REG_CCC_CTL, ccc_ctl | 1); + } +noccc: +#endif + /* + * Allocate per-port resources + * + * Ignore attach errors, leave the port intact for + * rescan and continue the loop. + */ + for (i = 0; error == 0 && i < AHCI_MAX_PORTS; i++) { + if ((pi & (1 << i)) == 0) { + /* dont allocate stuff if the port isnt implemented */ + continue; + } + error = ahci_port_alloc(sc, i); + if (error == 0) { + ahci_cam_attach(sc->sc_ports[i]); + } + if (error == ENODEV) + error = 0; + } + + /* + * Setup the interrupt vector and enable interrupts. Note that + * since the irq may be shared we do not set it up until we are + * ready to go. + */ + if (error == 0) { + error = bus_setup_intr(dev, sc->sc_irq, 0, ahci_intr, sc, + &sc->sc_irq_handle, &sc->sc_serializer); + } + + if (error) { + device_printf(dev, "unable to install interrupt\n"); + lwkt_serialize_exit(&sc->sc_serializer); + ahci_pci_detach(dev); + return (ENXIO); + } + ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); + lwkt_serialize_exit(&sc->sc_serializer); + + return(0); +} + +/* + * Device unload / detachment + */ +static int +ahci_pci_detach(device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + struct ahci_port *ap; + int i; + + /* + * Disable the controller and de-register the interrupt, if any. + * + * XXX interlock serializer against interrupt + */ + lwkt_serialize_handler_disable(&sc->sc_serializer); + if (sc->sc_regs) { + ahci_write(sc, AHCI_REG_GHC, 0); + } + if (sc->sc_irq_handle) { + bus_teardown_intr(dev, sc->sc_irq, sc->sc_irq_handle); + sc->sc_irq_handle = NULL; + } + + /* + * Free port structures and DMA memory + */ + for (i = 0; i < AHCI_MAX_PORTS; i++) { + ap = sc->sc_ports[i]; + if (ap) { + ahci_cam_detach(ap); + ahci_port_free(sc, i); + } + } + + /* + * Clean up the bus space + */ + if (sc->sc_irq) { + bus_release_resource(dev, SYS_RES_IRQ, + sc->sc_rid_irq, sc->sc_irq); + sc->sc_irq = NULL; + } + if (sc->sc_regs) { + bus_release_resource(dev, SYS_RES_MEMORY, + sc->sc_rid_regs, sc->sc_regs); + sc->sc_regs = NULL; + } + + if (sc->sc_tag_rfis) { + bus_dma_tag_destroy(sc->sc_tag_rfis); + sc->sc_tag_rfis = NULL; + } + if (sc->sc_tag_cmdh) { + bus_dma_tag_destroy(sc->sc_tag_cmdh); + sc->sc_tag_cmdh = NULL; + } + if (sc->sc_tag_cmdt) { + bus_dma_tag_destroy(sc->sc_tag_cmdt); + sc->sc_tag_cmdt = NULL; + } + if (sc->sc_tag_data) { + bus_dma_tag_destroy(sc->sc_tag_data); + sc->sc_tag_data = NULL; + } + + return (0); +} diff --git a/sys/dev/disk/ahci/ahci_cam.c b/sys/dev/disk/ahci/ahci_cam.c new file mode 100644 index 0000000000..934653dcd3 --- /dev/null +++ b/sys/dev/disk/ahci/ahci_cam.c @@ -0,0 +1,998 @@ +/* + * Copyright (c) 2009 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * + * Copyright (c) 2007 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $OpenBSD: atascsi.c,v 1.64 2009/02/16 21:19:06 miod Exp $ + * $DragonFly$ + */ +/* + * Implement each SATA port as its own SCSI bus on CAM. This way we can + * implement future port multiplier features as individual devices on the + * bus. + * + * Much of the cdb<->xa conversion code was taken from OpenBSD, the rest + * was written natively for DragonFly. + */ + +#include "ahci.h" + +static void ahci_xpt_action(struct cam_sim *sim, union ccb *ccb); +static void ahci_xpt_poll(struct cam_sim *sim); +static void ahci_xpt_scsi_disk_io(struct cam_sim *sim, union ccb *ccb); +static void ahci_xpt_scsi_atapi_io(struct cam_sim *sim, union ccb *ccb); + +static void ahci_ata_complete_disk_rw(struct ata_xfer *xa); +static void ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa); + +static int ahci_cam_probe(struct ahci_port *ap); +static void ata_fix_identify(struct ata_identify *id); +static void ahci_cam_rescan(struct ahci_port *ap); + +int +ahci_cam_attach(struct ahci_port *ap) +{ + struct cam_devq *devq; + struct cam_sim *sim; + int error; + int unit; + + unit = device_get_unit(ap->ap_sc->sc_dev); + devq = cam_simq_alloc(ap->ap_sc->sc_ncmds); + if (devq == NULL) { + return (ENOMEM); + } + sim = cam_sim_alloc(ahci_xpt_action, ahci_xpt_poll, "ahci", + (void *)ap, unit, &sim_mplock, 1, 1, devq); + cam_simq_release(devq); + if (sim == NULL) { + return (ENOMEM); + } + ap->ap_sim = sim; + error = xpt_bus_register(ap->ap_sim, ap->ap_num); + if (error != CAM_SUCCESS) { + ahci_cam_detach(ap); + return (EINVAL); + } + ap->ap_flags |= AP_F_BUS_REGISTERED; + error = xpt_create_path(&ap->ap_path, NULL, cam_sim_path(sim), + CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); + if (error != CAM_REQ_CMP) { + ahci_cam_detach(ap); + return (ENOMEM); + } + + error = ahci_cam_probe(ap); + if (error) { + ahci_cam_detach(ap); + return (EIO); + } + ap->ap_flags |= AP_F_CAM_ATTACHED; + + ahci_cam_rescan(ap); + + return(0); +} + +void +ahci_cam_changed(struct ahci_port *ap) +{ + if (ap->ap_sim == NULL) + return; + ahci_cam_probe(ap); + ahci_cam_rescan(ap); +} + +void +ahci_cam_detach(struct ahci_port *ap) +{ + int error; + + if ((ap->ap_flags & AP_F_CAM_ATTACHED) == 0) + return; + get_mplock(); + if (ap->ap_sim) { + xpt_freeze_simq(ap->ap_sim, 1); + } + if (ap->ap_path) { + xpt_free_path(ap->ap_path); + ap->ap_path = NULL; + } + if (ap->ap_flags & AP_F_BUS_REGISTERED) { + error = xpt_bus_deregister(cam_sim_path(ap->ap_sim)); + KKASSERT(error == CAM_REQ_CMP); + ap->ap_flags &= ~AP_F_BUS_REGISTERED; + } + if (ap->ap_sim) { + cam_sim_free(ap->ap_sim); + ap->ap_sim = NULL; + } + rel_mplock(); + ap->ap_flags &= ~AP_F_CAM_ATTACHED; +} + +/* + * Once the AHCI port has been attched we need to probe for a device or + * devices on the port and setup various options. + * + * XXX use this to allow re-probing and disconnect/reconnect + */ +static void +ahci_ata_dummy_done(struct ata_xfer *xa) +{ +} + +static int +ahci_cam_probe(struct ahci_port *ap) +{ + /*struct ahci_softc *sc = ap->ap_sc;*/ + struct ata_xfer *xa; + const char *devtype; + u_int64_t capacity; + u_int64_t capacity_bytes; + int model_len; + int status; + int devncqdepth; + int i; + + switch(ap->ap_ata.ap_type) { + case ATA_PORT_T_ATAPI: + devtype = "ATAPI"; + break; + case ATA_PORT_T_DISK: + devtype = "DISK"; + break; + default: + return(EIO); + } + + /* + * Issue identify, saving the result + */ + xa = ahci_ata_get_xfer(ap); + xa->complete = ahci_ata_dummy_done; + xa->data = &ap->ap_ata.ap_identify; + xa->datalen = sizeof(ap->ap_ata.ap_identify); + xa->fis->flags = ATA_H2D_FLAGS_CMD; + xa->fis->command = ATA_C_IDENTIFY; + xa->fis->features = 0; + xa->fis->device = 0; + xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL; + xa->timeout = hz; + + status = ahci_ata_cmd(xa); + if (status != ATA_COMPLETE) { + kprintf("%s: Detected %s device but unable to IDENTIFY\n", + PORTNAME(ap), devtype); + ahci_ata_put_xfer(xa); + return(EIO); + } + if (xa->state != ATA_S_COMPLETE) { + kprintf("%s: Detected %s device but unable to IDENTIFY " + " xa->state=%d\n", + PORTNAME(ap), devtype, xa->state); + ahci_ata_put_xfer(xa); + return(EIO); + } + ahci_ata_put_xfer(xa); + + ata_fix_identify(&ap->ap_ata.ap_identify); + + /* + * Read capacity using SATA probe info. + */ + if (le16toh(ap->ap_ata.ap_identify.cmdset83) & 0x0400) { + /* LBA48 feature set supported */ + capacity = 0; + for (i = 3; i >= 0; --i) { + capacity <<= 16; + capacity += + le16toh(ap->ap_ata.ap_identify.addrsecxt[i]); + } + } else { + capacity = le16toh(ap->ap_ata.ap_identify.addrsec[1]); + capacity <<= 16; + capacity += le16toh(ap->ap_ata.ap_identify.addrsec[0]); + } + ap->ap_ata.ap_capacity = capacity; + ap->ap_ata.ap_features |= ATA_PORT_F_PROBED; + + capacity_bytes = capacity * 512; + + /* + * Negotiate NCQ, throw away any ata_xfer's beyond the negotiated + * number of slots and limit the number of CAM ccb's to one less + * so we always have a slot available for recovery. + * + * NCQ is not used if ap_ncqdepth is 1 or the host controller does + * not support it, and in that case the driver can handle extra + * ccb's. + */ + if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) && + (le16toh(ap->ap_ata.ap_identify.satacap) & (1 << 8))) { + ap->ap_ata.ap_ncqdepth = (le16toh(ap->ap_ata.ap_identify.qdepth) & 0x1F) + 1; + devncqdepth = ap->ap_ata.ap_ncqdepth; + if (ap->ap_ata.ap_ncqdepth > ap->ap_sc->sc_ncmds) + ap->ap_ata.ap_ncqdepth = ap->ap_sc->sc_ncmds; + for (i = 0; i < ap->ap_sc->sc_ncmds; ++i) { + xa = ahci_ata_get_xfer(ap); + if (xa->tag < ap->ap_ata.ap_ncqdepth) { + xa->state = ATA_S_COMPLETE; + ahci_ata_put_xfer(xa); + } + } + if (ap->ap_ata.ap_ncqdepth > 1 && + ap->ap_ata.ap_ncqdepth >= ap->ap_sc->sc_ncmds) { + cam_devq_resize(ap->ap_sim->devq, + ap->ap_ata.ap_ncqdepth - 1); + } + } else { + devncqdepth = 0; + } + + for (model_len = 40; model_len; --model_len) { + if (ap->ap_ata.ap_identify.model[model_len-1] == ' ') + continue; + if (ap->ap_ata.ap_identify.model[model_len-1] == 0) + continue; + break; + } + kprintf("%s: Found %s \"%*.*s %8.8s\" serial=\"%20.20s\"\n" + "%s: tags=%d/%d satacaps=%04x satafeat=%04x " + "capacity=%lld.%02dMB\n" + "%s: f85=%04x f86=%04x f87=%04x\n", + PORTNAME(ap), + devtype, + model_len, model_len, + ap->ap_ata.ap_identify.model, + ap->ap_ata.ap_identify.firmware, + ap->ap_ata.ap_identify.serial, + + PORTNAME(ap), + devncqdepth, ap->ap_sc->sc_ncmds, + ap->ap_ata.ap_identify.satacap, + ap->ap_ata.ap_identify.satafsup, + (long long)capacity_bytes / (1024 * 1024), + (int)(capacity_bytes % (1024 * 1024)) * 100 / (1024 * 1024), + + PORTNAME(ap), + ap->ap_ata.ap_identify.features85, + ap->ap_ata.ap_identify.features86, + ap->ap_ata.ap_identify.features87 + ); + + /* + * DISKs only past this point + */ + if (ap->ap_ata.ap_type != ATA_PORT_T_DISK) + return (0); + + /* + * Enable write cache if supported + */ + if (ap->ap_ata.ap_identify.cmdset82 & ATA_IDENTIFY_WRITECACHE) { + xa = ahci_ata_get_xfer(ap); + xa->complete = ahci_ata_dummy_done; + xa->fis->command = ATA_C_SET_FEATURES; + xa->fis->features = ATA_SF_WRITECACHE_EN; + xa->fis->flags = ATA_H2D_FLAGS_CMD; + xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL; + xa->timeout = hz; + status = ahci_ata_cmd(xa); + if (status == ATA_COMPLETE) + ap->ap_ata.ap_features |= ATA_PORT_F_WCACHE; + ahci_ata_put_xfer(xa); + } + + /* + * Enable readahead if supported + */ + if (ap->ap_ata.ap_identify.cmdset82 & ATA_IDENTIFY_LOOKAHEAD) { + xa = ahci_ata_get_xfer(ap); + xa->complete = ahci_ata_dummy_done; + xa->fis->command = ATA_C_SET_FEATURES; + xa->fis->features = ATA_SF_LOOKAHEAD_EN; + xa->fis->flags = ATA_H2D_FLAGS_CMD; + xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL; + xa->timeout = hz; + status = ahci_ata_cmd(xa); + if (status == ATA_COMPLETE) + ap->ap_ata.ap_features |= ATA_PORT_F_RAHEAD; + ahci_ata_put_xfer(xa); + } + + /* + * FREEZE LOCK the device so malicious users can't lock it on us. + * As there is no harm in issuing this to devices that don't + * support the security feature set we just send it, and don't bother + * checking if the device sends a command abort to tell us it doesn't + * support it + */ + xa = ahci_ata_get_xfer(ap); + xa->complete = ahci_ata_dummy_done; + xa->fis->command = ATA_C_SEC_FREEZE_LOCK; + xa->fis->flags = ATA_H2D_FLAGS_CMD; + xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL; + xa->timeout = hz; + status = ahci_ata_cmd(xa); + if (status == ATA_COMPLETE) + ap->ap_ata.ap_features |= ATA_PORT_F_FRZLCK; + ahci_ata_put_xfer(xa); + + /* + * Attach as a SCSI disk + */ + + return (0); +} + +static void +ata_fix_identify(struct ata_identify *id) +{ + u_int16_t *swap; + int i; + + swap = (u_int16_t *)id->serial; + for (i = 0; i < sizeof(id->serial) / sizeof(u_int16_t); i++) + swap[i] = bswap16(swap[i]); + + swap = (u_int16_t *)id->firmware; + for (i = 0; i < sizeof(id->firmware) / sizeof(u_int16_t); i++) + swap[i] = bswap16(swap[i]); + + swap = (u_int16_t *)id->model; + for (i = 0; i < sizeof(id->model) / sizeof(u_int16_t); i++) + swap[i] = bswap16(swap[i]); +} + +/* + * Initiate bus scan. + */ +static void +ahci_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb) +{ + kfree(ccb, M_TEMP); +} + +static void +ahci_cam_rescan(struct ahci_port *ap) +{ + struct cam_path *path; + union ccb *ccb; + int status; + + ccb = kmalloc(sizeof(*ccb), M_TEMP, M_WAITOK | M_ZERO); + status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim), + CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); + if (status != CAM_REQ_CMP) + return; + + xpt_setup_ccb(&ccb->ccb_h, path, 5); /* 5 = low priority */ + ccb->ccb_h.func_code = XPT_SCAN_BUS; + ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback; + ccb->crcn.flags = CAM_FLAG_NONE; + xpt_action(ccb); + + /* scan is now underway */ +} + +/* + * Action function - dispatch command + */ +static +void +ahci_xpt_action(struct cam_sim *sim, union ccb *ccb) +{ + struct ahci_port *ap; + struct ccb_hdr *ccbh; + int unit; + + /* XXX lock */ + ap = cam_sim_softc(sim); + KKASSERT(ap != NULL); + ccbh = &ccb->ccb_h; + unit = cam_sim_unit(sim); + + /* + * Non-zero target and lun ids will be used for future + * port multiplication(?). A target wildcard indicates only + * the general bus is being probed. + * + * XXX What do we do with a LUN wildcard? + */ + if (ccbh->target_id != CAM_TARGET_WILDCARD) { + if (ap->ap_ata.ap_type == ATA_PORT_T_NONE) { + ccbh->status = CAM_REQ_INVALID; + xpt_done(ccb); + return; + } + if (ccbh->target_id) { + ccbh->status = CAM_DEV_NOT_THERE; + xpt_done(ccb); + return; + } + if (ccbh->target_lun != CAM_LUN_WILDCARD && ccbh->target_lun) { + ccbh->status = CAM_DEV_NOT_THERE; + xpt_done(ccb); + return; + } + } + + /* + * Switch on the meta XPT command + */ + switch(ccbh->func_code) { + case XPT_PATH_INQ: + ccb->cpi.version_num = 1; + ccb->cpi.hba_inquiry = 0; + ccb->cpi.target_sprt = 0; + ccb->cpi.hba_misc = 0; + ccb->cpi.hba_eng_cnt = 0; + bzero(ccb->cpi.vuhba_flags, sizeof(ccb->cpi.vuhba_flags)); + ccb->cpi.max_target = 7; + ccb->cpi.max_lun = 0; + ccb->cpi.async_flags = 0; + ccb->cpi.hpath_id = 0; + ccb->cpi.initiator_id = 7; + ccb->cpi.unit_number = cam_sim_unit(sim); + ccb->cpi.bus_id = cam_sim_bus(sim); + ccb->cpi.base_transfer_speed = 150000; + ccb->cpi.transport = XPORT_AHCI; + ccb->cpi.transport_version = 1; + ccb->cpi.protocol = PROTO_SCSI; + ccb->cpi.protocol_version = SCSI_REV_2; + + /* + * Non-zero target and lun ids will be used for future + * port multiplication(?). A target wildcard indicates only + * the general bus is being probed. + * + * XXX What do we do with a LUN wildcard? + */ + if (ccbh->target_id != CAM_TARGET_WILDCARD) { + switch(ahci_pread(ap, AHCI_PREG_SSTS) & + AHCI_PREG_SSTS_SPD) { + case AHCI_PREG_SSTS_SPD_GEN1: + ccb->cpi.base_transfer_speed = 150000; + break; + case AHCI_PREG_SSTS_SPD_GEN2: + ccb->cpi.base_transfer_speed = 300000; + break; + default: + /* unknown */ + ccb->cpi.base_transfer_speed = 1000; + break; + } + /* XXX check attached, set base xfer speed */ + } + ccbh->status = CAM_REQ_CMP; + xpt_done(ccb); + break; + case XPT_RESET_DEV: + ccbh->status = CAM_REQ_INVALID; + xpt_done(ccb); + break; + case XPT_RESET_BUS: + ccbh->status = CAM_REQ_INVALID; + xpt_done(ccb); + break; + case XPT_SET_TRAN_SETTINGS: + ccbh->status = CAM_FUNC_NOTAVAIL; + xpt_done(ccb); + break; + case XPT_GET_TRAN_SETTINGS: + ccb->cts.protocol = PROTO_SCSI; + ccb->cts.protocol_version = SCSI_REV_2; + ccb->cts.transport = XPORT_AHCI; + ccb->cts.transport_version = XPORT_VERSION_UNSPECIFIED; + ccb->cts.proto_specific.valid = 0; + ccb->cts.xport_specific.valid = 0; + ccbh->status = CAM_REQ_CMP; + xpt_done(ccb); + break; + case XPT_CALC_GEOMETRY: + cam_calc_geometry(&ccb->ccg, 1); + xpt_done(ccb); + break; + case XPT_SCSI_IO: + switch(ap->ap_ata.ap_type) { + case ATA_PORT_T_DISK: + ahci_xpt_scsi_disk_io(sim, ccb); + break; + case ATA_PORT_T_ATAPI: + kprintf("xpt_scsi_io - atapi\n"); + ahci_xpt_scsi_atapi_io(sim, ccb); + break; + default: + ccbh->status = CAM_REQ_INVALID; + xpt_done(ccb); + break; + } + break; + default: + kprintf("xpt_unknown\n"); + ccbh->status = CAM_REQ_INVALID; + xpt_done(ccb); + break; + } +} + +/* + * Poll function (unused?) + */ +static +void +ahci_xpt_poll(struct cam_sim *sim) +{ + /*struct ahci_port *ap = cam_sim_softc(sim);*/ + + kprintf("ahci_xpt_poll\n"); + /* XXX lock */ +} + +/* + * Convert the SCSI command in ccb to an ata_xfer command in xa, + * set the completion function to convert the response back, then + * dispatch to the OpenBSD AHCI layer. + * + * AHCI DISK commands only support a limited command set. + * + * AHCI ATAPI commands are basically a SCSI transport layer but + * with some munging. + */ +static +void +ahci_xpt_scsi_disk_io(struct cam_sim *sim, union ccb *ccb) +{ + struct ahci_port *ap; + struct ccb_hdr *ccbh; + struct ccb_scsiio *csio; + struct ata_xfer *xa; + struct ata_fis_h2d *fis; + scsi_cdb_t cdb; + union scsi_data *rdata; + int rdata_len; + u_int64_t capacity; + u_int64_t lba; + u_int32_t count; + + ap = cam_sim_softc(sim); + ccbh = &ccb->csio.ccb_h; + csio = &ccb->csio; + xa = ahci_ata_get_xfer(ap); + rdata = (void *)csio->data_ptr; + rdata_len = csio->dxfer_len; + + /* + * Build the FIS or process the csio to completion. + */ + cdb = (void *)((ccbh->flags & CAM_CDB_POINTER) ? + csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes); + + switch(cdb->generic.opcode) { + case REQUEST_SENSE: + /* + * Auto-sense everything, so explicit sense requests + * return no-sense. + */ + ccbh->status = CAM_SCSI_STATUS_ERROR; + break; + case INQUIRY: + /* + * Inquiry supported features + * + * [opcode, byte2, page_code, length, control] + */ + if (cdb->inquiry.byte2 & SI_EVPD) { + switch(cdb->inquiry.page_code) { + case SVPD_SUPPORTED_PAGE_LIST: + /* XXX atascsi_disk_vpd_supported */ + case SVPD_UNIT_SERIAL_NUMBER: + /* XXX atascsi_disk_vpd_serial */ + case SVPD_UNIT_DEVID: + /* XXX atascsi_disk_vpd_ident */ + default: + ccbh->status = CAM_FUNC_NOTAVAIL; + break; + } + } else { + bzero(rdata, rdata_len); + if (rdata_len < SHORT_INQUIRY_LENGTH) { + ccbh->status = CAM_CCB_LEN_ERR; + break; + } + if (rdata_len > sizeof(rdata->inquiry_data)) + rdata_len = sizeof(rdata->inquiry_data); + rdata->inquiry_data.device = T_DIRECT; + rdata->inquiry_data.version = SCSI_REV_SPC2; + rdata->inquiry_data.response_format = 2; + rdata->inquiry_data.additional_length = 32; + bcopy("SATA ", rdata->inquiry_data.vendor, 8); + bcopy(ap->ap_ata.ap_identify.model, + rdata->inquiry_data.product, + sizeof(rdata->inquiry_data.product)); + bcopy(ap->ap_ata.ap_identify.firmware, + rdata->inquiry_data.revision, + sizeof(rdata->inquiry_data.revision)); + ccbh->status = CAM_REQ_CMP; + } + break; + case READ_CAPACITY_16: + if (cdb->read_capacity_16.service_action != SRC16_SERVICE_ACTION) { + ccbh->status = CAM_REQ_INVALID; + break; + } + if (rdata_len < sizeof(rdata->read_capacity_data_16)) { + ccbh->status = CAM_CCB_LEN_ERR; + break; + } + /* fall through */ + case READ_CAPACITY: + if (rdata_len < sizeof(rdata->read_capacity_data)) { + ccbh->status = CAM_CCB_LEN_ERR; + break; + } + + capacity = ap->ap_ata.ap_capacity; + + bzero(rdata, rdata_len); + if (cdb->generic.opcode == READ_CAPACITY) { + rdata_len = sizeof(rdata->read_capacity_data); + if (capacity > 0xFFFFFFFFU) + capacity = 0xFFFFFFFFU; + bzero(&rdata->read_capacity_data, rdata_len); + scsi_ulto4b((u_int32_t)capacity - 1, + rdata->read_capacity_data.addr); + scsi_ulto4b(512, rdata->read_capacity_data.length); + } else { + rdata_len = sizeof(rdata->read_capacity_data_16); + bzero(&rdata->read_capacity_data_16, rdata_len); + scsi_u64to8b(capacity - 1, + rdata->read_capacity_data_16.addr); + scsi_ulto4b(512, rdata->read_capacity_data_16.length); + } + ccbh->status = CAM_REQ_CMP; + break; + case SYNCHRONIZE_CACHE: + /* + * Synchronize cache. Specification says this can take + * greater then 30 seconds so give it at least 45. + */ + fis = xa->fis; + xa->datalen = 0; + xa->flags = ATA_F_READ; + xa->complete = ahci_ata_complete_disk_synchronize_cache; + if (xa->timeout < 45 * hz) + xa->timeout = 45 * hz; + fis->flags = ATA_H2D_FLAGS_CMD; + fis->command = ATA_C_FLUSH_CACHE; + fis->device = 0; + break; + case TEST_UNIT_READY: + case START_STOP_UNIT: + case PREVENT_ALLOW: + /* + * Just silently return success + */ + ccbh->status = CAM_REQ_CMP; + rdata_len = 0; + break; + case ATA_PASS_12: + case ATA_PASS_16: + /* + * XXX implement pass-through + */ + ccbh->status = CAM_FUNC_NOTAVAIL; + break; + default: + switch(cdb->generic.opcode) { + case READ_6: + lba = scsi_3btoul(cdb->rw_6.addr) & 0x1FFFFF; + count = cdb->rw_6.length ? cdb->rw_6.length : 0x100; + xa->flags = ATA_F_READ; + break; + case READ_10: + lba = scsi_4btoul(cdb->rw_10.addr); + count = scsi_2btoul(cdb->rw_10.length); + xa->flags = ATA_F_READ; + break; + case READ_12: + lba = scsi_4btoul(cdb->rw_12.addr); + count = scsi_4btoul(cdb->rw_12.length); + xa->flags = ATA_F_READ; + break; + case READ_16: + lba = scsi_8btou64(cdb->rw_16.addr); + count = scsi_4btoul(cdb->rw_16.length); + xa->flags = ATA_F_READ; + break; + case WRITE_6: + lba = scsi_3btoul(cdb->rw_6.addr) & 0x1FFFFF; + count = cdb->rw_6.length ? cdb->rw_6.length : 0x100; + xa->flags = ATA_F_WRITE; + break; + case WRITE_10: + lba = scsi_4btoul(cdb->rw_10.addr); + count = scsi_2btoul(cdb->rw_10.length); + xa->flags = ATA_F_WRITE; + break; + case WRITE_12: + lba = scsi_4btoul(cdb->rw_12.addr); + count = scsi_4btoul(cdb->rw_12.length); + xa->flags = ATA_F_WRITE; + break; + case WRITE_16: + lba = scsi_8btou64(cdb->rw_16.addr); + count = scsi_4btoul(cdb->rw_16.length); + xa->flags = ATA_F_WRITE; + break; + default: + ccbh->status = CAM_REQ_INVALID; + break; + } + if (ccbh->status != CAM_REQ_INPROG) + break; + + fis = xa->fis; + fis->flags = ATA_H2D_FLAGS_CMD; + fis->lba_low = (u_int8_t)lba; + fis->lba_mid = (u_int8_t)(lba >> 8); + fis->lba_high = (u_int8_t)(lba >> 16); + fis->device = ATA_H2D_DEVICE_LBA; + + if (ap->ap_ata.ap_ncqdepth > 1 && + (ap->ap_sc->sc_cap & AHCI_REG_CAP_SNCQ) && + (ccbh->flags & CAM_POLLED) == 0) { + /* + * Use NCQ - always uses 48 bit addressing + */ + xa->flags |= ATA_F_NCQ; + fis->command = (xa->flags & ATA_F_WRITE) ? + ATA_C_WRITE_FPDMA : ATA_C_READ_FPDMA; + fis->lba_low_exp = (u_int8_t)(lba >> 24); + fis->lba_mid_exp = (u_int8_t)(lba >> 32); + fis->lba_high_exp = (u_int8_t)(lba >> 40); + fis->sector_count = xa->tag << 3; + fis->features = (u_int8_t)count; + fis->features_exp = (u_int8_t)(count >> 8); + } else if (count > 0x100 || lba > 0xFFFFFFFFU) { + /* + * Use LBA48 + */ + fis->command = (xa->flags & ATA_F_WRITE) ? + ATA_C_WRITEDMA_EXT : ATA_C_READDMA_EXT; + fis->lba_low_exp = (u_int8_t)(lba >> 24); + fis->lba_mid_exp = (u_int8_t)(lba >> 32); + fis->lba_high_exp = (u_int8_t)(lba >> 40); + fis->sector_count = (u_int8_t)count; + fis->sector_count_exp = (u_int8_t)(count >> 8); + } else { + /* + * Use LBA + * + * NOTE: 256 sectors is supported, stored as 0. + */ + fis->command = (xa->flags & ATA_F_WRITE) ? + ATA_C_WRITEDMA : ATA_C_READDMA; + fis->device |= (u_int8_t)(lba >> 24) & 0x0F; + fis->sector_count = (u_int8_t)count; + } + + xa->data = csio->data_ptr; + xa->datalen = csio->dxfer_len; + xa->complete = ahci_ata_complete_disk_rw; + xa->timeout = ccbh->timeout * hz / 1000; + if (ccbh->flags & CAM_POLLED) + xa->flags |= ATA_F_POLL; + break; + } + + /* + * If the request is still in progress the xa and FIS have + * been set up and must be dispatched. Otherwise the request + * is complete. + */ + if (ccbh->status == CAM_REQ_INPROG) { + KKASSERT(xa->complete != NULL); + xa->atascsi_private = ccb; + ccb->ccb_h.sim_priv.entries[0].ptr = ap; + lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_ata_cmd(xa); + lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + } else { + ahci_ata_put_xfer(xa); + xpt_done(ccb); + } +} + +static +void +ahci_xpt_scsi_atapi_io(struct cam_sim *sim, union ccb *ccb) +{ + struct ahci_port *ap; + struct ccb_hdr *ccbh; + struct ccb_scsiio *csio; + struct ata_xfer *xa; + struct ata_fis_h2d *fis; + scsi_cdb_t cdb; + + ap = cam_sim_softc(sim); + ccbh = &ccb->csio.ccb_h; + csio = &ccb->csio; + xa = ahci_ata_get_xfer(ap); + fis = xa->fis; + + cdb = (void *)((ccbh->flags & CAM_CDB_POINTER) ? + csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes); + + switch(cdb->generic.opcode) { + case REQUEST_SENSE: + /* + * Auto-sense everything, so explicit sense requests + * return no-sense. + */ + ccbh->status = CAM_SCSI_STATUS_ERROR; + break; + case TEST_UNIT_READY: + case SEND_DIAGNOSTIC: + case RECEIVE_DIAGNOSTIC: + case INQUIRY: + case MODE_SENSE_6: + case MODE_SENSE_10: + case MODE_SELECT_6: + case MODE_SELECT_10: + case LOG_SENSE: + case LOG_SELECT: + case RESERVE: + case RELEASE: + case PREVENT_ALLOW: + case SYNCHRONIZE_CACHE: + case CHANGE_DEFINITION: + case READ_BUFFER: + case WRITE_BUFFER: + + case READ_6: + case READ_10: + case READ_12: + case READ_16: + + case WRITE_6: + case WRITE_10: + case WRITE_12: + case WRITE_16: + + case START_STOP_UNIT: + case ATA_PASS_12: + case ATA_PASS_16: + + case READ_CAPACITY: + case REPORT_LUNS: + + case POSITION_TO_ELEMENT: + case READ_DEFECT_DATA_10: + case READ_CAPACITY_16: + case MOVE_MEDIUM: + case READ_ELEMENT_STATUS: + default: + ccbh->status = CAM_REQ_INVALID; + break; + } + + if (ccbh->status == CAM_REQ_INPROG) { + KKASSERT(xa->complete != NULL); + xa->atascsi_private = ccb; + ccb->ccb_h.sim_priv.entries[0].ptr = ap; + ahci_ata_cmd(xa); + } else { + ahci_ata_put_xfer(xa); + xpt_done(ccb); + } +} + +static +void +ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa) +{ + union ccb *ccb = xa->atascsi_private; + struct ccb_hdr *ccbh = &ccb->ccb_h; + struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr; + + switch(xa->state) { + case ATA_S_COMPLETE: + ccbh->status = CAM_REQ_CMP; + break; + case ATA_S_ERROR: + kprintf("%s: synchronize_cache: error\n", PORTNAME(ap)); + ccbh->status = CAM_REQ_CMP_ERR; + break; + case ATA_S_TIMEOUT: + kprintf("%s: synchronize_cache: timeout\n", PORTNAME(ap)); + ccbh->status = CAM_CMD_TIMEOUT; + break; + default: + kprintf("%s: synchronize_cache: unknown state %d\n", + PORTNAME(ap), xa->state); + ccbh->status = CAM_REQ_CMP_ERR; + break; + } + ahci_ata_put_xfer(xa); + lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + xpt_done(ccb); + lwkt_serialize_enter(&ap->ap_sc->sc_serializer); +} + +static +void +ahci_ata_complete_disk_rw(struct ata_xfer *xa) +{ + union ccb *ccb = xa->atascsi_private; + struct ccb_hdr *ccbh = &ccb->ccb_h; + struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr; + + switch(xa->state) { + case ATA_S_COMPLETE: + ccbh->status = CAM_REQ_CMP; + break; + case ATA_S_ERROR: + kprintf("%s: disk_rw: error\n", PORTNAME(ap)); + ccbh->status = CAM_REQ_CMP_ERR; + break; + case ATA_S_TIMEOUT: + kprintf("%s: disk_rw: timeout\n", PORTNAME(ap)); + ccbh->status = CAM_CMD_TIMEOUT; + break; + default: + kprintf("%s: disk_rw: unknown state %d\n", + PORTNAME(ap), xa->state); + ccbh->status = CAM_REQ_CMP_ERR; + break; + } + ccb->csio.resid = xa->resid; + ahci_ata_put_xfer(xa); + lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + xpt_done(ccb); + lwkt_serialize_enter(&ap->ap_sc->sc_serializer); +} diff --git a/sys/dev/disk/ahci/ahci_dragonfly.c b/sys/dev/disk/ahci/ahci_dragonfly.c new file mode 100644 index 0000000000..ae4ee856a0 --- /dev/null +++ b/sys/dev/disk/ahci/ahci_dragonfly.c @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2009 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * Primary device and CAM interface to OpenBSD AHCI driver, for DragonFly + */ + +#include "ahci.h" + +/* + * Device bus methods + */ + +static int ahci_probe (device_t dev); +static int ahci_attach (device_t dev); +static int ahci_detach (device_t dev); +#if 0 +static int ahci_shutdown (device_t dev); +static int ahci_suspend (device_t dev); +static int ahci_resume (device_t dev); +#endif + +static device_method_t ahci_methods[] = { + DEVMETHOD(device_probe, ahci_probe), + DEVMETHOD(device_attach, ahci_attach), + DEVMETHOD(device_detach, ahci_detach), +#if 0 + DEVMETHOD(device_shutdown, ahci_shutdown), + DEVMETHOD(device_suspend, ahci_suspend), + DEVMETHOD(device_resume, ahci_resume), +#endif + + DEVMETHOD(bus_print_child, bus_generic_print_child), + DEVMETHOD(bus_driver_added, bus_generic_driver_added), + {0, 0} +}; + +static devclass_t ahci_devclass; + +static driver_t ahci_driver = { + "ahci", + ahci_methods, + sizeof(struct ahci_softc) +}; + +MODULE_DEPEND(ahci, cam, 1, 1, 1); +DRIVER_MODULE(ahci, pci, ahci_driver, ahci_devclass, 0, 0); + +/* + * Device bus method procedures + */ +static int +ahci_probe (device_t dev) +{ + const struct ahci_device *ad; + + ad = ahci_lookup_device(dev); + if (ad) { + device_set_desc(dev, ad->name); + return(-5); /* higher priority the NATA */ + } + return(ENXIO); +} + +static int +ahci_attach (device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + int error; + + sc->sc_ad = ahci_lookup_device(dev); + if (sc->sc_ad == NULL) + return(ENXIO); + error = sc->sc_ad->ad_attach(dev); + return (error); +} + +static int +ahci_detach (device_t dev) +{ + struct ahci_softc *sc = device_get_softc(dev); + int error = 0; + + if (sc->sc_ad) { + error = sc->sc_ad->ad_detach(dev); + sc->sc_ad = NULL; + } + return(error); +} + +#if 0 + +static int +ahci_shutdown (device_t dev) +{ + return (0); +} + +static int +ahci_suspend (device_t dev) +{ + return (0); +} + +static int +ahci_resume (device_t dev) +{ + return (0); +} + +#endif diff --git a/sys/dev/disk/ahci/ahci_dragonfly.h b/sys/dev/disk/ahci/ahci_dragonfly.h new file mode 100644 index 0000000000..5058bb2f04 --- /dev/null +++ b/sys/dev/disk/ahci/ahci_dragonfly.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009 The DragonFly Project. All rights reserved. + * + * This code is derived from software contributed to The DragonFly Project + * by Matthew Dillon + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of The DragonFly Project nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific, prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include + +#include + +#define AHCI_CDEV_MAJOR 188 + +#define AHCI_IRQ_RID 0 diff --git a/sys/dev/disk/ahci/atascsi.h b/sys/dev/disk/ahci/atascsi.h new file mode 100644 index 0000000000..7950cfdb11 --- /dev/null +++ b/sys/dev/disk/ahci/atascsi.h @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2007 David Gwynne + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * $OpenBSD: atascsi.h,v 1.33 2009/02/16 21:19:06 miod Exp $ + */ + +struct atascsi; +struct scsi_link; + +/* + * ATA commands + */ + +#define ATA_C_READDMA_EXT 0x25 +#define ATA_C_READ_LOG_EXT 0x2f +#define ATA_C_WRITEDMA_EXT 0x35 +#define ATA_C_READ_FPDMA 0x60 +#define ATA_C_WRITE_FPDMA 0x61 +#define ATA_C_PACKET 0xa0 +#define ATA_C_READDMA 0xc8 +#define ATA_C_WRITEDMA 0xca +#define ATA_C_FLUSH_CACHE 0xe7 +#define ATA_C_FLUSH_CACHE_EXT 0xea /* lba48 */ +#define ATA_C_IDENTIFY 0xec +#define ATA_C_SET_FEATURES 0xef +#define ATA_C_SEC_FREEZE_LOCK 0xf5 + +/* + * ATA SET FEATURES subcommands + */ +#define ATA_SF_WRITECACHE_EN 0x02 +#define ATA_SF_LOOKAHEAD_EN 0xaa + +struct ata_identify { + u_int16_t config; /* 0 */ + u_int16_t ncyls; /* 1 */ + u_int16_t reserved1; /* 2 */ + u_int16_t nheads; /* 3 */ + u_int16_t track_size; /* 4 */ + u_int16_t sector_size; /* 5 */ + u_int16_t nsectors; /* 6 */ + u_int16_t reserved2[3]; /* 7 vendor unique */ + u_int8_t serial[20]; /* 10 */ + u_int16_t buffer_type; /* 20 */ + u_int16_t buffer_size; /* 21 */ + u_int16_t ecc; /* 22 */ + u_int8_t firmware[8]; /* 23 */ + u_int8_t model[40]; /* 27 */ + u_int16_t multi; /* 47 */ + u_int16_t dwcap; /* 48 */ + u_int16_t cap; /* 49 */ + u_int16_t reserved3; /* 50 */ + u_int16_t piomode; /* 51 */ + u_int16_t dmamode; /* 52 */ + u_int16_t validinfo; /* 53 */ + u_int16_t curcyls; /* 54 */ + u_int16_t curheads; /* 55 */ + u_int16_t cursectrk; /* 56 */ + u_int16_t curseccp[2]; /* 57 */ + u_int16_t mult2; /* 59 */ + u_int16_t addrsec[2]; /* 60 */ + u_int16_t worddma; /* 62 */ + u_int16_t dworddma; /* 63 */ + u_int16_t advpiomode; /* 64 */ + u_int16_t minmwdma; /* 65 */ + u_int16_t recmwdma; /* 66 */ + u_int16_t minpio; /* 67 */ + u_int16_t minpioflow; /* 68 */ + u_int16_t reserved4[2]; /* 69 */ + u_int16_t typtime[2]; /* 71 */ + u_int16_t reserved5[2]; /* 73 */ + u_int16_t qdepth; /* 75 */ + u_int16_t satacap; /* 76 */ + u_int16_t reserved6; /* 77 */ + u_int16_t satafsup; /* 78 */ + u_int16_t satafen; /* 79 */ + u_int16_t majver; /* 80 */ + u_int16_t minver; /* 81 */ + u_int16_t cmdset82; /* 82 */ + u_int16_t cmdset83; /* 83 */ + u_int16_t cmdset84; /* 84 */ + u_int16_t features85; /* 85 */ + u_int16_t features86; /* 86 */ + u_int16_t features87; /* 87 */ +#define ATA_ID_F87_WWN (1<<8) + u_int16_t ultradma; /* 88 */ + u_int16_t erasetime; /* 89 */ + u_int16_t erasetimex; /* 90 */ + u_int16_t apm; /* 91 */ + u_int16_t masterpw; /* 92 */ + u_int16_t hwreset; /* 93 */ + u_int16_t acoustic; /* 94 */ + u_int16_t stream_min; /* 95 */ + u_int16_t stream_xfer_d; /* 96 */ + u_int16_t stream_lat; /* 97 */ + u_int16_t streamperf[2]; /* 98 */ + u_int16_t addrsecxt[4]; /* 100 */ + u_int16_t stream_xfer_p; /* 104 */ + u_int16_t padding1; /* 105 */ + u_int16_t phys_sect_sz; /* 106 */ + u_int16_t seek_delay; /* 107 */ + u_int16_t naa_ieee_oui; /* 108 */ + u_int16_t ieee_oui_uid; /* 109 */ + u_int16_t uid_mid; /* 110 */ + u_int16_t uid_low; /* 111 */ + u_int16_t resv_wwn[4]; /* 112 */ + u_int16_t incits; /* 116 */ + u_int16_t words_lsec[2]; /* 117 */ + u_int16_t cmdset119; /* 119 */ + u_int16_t features120; /* 120 */ + u_int16_t padding2[6]; + u_int16_t rmsn; /* 127 */ + u_int16_t securestatus; /* 128 */ + u_int16_t vendor[31]; /* 129 */ + u_int16_t padding3[16]; /* 160 */ + u_int16_t curmedser[30]; /* 176 */ + u_int16_t sctsupport; /* 206 */ + u_int16_t padding4[48]; /* 207 */ + u_int16_t integrity; /* 255 */ +} __packed; + +/* + * IDENTIFY DEVICE data + */ +#define ATA_IDENTIFY_WRITECACHE (1 << 5) +#define ATA_IDENTIFY_LOOKAHEAD (1 << 6) + +/* + * Frame Information Structures + */ + +#define ATA_FIS_LENGTH 20 + +struct ata_fis_h2d { + u_int8_t type; +#define ATA_FIS_TYPE_H2D 0x27 + u_int8_t flags; +#define ATA_H2D_FLAGS_CMD (1<<7) + u_int8_t command; + u_int8_t features; +#define ATA_H2D_FEATURES_DMA (1<<0) +#define ATA_H2D_FEATURES_DIR (1<<2) +#define ATA_H2D_FEATURES_DIR_READ (1<<2) +#define ATA_H2D_FEATURES_DIR_WRITE (0<<2) + + u_int8_t lba_low; + u_int8_t lba_mid; + u_int8_t lba_high; + u_int8_t device; +#define ATA_H2D_DEVICE_LBA 0x40 + + u_int8_t lba_low_exp; + u_int8_t lba_mid_exp; + u_int8_t lba_high_exp; + u_int8_t features_exp; + + u_int8_t sector_count; + u_int8_t sector_count_exp; + u_int8_t reserved0; + u_int8_t control; + + u_int8_t reserved1; + u_int8_t reserved2; + u_int8_t reserved3; + u_int8_t reserved4; +} __packed; + +struct ata_fis_d2h { + u_int8_t type; +#define ATA_FIS_TYPE_D2H 0x34 + u_int8_t flags; +#define ATA_D2H_FLAGS_INTR (1<<6) + u_int8_t status; + u_int8_t error; + + u_int8_t lba_low; + u_int8_t lba_mid; + u_int8_t lba_high; + u_int8_t device; + + u_int8_t lba_low_exp; + u_int8_t lba_mid_exp; + u_int8_t lba_high_exp; + u_int8_t reserved0; + + u_int8_t sector_count; + u_int8_t sector_count_exp; + u_int8_t reserved1; + u_int8_t reserved2; + + u_int8_t reserved3; + u_int8_t reserved4; + u_int8_t reserved5; + u_int8_t reserved6; +} __packed; + +/* + * SATA log page 10h - + * looks like a D2H FIS, with errored tag number in first byte. + */ +struct ata_log_page_10h { + struct ata_fis_d2h err_regs; +#define ATA_LOG_10H_TYPE_NOTQUEUED 0x80 +#define ATA_LOG_10H_TYPE_TAG_MASK 0x1f + u_int8_t reserved[256 - sizeof(struct ata_fis_d2h)]; + u_int8_t vendor_specific[255]; + u_int8_t checksum; +} __packed; + +/* + * SATA registers + */ + +#define SATA_SStatus_DET 0x00f +#define SATA_SStatus_DET_NODEV 0x000 +#define SATA_SStatus_DET_NOPHY 0x001 +#define SATA_SStatus_DET_DEV 0x003 +#define SATA_SStatus_DET_OFFLINE 0x008 + +#define SATA_SStatus_SPD 0x0f0 +#define SATA_SStatus_SPD_NONE 0x000 +#define SATA_SStatus_SPD_1_5 0x010 +#define SATA_SStatus_SPD_3_0 0x020 + +#define SATA_SStatus_IPM 0xf00 +#define SATA_SStatus_IPM_NODEV 0x000 +#define SATA_SStatus_IPM_ACTIVE 0x100 +#define SATA_SStatus_IPM_PARTIAL 0x200 +#define SATA_SStatus_IPM_SLUMBER 0x600 + +#define SATA_SIGNATURE_PORT_MULTIPLIER 0x96690101 +#define SATA_SIGNATURE_ATAPI 0xeb140101 +#define SATA_SIGNATURE_DISK 0x00000101 + +/* + * ATA interface + */ + +struct ata_port { + struct ata_identify ap_identify; + struct atascsi *ap_as; + int ap_type; +#define ATA_PORT_T_NONE 0 +#define ATA_PORT_T_DISK 1 +#define ATA_PORT_T_ATAPI 2 + int ap_features; +#define ATA_PORT_F_PROBED (1 << 0) +#define ATA_PORT_F_WCACHE (1 << 1) +#define ATA_PORT_F_RAHEAD (1 << 2) +#define ATA_PORT_F_FRZLCK (1 << 3) + int ap_ncqdepth; + u_int64_t ap_capacity; +}; + +struct ata_xfer { + struct ata_fis_h2d *fis; + struct ata_fis_d2h rfis; + u_int8_t *packetcmd; + u_int8_t tag; + + void *data; + size_t datalen; + size_t resid; + + void (*complete)(struct ata_xfer *); + u_int timeout; + + int flags; +#define ATA_F_READ (1<<0) +#define ATA_F_WRITE (1<<1) +#define ATA_F_NOWAIT (1<<2) +#define ATA_F_POLL (1<<3) +#define ATA_F_PIO (1<<4) +#define ATA_F_PACKET (1<<5) +#define ATA_F_NCQ (1<<6) +#define ATA_F_TIMEOUT_RUNNING (1<<7) +#define ATA_FMT_FLAGS "\020" "\010TRUNNING" \ + "\007NCQ" "\006PACKET" \ + "\005PIO" "\004POLL" "\003NOWAIT" \ + "\002WRITE" "\001READ" + + volatile int state; +#define ATA_S_SETUP 0 +#define ATA_S_PENDING 1 +#define ATA_S_COMPLETE 2 +#define ATA_S_ERROR 3 +#define ATA_S_TIMEOUT 4 +#define ATA_S_ONCHIP 5 +#define ATA_S_PUT 6 + + void *atascsi_private; + + void (*ata_put_xfer)(struct ata_xfer *); +}; + +#define ATA_QUEUED 0 +#define ATA_COMPLETE 1 +#define ATA_ERROR 2 -- 2.41.0