From f4553de13c8796ae9536441d5de2afa0aa352699 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Fri, 12 Jun 2009 14:45:35 -0700 Subject: [PATCH] AHCI - Implement parallel port scan and thread each port interrupt. * Implement a thread helper for each port. The master interrupt will perform all actions which can be done without blocking and will delegate any remaining actions (typically error and timeout handling) to the port's thread helper. * The thread helper is responsible for the initial probe. Thus ALL AHCI SATA PORTS WILL NOW PROBE IN PARALLEL! Instead of 6 ports each taking 2 seconds to probe we now have 6 ports probing in a total of 2 seconds. * Multiple port multipliers will probe in parallel, but targets on each one have to be iterated. * The attach code waits for all ports to fully probe and then runs CAM attachments serially. This step goes very quickly since the ports have already probed. * Stalls on one physical port will no longer stall the rest of the ports. So, for example, stalls on the port connected to your port multiplier will not effect operations on, say, your internal SATA ports. --- sys/dev/disk/ahci/ahci.c | 273 ++++++++++++++++++++++------- sys/dev/disk/ahci/ahci.h | 26 ++- sys/dev/disk/ahci/ahci_attach.c | 56 ++++-- sys/dev/disk/ahci/ahci_cam.c | 100 +++++------ sys/dev/disk/ahci/ahci_dragonfly.c | 119 +++++++++++++ sys/dev/disk/ahci/ahci_dragonfly.h | 1 + sys/dev/disk/ahci/atascsi.h | 1 + 7 files changed, 437 insertions(+), 139 deletions(-) diff --git a/sys/dev/disk/ahci/ahci.c b/sys/dev/disk/ahci/ahci.c index 3e5527fd5e..7199aa7cef 100644 --- a/sys/dev/disk/ahci/ahci.c +++ b/sys/dev/disk/ahci/ahci.c @@ -51,10 +51,10 @@ #include "ahci.h" -int ahci_port_init(struct ahci_port *ap, struct ata_port *at); -int ahci_port_start(struct ahci_port *); -int ahci_port_stop(struct ahci_port *, int); -int ahci_port_clo(struct ahci_port *); +int ahci_port_start(struct ahci_port *ap); +int ahci_port_stop(struct ahci_port *ap, int stop_fis_rx); +int ahci_port_clo(struct ahci_port *ap); +void ahci_port_interrupt_enable(struct ahci_port *ap); int ahci_load_prdt(struct ahci_ccb *); void ahci_unload_prdt(struct ahci_ccb *); @@ -320,11 +320,12 @@ nomem: ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC); /* - * Do device-related port initialization. A failure here does not - * cause the port to be deallocated as we want to receive future - * hot-plug events. + * Start the port. The helper thread will call ahci_port_init() + * so the ports can all be started in parallel. A failure by + * ahci_port_init() does not deallocate the port since we still + * want hot-plug events. */ - ahci_port_init(ap, NULL); + ahci_os_start_port(ap); return(0); freeport: ahci_port_free(sc, port); @@ -348,9 +349,8 @@ reterr: * Returns 0 if a device is successfully detected. */ int -ahci_port_init(struct ahci_port *ap, struct ata_port *at) +ahci_port_init(struct ahci_port *ap, struct ata_port *atx) { - u_int32_t data; int rc; /* @@ -363,10 +363,13 @@ ahci_port_init(struct ahci_port *ap, struct ata_port *at) * Hard-reset the port. If a device is detected but it is busy * we try a second time, this time cycling the phy as well. */ - ap->ap_probe = ATA_PROBE_NEED_HARD_RESET; - rc = ahci_port_reset(ap, at, 1); + if (atx) + atx->at_probe = ATA_PROBE_NEED_HARD_RESET; + else + ap->ap_probe = ATA_PROBE_NEED_HARD_RESET; + rc = ahci_port_reset(ap, atx, 1); if (rc == EBUSY) { - rc = ahci_port_reset(ap, at, 2); + rc = ahci_port_reset(ap, atx, 2); } switch (rc) { @@ -399,7 +402,7 @@ ahci_port_init(struct ahci_port *ap, struct ata_port *at) kprintf("%s: Device on port is bricked, trying softreset\n", PORTNAME(ap)); - rc = ahci_port_reset(ap, at, 0); + rc = ahci_port_reset(ap, atx, 0); if (rc) { kprintf("%s: Unable unbrick device\n", PORTNAME(ap)); @@ -436,7 +439,7 @@ ahci_port_init(struct ahci_port *ap, struct ata_port *at) * * There's nothing to start for devices behind a port multiplier. */ - if (rc == 0 && at == NULL) { + if (rc == 0 && atx == NULL) { if (ahci_port_start(ap)) { kprintf("%s: failed to start command DMA on port, " "disabling\n", PORTNAME(ap)); @@ -450,26 +453,41 @@ ahci_port_init(struct ahci_port *ap, struct ata_port *at) * Enable interrupts on the port whether a device is sitting on * it or not, to handle hot-plug events. */ - if (at == NULL) { + if (atx == NULL) { ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); ahci_write(ap->ap_sc, AHCI_REG_IS, 1 << ap->ap_num); - data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | - AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | - AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE | - AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE | - AHCI_PREG_IE_DHRE; - if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) - data |= AHCI_PREG_IE_SDBE; -#ifdef AHCI_COALESCE - if (sc->sc_ccc_ports & (1 << port) - data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE); -#endif - ahci_pwrite(ap, AHCI_PREG_IE, data); + ahci_port_interrupt_enable(ap); } return(rc); } +/* + * Enable or re-enable interrupts on a port. + * + * This routine is called from the port initialization code or from the + * helper thread as the real interrupt may be forced to turn off certain + * interrupt sources. + */ +void +ahci_port_interrupt_enable(struct ahci_port *ap) +{ + u_int32_t data; + + data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE | + AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE | + AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE | + AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE | + AHCI_PREG_IE_DHRE; + if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF) + data |= AHCI_PREG_IE_SDBE; +#ifdef AHCI_COALESCE + if (sc->sc_ccc_ports & (1 << port) + data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE); +#endif + ahci_pwrite(ap, AHCI_PREG_IE, data); +} + /* * Run the port / target state machine from a main context. * @@ -504,7 +522,7 @@ ahci_port_state_machine(struct ahci_port *ap) if (ap->ap_type != ATA_PORT_T_PM) { if (ap->ap_probe == ATA_PROBE_FAILED) { ahci_cam_changed(ap, NULL, 0); - } else if (ap->ap_probe == ATA_PROBE_GOOD) { + } else if (ap->ap_probe >= ATA_PROBE_NEED_IDENT) { ahci_cam_changed(ap, NULL, 1); } return; @@ -574,7 +592,7 @@ ahci_port_state_machine(struct ahci_port *ap) if (data & (1 << target)) { kprintf("%s: HOTPLUG event, ", ATANAME(ap, at)); - if (at->at_probe == ATA_PROBE_GOOD) + if (at->at_probe >= ATA_PROBE_NEED_IDENT) kprintf("device inserted\n"); else kprintf("device removed\n"); @@ -593,7 +611,7 @@ ahci_port_state_machine(struct ahci_port *ap) at->at_features &= ~ATA_PORT_F_RESCAN; if (at->at_probe == ATA_PROBE_FAILED) { ahci_cam_changed(ap, at, 0); - } else if (at->at_probe == ATA_PROBE_GOOD) { + } else if (at->at_probe >= ATA_PROBE_NEED_IDENT) { ahci_cam_changed(ap, at, 1); } } @@ -616,6 +634,7 @@ ahci_port_free(struct ahci_softc *sc, u_int port) */ if (ap->ap_sc) { ahci_port_stop(ap, 1); + ahci_os_stop_port(ap); ahci_pwrite(ap, AHCI_PREG_CMD, 0); ahci_pwrite(ap, AHCI_PREG_IE, 0); ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS)); @@ -1210,7 +1229,6 @@ ahci_port_hardreset(struct ahci_port *ap, int hard) error = ahci_port_pmprobe(ap); if (error) { ap->ap_type = type; - ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET; error = 0; } else { ap->ap_type = ATA_PORT_T_PM; @@ -1228,8 +1246,10 @@ ahci_port_hardreset(struct ahci_port *ap, int hard) ahci_port_hardstop(ap); /* ap_probe set to failed */ } else { - ap->ap_probe = (ap->ap_type == ATA_PORT_T_PM) ? - ATA_PROBE_GOOD : ATA_PROBE_NEED_SOFT_RESET; + if (ap->ap_type == ATA_PORT_T_PM) + ap->ap_probe = ATA_PROBE_GOOD; + else + ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET; } return (error); } @@ -1740,6 +1760,7 @@ int ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) { struct ahci_port *ap = ccb->ccb_port; + int xtimeout = timeout * 2; if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) { ccb->ccb_xa.state = ATA_S_ERROR; @@ -1749,13 +1770,19 @@ ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)) ahci_start(ccb); do { - ahci_port_intr(ap); + ahci_port_intr(ap, 1); if (ccb->ccb_xa.state != ATA_S_ONCHIP && ccb->ccb_xa.state != ATA_S_PENDING) { crit_exit(); return (0); } ahci_os_sleep(100); + if (xtimeout < 0) { + kprintf("poll timeout %d xa.state = %d\n", timeout, ccb->ccb_xa.state); + Debugger("Excessive poll"); + break; + } + xtimeout -= 100; if (ccb->ccb_xa.state == ATA_S_ONCHIP) timeout -= 100; } while (timeout > 0); @@ -1926,11 +1953,17 @@ ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq) void ahci_intr(void *arg) { - struct ahci_softc *sc = arg; - u_int32_t is, ack = 0; - int port; + struct ahci_softc *sc = arg; + struct ahci_port *ap; + u_int32_t is, ack = 0; + int port; - /* Read global interrupt status */ + /* + * Check if the master enable is up, and whether any interrupts are + * pending. + */ + if ((sc->sc_flags & AHCI_F_INT_GOOD) == 0) + return; is = ahci_read(sc, AHCI_REG_IS); if (is == 0 || is == 0xffffffff) return; @@ -1946,11 +1979,21 @@ ahci_intr(void *arg) } #endif - /* Process interrupts for each port */ + /* + * Process interrupts for each port in a non-blocking fashion. + */ while (is) { port = ffs(is) - 1; - if (sc->sc_ports[port]) - ahci_port_intr(sc->sc_ports[port]); + ap = sc->sc_ports[port]; + if (ap) { + if (ahci_os_lock_port_nb(ap) == 0) { + ahci_port_intr(ap, 0); + ahci_os_unlock_port(ap); + } else { + ahci_pwrite(ap, AHCI_PREG_IE, 0); + ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); + } + } is &= ~(1 << port); } @@ -1958,48 +2001,128 @@ ahci_intr(void *arg) ahci_write(sc, AHCI_REG_IS, ack); } +/* + * Core called from helper thread. + */ void -ahci_port_intr(struct ahci_port *ap) +ahci_port_thread_core(struct ahci_port *ap, int mask) { - struct ahci_softc *sc = ap->ap_sc; - u_int32_t is, ci_saved, ci_masked; - int slot; - struct ahci_ccb *ccb = NULL; - struct ata_port *ccb_at = NULL; - volatile u_int32_t *active; + struct ahci_ccb *ccb; + int i; + + /* + * Process any expired timedouts. + */ + ahci_os_lock_port(ap); + if (mask & AP_SIGF_TIMEOUT) { + kprintf("%s: timeout", PORTNAME(ap)); + for (i = 0; i < ap->ap_sc->sc_ncmds; i++) { + ccb = &ap->ap_ccbs[i]; + if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_EXPIRED) { + kprintf("%s: timeout slot %d\n", + PORTNAME(ap), ccb->ccb_slot); + ahci_ata_cmd_timeout(ccb); + } + } + } + + /* + * Process port interrupts which require a higher level of + * intervention. + */ + if (mask & AP_SIGF_PORTINT) { + ahci_port_intr(ap, 1); + ahci_os_unlock_port(ap); + ahci_port_interrupt_enable(ap); + } else { + ahci_os_unlock_port(ap); + } +} + +/* + * Core per-port interrupt handler. + * + * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only + * deal with normal command completions which do not require blocking. + */ +void +ahci_port_intr(struct ahci_port *ap, int blockable) +{ + struct ahci_softc *sc = ap->ap_sc; + u_int32_t is, ci_saved, ci_masked; + int slot; + struct ahci_ccb *ccb = NULL; + struct ata_port *ccb_at = NULL; + volatile u_int32_t *active; #ifdef DIAGNOSTIC - u_int32_t tmp; + u_int32_t tmp; #endif + const u_int32_t blockable_mask = AHCI_PREG_IS_TFES | + AHCI_PREG_IS_IFS | + AHCI_PREG_IS_PCS | + AHCI_PREG_IS_PRCS | + AHCI_PREG_IS_HBFS | + AHCI_PREG_IS_OFS | + AHCI_PREG_IS_UFS; + enum { NEED_NOTHING, NEED_RESTART, NEED_HOTPLUG_INSERT, NEED_HOTPLUG_REMOVE } need = NEED_NOTHING; is = ahci_pread(ap, AHCI_PREG_IS); + + /* + * All basic command completions are always processed. + */ if (is & AHCI_PREG_IS_DPS) ahci_pwrite(ap, AHCI_PREG_IS, is & AHCI_PREG_IS_DPS); + /* + * If we can't block then we can't handle these here. Disable + * the interrupts in question so we don't live-lock, the helper + * thread will re-enable them. + * + * If the port is in a completely failed state we do not want + * to drop through to failed-command-processinf if blockable is 0, + * just let the thread deal with it all. + */ + if (blockable == 0) { + if (ap->ap_state == AP_S_FATAL_ERROR) { + ahci_pwrite(ap, AHCI_PREG_IE, + ahci_pread(ap, AHCI_PREG_IE) & ~is); + ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); + return; + } + if (is & blockable_mask) { + is &= blockable_mask | AHCI_PREG_IS_DHRS; + ahci_pwrite(ap, AHCI_PREG_IE, + ahci_pread(ap, AHCI_PREG_IE) & ~is); + ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT); + } + } + #if 0 kprintf("%s: INTERRUPT %b\n", PORTNAME(ap), is, AHCI_PFMT_IS); #endif /* - * Ack the port interrupt + * Either NCQ or non-NCQ commands will be active, never both. */ if (ap->ap_sactive) { - /* Active NCQ commands - use SActive instead of CI */ KKASSERT(ap->ap_active == 0); KKASSERT(ap->ap_active_cnt == 0); ci_saved = ahci_pread(ap, AHCI_PREG_SACT); active = &ap->ap_sactive; } else { - /* Save CI */ ci_saved = ahci_pread(ap, AHCI_PREG_CI); active = &ap->ap_active; } if (is & AHCI_PREG_IS_TFES) { /* - * Command failed. See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. + * Command failed (blockable). + * + * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2. * * This stops command processing. */ @@ -2145,6 +2268,8 @@ ahci_port_intr(struct ahci_port *ap) #endif } else if (is & AHCI_PREG_IS_DHRS) { /* + * Command posted D2H register FIS to the rfis (non-blocking). + * * Command posted D2H register FIS to the rfis. This * does NOT stop command processing and it is unclear * how we are supposed to deal with it other then using @@ -2174,7 +2299,7 @@ ahci_port_intr(struct ahci_port *ap) } /* - * Device notification to us. + * Device notification to us (non-blocking) * * NOTE! On some parts notification bits can get set without * generating an interrupt. It is unclear whether this is @@ -2207,6 +2332,8 @@ ahci_port_intr(struct ahci_port *ap) } /* + * Spurious IFS errors (blockable). + * * Spurious IFS errors can occur while we are doing a reset * sequence through a PM. Try to recover if we are being asked * to ignore IFS errors during these periods. @@ -2230,7 +2357,7 @@ ahci_port_intr(struct ahci_port *ap) } /* - * Port change (hot-plug). + * Port change (hot-plug) (blockable). * * A PCS interrupt will occur on hot-plug once communication is * established. @@ -2274,7 +2401,7 @@ ahci_port_intr(struct ahci_port *ap) } /* - * Check for remaining errors - they are fatal. + * Check for remaining errors - they are fatal. (blockable) */ if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) { @@ -2349,6 +2476,8 @@ failall: } /* + * CCB completion (non blocking). + * * CCB completion is detected by noticing its slot's bit in CI has * changed to zero some time after we activated it. * If we are polling, we may only be interested in particular slot(s). @@ -2386,6 +2515,9 @@ failall: ccb->ccb_done(ccb); } + /* + * Cleanup. Will not be set if non-blocking. + */ switch(need) { case NEED_RESTART: /* @@ -2869,6 +3001,7 @@ ahci_ata_cmd(struct ata_xfer *xa) } crit_enter(); + KKASSERT((xa->flags & ATA_F_TIMEOUT_EXPIRED) == 0); xa->flags |= ATA_F_TIMEOUT_DESIRED; ahci_start(ccb); crit_exit(); @@ -2891,11 +3024,12 @@ ahci_ata_cmd_done(struct ahci_ccb *ccb) xa->flags &= ~ATA_F_TIMEOUT_RUNNING; callout_stop(&ccb->ccb_timeout); } - xa->flags &= ~ATA_F_TIMEOUT_DESIRED; + xa->flags &= ~(ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_EXPIRED); - if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) + if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) { ahci_issue_pending_commands(ccb->ccb_port, - xa->flags & ATA_F_NCQ); + xa->flags & ATA_F_NCQ); + } ahci_unload_prdt(ccb); @@ -2911,15 +3045,22 @@ ahci_ata_cmd_done(struct ahci_ccb *ccb) xa->complete(xa); } +/* + * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags + * while the callout is runing. + * + * We can't safely get the port lock here or delay, we could block + * the callout thread. + */ static void ahci_ata_cmd_timeout_unserialized(void *arg) { struct ahci_ccb *ccb = arg; struct ahci_port *ap = ccb->ccb_port; - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); - ahci_ata_cmd_timeout(arg); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING; + ccb->ccb_xa.flags |= ATA_F_TIMEOUT_EXPIRED; + ahci_os_signal_port_thread(ap, AP_SIGF_TIMEOUT); } void @@ -2948,7 +3089,7 @@ ahci_ata_cmd_timeout(void *arg) */ KKASSERT(xa->flags & (ATA_F_POLL | ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_RUNNING)); - xa->flags &= ~ATA_F_TIMEOUT_RUNNING; + xa->flags &= ~(ATA_F_TIMEOUT_RUNNING | ATA_F_TIMEOUT_EXPIRED); ncq_cmd = (xa->flags & ATA_F_NCQ); active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active; @@ -3004,7 +3145,7 @@ ahci_ata_cmd_timeout(void *arg) "leave the port intact\n", ATANAME(ap, ccb->ccb_xa.at)); ccb->ccb_xa.at->at_probe = ATA_PROBE_FAILED; - ahci_port_intr(ap); + ahci_port_intr(ap, 1); ahci_port_stop(ap, 0); ahci_port_clo(ap); ahci_port_start(ap); @@ -3019,7 +3160,7 @@ ahci_ata_cmd_timeout(void *arg) "bricked on us\n", PORTNAME(ap)); ap->ap_state = AP_S_FATAL_ERROR; - ahci_port_intr(ap); + ahci_port_intr(ap, 1); status = 1; } else { status = 0; diff --git a/sys/dev/disk/ahci/ahci.h b/sys/dev/disk/ahci/ahci.h index 8c627a5ed9..a30e3785fd 100644 --- a/sys/dev/disk/ahci/ahci.h +++ b/sys/dev/disk/ahci/ahci.h @@ -418,9 +418,17 @@ struct ahci_port { #define AP_F_IN_RESET 0x0004 #define AP_F_SCAN_RUNNING 0x0008 #define AP_F_SCAN_REQUESTED 0x0010 -#define AP_F_IGNORE_IFS 0x0020 -#define AP_F_IFS_IGNORED 0x0040 -#define AP_F_IFS_OCCURED 0x0080 +#define AP_F_SCAN_COMPLETED 0x0020 +#define AP_F_IGNORE_IFS 0x0040 +#define AP_F_IFS_IGNORED 0x0080 +#define AP_F_IFS_OCCURED 0x0100 + int ap_signal; /* os per-port thread sig */ + thread_t ap_thread; /* os per-port thread */ + struct lock ap_lock; /* os per-port lock */ +#define AP_SIGF_INIT 0x0001 +#define AP_SIGF_TIMEOUT 0x0002 +#define AP_SIGF_PORTINT 0x0004 +#define AP_SIGF_STOP 0x8000 struct cam_sim *ap_sim; struct ahci_rfis *ap_rfis; @@ -465,7 +473,6 @@ struct ahci_port { struct ahci_softc { device_t sc_dev; const struct ahci_device *sc_ad; /* special casing */ - struct lwkt_serialize sc_serializer; struct resource *sc_irq; /* bus resources */ struct resource *sc_regs; /* bus resources */ @@ -486,6 +493,7 @@ struct ahci_softc { int sc_flags; #define AHCI_F_NO_NCQ (1<<0) #define AHCI_F_IGN_FR (1<<1) +#define AHCI_F_INT_GOOD (1<<2) u_int sc_ncmds; @@ -509,6 +517,7 @@ struct ahci_device { const struct ahci_device *ahci_lookup_device(device_t dev); int ahci_init(struct ahci_softc *); +int ahci_port_init(struct ahci_port *ap, struct ata_port *at); int ahci_port_alloc(struct ahci_softc *, u_int); void ahci_port_state_machine(struct ahci_port *ap); void ahci_port_free(struct ahci_softc *, u_int); @@ -522,7 +531,7 @@ void ahci_pwrite(struct ahci_port *, bus_size_t, u_int32_t); int ahci_pwait_eq(struct ahci_port *, int, bus_size_t, u_int32_t, u_int32_t); void ahci_intr(void *); -void ahci_port_intr(struct ahci_port *); +void ahci_port_intr(struct ahci_port *ap, int blockable); int ahci_cam_attach(struct ahci_port *ap); void ahci_cam_changed(struct ahci_port *ap, struct ata_port *at, int found); @@ -549,7 +558,14 @@ void ahci_put_ccb(struct ahci_ccb *ccb); int ahci_poll(struct ahci_ccb *ccb, int timeout, void (*timeout_fn)(void *)); int ahci_port_signature_detect(struct ahci_port *ap, struct ata_port *at); +void ahci_port_thread_core(struct ahci_port *ap, int mask); void ahci_os_sleep(int ticks); +void ahci_os_start_port(struct ahci_port *ap); +void ahci_os_stop_port(struct ahci_port *ap); +void ahci_os_signal_port_thread(struct ahci_port *ap, int mask); +void ahci_os_lock_port(struct ahci_port *ap); +int ahci_os_lock_port_nb(struct ahci_port *ap); +void ahci_os_unlock_port(struct ahci_port *ap); extern u_int32_t AhciForceGen1; diff --git a/sys/dev/disk/ahci/ahci_attach.c b/sys/dev/disk/ahci/ahci_attach.c index e3e42369bd..4cd670b49d 100644 --- a/sys/dev/disk/ahci/ahci_attach.c +++ b/sys/dev/disk/ahci/ahci_attach.c @@ -154,6 +154,7 @@ static int ahci_pci_attach(device_t dev) { struct ahci_softc *sc = device_get_softc(dev); + struct ahci_port *ap; const char *gen; u_int32_t cap, pi, reg; bus_addr_t addr; @@ -168,11 +169,8 @@ ahci_pci_attach(device_t dev) sc->sc_rid_irq = AHCI_IRQ_RID; sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_rid_irq, RF_SHAREABLE | RF_ACTIVE); - lwkt_serialize_init(&sc->sc_serializer); - lwkt_serialize_enter(&sc->sc_serializer); if (sc->sc_irq == NULL) { device_printf(dev, "unable to map interrupt\n"); - lwkt_serialize_exit(&sc->sc_serializer); ahci_pci_detach(dev); return (ENXIO); } @@ -187,7 +185,6 @@ ahci_pci_attach(device_t dev) &sc->sc_rid_regs, RF_ACTIVE); if (sc->sc_regs == NULL) { device_printf(dev, "unable to map registers\n"); - lwkt_serialize_exit(&sc->sc_serializer); ahci_pci_detach(dev); return (ENXIO); } @@ -199,7 +196,6 @@ ahci_pci_attach(device_t dev) */ error = ahci_init(sc); if (error) { - lwkt_serialize_exit(&sc->sc_serializer); ahci_pci_detach(dev); return (ENXIO); } @@ -287,7 +283,6 @@ ahci_pci_attach(device_t dev) if (error) { device_printf(dev, "unable to create dma tags\n"); - lwkt_serialize_exit(&sc->sc_serializer); ahci_pci_detach(dev); return (ENXIO); } @@ -376,6 +371,10 @@ noccc: * * Ignore attach errors, leave the port intact for * rescan and continue the loop. + * + * All ports are attached in parallel but the CAM scan-bus + * is held up until all ports are attached so we get a deterministic + * order. */ for (i = 0; error == 0 && i < AHCI_MAX_PORTS; i++) { if ((pi & (1 << i)) == 0) { @@ -383,12 +382,6 @@ noccc: continue; } error = ahci_port_alloc(sc, i); - if (error == 0) { - if (ahci_cam_attach(sc->sc_ports[i]) == 0) - ahci_cam_changed(sc->sc_ports[i], NULL, -1); - } - if (error == ENODEV) - error = 0; } /* @@ -398,17 +391,42 @@ noccc: */ if (error == 0) { error = bus_setup_intr(dev, sc->sc_irq, 0, ahci_intr, sc, - &sc->sc_irq_handle, &sc->sc_serializer); + &sc->sc_irq_handle, NULL); } if (error) { device_printf(dev, "unable to install interrupt\n"); - lwkt_serialize_exit(&sc->sc_serializer); ahci_pci_detach(dev); return (ENXIO); } + + /* + * Master interrupt enable, and call ahci_intr() in case we race + * our AHCI_F_INT_GOOD flag. + */ + crit_enter(); ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE | AHCI_REG_GHC_IE); - lwkt_serialize_exit(&sc->sc_serializer); + sc->sc_flags |= AHCI_F_INT_GOOD; + crit_exit(); + ahci_intr(sc); + + /* + * All ports are probing in parallel. Wait for them to finish + * and then issue the cam attachment and bus scan serially so + * the 'da' assignments are deterministic. + */ + for (i = 0; i < AHCI_MAX_PORTS; i++) { + if ((ap = sc->sc_ports[i]) != NULL) { + while (ap->ap_signal & AP_SIGF_INIT) + tsleep(&ap->ap_signal, 0, "ahprb1", hz); + if (ahci_cam_attach(ap) == 0) { + ahci_cam_changed(ap, NULL, -1); + while ((ap->ap_flags & AP_F_SCAN_COMPLETED) == 0) { + tsleep(&ap->ap_flags, 0, "ahprb2", hz); + } + } + } + } return(0); } @@ -426,12 +444,12 @@ ahci_pci_detach(device_t dev) /* * Disable the controller and de-register the interrupt, if any. * - * XXX interlock serializer against interrupt + * XXX interlock last interrupt? */ - lwkt_serialize_handler_disable(&sc->sc_serializer); - if (sc->sc_regs) { + sc->sc_flags &= ~AHCI_F_INT_GOOD; + if (sc->sc_regs) ahci_write(sc, AHCI_REG_GHC, 0); - } + if (sc->sc_irq_handle) { bus_teardown_intr(dev, sc->sc_irq, sc->sc_irq_handle); sc->sc_irq_handle = NULL; diff --git a/sys/dev/disk/ahci/ahci_cam.c b/sys/dev/disk/ahci/ahci_cam.c index 0e30e93239..1b965b6af5 100644 --- a/sys/dev/disk/ahci/ahci_cam.c +++ b/sys/dev/disk/ahci/ahci_cam.c @@ -259,6 +259,12 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx) error = EIO; + /* + * Delayed CAM attachment for initial probe, sim may be NULL + */ + if (ap->ap_sim == NULL) + return(0); + /* * A NULL atx indicates a probe of the directly connected device. * A non-NULL atx indicates a device connected via a port multiplier. @@ -270,9 +276,8 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx) if (atx == NULL) { at = ap->ap_ata; /* direct attached - device 0 */ if (ap->ap_type == ATA_PORT_T_PM) { - kprintf("%s: Found Port Multiplier\n", - ATANAME(ap, atx)); - ap->ap_probe = ATA_PROBE_GOOD; + kprintf("%s: Found Port Multiplier %d\n", + ATANAME(ap, atx), ap->ap_probe); return (0); } at->at_type = ap->ap_type; @@ -638,12 +643,16 @@ ahci_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb) { struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr; - ap->ap_flags &= ~AP_F_SCAN_RUNNING; - if (ap->ap_flags & AP_F_SCAN_REQUESTED) { - ap->ap_flags &= ~AP_F_SCAN_REQUESTED; - ahci_cam_rescan(ap); + if (ccb->ccb_h.func_code == XPT_SCAN_BUS) { + ap->ap_flags &= ~AP_F_SCAN_RUNNING; + if (ap->ap_flags & AP_F_SCAN_REQUESTED) { + ap->ap_flags &= ~AP_F_SCAN_REQUESTED; + ahci_cam_rescan(ap); + } + ap->ap_flags |= AP_F_SCAN_COMPLETED; + wakeup(&ap->ap_flags); } - kfree(ccb, M_TEMP); + xpt_free_ccb(ccb); } static void @@ -663,20 +672,18 @@ ahci_cam_rescan(struct ahci_port *ap) ap->ap_ata[i].at_features |= ATA_PORT_F_RESCAN; } - ccb = kmalloc(sizeof(*ccb), M_TEMP, M_WAITOK | M_ZERO); status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim), CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) return; + ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, path, 5); /* 5 = low priority */ ccb->ccb_h.func_code = XPT_ENG_EXEC; ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback; ccb->ccb_h.sim_priv.entries[0].ptr = ap; ccb->crcn.flags = CAM_FLAG_NONE; xpt_action_async(ccb); - - /* scan is now underway */ } static void @@ -690,29 +697,16 @@ ahci_xpt_rescan(struct ahci_port *ap) CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); if (status != CAM_REQ_CMP) return; - ccb = kmalloc(sizeof(*ccb), M_TEMP, M_WAITOK | M_ZERO); + + ccb = xpt_alloc_ccb(); xpt_setup_ccb(&ccb->ccb_h, path, 5); /* 5 = low priority */ ccb->ccb_h.func_code = XPT_SCAN_BUS; ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback; ccb->ccb_h.sim_priv.entries[0].ptr = ap; ccb->crcn.flags = CAM_FLAG_NONE; - xpt_action_async(ccb); + xpt_action(ccb); } -#if 0 - ccb = xpt_alloc_ccb(); - status = xpt_create_path(&ccb->ccb_h.path, xpt_periph, - cam_sim_path(ap->ap_sim), - CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD); - if (status == CAM_REQ_CMP) { - kprintf("RESCAN SCSI BUS %d\n", ccb->ccb_h.timeout); - ccb->crcn.flags = CAM_FLAG_NONE; - xpt_rescan(ccb); - } else { - xpt_free_ccb(ccb); - } -#endif - /* * Action function - dispatch command */ @@ -783,15 +777,10 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb) * probed. */ ccbh->status = CAM_REQ_CMP; - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); ahci_port_state_machine(ap); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); xpt_done(ccb); - - /* - * Rescanning the scsi bus should clean up the peripheral - * associations. - */ ahci_xpt_rescan(ap); break; case XPT_PATH_INQ: @@ -820,6 +809,8 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb) ccbh->status = CAM_REQ_CMP; if (ccbh->target_id != CAM_TARGET_WILDCARD) { + ahci_port_state_machine(ap); + switch(ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_SPD) { case AHCI_PREG_SSTS_SPD_GEN1: @@ -841,20 +832,20 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb) xpt_done(ccb); break; case XPT_RESET_DEV: - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); if (ap->ap_type == ATA_PORT_T_NONE) { ccbh->status = CAM_DEV_NOT_THERE; } else { ahci_port_reset(ap, atx, 0); ccbh->status = CAM_REQ_CMP; } - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); xpt_done(ccb); break; case XPT_RESET_BUS: - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); ahci_port_reset(ap, NULL, 1); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); ccbh->status = CAM_REQ_CMP; xpt_done(ccb); break; @@ -877,6 +868,17 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb) xpt_done(ccb); break; case XPT_SCSI_IO: + /* + * Our parallel startup code might have only probed through + * to the IDENT, so do the last step if necessary. + */ + if (at->at_probe == ATA_PROBE_NEED_IDENT) + ahci_cam_probe(ap, atx); + if (at->at_probe != ATA_PROBE_GOOD) { + ccbh->status = CAM_DEV_NOT_THERE; + xpt_done(ccb); + break; + } switch(at->at_type) { case ATA_PORT_T_DISK: ahci_xpt_scsi_disk_io(ap, atx, ccb); @@ -911,9 +913,9 @@ ahci_xpt_poll(struct cam_sim *sim) ap = cam_sim_softc(sim); crit_enter(); - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); - ahci_port_intr(ap); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); + ahci_port_intr(ap, 1); + ahci_os_unlock_port(ap); crit_exit(); } @@ -1192,10 +1194,10 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx, KKASSERT(xa->complete != NULL); xa->atascsi_private = ccb; ccb->ccb_h.sim_priv.entries[0].ptr = ap; - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); fis->flags |= at->at_target; ahci_ata_cmd(xa); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); } else { ahci_ata_put_xfer(xa); xpt_done(ccb); @@ -1380,9 +1382,9 @@ ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa) break; } ahci_ata_put_xfer(xa); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); xpt_done(ccb); - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); } /* @@ -1419,9 +1421,9 @@ ahci_ata_complete_disk_rw(struct ata_xfer *xa) } ccb->csio.resid = xa->resid; ahci_ata_put_xfer(xa); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); xpt_done(ccb); - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); } /* @@ -1466,9 +1468,9 @@ ahci_atapi_complete_cmd(struct ata_xfer *xa) } ccb->csio.resid = xa->resid; ahci_ata_put_xfer(xa); - lwkt_serialize_exit(&ap->ap_sc->sc_serializer); + ahci_os_unlock_port(ap); xpt_done(ccb); - lwkt_serialize_enter(&ap->ap_sc->sc_serializer); + ahci_os_lock_port(ap); } /* diff --git a/sys/dev/disk/ahci/ahci_dragonfly.c b/sys/dev/disk/ahci/ahci_dragonfly.c index f2e7bb6131..e68898c741 100644 --- a/sys/dev/disk/ahci/ahci_dragonfly.c +++ b/sys/dev/disk/ahci/ahci_dragonfly.c @@ -50,6 +50,8 @@ static int ahci_suspend (device_t dev); static int ahci_resume (device_t dev); #endif +static void ahci_port_thread(void *arg); + static device_method_t ahci_methods[] = { DEVMETHOD(device_probe, ahci_probe), DEVMETHOD(device_attach, ahci_attach), @@ -155,3 +157,120 @@ ahci_os_sleep(int ms) tsleep(&ticks, 0, "ahslp", ticks); } } + +/* + * Create the OS-specific port helper thread and per-port lock. + */ +void +ahci_os_start_port(struct ahci_port *ap) +{ + atomic_set_int(&ap->ap_signal, AP_SIGF_INIT); + lockinit(&ap->ap_lock, "ahcipo", 0, 0); + kthread_create(ahci_port_thread, ap, &ap->ap_thread, + "%s", PORTNAME(ap)); +} + +/* + * Stop the OS-specific port helper thread and kill the per-port lock. + */ +void +ahci_os_stop_port(struct ahci_port *ap) +{ + if (ap->ap_thread) { + ahci_os_signal_port_thread(ap, AP_SIGF_STOP); + ahci_os_sleep(10); + if (ap->ap_thread) { + kprintf("%s: Waiting for thread to terminate\n", + PORTNAME(ap)); + while (ap->ap_thread) + ahci_os_sleep(100); + kprintf("%s: thread terminated\n", + PORTNAME(ap)); + } + } + lockuninit(&ap->ap_lock); +} + +/* + * Add (mask) to the set of bits being sent to the per-port thread helper + * and wake the helper up if necessary. + */ +void +ahci_os_signal_port_thread(struct ahci_port *ap, int mask) +{ + atomic_set_int(&ap->ap_signal, mask); + wakeup(&ap->ap_thread); +} + +/* + * Unconditionally lock the port structure for access. + */ +void +ahci_os_lock_port(struct ahci_port *ap) +{ + lockmgr(&ap->ap_lock, LK_EXCLUSIVE); +} + +/* + * Conditionally lock the port structure for access. + * + * Returns 0 on success, non-zero on failure. + */ +int +ahci_os_lock_port_nb(struct ahci_port *ap) +{ + return (lockmgr(&ap->ap_lock, LK_EXCLUSIVE | LK_NOWAIT)); +} + +/* + * Unlock a previously locked port. + */ +void +ahci_os_unlock_port(struct ahci_port *ap) +{ + lockmgr(&ap->ap_lock, LK_RELEASE); +} + +/* + * Per-port thread helper. This helper thread is responsible for + * atomically retrieving and clearing the signal mask and calling + * the machine-independant driver core. + */ +static +void +ahci_port_thread(void *arg) +{ + struct ahci_port *ap = arg; + int mask; + + /* + * The helper thread is responsible for the initial port init, + * so all the ports can be inited in parallel. + * + * We also run the state machine which should do all probes. + * Since CAM is not attached yet we will not get out-of-order + * SCSI attachments. + */ + ahci_os_lock_port(ap); + ahci_port_init(ap, NULL); + ahci_port_state_machine(ap); + ahci_os_unlock_port(ap); + atomic_clear_int(&ap->ap_signal, AP_SIGF_INIT); + wakeup(&ap->ap_signal); + + /* + * Then loop on the helper core. + */ + mask = ap->ap_signal; + while ((mask & AP_SIGF_STOP) == 0) { + atomic_clear_int(&ap->ap_signal, mask); + ahci_port_thread_core(ap, mask); + crit_enter(); + tsleep_interlock(&ap->ap_thread); + if (ap->ap_signal == 0) + tsleep(&ap->ap_thread, 0, "ahport", 0); + crit_exit(); + mask = ap->ap_signal; + } + ap->ap_thread = NULL; +} diff --git a/sys/dev/disk/ahci/ahci_dragonfly.h b/sys/dev/disk/ahci/ahci_dragonfly.h index 2937303978..ec2be679d1 100644 --- a/sys/dev/disk/ahci/ahci_dragonfly.h +++ b/sys/dev/disk/ahci/ahci_dragonfly.h @@ -48,6 +48,7 @@ #include #include #include +#include #include #include diff --git a/sys/dev/disk/ahci/atascsi.h b/sys/dev/disk/ahci/atascsi.h index 10b770020b..949131470a 100644 --- a/sys/dev/disk/ahci/atascsi.h +++ b/sys/dev/disk/ahci/atascsi.h @@ -318,6 +318,7 @@ struct ata_xfer { #define ATA_F_NCQ (1<<6) #define ATA_F_TIMEOUT_RUNNING (1<<7) #define ATA_F_TIMEOUT_DESIRED (1<<8) +#define ATA_F_TIMEOUT_EXPIRED (1<<9) #define ATA_FMT_FLAGS "\020" "\010TRUNNING" \ "\007NCQ" "\006PACKET" \ "\005PIO" "\004POLL" "\003NOWAIT" \ -- 2.41.0