AHCI - Port multiplier plug/unplug work, tuning, cleanup, bug fixes.
[dragonfly.git] / sys / dev / disk / ahci / ahci.c
CommitLineData
258223a3
MD
1/*
2 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 *
16 *
17 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
18 *
19 * This code is derived from software contributed to The DragonFly Project
20 * by Matthew Dillon <dillon@backplane.com>
21 *
22 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions
24 * are met:
25 *
26 * 1. Redistributions of source code must retain the above copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in
30 * the documentation and/or other materials provided with the
31 * distribution.
32 * 3. Neither the name of The DragonFly Project nor the names of its
33 * contributors may be used to endorse or promote products derived
34 * from this software without specific, prior written permission.
35 *
36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
39 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
40 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
41 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
42 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
43 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
44 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
45 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
46 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
47 * SUCH DAMAGE.
48 *
49 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $
50 */
51
52#include "ahci.h"
53
f4553de1
MD
54int ahci_port_start(struct ahci_port *ap);
55int ahci_port_stop(struct ahci_port *ap, int stop_fis_rx);
56int ahci_port_clo(struct ahci_port *ap);
57void ahci_port_interrupt_enable(struct ahci_port *ap);
258223a3
MD
58
59int ahci_load_prdt(struct ahci_ccb *);
60void ahci_unload_prdt(struct ahci_ccb *);
61static void ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs,
62 int nsegs, int error);
258223a3 63void ahci_start(struct ahci_ccb *);
17eab71e 64int ahci_port_softreset(struct ahci_port *ap);
1980eff3
MD
65int ahci_port_pmprobe(struct ahci_port *ap);
66int ahci_port_hardreset(struct ahci_port *ap, int hard);
cf5f3a81
MD
67void ahci_port_hardstop(struct ahci_port *ap);
68void ahci_flush_tfd(struct ahci_port *ap);
258223a3 69
831bc9e3
MD
70static void ahci_ata_cmd_timeout_unserialized(void *);
71void ahci_quick_timeout(struct ahci_ccb *ccb);
72void ahci_check_active_timeouts(struct ahci_port *ap);
258223a3 73
831bc9e3
MD
74void ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at);
75void ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at);
258223a3
MD
76void ahci_issue_pending_ncq_commands(struct ahci_port *);
77void ahci_issue_pending_commands(struct ahci_port *, int);
78
258223a3
MD
79struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
80void ahci_put_err_ccb(struct ahci_ccb *);
81
82int ahci_port_read_ncq_error(struct ahci_port *, int *);
83
84struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, bus_dma_tag_t tag);
85void ahci_dmamem_free(struct ahci_softc *, struct ahci_dmamem *);
86static void ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error);
87
88void ahci_empty_done(struct ahci_ccb *ccb);
89void ahci_ata_cmd_done(struct ahci_ccb *ccb);
90
91/* Wait for all bits in _b to be cleared */
cec85a37
MD
92#define ahci_pwait_clr(_ap, _r, _b) \
93 ahci_pwait_eq((_ap), AHCI_PWAIT_TIMEOUT, (_r), (_b), 0)
94#define ahci_pwait_clr_to(_ap, _to, _r, _b) \
95 ahci_pwait_eq((_ap), _to, (_r), (_b), 0)
258223a3
MD
96
97/* Wait for all bits in _b to be set */
cec85a37
MD
98#define ahci_pwait_set(_ap, _r, _b) \
99 ahci_pwait_eq((_ap), AHCI_PWAIT_TIMEOUT, (_r), (_b), (_b))
100#define ahci_pwait_set_to(_ap, _to, _r, _b) \
101 ahci_pwait_eq((_ap), _to, (_r), (_b), (_b))
102
103#define AHCI_PWAIT_TIMEOUT 1000
258223a3 104
fd8bd957
MD
105/*
106 * Initialize the global AHCI hardware. This code does not set up any of
107 * its ports.
108 */
258223a3
MD
109int
110ahci_init(struct ahci_softc *sc)
111{
112 u_int32_t cap, pi;
831bc9e3
MD
113 int i;
114 struct ahci_port *ap;
258223a3
MD
115
116 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b",
117 ahci_read(sc, AHCI_REG_GHC), AHCI_FMT_GHC);
118
119 /* save BIOS initialised parameters, enable staggered spin up */
120 cap = ahci_read(sc, AHCI_REG_CAP);
121 cap &= AHCI_REG_CAP_SMPS;
122 cap |= AHCI_REG_CAP_SSS;
123 pi = ahci_read(sc, AHCI_REG_PI);
124
831bc9e3
MD
125#if 1
126 /*
127 * This is a hack that currently does not appear to have
128 * a significant effect, but I noticed the port registers
129 * do not appear to be completely cleared after the host
130 * controller is reset.
131 */
132 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO);
133 ap->ap_sc = sc;
134 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
135 if ((pi & (1 << i)) == 0)
136 continue;
137 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
138 AHCI_PORT_REGION(i), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
139 device_printf(sc->sc_dev, "can't map port\n");
140 return (1);
141 }
142 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED |
143 AHCI_PREG_SCTL_DET_DISABLE);
144 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
145 ahci_pwrite(ap, AHCI_PREG_IE, 0);
146 ahci_pwrite(ap, AHCI_PREG_CMD, 0);
147 ahci_pwrite(ap, AHCI_PREG_IS, 0);
148 }
149 kfree(ap, M_DEVBUF);
150#endif
151
17eab71e
MD
152 /*
153 * Unconditionally reset the controller, do not conditionalize on
154 * trying to figure it if it was previously active or not.
831bc9e3
MD
155 *
156 * NOTE BRICKS (1)
157 *
158 * If you have a port multiplier and it does not have a device
159 * in target 0, and it probes normally, but a later operation
160 * mis-probes a target behind that PM, it is possible for the
161 * port to brick such that only (a) a power cycle of the host
162 * or (b) placing a device in target 0 will fix the problem.
163 * Power cycling the PM has no effect (it works fine on another
164 * host port). This issue is unrelated to CLO.
17eab71e
MD
165 */
166 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
831bc9e3
MD
167 if (ahci_wait_ne(sc, AHCI_REG_GHC,
168 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) {
17eab71e
MD
169 device_printf(sc->sc_dev,
170 "unable to reset controller\n");
171 return (1);
258223a3 172 }
831bc9e3 173 ahci_os_sleep(100);
258223a3
MD
174
175 /* enable ahci (global interrupts disabled) */
176 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
177
178 /* restore parameters */
179 ahci_write(sc, AHCI_REG_CAP, cap);
180 ahci_write(sc, AHCI_REG_PI, pi);
181
182 return (0);
183}
184
fd8bd957
MD
185/*
186 * Allocate and initialize an AHCI port.
187 */
258223a3
MD
188int
189ahci_port_alloc(struct ahci_softc *sc, u_int port)
190{
1980eff3
MD
191 struct ahci_port *ap;
192 struct ata_port *at;
193 struct ahci_ccb *ccb;
194 u_int64_t dva;
195 u_int32_t cmd;
196 struct ahci_cmd_hdr *hdr;
197 struct ahci_cmd_table *table;
258223a3
MD
198 int rc = ENOMEM;
199 int error;
200 int i;
201
202 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO);
258223a3
MD
203
204 ksnprintf(ap->ap_name, sizeof(ap->ap_name), "%s%d.%d",
205 device_get_name(sc->sc_dev),
206 device_get_unit(sc->sc_dev),
207 port);
208 sc->sc_ports[port] = ap;
209
1980eff3
MD
210 /*
211 * Allocate enough so we never have to reallocate, it makes
212 * it easier.
213 *
214 * ap_pmcount will be reduced by the scan if we encounter the
215 * port multiplier port prior to target 15.
216 */
217 if (ap->ap_ata == NULL) {
218 ap->ap_ata = kmalloc(sizeof(*ap->ap_ata) * AHCI_MAX_PMPORTS,
219 M_DEVBUF, M_INTWAIT | M_ZERO);
220 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
221 at = &ap->ap_ata[i];
222 at->at_ahci_port = ap;
223 at->at_target = i;
3209f581 224 at->at_probe = ATA_PROBE_NEED_INIT;
831bc9e3 225 at->at_features |= ATA_PORT_F_RESCAN;
1980eff3
MD
226 ksnprintf(at->at_name, sizeof(at->at_name),
227 "%s.%d", ap->ap_name, i);
228 }
229 }
258223a3
MD
230 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
231 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
232 device_printf(sc->sc_dev,
233 "unable to create register window for port %d\n",
234 port);
235 goto freeport;
236 }
237
238 ap->ap_sc = sc;
239 ap->ap_num = port;
3209f581 240 ap->ap_probe = ATA_PROBE_NEED_INIT;
258223a3
MD
241 TAILQ_INIT(&ap->ap_ccb_free);
242 TAILQ_INIT(&ap->ap_ccb_pending);
243 lockinit(&ap->ap_ccb_lock, "ahcipo", 0, 0);
244
245 /* Disable port interrupts */
246 ahci_pwrite(ap, AHCI_PREG_IE, 0);
831bc9e3 247 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3 248
17eab71e
MD
249 /*
250 * Sec 10.1.2 - deinitialise port if it is already running
251 */
258223a3
MD
252 cmd = ahci_pread(ap, AHCI_PREG_CMD);
253 if ((cmd & (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
17eab71e 254 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
258223a3
MD
255 (ahci_pread(ap, AHCI_PREG_SCTL) & AHCI_PREG_SCTL_DET)) {
256 int r;
257
258 r = ahci_port_stop(ap, 1);
259 if (r) {
260 device_printf(sc->sc_dev,
261 "unable to disable %s, ignoring port %d\n",
262 ((r == 2) ? "CR" : "FR"), port);
263 rc = ENXIO;
264 goto freeport;
265 }
266
267 /* Write DET to zero */
cf5f3a81 268 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED);
258223a3
MD
269 }
270
271 /* Allocate RFIS */
272 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sc->sc_tag_rfis);
273 if (ap->ap_dmamem_rfis == NULL) {
cf5f3a81 274 kprintf("%s: NORFIS\n", PORTNAME(ap));
258223a3
MD
275 goto nomem;
276 }
277
278 /* Setup RFIS base address */
279 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
280 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
281 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
282 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
283
831bc9e3
MD
284 /* Clear SERR before starting FIS reception or ST or anything */
285 ahci_flush_tfd(ap);
286 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
287
258223a3
MD
288 /* Enable FIS reception and activate port. */
289 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1980eff3 290 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA);
258223a3
MD
291 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
292 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
293
294 /* Check whether port activated. Skip it if not. */
295 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
296 if ((cmd & AHCI_PREG_CMD_FRE) == 0) {
cf5f3a81 297 kprintf("%s: NOT-ACTIVATED\n", PORTNAME(ap));
258223a3
MD
298 rc = ENXIO;
299 goto freeport;
300 }
301
302 /* Allocate a CCB for each command slot */
303 ap->ap_ccbs = kmalloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF,
304 M_WAITOK | M_ZERO);
305 if (ap->ap_ccbs == NULL) {
306 device_printf(sc->sc_dev,
307 "unable to allocate command list for port %d\n",
308 port);
309 goto freeport;
310 }
311
312 /* Command List Structures and Command Tables */
313 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, sc->sc_tag_cmdh);
314 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, sc->sc_tag_cmdt);
315 if (ap->ap_dmamem_cmd_table == NULL ||
316 ap->ap_dmamem_cmd_list == NULL) {
317nomem:
318 device_printf(sc->sc_dev,
319 "unable to allocate DMA memory for port %d\n",
320 port);
321 goto freeport;
322 }
323
324 /* Setup command list base address */
325 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
326 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
327 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
328
329 /* Split CCB allocation into CCBs and assign to command header/table */
330 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
331 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
332 for (i = 0; i < sc->sc_ncmds; i++) {
333 ccb = &ap->ap_ccbs[i];
334
335 error = bus_dmamap_create(sc->sc_tag_data, BUS_DMA_ALLOCNOW,
336 &ccb->ccb_dmamap);
337 if (error) {
338 device_printf(sc->sc_dev,
339 "unable to create dmamap for port %d "
340 "ccb %d\n", port, i);
341 goto freeport;
342 }
343
344 callout_init(&ccb->ccb_timeout);
345 ccb->ccb_slot = i;
346 ccb->ccb_port = ap;
347 ccb->ccb_cmd_hdr = &hdr[i];
348 ccb->ccb_cmd_table = &table[i];
349 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
350 ccb->ccb_slot * sizeof(struct ahci_cmd_table);
351 ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32));
352 ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva);
353
354 ccb->ccb_xa.fis =
355 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
356 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
357 ccb->ccb_xa.tag = i;
358
258223a3
MD
359 ccb->ccb_xa.state = ATA_S_COMPLETE;
360 ahci_put_ccb(ccb);
361 }
362
363 /* Wait for ICC change to complete */
364 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC);
365
fd8bd957 366 /*
f4553de1
MD
367 * Start the port. The helper thread will call ahci_port_init()
368 * so the ports can all be started in parallel. A failure by
369 * ahci_port_init() does not deallocate the port since we still
370 * want hot-plug events.
fd8bd957 371 */
f4553de1 372 ahci_os_start_port(ap);
fd8bd957
MD
373 return(0);
374freeport:
375 ahci_port_free(sc, port);
fd8bd957
MD
376 return (rc);
377}
378
379/*
380 * [re]initialize an idle port. No CCBs should be active.
381 *
1980eff3
MD
382 * If at is NULL we are initializing a directly connected port, otherwise
383 * we are indirectly initializing a port multiplier port.
384 *
fd8bd957
MD
385 * This function is called during the initial port allocation sequence
386 * and is also called on hot-plug insertion. We take no chances and
387 * use a portreset instead of a softreset.
388 *
22181ab7
MD
389 * This function is the only way to move a failed port back to active
390 * status.
391 *
fd8bd957
MD
392 * Returns 0 if a device is successfully detected.
393 */
394int
f4553de1 395ahci_port_init(struct ahci_port *ap, struct ata_port *atx)
fd8bd957
MD
396{
397 int rc;
398
399 /*
1980eff3 400 * Clear all notification bits
fd8bd957 401 */
1980eff3
MD
402 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
403 ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
404
405 /*
406 * Hard-reset the port. If a device is detected but it is busy
407 * we try a second time, this time cycling the phy as well.
408 */
f4553de1
MD
409 if (atx)
410 atx->at_probe = ATA_PROBE_NEED_HARD_RESET;
411 else
412 ap->ap_probe = ATA_PROBE_NEED_HARD_RESET;
831bc9e3
MD
413 rc = ahci_port_reset(ap, atx, 2);
414#if 0
f4553de1 415 rc = ahci_port_reset(ap, atx, 1);
1980eff3 416 if (rc == EBUSY) {
f4553de1 417 rc = ahci_port_reset(ap, atx, 2);
17eab71e 418 }
831bc9e3 419#endif
fd8bd957 420
258223a3
MD
421 switch (rc) {
422 case ENODEV:
fd8bd957
MD
423 /*
424 * We had problems talking to the device on the port.
425 */
258223a3
MD
426 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
427 case AHCI_PREG_SSTS_DET_DEV_NE:
419cb1ab 428 kprintf("%s: Device not communicating\n", PORTNAME(ap));
258223a3
MD
429 break;
430 case AHCI_PREG_SSTS_DET_PHYOFFLINE:
419cb1ab 431 kprintf("%s: PHY offline\n", PORTNAME(ap));
258223a3
MD
432 break;
433 default:
419cb1ab 434 kprintf("%s: No device detected\n", PORTNAME(ap));
258223a3
MD
435 break;
436 }
437 break;
438
439 case EBUSY:
fd8bd957 440 /*
17eab71e
MD
441 * The device on the port is still telling us its busy,
442 * which means that it is not properly handling a SATA
443 * port COMRESET.
fd8bd957 444 *
17eab71e
MD
445 * It may be possible to softreset the device using CLO
446 * and a device reset command.
fd8bd957 447 */
17eab71e
MD
448 kprintf("%s: Device on port is bricked, trying softreset\n",
449 PORTNAME(ap));
258223a3 450
f4553de1 451 rc = ahci_port_reset(ap, atx, 0);
258223a3 452 if (rc) {
17eab71e 453 kprintf("%s: Unable unbrick device\n",
fd8bd957
MD
454 PORTNAME(ap));
455 } else {
17eab71e 456 kprintf("%s: Successfully unbricked\n",
fd8bd957 457 PORTNAME(ap));
258223a3
MD
458 }
459 break;
460
461 default:
462 break;
463 }
464
465 /*
17eab71e
MD
466 * Command transfers can only be enabled if a device was successfully
467 * detected.
1980eff3
MD
468 *
469 * Allocate or deallocate the ap_ata array here too.
258223a3 470 */
1980eff3
MD
471 switch(ap->ap_type) {
472 case ATA_PORT_T_NONE:
473 ap->ap_pmcount = 0;
474 break;
475 case ATA_PORT_T_PM:
476 /* already set */
477 break;
478 default:
479 ap->ap_pmcount = 1;
480 break;
481 }
482
483 /*
484 * Start the port if we succeeded.
485 *
486 * There's nothing to start for devices behind a port multiplier.
487 */
f4553de1 488 if (rc == 0 && atx == NULL) {
17eab71e 489 if (ahci_port_start(ap)) {
fd8bd957
MD
490 kprintf("%s: failed to start command DMA on port, "
491 "disabling\n", PORTNAME(ap));
258223a3
MD
492 rc = ENXIO; /* couldn't start port */
493 }
494 }
495
17eab71e 496 /*
3209f581 497 * Flush interrupts on the port. XXX
1980eff3
MD
498 *
499 * Enable interrupts on the port whether a device is sitting on
500 * it or not, to handle hot-plug events.
17eab71e 501 */
f4553de1 502 if (atx == NULL) {
1980eff3
MD
503 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
504 ahci_write(ap->ap_sc, AHCI_REG_IS, 1 << ap->ap_num);
505
f4553de1 506 ahci_port_interrupt_enable(ap);
1980eff3 507 }
fd8bd957 508 return(rc);
258223a3
MD
509}
510
fd8bd957 511/*
f4553de1
MD
512 * Enable or re-enable interrupts on a port.
513 *
514 * This routine is called from the port initialization code or from the
515 * helper thread as the real interrupt may be forced to turn off certain
516 * interrupt sources.
517 */
518void
519ahci_port_interrupt_enable(struct ahci_port *ap)
520{
521 u_int32_t data;
522
523 data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
524 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE |
525 AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE |
526 AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE |
527 AHCI_PREG_IE_DHRE;
528 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
529 data |= AHCI_PREG_IE_SDBE;
530#ifdef AHCI_COALESCE
531 if (sc->sc_ccc_ports & (1 << port)
532 data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE);
533#endif
534 ahci_pwrite(ap, AHCI_PREG_IE, data);
535}
536
537/*
3209f581
MD
538 * Run the port / target state machine from a main context.
539 *
540 * The state machine for the port is always run.
541 *
542 * If atx is non-NULL run the state machine for a particular target.
543 * If atx is NULL run the state machine for all targets.
544 */
545void
831bc9e3 546ahci_port_state_machine(struct ahci_port *ap, int initial)
3209f581
MD
547{
548 struct ata_port *at;
549 u_int32_t data;
550 int target;
551 int didsleep;
831bc9e3 552 int loop;
3209f581 553
831bc9e3
MD
554 /*
555 * State machine for port. Note that CAM is not yet associated
556 * during the initial parallel probe and the port's probe state
557 * will not get past ATA_PROBE_NEED_IDENT.
558 */
3209f581
MD
559 if (ap->ap_type == ATA_PORT_T_NONE) {
560 if (ap->ap_probe == ATA_PROBE_NEED_INIT) {
561 for (target = 0; target < AHCI_MAX_PMPORTS; ++target) {
562 at = &ap->ap_ata[target];
563 at->at_probe = ATA_PROBE_NEED_INIT;
831bc9e3 564 at->at_features |= ATA_PORT_F_RESCAN;
3209f581
MD
565 }
566 ahci_port_init(ap, NULL);
567 }
568 if (ap->ap_probe == ATA_PROBE_NEED_HARD_RESET)
569 ahci_port_reset(ap, NULL, 1);
570 if (ap->ap_probe == ATA_PROBE_NEED_SOFT_RESET)
571 ahci_port_reset(ap, NULL, 0);
572 if (ap->ap_probe == ATA_PROBE_NEED_IDENT)
573 ahci_cam_probe(ap, NULL);
574 }
575 if (ap->ap_type != ATA_PORT_T_PM) {
576 if (ap->ap_probe == ATA_PROBE_FAILED) {
577 ahci_cam_changed(ap, NULL, 0);
f4553de1 578 } else if (ap->ap_probe >= ATA_PROBE_NEED_IDENT) {
3209f581
MD
579 ahci_cam_changed(ap, NULL, 1);
580 }
581 return;
582 }
583
831bc9e3
MD
584 /*
585 * Port Multiplier state machine.
586 *
587 * Get a mask of changed targets and combine with any runnable
588 * states already present.
589 */
590 for (loop = 0; ;++loop) {
3209f581
MD
591 if (ahci_pm_read(ap, 15, AHCI_PMREG_EINFO, &data)) {
592 kprintf("%s: PM unable to read hot-plug bitmap\n",
593 PORTNAME(ap));
594 break;
595 }
596 data &= (1 << ap->ap_pmcount) - 1;
597
598 /*
831bc9e3
MD
599 * Do at least one loop, then stop if no more state changes
600 * have occured. The PM might not generate a new
601 * notification until we clear the entire bitmap.
3209f581 602 */
831bc9e3 603 if (loop && data == 0)
3209f581
MD
604 break;
605
606 /*
607 * New devices showing up in the bitmap require some spin-up
608 * time before we start probing them. Reset didsleep. The
609 * first new device we detect will sleep before probing.
831bc9e3
MD
610 *
611 * This only applies to devices whos change bit is set in
612 * the data, and does not apply to the initial boot-time
613 * probe.
3209f581
MD
614 */
615 didsleep = 0;
616
617 for (target = 0; target < ap->ap_pmcount; ++target) {
618 at = &ap->ap_ata[target];
619
620 /*
621 * Check the target state for targets behind the PM
622 * which have changed state. This will adjust
623 * at_probe and set ATA_PORT_F_RESCAN
624 *
625 * We want to wait at least 4 seconds before probing
626 * a newly inserted device. If the check status
627 * indicates a device is present and in need of a
628 * hard reset, we make sure we have slept before
629 * continuing.
831bc9e3
MD
630 *
631 * NOTE: When pm_check_good finds a good port it
632 * typically starts us in probe state
633 * NEED_HARD_RESET rather than INIT.
3209f581
MD
634 */
635 if (data & (1 << target)) {
636 ahci_pm_check_good(ap, target);
831bc9e3
MD
637 if (initial == 0 && didsleep == 0 &&
638 at->at_probe <= ATA_PROBE_NEED_HARD_RESET
639 ) {
640 didsleep = 1;
641 ahci_os_sleep(4000);
642 }
643 }
644
645 /*
646 * Report hot-plug events before the probe state
647 * really gets hot. Only actual events are reported
648 * here to reduce spew.
649 */
650 if (data & (1 << target)) {
651 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap, at));
652 switch(at->at_probe) {
653 case ATA_PROBE_NEED_INIT:
654 case ATA_PROBE_NEED_HARD_RESET:
655 kprintf("Device inserted\n");
656 break;
657 case ATA_PROBE_FAILED:
658 kprintf("Device removed\n");
659 break;
660 default:
661 kprintf("Device probe in progress\n");
662 break;
3209f581
MD
663 }
664 }
665
666 /*
831bc9e3
MD
667 * Run through the state machine as necessary if
668 * the port is not marked failed.
669 *
670 * The state machine may stop at NEED_IDENT if
671 * CAM is not yet attached.
672 *
673 * Acquire exclusive access to the port while we
674 * are doing this. This prevents command-completion
675 * from queueing commands for non-polled targets
676 * inbetween our probe steps. We need to do this
677 * because the reset probes can generate severe PHY
678 * and protocol errors and soft-brick the port.
3209f581 679 */
831bc9e3
MD
680 if (at->at_probe != ATA_PROBE_FAILED &&
681 at->at_probe != ATA_PROBE_GOOD) {
682 ahci_beg_exclusive_access(ap, at);
3209f581
MD
683 if (at->at_probe == ATA_PROBE_NEED_INIT)
684 ahci_port_init(ap, at);
685 if (at->at_probe == ATA_PROBE_NEED_HARD_RESET)
686 ahci_port_reset(ap, at, 1);
687 if (at->at_probe == ATA_PROBE_NEED_SOFT_RESET)
688 ahci_port_reset(ap, at, 0);
689 if (at->at_probe == ATA_PROBE_NEED_IDENT)
690 ahci_cam_probe(ap, at);
831bc9e3 691 ahci_end_exclusive_access(ap, at);
3209f581
MD
692 }
693
694 /*
831bc9e3 695 * Add or remove from CAM
3209f581
MD
696 */
697 if (at->at_features & ATA_PORT_F_RESCAN) {
698 at->at_features &= ~ATA_PORT_F_RESCAN;
699 if (at->at_probe == ATA_PROBE_FAILED) {
700 ahci_cam_changed(ap, at, 0);
f4553de1 701 } else if (at->at_probe >= ATA_PROBE_NEED_IDENT) {
3209f581
MD
702 ahci_cam_changed(ap, at, 1);
703 }
704 }
705 }
706 }
707}
708
709
710/*
fd8bd957
MD
711 * De-initialize and detach a port.
712 */
258223a3
MD
713void
714ahci_port_free(struct ahci_softc *sc, u_int port)
715{
716 struct ahci_port *ap = sc->sc_ports[port];
717 struct ahci_ccb *ccb;
718
17eab71e
MD
719 /*
720 * Ensure port is disabled and its interrupts are all flushed.
721 */
258223a3 722 if (ap->ap_sc) {
17eab71e 723 ahci_port_stop(ap, 1);
f4553de1 724 ahci_os_stop_port(ap);
258223a3
MD
725 ahci_pwrite(ap, AHCI_PREG_CMD, 0);
726 ahci_pwrite(ap, AHCI_PREG_IE, 0);
727 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
728 ahci_write(sc, AHCI_REG_IS, 1 << port);
729 }
730
731 if (ap->ap_ccbs) {
732 while ((ccb = ahci_get_ccb(ap)) != NULL) {
733 if (ccb->ccb_dmamap) {
734 bus_dmamap_destroy(sc->sc_tag_data,
735 ccb->ccb_dmamap);
736 ccb->ccb_dmamap = NULL;
737 }
738 }
739 kfree(ap->ap_ccbs, M_DEVBUF);
740 ap->ap_ccbs = NULL;
741 }
742
743 if (ap->ap_dmamem_cmd_list) {
744 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
745 ap->ap_dmamem_cmd_list = NULL;
746 }
747 if (ap->ap_dmamem_rfis) {
748 ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
749 ap->ap_dmamem_rfis = NULL;
750 }
751 if (ap->ap_dmamem_cmd_table) {
752 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
753 ap->ap_dmamem_cmd_table = NULL;
754 }
1980eff3
MD
755 if (ap->ap_ata) {
756 kfree(ap->ap_ata, M_DEVBUF);
757 ap->ap_ata = NULL;
758 }
258223a3
MD
759
760 /* bus_space(9) says we dont free the subregions handle */
761
762 kfree(ap, M_DEVBUF);
763 sc->sc_ports[port] = NULL;
764}
765
fd8bd957
MD
766/*
767 * Start high-level command processing on the port
768 */
258223a3 769int
17eab71e 770ahci_port_start(struct ahci_port *ap)
258223a3 771{
8bf6a3ff 772 u_int32_t r, oldr, s, olds, is, oldis, tfd, oldtfd;
258223a3 773
17eab71e
MD
774 /*
775 * FRE must be turned on before ST. Wait for FR to go active
776 * before turning on ST. The spec doesn't seem to think this
777 * is necessary but waiting here avoids an on-off race in the
778 * ahci_port_stop() code.
779 */
cec07d75 780 /* XXX REMOVE ME */
1980eff3
MD
781 olds = ahci_pread(ap, AHCI_PREG_SERR);
782 oldis= ahci_pread(ap, AHCI_PREG_IS);
8bf6a3ff 783 oldtfd = ahci_pread(ap, AHCI_PREG_TFD);
1980eff3 784 oldr = r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
17eab71e
MD
785 if ((r & AHCI_PREG_CMD_FRE) == 0) {
786 r |= AHCI_PREG_CMD_FRE;
787 ahci_pwrite(ap, AHCI_PREG_CMD, r);
788 }
789 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) {
790 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) {
791 kprintf("%s: Cannot start FIS reception\n",
792 PORTNAME(ap));
793 return (2);
794 }
795 }
796
797 /*
798 * Turn on ST, wait for CR to come up.
799 */
800 r |= AHCI_PREG_CMD_ST;
258223a3 801 ahci_pwrite(ap, AHCI_PREG_CMD, r);
17eab71e 802 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) {
8bf6a3ff
MD
803 s = ahci_pread(ap, AHCI_PREG_SERR);
804 is = ahci_pread(ap, AHCI_PREG_IS);
805 tfd = ahci_pread(ap, AHCI_PREG_TFD);
1980eff3
MD
806 kprintf("%s: Cannot start command DMA\n"
807 "OCMD=%b OSERR=%b\n"
808 "NCMP=%b NSERR=%b\n"
8bf6a3ff
MD
809 "OLDIS=%b\nNEWIS=%b\n"
810 "OLDTFD=%b\nNEWTFD=%b\n",
1980eff3
MD
811 PORTNAME(ap),
812 oldr, AHCI_PFMT_CMD, olds, AHCI_PFMT_SERR,
813 r, AHCI_PFMT_CMD, s, AHCI_PFMT_SERR,
8bf6a3ff
MD
814 oldis, AHCI_PFMT_IS, is, AHCI_PFMT_IS,
815 oldtfd, AHCI_PFMT_TFD_STS, tfd, AHCI_PFMT_TFD_STS);
17eab71e
MD
816 return (1);
817 }
258223a3
MD
818
819#ifdef AHCI_COALESCE
17eab71e
MD
820 /*
821 * (Re-)enable coalescing on the port.
822 */
258223a3
MD
823 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
824 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
825 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
826 ap->ap_sc->sc_ccc_ports_cur);
827 }
828#endif
829
258223a3
MD
830 return (0);
831}
832
fd8bd957
MD
833/*
834 * Stop high-level command processing on a port
835 */
258223a3
MD
836int
837ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
838{
839 u_int32_t r;
840
841#ifdef AHCI_COALESCE
17eab71e
MD
842 /*
843 * Disable coalescing on the port while it is stopped.
844 */
258223a3
MD
845 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
846 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
847 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
848 ap->ap_sc->sc_ccc_ports_cur);
849 }
850#endif
851
17eab71e
MD
852 /*
853 * Turn off ST, then wait for CR to go off.
854 */
258223a3
MD
855 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
856 r &= ~AHCI_PREG_CMD_ST;
258223a3
MD
857 ahci_pwrite(ap, AHCI_PREG_CMD, r);
858
17eab71e
MD
859 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) {
860 kprintf("%s: Port bricked, unable to stop (ST)\n",
861 PORTNAME(ap));
258223a3 862 return (1);
17eab71e 863 }
258223a3 864
1980eff3 865#if 0
17eab71e
MD
866 /*
867 * Turn off FRE, then wait for FR to go off. FRE cannot
868 * be turned off until CR transitions to 0.
869 */
1980eff3
MD
870 if ((r & AHCI_PREG_CMD_FR) == 0) {
871 kprintf("%s: FR stopped, clear FRE for next start\n",
872 PORTNAME(ap));
873 stop_fis_rx = 2;
874 }
875#endif
17eab71e
MD
876 if (stop_fis_rx) {
877 r &= ~AHCI_PREG_CMD_FRE;
878 ahci_pwrite(ap, AHCI_PREG_CMD, r);
879 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) {
880 kprintf("%s: Port bricked, unable to stop (FRE)\n",
881 PORTNAME(ap));
882 return (2);
883 }
884 }
258223a3
MD
885
886 return (0);
887}
888
fd8bd957
MD
889/*
890 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ}
891 */
258223a3
MD
892int
893ahci_port_clo(struct ahci_port *ap)
894{
895 struct ahci_softc *sc = ap->ap_sc;
896 u_int32_t cmd;
897
898 /* Only attempt CLO if supported by controller */
899 if ((ahci_read(sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) == 0)
900 return (1);
901
902 /* Issue CLO */
903 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
904#ifdef DIAGNOSTIC
905 if (cmd & AHCI_PREG_CMD_ST) {
906 kprintf("%s: CLO requested while port running\n",
907 PORTNAME(ap));
908 }
909#endif
910 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
911
912 /* Wait for completion */
913 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) {
914 kprintf("%s: CLO did not complete\n", PORTNAME(ap));
915 return (1);
916 }
917
918 return (0);
919}
920
fd8bd957 921/*
1980eff3
MD
922 * Reset a port.
923 *
924 * If hard is 0 perform a softreset of the port.
925 * If hard is 1 perform a hard reset of the port.
926 * If hard is 2 perform a hard reset of the port and cycle the phy.
927 *
928 * If at is non-NULL an indirect port via a port-multiplier is being
929 * reset, otherwise a direct port is being reset.
17eab71e 930 *
1980eff3 931 * NOTE: Indirect ports can only be soft-reset.
17eab71e
MD
932 */
933int
1980eff3 934ahci_port_reset(struct ahci_port *ap, struct ata_port *at, int hard)
17eab71e
MD
935{
936 int rc;
937
938 if (hard) {
1980eff3
MD
939 if (at)
940 rc = ahci_pm_hardreset(ap, at->at_target, hard);
941 else
942 rc = ahci_port_hardreset(ap, hard);
17eab71e 943 } else {
1980eff3
MD
944 if (at)
945 rc = ahci_pm_softreset(ap, at->at_target);
946 else
947 rc = ahci_port_softreset(ap);
17eab71e
MD
948 }
949 return(rc);
950}
951
952/*
fd8bd957
MD
953 * AHCI soft reset, Section 10.4.1
954 *
1980eff3
MD
955 * (at) will be NULL when soft-resetting a directly-attached device, and
956 * non-NULL when soft-resetting a device through a port multiplier.
957 *
fd8bd957 958 * This function keeps port communications intact and attempts to generate
1980eff3 959 * a reset to the connected device using device commands.
fd8bd957 960 */
258223a3
MD
961int
962ahci_port_softreset(struct ahci_port *ap)
963{
1980eff3
MD
964 struct ahci_ccb *ccb = NULL;
965 struct ahci_cmd_hdr *cmd_slot;
966 u_int8_t *fis;
3209f581 967 int error;
1980eff3
MD
968 u_int32_t cmd;
969
3209f581 970 error = EIO;
1980eff3
MD
971
972 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap),
973 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD);
258223a3
MD
974
975 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
976
977 crit_enter();
1980eff3
MD
978 ap->ap_flags |= AP_F_IN_RESET;
979 ap->ap_state = AP_S_NORMAL;
258223a3 980
1980eff3
MD
981 /*
982 * Remember port state in cmd (main to restore start/stop)
983 *
984 * Idle port.
985 */
986 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
258223a3
MD
987 if (ahci_port_stop(ap, 0)) {
988 kprintf("%s: failed to stop port, cannot softreset\n",
989 PORTNAME(ap));
990 goto err;
991 }
cf5f3a81
MD
992
993 /*
1980eff3 994 * Request CLO if device appears hung.
cf5f3a81 995 */
258223a3 996 if (ahci_pread(ap, AHCI_PREG_TFD) &
1980eff3 997 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
998 ahci_port_clo(ap);
999 }
1000
1980eff3
MD
1001 /*
1002 * This is an attempt to clear errors so a new signature will
1003 * be latched. It isn't working properly. XXX
1004 */
cf5f3a81 1005 ahci_flush_tfd(ap);
1980eff3 1006 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3
MD
1007
1008 /* Restart port */
17eab71e 1009 if (ahci_port_start(ap)) {
258223a3
MD
1010 kprintf("%s: failed to start port, cannot softreset\n",
1011 PORTNAME(ap));
1012 goto err;
1013 }
1014
1015 /* Check whether CLO worked */
1016 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1980eff3 1017 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
1018 kprintf("%s: CLO %s, need port reset\n",
1019 PORTNAME(ap),
1020 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO)
1021 ? "failed" : "unsupported");
3209f581 1022 error = EBUSY;
258223a3
MD
1023 goto err;
1024 }
1025
cec85a37
MD
1026 /*
1027 * Prep first D2H command with SRST feature & clear busy/reset flags
1028 *
1029 * It is unclear which other fields in the FIS are used. Just zero
1030 * everything.
1031 */
258223a3 1032 ccb = ahci_get_err_ccb(ap);
1980eff3 1033 ccb->ccb_xa.at = NULL;
258223a3 1034 cmd_slot = ccb->ccb_cmd_hdr;
258223a3
MD
1035
1036 fis = ccb->ccb_cmd_table->cfis;
cec85a37 1037 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1980eff3
MD
1038 fis[0] = ATA_FIS_TYPE_H2D;
1039 fis[15] = ATA_FIS_CONTROL_SRST|ATA_FIS_CONTROL_4BIT;
258223a3
MD
1040
1041 cmd_slot->prdtl = 0;
1042 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1043 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
1044 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
258223a3
MD
1045
1046 ccb->ccb_xa.state = ATA_S_PENDING;
5f8c1efd 1047 ccb->ccb_xa.flags = 0;
831bc9e3 1048 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
5f8c1efd 1049 kprintf("%s: First FIS failed\n", PORTNAME(ap));
258223a3 1050 goto err;
cec85a37 1051 }
258223a3 1052
cec85a37 1053 /*
831bc9e3
MD
1054 * WARNING! TIME SENSITIVE SPACE! WARNING!
1055 *
1056 * The two FISes are supposed to be back to back. Don't issue other
1057 * commands or even delay if we can help it.
1980eff3 1058 */
1980eff3
MD
1059
1060 /*
cec85a37
MD
1061 * Prep second D2H command to read status and complete reset sequence
1062 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1063 * Rev 2.6 and it is unclear how the second FIS should be set up
1064 * from the AHCI document.
1065 *
b089d0bf 1066 * Give the device 3ms before sending the second FIS.
cec85a37
MD
1067 *
1068 * It is unclear which other fields in the FIS are used. Just zero
1069 * everything.
1070 */
1071 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1980eff3
MD
1072 fis[0] = ATA_FIS_TYPE_H2D;
1073 fis[15] = ATA_FIS_CONTROL_4BIT;
258223a3
MD
1074
1075 cmd_slot->prdtl = 0;
1076 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
258223a3
MD
1077
1078 ccb->ccb_xa.state = ATA_S_PENDING;
5f8c1efd 1079 ccb->ccb_xa.flags = 0;
831bc9e3 1080 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
5f8c1efd 1081 kprintf("%s: Second FIS failed\n", PORTNAME(ap));
258223a3 1082 goto err;
cec85a37 1083 }
258223a3 1084
1980eff3
MD
1085 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1086 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
1087 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1088 PORTNAME(ap),
1089 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
3209f581 1090 error = EBUSY;
258223a3
MD
1091 goto err;
1092 }
3209f581 1093 ahci_os_sleep(10);
258223a3 1094
fd8bd957
MD
1095 /*
1096 * If the softreset is trying to clear a BSY condition after a
1097 * normal portreset we assign the port type.
1098 *
1099 * If the softreset is being run first as part of the ccb error
1100 * processing code then report if the device signature changed
1101 * unexpectedly.
1102 */
1980eff3
MD
1103 if (ap->ap_type == ATA_PORT_T_NONE) {
1104 ap->ap_type = ahci_port_signature_detect(ap, NULL);
fd8bd957 1105 } else {
1980eff3
MD
1106 if (ahci_port_signature_detect(ap, NULL) != ap->ap_type) {
1107 kprintf("%s: device signature unexpectedly "
1108 "changed\n", PORTNAME(ap));
3209f581 1109 error = EBUSY; /* XXX */
fd8bd957
MD
1110 }
1111 }
3209f581 1112 error = 0;
1980eff3 1113
3209f581 1114 ahci_os_sleep(3);
258223a3
MD
1115err:
1116 if (ccb != NULL) {
258223a3 1117 ahci_put_err_ccb(ccb);
1980eff3
MD
1118
1119 /*
1120 * If the target is busy use CLO to clear the busy
1121 * condition. The BSY should be cleared on the next
1122 * start.
1123 */
1124 if (ahci_pread(ap, AHCI_PREG_TFD) &
1125 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1126 ahci_port_clo(ap);
1127 }
258223a3
MD
1128 }
1129
cf5f3a81
MD
1130 /*
1131 * If we failed to softreset make the port quiescent, otherwise
1132 * make sure the port's start/stop state matches what it was on
1133 * entry.
1980eff3
MD
1134 *
1135 * Don't kill the port if the softreset is on a port multiplier
1136 * target, that would kill all the targets!
cf5f3a81 1137 */
3209f581 1138 if (error) {
cf5f3a81 1139 ahci_port_hardstop(ap);
3209f581 1140 /* ap_probe set to failed */
cf5f3a81 1141 } else if (cmd & AHCI_PREG_CMD_ST) {
3209f581 1142 ap->ap_probe = ATA_PROBE_NEED_IDENT;
1980eff3 1143 kprintf("%s: STARTING PORT\n", PORTNAME(ap));
cf5f3a81
MD
1144 ahci_port_start(ap);
1145 } else {
3209f581 1146 ap->ap_probe = ATA_PROBE_NEED_IDENT;
1980eff3 1147 kprintf("%s: STOPPING PORT\n", PORTNAME(ap));
cf5f3a81
MD
1148 ahci_port_stop(ap, !(cmd & AHCI_PREG_CMD_FRE));
1149 }
3209f581 1150 ap->ap_flags &= ~AP_F_IN_RESET;
258223a3
MD
1151 crit_exit();
1152
1980eff3 1153 kprintf("%s: END SOFTRESET\n", PORTNAME(ap));
1980eff3 1154
3209f581 1155 return (error);
258223a3
MD
1156}
1157
fd8bd957
MD
1158/*
1159 * AHCI port reset, Section 10.4.2
1160 *
1161 * This function does a hard reset of the port. Note that the device
1162 * connected to the port could still end-up hung.
1163 */
258223a3 1164int
1980eff3 1165ahci_port_hardreset(struct ahci_port *ap, int hard)
258223a3 1166{
1980eff3 1167 u_int32_t cmd, r;
3209f581 1168 int error;
1980eff3
MD
1169 int loop;
1170 int type;
258223a3
MD
1171
1172 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1173
1980eff3
MD
1174 ap->ap_flags |= AP_F_IN_RESET;
1175
1176 /*
1177 * Idle the port,
1178 */
258223a3 1179 ahci_port_stop(ap, 0);
cf5f3a81 1180 ap->ap_state = AP_S_NORMAL;
3209f581 1181 error = 0;
cf5f3a81
MD
1182
1183 /*
1980eff3
MD
1184 * The port may have been quiescent with its SUD bit cleared, so
1185 * set the SUD (spin up device).
cf5f3a81
MD
1186 */
1187 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1188 cmd |= AHCI_PREG_CMD_SUD;
1189 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
258223a3 1190
1980eff3
MD
1191 /*
1192 * Perform device detection. Cycle the PHY off, wait 10ms.
1193 * This simulates the SATA cable being physically unplugged.
1194 */
1195 ap->ap_type = ATA_PORT_T_NONE;
1196
1197 r = AHCI_PREG_SCTL_IPM_DISABLED;
1198 if (hard == 2)
1199 r |= AHCI_PREG_SCTL_DET_DISABLE;
1200 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
3209f581 1201 ahci_os_sleep(10);
258223a3 1202
1980eff3
MD
1203 /*
1204 * Start transmitting COMRESET. COMRESET must be sent for at
1205 * least 1ms.
1206 */
1207 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
258223a3
MD
1208 if (AhciForceGen1 & (1 << ap->ap_num)) {
1209 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap));
1210 r |= AHCI_PREG_SCTL_SPD_GEN1;
1211 } else {
1212 r |= AHCI_PREG_SCTL_SPD_ANY;
1213 }
1214 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
831bc9e3
MD
1215
1216 /*
1217 * Through trial and error it seems to take around 100ms
1218 * for the detect logic to settle down. If this is too
1219 * short the softreset code will fail.
1220 */
1221 ahci_os_sleep(100);
cf5f3a81
MD
1222
1223 /*
1224 * Only SERR_DIAG_X needs to be cleared for TFD updates, but
1225 * since we are hard-resetting the port we might as well clear
1226 * the whole enchillada
1227 */
1228 ahci_flush_tfd(ap);
1229 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3
MD
1230 r &= ~AHCI_PREG_SCTL_DET_INIT;
1231 r |= AHCI_PREG_SCTL_DET_NONE;
1232 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
258223a3 1233
1980eff3
MD
1234 /*
1235 * Try to determine if there is a device on the port.
1236 *
1237 * Give the device 3/10 second to at least be detected.
1238 * If we fail clear PRCS (phy detect) since we may cycled
1239 * the phy and probably caused another PRCS interrupt.
1240 */
1241 for (loop = 30; loop; --loop) {
1242 r = ahci_pread(ap, AHCI_PREG_SSTS);
1243 if (r & AHCI_PREG_SSTS_DET)
1244 break;
3209f581 1245 ahci_os_sleep(10);
1980eff3
MD
1246 }
1247 if (loop == 0) {
1248 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS);
1249 kprintf("%s: Port appears to be unplugged\n",
1250 PORTNAME(ap));
3209f581 1251 error = ENODEV;
1980eff3
MD
1252 }
1253
1254 /*
1255 * There is something on the port. Give the device 3 seconds
1256 * to fully negotiate.
1257 */
3209f581 1258 if (error == 0 &&
1980eff3
MD
1259 ahci_pwait_eq(ap, 3000, AHCI_PREG_SSTS,
1260 AHCI_PREG_SSTS_DET, AHCI_PREG_SSTS_DET_DEV)) {
1261 kprintf("%s: Device may be powered down\n",
1262 PORTNAME(ap));
3209f581 1263 error = ENODEV;
258223a3
MD
1264 }
1265
cec85a37 1266 /*
1980eff3 1267 * Wait for the device to become ready.
cec85a37 1268 *
b089d0bf
MD
1269 * This can take more then a second, give it 3 seconds. If we
1270 * succeed give the device another 3ms after that.
1980eff3 1271 *
3209f581 1272 * NOTE: Port multipliers can do two things here. First they can
1980eff3
MD
1273 * return device-ready if a device is on target 0 and also
1274 * return the signature for that device. If there is no
1275 * device on target 0 then BSY/DRQ is never cleared and
1276 * it never comes ready.
cec85a37 1277 */
3209f581 1278 if (error == 0 &&
1980eff3
MD
1279 ahci_pwait_clr_to(ap, 3000, AHCI_PREG_TFD,
1280 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1281 /*
1282 * The device is bricked or its a port multiplier and will
1283 * not unbusy until we do the pmprobe CLO softreset sequence.
1284 */
3209f581
MD
1285 error = ahci_port_pmprobe(ap);
1286 if (error) {
1980eff3
MD
1287 kprintf("%s: Device will not come ready 0x%b\n",
1288 PORTNAME(ap),
1289 ahci_pread(ap, AHCI_PREG_TFD),
1290 AHCI_PFMT_TFD_STS);
1291 } else {
1292 ap->ap_type = ATA_PORT_T_PM;
1980eff3 1293 }
3209f581 1294 } else if (error == 0) {
1980eff3
MD
1295 /*
1296 * We generally will not get a port multiplier signature in
1297 * this case even if this is a port multiplier, because of
1298 * Intel's stupidity. We almost certainly got target 0
1299 * behind the PM, if there is a PM.
1300 *
1301 * Save the signature and probe for a PM. If we do not
1302 * find a PM then use the saved signature and return
1303 * success.
1304 */
1305 type = ahci_port_signature_detect(ap, NULL);
3209f581
MD
1306 error = ahci_port_pmprobe(ap);
1307 if (error) {
1980eff3 1308 ap->ap_type = type;
3209f581 1309 error = 0;
1980eff3
MD
1310 } else {
1311 ap->ap_type = ATA_PORT_T_PM;
3209f581 1312 kprintf("%s: Port multiplier detected\n",
1980eff3
MD
1313 PORTNAME(ap));
1314 }
1315 }
1316
1317 /*
1318 * hard-stop the port if we failed. This will set ap_probe
1319 * to FAILED.
1320 */
1980eff3 1321 ap->ap_flags &= ~AP_F_IN_RESET;
3209f581
MD
1322 if (error) {
1323 ahci_port_hardstop(ap);
1324 /* ap_probe set to failed */
1325 } else {
f4553de1
MD
1326 if (ap->ap_type == ATA_PORT_T_PM)
1327 ap->ap_probe = ATA_PROBE_GOOD;
1328 else
1329 ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET;
3209f581
MD
1330 }
1331 return (error);
1980eff3
MD
1332}
1333
1334/*
1335 * AHCI port multiplier probe. This routine is run by the hardreset code
1336 * if it gets past the device detect, whether or not BSY is found to be
1337 * stuck.
1338 *
1339 * We MUST use CLO to properly probe whether the port multiplier exists
1340 * or not.
1341 *
1342 * Return 0 on success, non-zero on failure.
1343 */
1344int
1345ahci_port_pmprobe(struct ahci_port *ap)
1346{
1347 struct ahci_cmd_hdr *cmd_slot;
1348 struct ahci_ccb *ccb = NULL;
1349 u_int8_t *fis = NULL;
831bc9e3 1350 int error = EIO;
1980eff3
MD
1351 u_int32_t cmd;
1352 int count;
1353
1354 /*
1355 * If we don't support port multipliers don't try to detect one.
1356 */
1357 if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SPM) == 0)
1358 return (ENODEV);
1359
1360 count = 2;
831bc9e3 1361#if 1
1980eff3
MD
1362 kprintf("%s: START PMPROBE\n", PORTNAME(ap));
1363#endif
1364retry:
1365 /*
1366 * This code is only called from hardreset, which does not
1367 * high level command processing. The port should be stopped.
1368 *
1369 * Set PMA mode while the port is stopped.
1370 *
1371 * NOTE: On retry the port might be running, stopped, or failed.
1372 */
1373 ahci_port_stop(ap, 0);
1374 ap->ap_state = AP_S_NORMAL;
1375 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
831bc9e3
MD
1376 if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
1377 cmd |= AHCI_PREG_CMD_PMA;
1378 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1379 }
1980eff3
MD
1380
1381 /*
1382 * Flush any errors and request CLO unconditionally, then start
1383 * the port.
1384 */
1385 ahci_flush_tfd(ap);
1386 ahci_port_clo(ap);
1387 if (ahci_port_start(ap)) {
1388 kprintf("%s: PMPROBE failed to start port, cannot softreset\n",
1389 PORTNAME(ap));
1390 goto err;
1391 }
1392
1393 /*
1394 * Check whether CLO worked
1395 */
1396 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1397 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1398 kprintf("%s: PMPROBE CLO %s, need port reset\n",
258223a3 1399 PORTNAME(ap),
1980eff3
MD
1400 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO)
1401 ? "failed" : "unsupported");
831bc9e3 1402 error = EBUSY;
258223a3
MD
1403 goto err;
1404 }
1405
1980eff3
MD
1406 /*
1407 * Prep the first H2D command with SRST feature & clear busy/reset
1408 * flags.
1409 */
1410 ccb = ahci_get_err_ccb(ap);
1411 cmd_slot = ccb->ccb_cmd_hdr;
1412
1413 fis = ccb->ccb_cmd_table->cfis;
1414 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1415 fis[0] = ATA_FIS_TYPE_H2D;
1416 fis[1] = 0x0F; /* Target 15 */
1417 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1418
1419 cmd_slot->prdtl = 0;
1420 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1421 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
1422 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
1423 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
1424
1425 ccb->ccb_xa.state = ATA_S_PENDING;
1426 ccb->ccb_xa.flags = 0;
1427
831bc9e3 1428 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
1980eff3
MD
1429 kprintf("%s: PMPROBE First FIS failed\n", PORTNAME(ap));
1430 if (--count) {
1980eff3
MD
1431 ahci_put_err_ccb(ccb);
1432 goto retry;
1433 }
1434 goto err;
1435 }
1436 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1437 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1438 kprintf("%s: PMPROBE Busy after first FIS\n", PORTNAME(ap));
1439 }
1440
1441 /*
1442 * The device may have muffed up the PHY when it reset.
1443 */
831bc9e3 1444 ahci_os_sleep(100);
1980eff3
MD
1445 ahci_flush_tfd(ap);
1446 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1447 /* ahci_pm_phy_status(ap, 15, &cmd); */
1448
1449 /*
1450 * Prep second D2H command to read status and complete reset sequence
1451 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1452 * Rev 2.6 and it is unclear how the second FIS should be set up
1453 * from the AHCI document.
1454 *
1455 * Give the device 3ms before sending the second FIS.
1456 *
1457 * It is unclear which other fields in the FIS are used. Just zero
1458 * everything.
1459 */
1460 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1461 fis[0] = ATA_FIS_TYPE_H2D;
1462 fis[1] = 0x0F;
1463 fis[15] = ATA_FIS_CONTROL_4BIT;
1464
1465 cmd_slot->prdtl = 0;
1466 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1467 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
1468
1469 ccb->ccb_xa.state = ATA_S_PENDING;
1470 ccb->ccb_xa.flags = 0;
1471
831bc9e3 1472 if (ahci_poll(ccb, 5000, ahci_quick_timeout) != ATA_S_COMPLETE) {
1980eff3
MD
1473 kprintf("%s: PMPROBE Second FIS failed\n", PORTNAME(ap));
1474 if (--count) {
1980eff3
MD
1475 ahci_put_err_ccb(ccb);
1476 goto retry;
1477 }
1478 goto err;
1479 }
1480
1481 /*
1482 * What? We succeeded? Yup, but for some reason the signature
1483 * is still latched from the original detect (that saw target 0
1484 * behind the PM), and I don't know how to clear the condition
1485 * other then by retrying the whole reset sequence.
1486 */
1487 if (--count) {
1488 fis[15] = 0;
1489 ahci_put_err_ccb(ccb);
1490 goto retry;
1491 }
1492
1493 /*
1494 * Get the signature. The caller sets the ap fields.
1495 */
1496 if (ahci_port_signature_detect(ap, NULL) == ATA_PORT_T_PM) {
1497 ap->ap_ata[15].at_probe = ATA_PROBE_GOOD;
831bc9e3 1498 error = 0;
1980eff3 1499 } else {
831bc9e3 1500 error = EBUSY;
1980eff3 1501 }
cf5f3a81
MD
1502
1503 /*
1980eff3 1504 * Fall through / clean up the CCB and perform error processing.
cf5f3a81 1505 */
258223a3 1506err:
831bc9e3 1507 if (ccb != NULL)
1980eff3 1508 ahci_put_err_ccb(ccb);
1980eff3 1509
831bc9e3 1510 if (error == 0 && ahci_pm_identify(ap)) {
3209f581
MD
1511 kprintf("%s: PM - cannot identify port multiplier\n",
1512 PORTNAME(ap));
831bc9e3 1513 error = EBUSY;
3209f581
MD
1514 }
1515#if 0
831bc9e3 1516 if (error == 0 && ahci_pm_set_feature(ap, ATA_SATAFT_ASYNCNOTIFY, 1)) {
3209f581
MD
1517 kprintf("%s: PM - Warning, cannot enable async notify\n",
1518 PORTNAME(ap));
1519 /* ignore error */
1520 }
831bc9e3 1521 if (error == 0) {
3209f581
MD
1522 u_int32_t data;
1523 if (ahci_pm_read(ap, 2, 4, &data))
1524 kprintf("Cannot read snotify\n");
1525 else
1526 kprintf("Read snotify %08x\n", data);
1527 }
1528#endif
1529
1980eff3
MD
1530 /*
1531 * If we failed turn off PMA, otherwise identify the port multiplier.
1532 * CAM will iterate the devices.
1533 */
831bc9e3 1534 if (error) {
1980eff3
MD
1535 ahci_port_stop(ap, 0);
1536 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1537 cmd &= ~AHCI_PREG_CMD_PMA;
1538 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1980eff3
MD
1539 }
1540 ahci_port_stop(ap, 0);
1541
831bc9e3 1542 return(error);
cf5f3a81
MD
1543}
1544
1980eff3 1545
cf5f3a81
MD
1546/*
1547 * Hard-stop on hot-swap device removal. See 10.10.1
1548 *
1549 * Place the port in a mode that will allow it to detect hot-swap insertions.
1550 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't
1551 * seem to do the job.
1552 */
1553void
1554ahci_port_hardstop(struct ahci_port *ap)
1555{
1980eff3 1556 struct ata_port *at;
cf5f3a81
MD
1557 u_int32_t r;
1558 u_int32_t cmd;
1980eff3 1559 int i;
cf5f3a81
MD
1560
1561 /*
1562 * Stop the port. We can't modify things like SUD if the port
1563 * is running.
1564 */
1565 ap->ap_state = AP_S_FATAL_ERROR;
1980eff3
MD
1566 ap->ap_probe = ATA_PROBE_FAILED;
1567 ap->ap_type = ATA_PORT_T_NONE;
cf5f3a81
MD
1568 ahci_port_stop(ap, 0);
1569 cmd = ahci_pread(ap, AHCI_PREG_CMD);
1570
1571 /*
1980eff3
MD
1572 * Clean up AT sub-ports on SATA port.
1573 */
1574 for (i = 0; ap->ap_ata && i < AHCI_MAX_PMPORTS; ++i) {
1575 at = &ap->ap_ata[i];
1576 at->at_type = ATA_PORT_T_NONE;
3209f581 1577 at->at_probe = ATA_PROBE_FAILED;
1980eff3
MD
1578 }
1579
1580 /*
1581 * Turn off port-multiplier control bit
1582 */
1583 if (cmd & AHCI_PREG_CMD_PMA) {
1584 cmd &= ~AHCI_PREG_CMD_PMA;
1585 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1586 }
1587
1588 /*
cf5f3a81
MD
1589 * Make sure FRE is active. There isn't anything we can do if it
1590 * fails so just ignore errors.
1591 */
1592 if ((cmd & AHCI_PREG_CMD_FRE) == 0) {
1593 cmd |= AHCI_PREG_CMD_FRE;
1594 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1595 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0)
1596 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR);
1597 }
1598
1599 /*
1600 * 10.10.3 DET must be set to 0 before setting SUD to 0.
1601 * 10.10.1 place us in the Listen state.
1602 *
1603 * Deactivating SUD only applies if the controller supports SUD.
1604 */
1605 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED);
3209f581 1606 ahci_os_sleep(1);
cf5f3a81
MD
1607 if (cmd & AHCI_PREG_CMD_SUD) {
1608 cmd &= ~AHCI_PREG_CMD_SUD;
1609 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1610 }
3209f581 1611 ahci_os_sleep(1);
cf5f3a81
MD
1612
1613 /*
1614 * Transition su to the spin-up state. HVA shall send COMRESET and
1615 * begin initialization sequence (whatever that means).
1616 *
1617 * This only applies if the controller supports SUD.
1618 */
1619 cmd |= AHCI_PREG_CMD_SUD;
258223a3 1620 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
3209f581 1621 ahci_os_sleep(1);
258223a3 1622
cf5f3a81
MD
1623 /*
1624 * Transition us to the Reset state. Theoretically we send a
1625 * continuous stream of COMRESETs in this state.
1626 */
1627 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1628 if (AhciForceGen1 & (1 << ap->ap_num)) {
1629 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap));
1630 r |= AHCI_PREG_SCTL_SPD_GEN1;
1631 } else {
1632 r |= AHCI_PREG_SCTL_SPD_ANY;
1633 }
1634 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
3209f581 1635 ahci_os_sleep(1);
cf5f3a81
MD
1636
1637 /*
1638 * Flush SERR_DIAG_X so the TFD can update.
1639 */
1640 ahci_flush_tfd(ap);
1641
1642 /*
1643 * Leave us in COMRESET (both SUD and INIT active), the HBA should
1644 * hopefully send us a DIAG_X-related interrupt if it receives
1645 * a COMINIT, and if not that then at least a Phy transition
1646 * interrupt.
1647 *
1648 * If we transition INIT from 1->0 to begin the initalization
1649 * sequence it is unclear if that sequence will remain active
1650 * until the next device insertion.
1651 *
1652 * If we go back to the listen state it is unclear if the
1653 * device will actually send us a COMINIT, since we aren't
1654 * sending any COMRESET's
1655 */
1656 /* NOP */
1657}
1658
1659/*
1660 * Multiple events may have built up in the TFD. The spec is not very
1661 * clear on this but it does seem to serialize events so clearing DIAG_X
1662 * just once might not do the job during a reset sequence.
831bc9e3
MD
1663 *
1664 * XXX this probably isn't right.
cf5f3a81
MD
1665 */
1666void
1667ahci_flush_tfd(struct ahci_port *ap)
1668{
1669 u_int32_t r;
1670
1671 r = ahci_pread(ap, AHCI_PREG_SERR);
1980eff3
MD
1672 while (r & AHCI_PREG_SERR_DIAG_X) {
1673 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
3209f581 1674 ahci_os_sleep(1);
cf5f3a81
MD
1675 r = ahci_pread(ap, AHCI_PREG_SERR);
1676 }
258223a3
MD
1677}
1678
fd8bd957
MD
1679/*
1680 * Figure out what type of device is connected to the port, ATAPI or
1681 * DISK.
1682 */
1683int
1980eff3 1684ahci_port_signature_detect(struct ahci_port *ap, struct ata_port *at)
fd8bd957
MD
1685{
1686 u_int32_t sig;
1687
1688 sig = ahci_pread(ap, AHCI_PREG_SIG);
1980eff3 1689 kprintf("%s: sig %08x\n", ATANAME(ap, at), sig);
fd8bd957
MD
1690 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) {
1691 return(ATA_PORT_T_ATAPI);
1980eff3
MD
1692 } else if ((sig & 0xffff0000) ==
1693 (SATA_SIGNATURE_PORT_MULTIPLIER & 0xffff0000)) {
1980eff3 1694 return(ATA_PORT_T_PM);
fd8bd957
MD
1695 } else {
1696 return(ATA_PORT_T_DISK);
1697 }
1698}
1699
1700/*
1701 * Load the DMA descriptor table for a CCB's buffer.
1702 */
258223a3
MD
1703int
1704ahci_load_prdt(struct ahci_ccb *ccb)
1705{
1706 struct ahci_port *ap = ccb->ccb_port;
1707 struct ahci_softc *sc = ap->ap_sc;
1708 struct ata_xfer *xa = &ccb->ccb_xa;
1709 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1710 bus_dmamap_t dmap = ccb->ccb_dmamap;
1711 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1712 int error;
1713
1714 if (xa->datalen == 0) {
1715 ccb->ccb_cmd_hdr->prdtl = 0;
1716 return (0);
1717 }
1718
1719 error = bus_dmamap_load(sc->sc_tag_data, dmap,
1720 xa->data, xa->datalen,
1721 ahci_load_prdt_callback,
1722 &prdt,
1723 ((xa->flags & ATA_F_NOWAIT) ?
1724 BUS_DMA_NOWAIT : BUS_DMA_WAITOK));
1725 if (error != 0) {
1726 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1727 return (1);
1728 }
1729 if (xa->flags & ATA_F_PIO)
1730 prdt->flags |= htole32(AHCI_PRDT_FLAG_INTR);
1731
1732 cmd_slot->prdtl = htole16(prdt - ccb->ccb_cmd_table->prdt + 1);
1733
1734 bus_dmamap_sync(sc->sc_tag_data, dmap,
1735 (xa->flags & ATA_F_READ) ?
1736 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
1737
1738 return (0);
1739
1740#ifdef DIAGNOSTIC
1741diagerr:
1742 bus_dmamap_unload(sc->sc_tag_data, dmap);
1743 return (1);
1744#endif
1745}
1746
1747/*
1748 * Callback from BUSDMA system to load the segment list. The passed segment
1749 * list is a temporary structure.
1750 */
1751static
1752void
1753ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, int nsegs,
1754 int error)
1755{
1756 struct ahci_prdt *prd = *(void **)info;
1757 u_int64_t addr;
1758
1759 KKASSERT(nsegs <= AHCI_MAX_PRDT);
1760
1761 while (nsegs) {
1762 addr = segs->ds_addr;
1763 prd->dba_hi = htole32((u_int32_t)(addr >> 32));
1764 prd->dba_lo = htole32((u_int32_t)addr);
1765#ifdef DIAGNOSTIC
1766 KKASSERT((addr & 1) == 0);
1767 KKASSERT((segs->ds_len & 1) == 0);
1768#endif
1769 prd->flags = htole32(segs->ds_len - 1);
1770 --nsegs;
1771 if (nsegs)
1772 ++prd;
1773 ++segs;
1774 }
1775 *(void **)info = prd; /* return last valid segment */
1776}
1777
1778void
1779ahci_unload_prdt(struct ahci_ccb *ccb)
1780{
1781 struct ahci_port *ap = ccb->ccb_port;
1782 struct ahci_softc *sc = ap->ap_sc;
1783 struct ata_xfer *xa = &ccb->ccb_xa;
1784 bus_dmamap_t dmap = ccb->ccb_dmamap;
1785
1786 if (xa->datalen != 0) {
1787 bus_dmamap_sync(sc->sc_tag_data, dmap,
1788 (xa->flags & ATA_F_READ) ?
1789 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1790
1791 bus_dmamap_unload(sc->sc_tag_data, dmap);
1792
1793 if (ccb->ccb_xa.flags & ATA_F_NCQ)
1794 xa->resid = 0;
1795 else
1796 xa->resid = xa->datalen -
1797 le32toh(ccb->ccb_cmd_hdr->prdbc);
1798 }
1799}
1800
5f8c1efd
MD
1801/*
1802 * Start a command and poll for completion.
1803 *
3209f581
MD
1804 * timeout is in ms and only counts once the command gets on-chip.
1805 *
831bc9e3
MD
1806 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine
1807 * that no error occured.
1808 *
5f8c1efd
MD
1809 * NOTE: If the caller specifies a NULL timeout function the caller is
1810 * responsible for clearing hardware state on failure, but we will
1811 * deal with removing the ccb from any pending queue.
1812 *
1813 * NOTE: NCQ should never be used with this function.
cf5f3a81
MD
1814 *
1815 * NOTE: If the port is in a failed state and stopped we do not try
1816 * to activate the ccb.
5f8c1efd 1817 */
258223a3 1818int
831bc9e3
MD
1819ahci_poll(struct ahci_ccb *ccb, int timeout,
1820 void (*timeout_fn)(struct ahci_ccb *))
258223a3 1821{
5f8c1efd 1822 struct ahci_port *ap = ccb->ccb_port;
258223a3 1823
cf5f3a81
MD
1824 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) {
1825 ccb->ccb_xa.state = ATA_S_ERROR;
831bc9e3 1826 return(ccb->ccb_xa.state);
cf5f3a81 1827 }
258223a3
MD
1828 crit_enter();
1829 ahci_start(ccb);
1980eff3 1830
258223a3 1831 do {
f4553de1 1832 ahci_port_intr(ap, 1);
831bc9e3
MD
1833 switch(ccb->ccb_xa.state) {
1834 case ATA_S_ONCHIP:
1835 timeout -= ahci_os_softsleep();
1836 break;
1837 case ATA_S_PENDING:
1838 ahci_os_softsleep();
1839 ahci_check_active_timeouts(ap);
f4553de1 1840 break;
831bc9e3
MD
1841 default:
1842 crit_exit();
1843 return (ccb->ccb_xa.state);
f4553de1 1844 }
3209f581 1845 } while (timeout > 0);
5f8c1efd 1846
831bc9e3
MD
1847 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n",
1848 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_slot,
1849 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD,
1850 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS,
1851 ahci_pread(ap, AHCI_PREG_SERR), AHCI_PFMT_SERR);
1852
1853 timeout_fn(ccb);
5f8c1efd 1854
258223a3
MD
1855 crit_exit();
1856
831bc9e3
MD
1857 return(ccb->ccb_xa.state);
1858}
1859
1860/*
1861 * When polling we have to check if the currently active CCB(s)
1862 * have timed out as the callout will be deadlocked while we
1863 * hold the port lock.
1864 */
1865void
1866ahci_check_active_timeouts(struct ahci_port *ap)
1867{
1868 struct ahci_ccb *ccb;
1869 u_int32_t mask;
1870 int tag;
1871
1872 mask = ap->ap_active | ap->ap_sactive;
1873 while (mask) {
1874 tag = ffs(mask) - 1;
1875 mask &= ~(1 << tag);
1876 ccb = &ap->ap_ccbs[tag];
1877 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_EXPIRED) {
1878 ahci_ata_cmd_timeout(ccb);
1879 }
1880 }
258223a3
MD
1881}
1882
3209f581
MD
1883static
1884__inline
1885void
1886ahci_start_timeout(struct ahci_ccb *ccb)
1887{
1888 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_DESIRED) {
1889 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_RUNNING;
1890 callout_reset(&ccb->ccb_timeout,
1891 (ccb->ccb_xa.timeout * hz + 999) / 1000,
1892 ahci_ata_cmd_timeout_unserialized, ccb);
1893 }
1894}
1895
258223a3
MD
1896void
1897ahci_start(struct ahci_ccb *ccb)
1898{
1899 struct ahci_port *ap = ccb->ccb_port;
1900 struct ahci_softc *sc = ap->ap_sc;
1901
1902 KKASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
1903
1904 /* Zero transferred byte count before transfer */
1905 ccb->ccb_cmd_hdr->prdbc = 0;
1906
1907 /* Sync command list entry and corresponding command table entry */
1908 bus_dmamap_sync(sc->sc_tag_cmdh,
1909 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1910 BUS_DMASYNC_PREWRITE);
1911 bus_dmamap_sync(sc->sc_tag_cmdt,
1912 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1913 BUS_DMASYNC_PREWRITE);
1914
1915 /* Prepare RFIS area for write by controller */
1916 bus_dmamap_sync(sc->sc_tag_rfis,
1917 AHCI_DMA_MAP(ap->ap_dmamem_rfis),
1918 BUS_DMASYNC_PREREAD);
1919
1920 if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1980eff3
MD
1921 /*
1922 * Issue NCQ commands only when there are no outstanding
1923 * standard commands.
1924 */
1925 if (ap->ap_active || TAILQ_FIRST(&ap->ap_ccb_pending)) {
258223a3 1926 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1980eff3 1927 } else {
3209f581 1928 ahci_start_timeout(ccb);
258223a3
MD
1929 KKASSERT(ap->ap_active_cnt == 0);
1930 ap->ap_sactive |= (1 << ccb->ccb_slot);
1931 ccb->ccb_xa.state = ATA_S_ONCHIP;
1932 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1933 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1934 }
1935 } else {
5f8c1efd
MD
1936 /*
1937 * Wait for all NCQ commands to finish before issuing standard
1980eff3
MD
1938 * command. Allow up to <limit> non-NCQ commands to be active.
1939 *
1940 * XXX If ap is a port multiplier only allow 1. At least the
1941 * NVidia-MCP77 part seems to barf if more then one
1942 * command is activated, even though it isn't NCQ.
1943 *
1944 * If I set up more then one I get phy errors and the
1945 * port fails.
5f8c1efd 1946 */
1980eff3
MD
1947 int limit = (ap->ap_type == ATA_PORT_T_PM) ? 1 : 2;
1948 if (ap->ap_sactive || ap->ap_active_cnt >= limit) {
258223a3 1949 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1980eff3 1950 } else {
3209f581 1951 ahci_start_timeout(ccb);
258223a3
MD
1952 ap->ap_active |= 1 << ccb->ccb_slot;
1953 ccb->ccb_xa.state = ATA_S_ONCHIP;
1954 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1955 ap->ap_active_cnt++;
1956 }
1957 }
1958}
1959
831bc9e3
MD
1960/*
1961 * While holding the port lock acquire exclusive access to the port.
1962 *
1963 * This is used when running the state machine to initialize and identify
1964 * targets over a port multiplier. Setting exclusive access prevents
1965 * ahci_port_intr() from activating any requests sitting on the pending
1966 * queue.
1967 */
1968void
1969ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at)
1970{
1971 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) == 0);
1972 ap->ap_flags |= AP_F_EXCLUSIVE_ACCESS;
1973 while (ap->ap_active || ap->ap_sactive) {
1974 ahci_port_intr(ap, 1);
1975 ahci_os_softsleep();
1976 }
1977}
1978
1979void
1980ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at)
1981{
1982 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) != 0);
1983 ap->ap_flags &= ~AP_F_EXCLUSIVE_ACCESS;
1984 if (ap->ap_active == 0 && ap->ap_sactive == 0)
1985 ahci_issue_pending_commands(ap, 0);
1986}
1987
258223a3
MD
1988void
1989ahci_issue_pending_ncq_commands(struct ahci_port *ap)
1990{
1991 struct ahci_ccb *nextccb;
1992 u_int32_t sact_change = 0;
1993
1994 KKASSERT(ap->ap_active_cnt == 0);
1995
1996 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1997 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
1998 return;
831bc9e3
MD
1999 if (ap->ap_flags & AP_F_EXCLUSIVE_ACCESS)
2000 return;
258223a3
MD
2001
2002 /* Start all the NCQ commands at the head of the pending list. */
2003 do {
2004 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
3209f581 2005 ahci_start_timeout(nextccb);
258223a3
MD
2006 sact_change |= 1 << nextccb->ccb_slot;
2007 nextccb->ccb_xa.state = ATA_S_ONCHIP;
2008 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
2009 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ));
2010
2011 ap->ap_sactive |= sact_change;
2012 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
2013 ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
2014
2015 return;
2016}
2017
2018void
2019ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
2020{
2021 struct ahci_ccb *nextccb;
2022
2023 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
831bc9e3
MD
2024 if (nextccb == NULL)
2025 return;
2026 if (ap->ap_flags & AP_F_EXCLUSIVE_ACCESS)
2027 return;
2028
2029 if (nextccb->ccb_xa.flags & ATA_F_NCQ) {
258223a3
MD
2030 KKASSERT(last_was_ncq == 0); /* otherwise it should have
2031 * been started already. */
2032
1980eff3
MD
2033 /*
2034 * Issue NCQ commands only when there are no outstanding
2035 * standard commands.
2036 */
258223a3
MD
2037 if (ap->ap_active == 0)
2038 ahci_issue_pending_ncq_commands(ap);
2039 else
1980eff3 2040 KKASSERT(ap->ap_active_cnt > 0);
831bc9e3 2041 } else {
1980eff3 2042 if (ap->ap_sactive || last_was_ncq)
258223a3
MD
2043 KKASSERT(ap->ap_active_cnt == 0);
2044
1980eff3
MD
2045 /*
2046 * Wait for all NCQ commands to finish before issuing standard
2047 * command. Then keep up to 2 standard commands on-chip at
2048 * a time.
2049 */
2050 if (ap->ap_sactive)
258223a3
MD
2051 return;
2052
1980eff3
MD
2053 while (ap->ap_active_cnt < 2 &&
2054 nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) == 0) {
258223a3 2055 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
3209f581 2056 ahci_start_timeout(nextccb);
258223a3
MD
2057 ap->ap_active |= 1 << nextccb->ccb_slot;
2058 nextccb->ccb_xa.state = ATA_S_ONCHIP;
2059 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1980eff3 2060 ap->ap_active_cnt++;
258223a3 2061 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1980eff3 2062 }
258223a3
MD
2063 }
2064}
2065
2066void
2067ahci_intr(void *arg)
2068{
f4553de1
MD
2069 struct ahci_softc *sc = arg;
2070 struct ahci_port *ap;
2071 u_int32_t is, ack = 0;
2072 int port;
258223a3 2073
f4553de1
MD
2074 /*
2075 * Check if the master enable is up, and whether any interrupts are
2076 * pending.
2077 */
2078 if ((sc->sc_flags & AHCI_F_INT_GOOD) == 0)
2079 return;
258223a3
MD
2080 is = ahci_read(sc, AHCI_REG_IS);
2081 if (is == 0 || is == 0xffffffff)
2082 return;
2083 ack = is;
2084
2085#ifdef AHCI_COALESCE
2086 /* Check coalescing interrupt first */
2087 if (is & sc->sc_ccc_mask) {
2088 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2089 DEVNAME(sc));
2090 is &= ~sc->sc_ccc_mask;
2091 is |= sc->sc_ccc_ports_cur;
2092 }
2093#endif
2094
f4553de1
MD
2095 /*
2096 * Process interrupts for each port in a non-blocking fashion.
2097 */
258223a3
MD
2098 while (is) {
2099 port = ffs(is) - 1;
f4553de1
MD
2100 ap = sc->sc_ports[port];
2101 if (ap) {
2102 if (ahci_os_lock_port_nb(ap) == 0) {
2103 ahci_port_intr(ap, 0);
2104 ahci_os_unlock_port(ap);
2105 } else {
2106 ahci_pwrite(ap, AHCI_PREG_IE, 0);
2107 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2108 }
2109 }
258223a3
MD
2110 is &= ~(1 << port);
2111 }
2112
2113 /* Finally, acknowledge global interrupt */
2114 ahci_write(sc, AHCI_REG_IS, ack);
2115}
2116
f4553de1
MD
2117/*
2118 * Core called from helper thread.
2119 */
3209f581 2120void
f4553de1 2121ahci_port_thread_core(struct ahci_port *ap, int mask)
258223a3 2122{
f4553de1
MD
2123 /*
2124 * Process any expired timedouts.
2125 */
2126 ahci_os_lock_port(ap);
2127 if (mask & AP_SIGF_TIMEOUT) {
831bc9e3 2128 ahci_check_active_timeouts(ap);
f4553de1
MD
2129 }
2130
2131 /*
2132 * Process port interrupts which require a higher level of
2133 * intervention.
2134 */
2135 if (mask & AP_SIGF_PORTINT) {
2136 ahci_port_intr(ap, 1);
f4553de1 2137 ahci_port_interrupt_enable(ap);
831bc9e3 2138 ahci_os_unlock_port(ap);
f4553de1
MD
2139 } else {
2140 ahci_os_unlock_port(ap);
2141 }
2142}
2143
2144/*
2145 * Core per-port interrupt handler.
2146 *
2147 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only
2148 * deal with normal command completions which do not require blocking.
2149 */
2150void
2151ahci_port_intr(struct ahci_port *ap, int blockable)
2152{
2153 struct ahci_softc *sc = ap->ap_sc;
2154 u_int32_t is, ci_saved, ci_masked;
2155 int slot;
2156 struct ahci_ccb *ccb = NULL;
2157 struct ata_port *ccb_at = NULL;
2158 volatile u_int32_t *active;
258223a3 2159#ifdef DIAGNOSTIC
f4553de1 2160 u_int32_t tmp;
258223a3 2161#endif
f4553de1
MD
2162 const u_int32_t blockable_mask = AHCI_PREG_IS_TFES |
2163 AHCI_PREG_IS_IFS |
2164 AHCI_PREG_IS_PCS |
2165 AHCI_PREG_IS_PRCS |
2166 AHCI_PREG_IS_HBFS |
2167 AHCI_PREG_IS_OFS |
2168 AHCI_PREG_IS_UFS;
2169
22181ab7
MD
2170 enum { NEED_NOTHING, NEED_RESTART, NEED_HOTPLUG_INSERT,
2171 NEED_HOTPLUG_REMOVE } need = NEED_NOTHING;
258223a3
MD
2172
2173 is = ahci_pread(ap, AHCI_PREG_IS);
f4553de1
MD
2174
2175 /*
2176 * All basic command completions are always processed.
2177 */
cec07d75
MD
2178 if (is & AHCI_PREG_IS_DPS)
2179 ahci_pwrite(ap, AHCI_PREG_IS, is & AHCI_PREG_IS_DPS);
258223a3 2180
f4553de1
MD
2181 /*
2182 * If we can't block then we can't handle these here. Disable
2183 * the interrupts in question so we don't live-lock, the helper
2184 * thread will re-enable them.
2185 *
2186 * If the port is in a completely failed state we do not want
dbef6246 2187 * to drop through to failed-command-processing if blockable is 0,
f4553de1 2188 * just let the thread deal with it all.
dbef6246
MD
2189 *
2190 * Otherwise we fall through and still handle DHRS and any commands
2191 * which completed normally. Even if we are errored we haven't
2192 * stopped the port yet so CI/SACT are still good.
f4553de1
MD
2193 */
2194 if (blockable == 0) {
2195 if (ap->ap_state == AP_S_FATAL_ERROR) {
2196 ahci_pwrite(ap, AHCI_PREG_IE,
2197 ahci_pread(ap, AHCI_PREG_IE) & ~is);
2198 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2199 return;
2200 }
2201 if (is & blockable_mask) {
f4553de1 2202 ahci_pwrite(ap, AHCI_PREG_IE,
dbef6246
MD
2203 ahci_pread(ap, AHCI_PREG_IE) & ~blockable_mask);
2204 is &= ~blockable_mask;
f4553de1
MD
2205 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2206 }
2207 }
2208
1980eff3
MD
2209#if 0
2210 kprintf("%s: INTERRUPT %b\n", PORTNAME(ap),
2211 is, AHCI_PFMT_IS);
2212#endif
2213
3209f581 2214 /*
f4553de1 2215 * Either NCQ or non-NCQ commands will be active, never both.
3209f581 2216 */
258223a3 2217 if (ap->ap_sactive) {
258223a3
MD
2218 KKASSERT(ap->ap_active == 0);
2219 KKASSERT(ap->ap_active_cnt == 0);
2220 ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
2221 active = &ap->ap_sactive;
2222 } else {
258223a3
MD
2223 ci_saved = ahci_pread(ap, AHCI_PREG_CI);
2224 active = &ap->ap_active;
2225 }
2226
258223a3 2227 if (is & AHCI_PREG_IS_TFES) {
1980eff3 2228 /*
f4553de1
MD
2229 * Command failed (blockable).
2230 *
2231 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2.
1980eff3
MD
2232 *
2233 * This stops command processing.
2234 */
2235 u_int32_t tfd, serr;
2236 int err_slot;
258223a3
MD
2237
2238 tfd = ahci_pread(ap, AHCI_PREG_TFD);
2239 serr = ahci_pread(ap, AHCI_PREG_SERR);
2240
cf5f3a81
MD
2241 /*
2242 * If no NCQ commands are active the error slot is easily
2243 * determined, otherwise we have to extract the error
2244 * from the log page.
2245 */
258223a3 2246 if (ap->ap_sactive == 0) {
cf5f3a81
MD
2247 err_slot = AHCI_PREG_CMD_CCS(
2248 ahci_pread(ap, AHCI_PREG_CMD));
258223a3 2249 ccb = &ap->ap_ccbs[err_slot];
1980eff3 2250 ccb_at = ccb->ccb_xa.at; /* can be NULL */
258223a3
MD
2251
2252 /* Preserve received taskfile data from the RFIS. */
2253 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
cf5f3a81
MD
2254 sizeof(struct ata_fis_d2h));
2255 } else {
2256 err_slot = -1;
2257 }
258223a3 2258
1980eff3
MD
2259 DPRINTF(AHCI_D_VERBOSE, "%s: errd slot %d, TFD: %b, SERR: %b\n",
2260 PORTNAME(ap), err_slot,
2261 tfd, AHCI_PFMT_TFD_STS,
2262 serr, AHCI_PFMT_SERR);
258223a3 2263
cf5f3a81 2264 /* Stopping the port clears CI and SACT */
258223a3 2265 ahci_port_stop(ap, 0);
22181ab7 2266 need = NEED_RESTART;
258223a3 2267
cf5f3a81
MD
2268 /*
2269 * Clear SERR (primarily DIAG_X) to enable capturing of the
2270 * next error.
2271 */
258223a3
MD
2272 ahci_pwrite(ap, AHCI_PREG_SERR, serr);
2273
2274 /* Acknowledge the interrupts we can recover from. */
cf5f3a81 2275 ahci_pwrite(ap, AHCI_PREG_IS,
cec07d75 2276 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_IFS));
1980eff3 2277 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_IFS);
258223a3
MD
2278
2279 /* If device hasn't cleared its busy status, try to idle it. */
2280 if (tfd & (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1980eff3
MD
2281 kprintf("%s: Attempting to idle device ccb=%p\n",
2282 PORTNAME(ap), ccb_at);
2283 if (ap->ap_flags & AP_F_IN_RESET)
2284 goto fatal;
2285 /*
2286 * XXX how do we unbrick a PM target (ccb_at != NULL).
2287 *
2288 * For now fail the target and use CLO to clear the
2289 * busy condition and make the ahci port usable for
2290 * the remaining devices.
2291 */
2292 if (ccb_at) {
2293 ccb_at->at_probe = ATA_PROBE_FAILED;
2294 ahci_port_clo(ap);
2295 } else if (ahci_port_reset(ap, ccb_at, 0)) {
17eab71e
MD
2296 kprintf("%s: Unable to idle device, port "
2297 "bricked on us\n",
258223a3 2298 PORTNAME(ap));
17eab71e 2299 goto fatal;
258223a3
MD
2300 }
2301
2302 /* Had to reset device, can't gather extended info. */
2303 } else if (ap->ap_sactive) {
1980eff3
MD
2304 /*
2305 * Recover the NCQ error from log page 10h.
2306 *
2307 * XXX NCQ currently not supported with port
2308 * multiplier.
2309 */
258223a3 2310 ahci_port_read_ncq_error(ap, &err_slot);
cec07d75
MD
2311 kprintf("recover from NCQ error err_slot %d\n",
2312 err_slot);
258223a3
MD
2313 if (err_slot < 0)
2314 goto failall;
2315
2316 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2317 PORTNAME(ap), err_slot);
2318
2319 ccb = &ap->ap_ccbs[err_slot];
2320 } else {
2321 /* Didn't reset, could gather extended info from log. */
1980eff3
MD
2322 kprintf("%s: didn't reset err_slot %d "
2323 "sact=%08x act=%08x\n",
2324 PORTNAME(ap),
cf5f3a81 2325 err_slot, ap->ap_sactive, ap->ap_active);
258223a3
MD
2326 }
2327
2328 /*
2329 * If we couldn't determine the errored slot, reset the port
2330 * and fail all the active slots.
2331 */
2332 if (err_slot == -1) {
1980eff3
MD
2333 if (ap->ap_flags & AP_F_IN_RESET)
2334 goto fatal;
2335 /*
2336 * XXX how do we unbrick a PM target (ccb_at != NULL).
2337 *
2338 * For now fail the target and use CLO to clear the
2339 * busy condition and make the ahci port usable for
2340 * the remaining devices.
2341 */
2342 if (ccb_at) {
2343 ccb_at->at_probe = ATA_PROBE_FAILED;
2344 ahci_port_clo(ap);
2345 } else if (ahci_port_reset(ap, ccb_at, 0)) {
17eab71e
MD
2346 kprintf("%s: Unable to idle device after "
2347 "NCQ error, port bricked on us\n",
258223a3
MD
2348 PORTNAME(ap));
2349 goto fatal;
2350 }
2351 kprintf("%s: couldn't recover NCQ error, failing "
2352 "all outstanding commands.\n",
2353 PORTNAME(ap));
2354 goto failall;
2355 }
2356
2357 /* Clear the failed command in saved CI so completion runs. */
2358 ci_saved &= ~(1 << err_slot);
2359
2360 /* Note the error in the ata_xfer. */
2361 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2362 ccb->ccb_xa.state = ATA_S_ERROR;
2363
2364#ifdef DIAGNOSTIC
2365 /* There may only be one outstanding standard command now. */
2366 if (ap->ap_sactive == 0) {
2367 tmp = ci_saved;
2368 if (tmp) {
2369 slot = ffs(tmp) - 1;
2370 tmp &= ~(1 << slot);
2371 KKASSERT(tmp == 0);
2372 }
2373 }
2374#endif
1980eff3
MD
2375 } else if (is & AHCI_PREG_IS_DHRS) {
2376 /*
f4553de1
MD
2377 * Command posted D2H register FIS to the rfis (non-blocking).
2378 *
1980eff3 2379 * Command posted D2H register FIS to the rfis. This
8bf6a3ff
MD
2380 * does NOT stop command processing and it is unclear
2381 * how we are supposed to deal with it other then using
2382 * only a queue of 1.
2383 *
2384 * We must copy the port rfis to the ccb and restart
2385 * command processing. ahci_pm_read() does not function
2386 * without this support.
1980eff3
MD
2387 */
2388 int err_slot;
2389
2390 if (ap->ap_sactive == 0) {
2391 err_slot = AHCI_PREG_CMD_CCS(
2392 ahci_pread(ap, AHCI_PREG_CMD));
2393 ccb = &ap->ap_ccbs[err_slot];
2394 ccb_at = ccb->ccb_xa.at; /* can be NULL */
2395
2396 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
2397 sizeof(struct ata_fis_d2h));
2398 } else {
2399 kprintf("%s: Unexpected DHRS posted while "
2400 "NCQ running\n", PORTNAME(ap));
2401 err_slot = -1;
2402 }
cec07d75
MD
2403 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
2404 is &= ~AHCI_PREG_IS_DHRS;
1980eff3
MD
2405 }
2406
2407 /*
f4553de1 2408 * Device notification to us (non-blocking)
1980eff3 2409 *
cec07d75
MD
2410 * NOTE! On some parts notification bits can get set without
2411 * generating an interrupt. It is unclear whether this is
2412 * a bug in the PM (sending a DTOH device setbits with 'N' set
2413 * and 'I' not set), or a bug in the host controller.
2414 *
2415 * It only seems to occur under load.
1980eff3 2416 */
cec07d75 2417 if (/*(is & AHCI_PREG_IS_SDBS) &&*/ (sc->sc_cap & AHCI_REG_CAP_SSNTF)) {
1980eff3 2418 u_int32_t data;
cec07d75 2419 const char *xstr;
1980eff3
MD
2420
2421 data = ahci_pread(ap, AHCI_PREG_SNTF);
cec07d75
MD
2422 if (is & AHCI_PREG_IS_SDBS) {
2423 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_SDBS);
2424 is &= ~AHCI_PREG_IS_SDBS;
2425 xstr = " (no SDBS!)";
2426 } else {
2427 xstr = "";
2428 }
1980eff3 2429 if (data) {
1980eff3 2430 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_SDBS);
cec07d75
MD
2431
2432 kprintf("%s: NOTIFY %08x%s\n",
2433 PORTNAME(ap), data, xstr);
2434 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_N);
3209f581
MD
2435 ahci_pwrite(ap, AHCI_PREG_SNTF, data);
2436 ahci_cam_changed(ap, NULL, -1);
1980eff3
MD
2437 }
2438 }
3209f581
MD
2439
2440 /*
f4553de1
MD
2441 * Spurious IFS errors (blockable).
2442 *
3209f581
MD
2443 * Spurious IFS errors can occur while we are doing a reset
2444 * sequence through a PM. Try to recover if we are being asked
2445 * to ignore IFS errors during these periods.
2446 */
2447 if ((is & AHCI_PREG_IS_IFS) && (ap->ap_flags & AP_F_IGNORE_IFS)) {
1980eff3 2448 u_int32_t serr = ahci_pread(ap, AHCI_PREG_SERR);
3209f581
MD
2449 if ((ap->ap_flags & AP_F_IFS_IGNORED) == 0) {
2450 kprintf("%s: Ignoring IFS (XXX) (IS: %b, SERR: %b)\n",
2451 PORTNAME(ap),
2452 is, AHCI_PFMT_IS,
2453 serr, AHCI_PFMT_SERR);
2454 ap->ap_flags |= AP_F_IFS_IGNORED;
2455 }
2456 ap->ap_flags |= AP_F_IFS_OCCURED;
1980eff3
MD
2457 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
2458 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
2459 is &= ~AHCI_PREG_IS_IFS;
2460 ahci_port_stop(ap, 0);
2461 ahci_port_start(ap);
2462 need = NEED_RESTART;
258223a3
MD
2463 }
2464
2465 /*
f4553de1 2466 * Port change (hot-plug) (blockable).
258223a3
MD
2467 *
2468 * A PCS interrupt will occur on hot-plug once communication is
2469 * established.
2470 *
2471 * A PRCS interrupt will occur on hot-unplug (and possibly also
2472 * on hot-plug).
2473 *
22181ab7
MD
2474 * XXX We can then check the CPS (Cold Presence State) bit, if
2475 * supported, to determine if a device is plugged in or not and do
2476 * the right thing.
2477 *
2478 * WARNING: A PCS interrupt is cleared by clearing DIAG_X, and
2479 * can also occur if an unsolicited COMINIT is received.
2480 * If this occurs command processing is automatically
2481 * stopped (CR goes inactive) and the port must be stopped
2482 * and restarted.
258223a3
MD
2483 */
2484 if (is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)) {
cec07d75
MD
2485 ahci_pwrite(ap, AHCI_PREG_IS,
2486 is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS));
2487 is &= ~(AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS);
258223a3 2488 ahci_pwrite(ap, AHCI_PREG_SERR,
1980eff3 2489 (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X));
22181ab7 2490 ahci_port_stop(ap, 0);
258223a3
MD
2491 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
2492 case AHCI_PREG_SSTS_DET_DEV:
1980eff3 2493 if (ap->ap_type == ATA_PORT_T_NONE) {
22181ab7
MD
2494 need = NEED_HOTPLUG_INSERT;
2495 goto fatal;
258223a3 2496 }
22181ab7 2497 need = NEED_RESTART;
258223a3
MD
2498 break;
2499 default:
1980eff3 2500 if (ap->ap_type != ATA_PORT_T_NONE) {
22181ab7
MD
2501 need = NEED_HOTPLUG_REMOVE;
2502 goto fatal;
258223a3 2503 }
22181ab7 2504 need = NEED_RESTART;
258223a3
MD
2505 break;
2506 }
2507 }
2508
22181ab7 2509 /*
f4553de1 2510 * Check for remaining errors - they are fatal. (blockable)
22181ab7 2511 */
258223a3
MD
2512 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
2513 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
cec07d75
MD
2514 u_int32_t serr;
2515
2516 ahci_pwrite(ap, AHCI_PREG_IS,
2517 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2518 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2519 AHCI_PREG_IS_UFS));
cec07d75 2520 serr = ahci_pread(ap, AHCI_PREG_SERR);
831bc9e3 2521 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), "
4444122d
MD
2522 "disabling port.\n",
2523 PORTNAME(ap),
2524 is, AHCI_PFMT_IS,
1980eff3 2525 serr, AHCI_PFMT_SERR
4444122d 2526 );
831bc9e3
MD
2527 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2528 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2529 AHCI_PREG_IS_UFS);
258223a3
MD
2530 /* XXX try recovery first */
2531 goto fatal;
2532 }
2533
22181ab7
MD
2534 /*
2535 * Fail all outstanding commands if we know the port won't recover.
1980eff3
MD
2536 *
2537 * We may have a ccb_at if the failed command is known and was
2538 * being sent to a device over a port multiplier (PM). In this
2539 * case if the port itself has not completely failed we fail just
2540 * the commands related to that target.
22181ab7 2541 */
258223a3
MD
2542 if (ap->ap_state == AP_S_FATAL_ERROR) {
2543fatal:
2544 ap->ap_state = AP_S_FATAL_ERROR;
2545failall:
2546
cf5f3a81
MD
2547 /* Stopping the port clears CI/SACT */
2548 ahci_port_stop(ap, 0);
258223a3 2549
1980eff3
MD
2550 /*
2551 * Error all the active slots. If running across a PM
2552 * try to error out just the slots related to the target.
2553 */
258223a3
MD
2554 ci_masked = ci_saved & *active;
2555 while (ci_masked) {
2556 slot = ffs(ci_masked) - 1;
2557 ccb = &ap->ap_ccbs[slot];
1980eff3
MD
2558 if (ccb_at == ccb->ccb_xa.at ||
2559 ap->ap_state == AP_S_FATAL_ERROR) {
2560 ci_masked &= ~(1 << slot);
2561 ccb->ccb_xa.state = ATA_S_ERROR;
2562 }
258223a3
MD
2563 }
2564
2565 /* Run completion for all active slots. */
2566 ci_saved &= ~*active;
2567
2568 /*
2569 * Don't restart the port if our problems were deemed fatal.
2570 *
2571 * Also acknowlege all fatal interrupt sources to prevent
2572 * a livelock.
2573 */
2574 if (ap->ap_state == AP_S_FATAL_ERROR) {
22181ab7
MD
2575 if (need == NEED_RESTART)
2576 need = NEED_NOTHING;
258223a3
MD
2577 ahci_pwrite(ap, AHCI_PREG_IS,
2578 AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2579 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2580 AHCI_PREG_IS_UFS);
2581 }
2582 }
2583
2584 /*
f4553de1
MD
2585 * CCB completion (non blocking).
2586 *
258223a3
MD
2587 * CCB completion is detected by noticing its slot's bit in CI has
2588 * changed to zero some time after we activated it.
2589 * If we are polling, we may only be interested in particular slot(s).
cf5f3a81
MD
2590 *
2591 * Any active bits not saved are completed within the restrictions
2592 * imposed by the caller.
258223a3 2593 */
3209f581 2594 ci_masked = ~ci_saved & *active;
258223a3
MD
2595 while (ci_masked) {
2596 slot = ffs(ci_masked) - 1;
2597 ccb = &ap->ap_ccbs[slot];
2598 ci_masked &= ~(1 << slot);
2599
2600 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2601 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2602 " (error)" : "");
2603
2604 bus_dmamap_sync(sc->sc_tag_cmdh,
2605 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
2606 BUS_DMASYNC_POSTWRITE);
2607
2608 bus_dmamap_sync(sc->sc_tag_cmdt,
2609 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
2610 BUS_DMASYNC_POSTWRITE);
2611
2612 bus_dmamap_sync(sc->sc_tag_rfis,
2613 AHCI_DMA_MAP(ap->ap_dmamem_rfis),
2614 BUS_DMASYNC_POSTREAD);
2615
2616 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
2617 if (active == &ap->ap_active) {
2618 KKASSERT(ap->ap_active_cnt > 0);
2619 --ap->ap_active_cnt;
2620 }
258223a3 2621 ccb->ccb_done(ccb);
258223a3
MD
2622 }
2623
f4553de1
MD
2624 /*
2625 * Cleanup. Will not be set if non-blocking.
2626 */
22181ab7
MD
2627 switch(need) {
2628 case NEED_RESTART:
2629 /*
2630 * A recoverable error occured and we can restart outstanding
2631 * commands on the port.
2632 */
17eab71e 2633 ahci_port_start(ap);
258223a3 2634
258223a3
MD
2635 if (ci_saved) {
2636#ifdef DIAGNOSTIC
2637 tmp = ci_saved;
2638 while (tmp) {
2639 slot = ffs(tmp) - 1;
2640 tmp &= ~(1 << slot);
2641 ccb = &ap->ap_ccbs[slot];
2642 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2643 KKASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
2644 (!!ap->ap_sactive));
2645 }
2646#endif
2647 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2648 "re-enabling%s slots %08x\n", PORTNAME(ap),
2649 ap->ap_sactive ? " NCQ" : "", ci_saved);
2650
2651 if (ap->ap_sactive)
2652 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
2653 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
2654 }
22181ab7
MD
2655 break;
2656 case NEED_HOTPLUG_INSERT:
2657 /*
cf5f3a81
MD
2658 * A hot-plug insertion event has occured and all
2659 * outstanding commands have already been revoked.
1980eff3
MD
2660 *
2661 * Don't recurse if this occurs while we are
2662 * resetting the port.
22181ab7 2663 */
1980eff3
MD
2664 if ((ap->ap_flags & AP_F_IN_RESET) == 0) {
2665 kprintf("%s: HOTPLUG - Device inserted\n",
2666 PORTNAME(ap));
3209f581
MD
2667 ap->ap_probe = ATA_PROBE_NEED_INIT;
2668 ahci_cam_changed(ap, NULL, -1);
1980eff3 2669 }
22181ab7
MD
2670 break;
2671 case NEED_HOTPLUG_REMOVE:
cf5f3a81
MD
2672 /*
2673 * A hot-plug removal event has occured and all
2674 * outstanding commands have already been revoked.
1980eff3
MD
2675 *
2676 * Don't recurse if this occurs while we are
2677 * resetting the port.
cf5f3a81 2678 */
1980eff3
MD
2679 if ((ap->ap_flags & AP_F_IN_RESET) == 0) {
2680 kprintf("%s: HOTPLUG - Device removed\n",
2681 PORTNAME(ap));
2682 ahci_port_hardstop(ap);
3209f581
MD
2683 /* ap_probe set to failed */
2684 ahci_cam_changed(ap, NULL, -1);
1980eff3 2685 }
22181ab7
MD
2686 break;
2687 default:
2688 break;
258223a3 2689 }
258223a3
MD
2690}
2691
2692struct ahci_ccb *
2693ahci_get_ccb(struct ahci_port *ap)
2694{
2695 struct ahci_ccb *ccb;
2696
2697 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE);
2698 ccb = TAILQ_FIRST(&ap->ap_ccb_free);
2699 if (ccb != NULL) {
2700 KKASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2701 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2702 ccb->ccb_xa.state = ATA_S_SETUP;
1980eff3 2703 ccb->ccb_xa.at = NULL;
258223a3
MD
2704 }
2705 lockmgr(&ap->ap_ccb_lock, LK_RELEASE);
2706
2707 return (ccb);
2708}
2709
2710void
2711ahci_put_ccb(struct ahci_ccb *ccb)
2712{
2713 struct ahci_port *ap = ccb->ccb_port;
2714
2715#ifdef DIAGNOSTIC
2716 if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
2717 ccb->ccb_xa.state != ATA_S_TIMEOUT &&
2718 ccb->ccb_xa.state != ATA_S_ERROR) {
2719 kprintf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2720 "slot %d\n",
2721 PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
2722 ccb->ccb_slot);
2723 }
2724#endif
2725
2726 ccb->ccb_xa.state = ATA_S_PUT;
2727 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE);
2728 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2729 lockmgr(&ap->ap_ccb_lock, LK_RELEASE);
2730}
2731
2732struct ahci_ccb *
2733ahci_get_err_ccb(struct ahci_port *ap)
2734{
2735 struct ahci_ccb *err_ccb;
2736 u_int32_t sact;
2737
2738 /* No commands may be active on the chip. */
2739 sact = ahci_pread(ap, AHCI_PREG_SACT);
2740 if (sact != 0)
2741 kprintf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2742 KKASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2743
2744#ifdef DIAGNOSTIC
2745 KKASSERT(ap->ap_err_busy == 0);
2746 ap->ap_err_busy = 1;
2747#endif
2748 /* Save outstanding command state. */
2749 ap->ap_err_saved_active = ap->ap_active;
2750 ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2751 ap->ap_err_saved_sactive = ap->ap_sactive;
2752
2753 /*
2754 * Pretend we have no commands outstanding, so that completions won't
2755 * run prematurely.
2756 */
2757 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2758
2759 /*
2760 * Grab a CCB to use for error recovery. This should never fail, as
2761 * we ask atascsi to reserve one for us at init time.
2762 */
2763 err_ccb = ahci_get_ccb(ap);
2764 KKASSERT(err_ccb != NULL);
2765 err_ccb->ccb_xa.flags = 0;
2766 err_ccb->ccb_done = ahci_empty_done;
2767
2768 return err_ccb;
2769}
2770
2771void
2772ahci_put_err_ccb(struct ahci_ccb *ccb)
2773{
2774 struct ahci_port *ap = ccb->ccb_port;
2775 u_int32_t sact;
5f8c1efd 2776 u_int32_t ci;
258223a3
MD
2777
2778#ifdef DIAGNOSTIC
2779 KKASSERT(ap->ap_err_busy);
2780#endif
5f8c1efd
MD
2781 /*
2782 * No commands may be active on the chip
2783 */
258223a3 2784 sact = ahci_pread(ap, AHCI_PREG_SACT);
5f8c1efd
MD
2785 if (sact) {
2786 panic("ahci_port_err_ccb(%d) but SACT %08x != 0\n",
2787 ccb->ccb_slot, sact);
2788 }
2789 ci = ahci_pread(ap, AHCI_PREG_CI);
2790 if (ci) {
cf5f3a81
MD
2791 panic("ahci_put_err_ccb(%d) but CI %08x != 0 "
2792 "(act=%08x sact=%08x)\n",
2793 ccb->ccb_slot, ci,
2794 ap->ap_active, ap->ap_sactive);
258223a3 2795 }
258223a3
MD
2796
2797 /* Done with the CCB */
2798 ahci_put_ccb(ccb);
2799
2800 /* Restore outstanding command state */
2801 ap->ap_sactive = ap->ap_err_saved_sactive;
2802 ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2803 ap->ap_active = ap->ap_err_saved_active;
2804
2805#ifdef DIAGNOSTIC
2806 ap->ap_err_busy = 0;
2807#endif
2808}
2809
1980eff3
MD
2810/*
2811 * Read log page to get NCQ error.
2812 *
2813 * NOTE: NCQ not currently supported on port multipliers. XXX
2814 */
258223a3
MD
2815int
2816ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp)
2817{
2818 struct ahci_ccb *ccb;
2819 struct ahci_cmd_hdr *cmd_slot;
2820 u_int32_t cmd;
2821 struct ata_fis_h2d *fis;
2822 int rc = EIO;
2823
2824 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2825
2826 /* Save command register state. */
2827 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
2828
2829 /* Port should have been idled already. Start it. */
2830 KKASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
17eab71e 2831 ahci_port_start(ap);
258223a3
MD
2832
2833 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2834 ccb = ahci_get_err_ccb(ap);
2835 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
2836 ccb->ccb_xa.data = ap->ap_err_scratch;
2837 ccb->ccb_xa.datalen = 512;
2838 cmd_slot = ccb->ccb_cmd_hdr;
2839 bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
2840
2841 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2842 fis->type = ATA_FIS_TYPE_H2D;
2843 fis->flags = ATA_H2D_FLAGS_CMD;
2844 fis->command = ATA_C_READ_LOG_EXT;
2845 fis->lba_low = 0x10; /* queued error log page (10h) */
2846 fis->sector_count = 1; /* number of sectors (1) */
2847 fis->sector_count_exp = 0;
2848 fis->lba_mid = 0; /* starting offset */
2849 fis->lba_mid_exp = 0;
2850 fis->device = 0;
2851
2852 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
2853
2854 if (ahci_load_prdt(ccb) != 0) {
2855 rc = ENOMEM; /* XXX caller must abort all commands */
2856 goto err;
2857 }
2858
2859 ccb->ccb_xa.state = ATA_S_PENDING;
831bc9e3 2860 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != 0)
258223a3
MD
2861 goto err;
2862
2863 rc = 0;
2864err:
2865 /* Abort our command, if it failed, by stopping command DMA. */
831bc9e3 2866 if (rc) {
258223a3
MD
2867 kprintf("%s: log page read failed, slot %d was still active.\n",
2868 PORTNAME(ap), ccb->ccb_slot);
258223a3
MD
2869 }
2870
2871 /* Done with the error CCB now. */
2872 ahci_unload_prdt(ccb);
2873 ahci_put_err_ccb(ccb);
2874
2875 /* Extract failed register set and tags from the scratch space. */
2876 if (rc == 0) {
2877 struct ata_log_page_10h *log;
2878 int err_slot;
2879
2880 log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2881 if (log->err_regs.type & ATA_LOG_10H_TYPE_NOTQUEUED) {
2882 /* Not queued bit was set - wasn't an NCQ error? */
2883 kprintf("%s: read NCQ error page, but not an NCQ "
2884 "error?\n",
2885 PORTNAME(ap));
2886 rc = ESRCH;
2887 } else {
2888 /* Copy back the log record as a D2H register FIS. */
2889 *err_slotp = err_slot = log->err_regs.type &
2890 ATA_LOG_10H_TYPE_TAG_MASK;
2891
2892 ccb = &ap->ap_ccbs[err_slot];
2893 memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2894 sizeof(struct ata_fis_d2h));
2895 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2896 ccb->ccb_xa.rfis.flags = 0;
2897 }
2898 }
2899
2900 /* Restore saved CMD register state */
2901 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2902
2903 return (rc);
2904}
2905
2906/*
2907 * Allocate memory for various structures DMAd by hardware. The maximum
2908 * number of segments for these tags is 1 so the DMA memory will have a
2909 * single physical base address.
2910 */
2911struct ahci_dmamem *
2912ahci_dmamem_alloc(struct ahci_softc *sc, bus_dma_tag_t tag)
2913{
2914 struct ahci_dmamem *adm;
2915 int error;
2916
2917 adm = kmalloc(sizeof(*adm), M_DEVBUF, M_INTWAIT | M_ZERO);
2918
2919 error = bus_dmamem_alloc(tag, (void **)&adm->adm_kva,
2920 BUS_DMA_ZERO, &adm->adm_map);
2921 if (error == 0) {
2922 adm->adm_tag = tag;
2923 error = bus_dmamap_load(tag, adm->adm_map,
2924 adm->adm_kva,
2925 bus_dma_tag_getmaxsize(tag),
2926 ahci_dmamem_saveseg, &adm->adm_busaddr,
2927 0);
2928 }
2929 if (error) {
2930 if (adm->adm_map) {
2931 bus_dmamap_destroy(tag, adm->adm_map);
2932 adm->adm_map = NULL;
2933 adm->adm_tag = NULL;
2934 adm->adm_kva = NULL;
2935 }
2936 kfree(adm, M_DEVBUF);
2937 adm = NULL;
2938 }
2939 return (adm);
2940}
2941
2942static
2943void
2944ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error)
2945{
2946 KKASSERT(error == 0);
2947 KKASSERT(nsegs == 1);
2948 *(bus_addr_t *)info = segs->ds_addr;
2949}
2950
2951
2952void
2953ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2954{
2955 if (adm->adm_map) {
2956 bus_dmamap_unload(adm->adm_tag, adm->adm_map);
2957 bus_dmamap_destroy(adm->adm_tag, adm->adm_map);
2958 adm->adm_map = NULL;
2959 adm->adm_tag = NULL;
2960 adm->adm_kva = NULL;
2961 }
2962 kfree(adm, M_DEVBUF);
2963}
2964
2965u_int32_t
2966ahci_read(struct ahci_softc *sc, bus_size_t r)
2967{
2968 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2969 BUS_SPACE_BARRIER_READ);
2970 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2971}
2972
2973void
2974ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2975{
2976 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2977 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2978 BUS_SPACE_BARRIER_WRITE);
2979}
2980
258223a3
MD
2981u_int32_t
2982ahci_pread(struct ahci_port *ap, bus_size_t r)
2983{
2984 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2985 BUS_SPACE_BARRIER_READ);
2986 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
2987}
2988
2989void
2990ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
2991{
2992 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
2993 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
2994 BUS_SPACE_BARRIER_WRITE);
2995}
2996
831bc9e3
MD
2997/*
2998 * Wait up to (timeout) milliseconds for the masked port register to
2999 * match the target.
3000 *
3001 * Timeout is in milliseconds.
3002 */
258223a3 3003int
cec85a37
MD
3004ahci_pwait_eq(struct ahci_port *ap, int timeout,
3005 bus_size_t r, u_int32_t mask, u_int32_t target)
258223a3 3006{
831bc9e3 3007 int t;
258223a3 3008
831bc9e3
MD
3009 /*
3010 * Loop hard up to 100uS
3011 */
3012 for (t = 0; t < 100; ++t) {
258223a3
MD
3013 if ((ahci_pread(ap, r) & mask) == target)
3014 return (0);
831bc9e3 3015 ahci_os_hardsleep(1); /* us */
258223a3
MD
3016 }
3017
831bc9e3
MD
3018 do {
3019 timeout -= ahci_os_softsleep();
3020 if ((ahci_pread(ap, r) & mask) == target)
3021 return (0);
3022 } while (timeout > 0);
258223a3
MD
3023 return (1);
3024}
3025
831bc9e3
MD
3026int
3027ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
3028 u_int32_t target)
3029{
3030 int t;
3031
3032 /*
3033 * Loop hard up to 100uS
3034 */
3035 for (t = 0; t < 100; ++t) {
3036 if ((ahci_read(sc, r) & mask) != target)
3037 return (0);
3038 ahci_os_hardsleep(1); /* us */
3039 }
3040
3041 /*
3042 * And one millisecond the slow way
3043 */
3044 t = 1000;
3045 do {
3046 t -= ahci_os_softsleep();
3047 if ((ahci_read(sc, r) & mask) != target)
3048 return (0);
3049 } while (t > 0);
3050
3051 return (1);
3052}
3053
3054
1980eff3
MD
3055/*
3056 * Acquire an ata transfer.
3057 *
3058 * Pass a NULL at for direct-attached transfers, and a non-NULL at for
3059 * targets that go through the port multiplier.
3060 */
258223a3 3061struct ata_xfer *
1980eff3 3062ahci_ata_get_xfer(struct ahci_port *ap, struct ata_port *at)
258223a3 3063{
258223a3
MD
3064 struct ahci_ccb *ccb;
3065
3066 ccb = ahci_get_ccb(ap);
3067 if (ccb == NULL) {
3068 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
3069 PORTNAME(ap));
3070 return (NULL);
3071 }
3072
3073 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
3074 PORTNAME(ap), ccb->ccb_slot);
3075
1980eff3 3076 ccb->ccb_xa.at = at;
258223a3
MD
3077 ccb->ccb_xa.fis->type = ATA_FIS_TYPE_H2D;
3078
3079 return (&ccb->ccb_xa);
3080}
3081
3082void
3083ahci_ata_put_xfer(struct ata_xfer *xa)
3084{
3085 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
3086
3087 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
3088
3089 ahci_put_ccb(ccb);
3090}
3091
3092int
3093ahci_ata_cmd(struct ata_xfer *xa)
3094{
3095 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
3096 struct ahci_cmd_hdr *cmd_slot;
3097
3098 KKASSERT(xa->state == ATA_S_SETUP);
3099
3100 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
3101 goto failcmd;
1980eff3
MD
3102#if 0
3103 kprintf("%s: started std command %b ccb %d ccb_at %p %d\n",
3104 ATANAME(ccb->ccb_port, ccb->ccb_xa.at),
3105 ahci_pread(ccb->ccb_port, AHCI_PREG_CMD), AHCI_PFMT_CMD,
3106 ccb->ccb_slot,
3107 ccb->ccb_xa.at,
3108 ccb->ccb_xa.at ? ccb->ccb_xa.at->at_target : -1);
3109#endif
258223a3
MD
3110
3111 ccb->ccb_done = ahci_ata_cmd_done;
3112
3113 cmd_slot = ccb->ccb_cmd_hdr;
3114 cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */
1980eff3
MD
3115 if (ccb->ccb_xa.at) {
3116 cmd_slot->flags |= htole16(ccb->ccb_xa.at->at_target <<
3117 AHCI_CMD_LIST_FLAG_PMP_SHIFT);
3118 }
258223a3
MD
3119
3120 if (xa->flags & ATA_F_WRITE)
3121 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
3122
3123 if (xa->flags & ATA_F_PACKET)
3124 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A);
3125
3126 if (ahci_load_prdt(ccb) != 0)
3127 goto failcmd;
3128
3129 xa->state = ATA_S_PENDING;
3130
831bc9e3
MD
3131 if (xa->flags & ATA_F_POLL)
3132 return (ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout));
258223a3
MD
3133
3134 crit_enter();
f4553de1 3135 KKASSERT((xa->flags & ATA_F_TIMEOUT_EXPIRED) == 0);
3209f581 3136 xa->flags |= ATA_F_TIMEOUT_DESIRED;
258223a3
MD
3137 ahci_start(ccb);
3138 crit_exit();
831bc9e3 3139 return (xa->state);
258223a3
MD
3140
3141failcmd:
3142 crit_enter();
3143 xa->state = ATA_S_ERROR;
3144 xa->complete(xa);
3145 crit_exit();
831bc9e3 3146 return (ATA_S_ERROR);
258223a3
MD
3147}
3148
3149void
3150ahci_ata_cmd_done(struct ahci_ccb *ccb)
3151{
3152 struct ata_xfer *xa = &ccb->ccb_xa;
3153
831bc9e3
MD
3154 /*
3155 * NOTE: callout does not lock port and may race us modifying
3156 * the flags, so make sure its stopped.
3157 */
258223a3 3158 if (xa->flags & ATA_F_TIMEOUT_RUNNING) {
258223a3 3159 callout_stop(&ccb->ccb_timeout);
831bc9e3 3160 xa->flags &= ~ATA_F_TIMEOUT_RUNNING;
258223a3 3161 }
f4553de1 3162 xa->flags &= ~(ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_EXPIRED);
258223a3 3163
f4553de1 3164 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) {
258223a3 3165 ahci_issue_pending_commands(ccb->ccb_port,
f4553de1
MD
3166 xa->flags & ATA_F_NCQ);
3167 }
258223a3
MD
3168
3169 ahci_unload_prdt(ccb);
3170
3171 if (xa->state == ATA_S_ONCHIP)
3172 xa->state = ATA_S_COMPLETE;
3173#ifdef DIAGNOSTIC
3174 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
3175 kprintf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
3176 "slot %d\n",
3177 PORTNAME(ccb->ccb_port), xa->state, ccb->ccb_slot);
3178#endif
3179 if (xa->state != ATA_S_TIMEOUT)
3180 xa->complete(xa);
3181}
3182
f4553de1
MD
3183/*
3184 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags
3185 * while the callout is runing.
3186 *
3187 * We can't safely get the port lock here or delay, we could block
3188 * the callout thread.
3189 */
258223a3
MD
3190static void
3191ahci_ata_cmd_timeout_unserialized(void *arg)
3192{
3193 struct ahci_ccb *ccb = arg;
3194 struct ahci_port *ap = ccb->ccb_port;
3195
f4553de1
MD
3196 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING;
3197 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_EXPIRED;
3198 ahci_os_signal_port_thread(ap, AP_SIGF_TIMEOUT);
258223a3
MD
3199}
3200
1980eff3 3201void
831bc9e3 3202ahci_ata_cmd_timeout(struct ahci_ccb *ccb)
258223a3 3203{
258223a3
MD
3204 struct ata_xfer *xa = &ccb->ccb_xa;
3205 struct ahci_port *ap = ccb->ccb_port;
3206 volatile u_int32_t *active;
3207 int ccb_was_started, ncq_cmd;
131be210 3208 int status;
258223a3
MD
3209
3210 crit_enter();
831bc9e3 3211 kprintf("%s: CMD TIMEOUT state=%d cmd-reg 0x%b\n"
cf5f3a81
MD
3212 "\tsactive=%08x active=%08x\n"
3213 "\t sact=%08x ci=%08x\n",
1980eff3 3214 ATANAME(ap, ccb->ccb_xa.at),
831bc9e3 3215 ccb->ccb_xa.state,
258223a3 3216 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD,
cf5f3a81 3217 ap->ap_sactive, ap->ap_active,
258223a3
MD
3218 ahci_pread(ap, AHCI_PREG_SACT),
3219 ahci_pread(ap, AHCI_PREG_CI));
3220
9e145b23
MD
3221 /*
3222 * NOTE: Timeout will not be running if the command was polled.
3209f581 3223 * If we got here at least one of these flags should be set.
9e145b23 3224 */
3209f581
MD
3225 KKASSERT(xa->flags & (ATA_F_POLL | ATA_F_TIMEOUT_DESIRED |
3226 ATA_F_TIMEOUT_RUNNING));
f4553de1 3227 xa->flags &= ~(ATA_F_TIMEOUT_RUNNING | ATA_F_TIMEOUT_EXPIRED);
258223a3
MD
3228 ncq_cmd = (xa->flags & ATA_F_NCQ);
3229 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
3230
3231 if (ccb->ccb_xa.state == ATA_S_PENDING) {
3232 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
3233 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
3234 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
3235 ccb_was_started = 0;
258223a3
MD
3236 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
3237 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
3238 "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
3239 (*active & (1 << ccb->ccb_slot)) ?
3240 " but slot is still active?" : ".");
3241 goto ret;
3242 } else if ((ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : AHCI_PREG_CI) &
3243 (1 << ccb->ccb_slot)) == 0 &&
3244 (*active & (1 << ccb->ccb_slot))) {
831bc9e3
MD
3245 kprintf("%s: ahci_port_intr() failed to detect "
3246 "completed slot\n", ATANAME(ap, ccb->ccb_xa.at));
258223a3 3247 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
3248 if (ncq_cmd == 0) {
3249 KKASSERT(ap->ap_active_cnt > 0);
3250 --ap->ap_active_cnt;
3251 }
258223a3
MD
3252 ccb->ccb_done(ccb);
3253 goto ret;
3254 } else {
258223a3
MD
3255 ccb_was_started = 1;
3256 }
3257
3258 /* Complete the slot with a timeout error. */
3259 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3260 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
3261 if (ncq_cmd == 0) {
3262 KKASSERT(ap->ap_active_cnt > 0);
3263 --ap->ap_active_cnt;
3264 }
258223a3
MD
3265 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
3266 ccb->ccb_done(ccb); /* This won't issue pending commands or run the
3267 atascsi completion. */
3268
3269 /* Reset port to abort running command. */
3270 if (ccb_was_started) {
3271 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
3272 "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ"
3273 : "", ccb->ccb_slot, *active);
1980eff3 3274 /* XXX */
3209f581 3275 if (ccb->ccb_xa.at && ap->ap_type == ATA_PORT_T_PM) {
1980eff3 3276 /* XXX how do we unbrick a PM target? */
131be210
MD
3277 kprintf("%s: PM target bricked and timed-out, "
3278 "disabling PM target but trying to "
3279 "leave the port intact\n",
3280 ATANAME(ap, ccb->ccb_xa.at));
3281 ccb->ccb_xa.at->at_probe = ATA_PROBE_FAILED;
f4553de1 3282 ahci_port_intr(ap, 1);
131be210
MD
3283 ahci_port_stop(ap, 0);
3284 ahci_port_clo(ap);
3285 ahci_port_start(ap);
3286 status = 0;
1980eff3 3287 } else if (ahci_port_reset(ap, ccb->ccb_xa.at, 0)) {
cf5f3a81
MD
3288 /*
3289 * If the softreset failed place the port in a
3290 * failed state and use ahci_port_intr() to cancel
3291 * any remaining commands.
3292 */
17eab71e
MD
3293 kprintf("%s: Unable to reset during timeout, port "
3294 "bricked on us\n",
258223a3
MD
3295 PORTNAME(ap));
3296 ap->ap_state = AP_S_FATAL_ERROR;
f4553de1 3297 ahci_port_intr(ap, 1);
131be210 3298 status = 1;
cf5f3a81 3299 } else {
131be210
MD
3300 status = 0;
3301 }
3302 if (status == 0) {
cf5f3a81
MD
3303 /*
3304 * Restart any other commands that were aborted
3305 * by the reset.
3306 */
3307 if (*active) {
3308 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
3309 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
3310 *active);
3311 if (ncq_cmd)
3312 ahci_pwrite(ap, AHCI_PREG_SACT, *active);
3313 ahci_pwrite(ap, AHCI_PREG_CI, *active);
3314 }
258223a3
MD
3315 }
3316 }
3317
3318 /* Issue any pending commands now. */
3319 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
3320 if (ccb_was_started)
3321 ahci_issue_pending_commands(ap, ncq_cmd);
3322 else if (ap->ap_active == 0)
3323 ahci_issue_pending_ncq_commands(ap);
3324
3325 /* Complete the timed out ata_xfer I/O (may generate new I/O). */
3326 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
3327 xa->complete(xa);
3328
3329 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
3330ret:
3331 crit_exit();
3332}
3333
831bc9e3
MD
3334/*
3335 * Used by the softreset, pmprobe, and read_ncq_error only, in very
3336 * specialized, controlled circumstances.
3337 *
3338 * Only one command may be pending.
3339 */
3340void
3341ahci_quick_timeout(struct ahci_ccb *ccb)
3342{
3343 struct ahci_port *ap = ccb->ccb_port;
3344
3345 switch (ccb->ccb_xa.state) {
3346 case ATA_S_PENDING:
3347 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
3348 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3349 break;
3350 case ATA_S_ONCHIP:
3351 KKASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
3352 ap->ap_sactive == 0);
3353 ahci_port_stop(ap, 0);
3354 ahci_port_start(ap);
3355
3356 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3357 ap->ap_active &= ~(1 << ccb->ccb_slot);
3358 KKASSERT(ap->ap_active_cnt > 0);
3359 --ap->ap_active_cnt;
3360 break;
3361 default:
3362 panic("%s: ahci_quick_timeout: ccb in bad state %d",
3363 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_xa.state);
3364 }
3365}
3366
258223a3
MD
3367void
3368ahci_empty_done(struct ahci_ccb *ccb)
3369{
831bc9e3
MD
3370 if (ccb->ccb_xa.state == ATA_S_ONCHIP)
3371 ccb->ccb_xa.state = ATA_S_COMPLETE;
258223a3 3372}