AHCI - fixes to ahci_port_init()
[dragonfly.git] / sys / dev / disk / ahci / ahci.c
CommitLineData
258223a3
MD
1/*
2 * Copyright (c) 2006 David Gwynne <dlg@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 *
16 *
17 * Copyright (c) 2009 The DragonFly Project. All rights reserved.
18 *
19 * This code is derived from software contributed to The DragonFly Project
20 * by Matthew Dillon <dillon@backplane.com>
21 *
22 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions
24 * are met:
25 *
26 * 1. Redistributions of source code must retain the above copyright
27 * notice, this list of conditions and the following disclaimer.
28 * 2. Redistributions in binary form must reproduce the above copyright
29 * notice, this list of conditions and the following disclaimer in
30 * the documentation and/or other materials provided with the
31 * distribution.
32 * 3. Neither the name of The DragonFly Project nor the names of its
33 * contributors may be used to endorse or promote products derived
34 * from this software without specific, prior written permission.
35 *
36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
39 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
40 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
41 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
42 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
43 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
44 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
45 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
46 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
47 * SUCH DAMAGE.
48 *
49 * $OpenBSD: ahci.c,v 1.147 2009/02/16 21:19:07 miod Exp $
50 */
51
52#include "ahci.h"
53
f4553de1
MD
54int ahci_port_start(struct ahci_port *ap);
55int ahci_port_stop(struct ahci_port *ap, int stop_fis_rx);
56int ahci_port_clo(struct ahci_port *ap);
57void ahci_port_interrupt_enable(struct ahci_port *ap);
258223a3
MD
58
59int ahci_load_prdt(struct ahci_ccb *);
60void ahci_unload_prdt(struct ahci_ccb *);
61static void ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs,
62 int nsegs, int error);
258223a3 63void ahci_start(struct ahci_ccb *);
17eab71e 64int ahci_port_softreset(struct ahci_port *ap);
1980eff3
MD
65int ahci_port_pmprobe(struct ahci_port *ap);
66int ahci_port_hardreset(struct ahci_port *ap, int hard);
cf5f3a81
MD
67void ahci_port_hardstop(struct ahci_port *ap);
68void ahci_flush_tfd(struct ahci_port *ap);
258223a3 69
831bc9e3
MD
70static void ahci_ata_cmd_timeout_unserialized(void *);
71void ahci_quick_timeout(struct ahci_ccb *ccb);
72void ahci_check_active_timeouts(struct ahci_port *ap);
258223a3 73
831bc9e3
MD
74void ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at);
75void ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at);
258223a3
MD
76void ahci_issue_pending_ncq_commands(struct ahci_port *);
77void ahci_issue_pending_commands(struct ahci_port *, int);
78
258223a3
MD
79struct ahci_ccb *ahci_get_err_ccb(struct ahci_port *);
80void ahci_put_err_ccb(struct ahci_ccb *);
81
82int ahci_port_read_ncq_error(struct ahci_port *, int *);
83
84struct ahci_dmamem *ahci_dmamem_alloc(struct ahci_softc *, bus_dma_tag_t tag);
85void ahci_dmamem_free(struct ahci_softc *, struct ahci_dmamem *);
86static void ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error);
87
88void ahci_empty_done(struct ahci_ccb *ccb);
89void ahci_ata_cmd_done(struct ahci_ccb *ccb);
90
91/* Wait for all bits in _b to be cleared */
cec85a37
MD
92#define ahci_pwait_clr(_ap, _r, _b) \
93 ahci_pwait_eq((_ap), AHCI_PWAIT_TIMEOUT, (_r), (_b), 0)
94#define ahci_pwait_clr_to(_ap, _to, _r, _b) \
95 ahci_pwait_eq((_ap), _to, (_r), (_b), 0)
258223a3
MD
96
97/* Wait for all bits in _b to be set */
cec85a37
MD
98#define ahci_pwait_set(_ap, _r, _b) \
99 ahci_pwait_eq((_ap), AHCI_PWAIT_TIMEOUT, (_r), (_b), (_b))
100#define ahci_pwait_set_to(_ap, _to, _r, _b) \
101 ahci_pwait_eq((_ap), _to, (_r), (_b), (_b))
102
103#define AHCI_PWAIT_TIMEOUT 1000
258223a3 104
fd8bd957
MD
105/*
106 * Initialize the global AHCI hardware. This code does not set up any of
107 * its ports.
108 */
258223a3
MD
109int
110ahci_init(struct ahci_softc *sc)
111{
112 u_int32_t cap, pi;
831bc9e3
MD
113 int i;
114 struct ahci_port *ap;
258223a3
MD
115
116 DPRINTF(AHCI_D_VERBOSE, " GHC 0x%b",
117 ahci_read(sc, AHCI_REG_GHC), AHCI_FMT_GHC);
118
119 /* save BIOS initialised parameters, enable staggered spin up */
120 cap = ahci_read(sc, AHCI_REG_CAP);
121 cap &= AHCI_REG_CAP_SMPS;
122 cap |= AHCI_REG_CAP_SSS;
123 pi = ahci_read(sc, AHCI_REG_PI);
124
831bc9e3
MD
125#if 1
126 /*
127 * This is a hack that currently does not appear to have
128 * a significant effect, but I noticed the port registers
129 * do not appear to be completely cleared after the host
130 * controller is reset.
131 */
132 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO);
133 ap->ap_sc = sc;
134 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
135 if ((pi & (1 << i)) == 0)
136 continue;
137 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
138 AHCI_PORT_REGION(i), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
139 device_printf(sc->sc_dev, "can't map port\n");
140 return (1);
141 }
142 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED |
143 AHCI_PREG_SCTL_DET_DISABLE);
144 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
145 ahci_pwrite(ap, AHCI_PREG_IE, 0);
146 ahci_pwrite(ap, AHCI_PREG_CMD, 0);
147 ahci_pwrite(ap, AHCI_PREG_IS, 0);
148 }
149 kfree(ap, M_DEVBUF);
150#endif
151
17eab71e
MD
152 /*
153 * Unconditionally reset the controller, do not conditionalize on
154 * trying to figure it if it was previously active or not.
831bc9e3
MD
155 *
156 * NOTE BRICKS (1)
157 *
158 * If you have a port multiplier and it does not have a device
159 * in target 0, and it probes normally, but a later operation
160 * mis-probes a target behind that PM, it is possible for the
161 * port to brick such that only (a) a power cycle of the host
162 * or (b) placing a device in target 0 will fix the problem.
163 * Power cycling the PM has no effect (it works fine on another
164 * host port). This issue is unrelated to CLO.
17eab71e
MD
165 */
166 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_HR);
831bc9e3
MD
167 if (ahci_wait_ne(sc, AHCI_REG_GHC,
168 AHCI_REG_GHC_HR, AHCI_REG_GHC_HR) != 0) {
17eab71e
MD
169 device_printf(sc->sc_dev,
170 "unable to reset controller\n");
171 return (1);
258223a3 172 }
831bc9e3 173 ahci_os_sleep(100);
258223a3
MD
174
175 /* enable ahci (global interrupts disabled) */
176 ahci_write(sc, AHCI_REG_GHC, AHCI_REG_GHC_AE);
177
178 /* restore parameters */
179 ahci_write(sc, AHCI_REG_CAP, cap);
180 ahci_write(sc, AHCI_REG_PI, pi);
181
182 return (0);
183}
184
fd8bd957
MD
185/*
186 * Allocate and initialize an AHCI port.
187 */
258223a3
MD
188int
189ahci_port_alloc(struct ahci_softc *sc, u_int port)
190{
1980eff3
MD
191 struct ahci_port *ap;
192 struct ata_port *at;
193 struct ahci_ccb *ccb;
194 u_int64_t dva;
195 u_int32_t cmd;
196 struct ahci_cmd_hdr *hdr;
197 struct ahci_cmd_table *table;
258223a3
MD
198 int rc = ENOMEM;
199 int error;
200 int i;
201
202 ap = kmalloc(sizeof(*ap), M_DEVBUF, M_WAITOK | M_ZERO);
258223a3
MD
203
204 ksnprintf(ap->ap_name, sizeof(ap->ap_name), "%s%d.%d",
205 device_get_name(sc->sc_dev),
206 device_get_unit(sc->sc_dev),
207 port);
208 sc->sc_ports[port] = ap;
209
1980eff3
MD
210 /*
211 * Allocate enough so we never have to reallocate, it makes
212 * it easier.
213 *
214 * ap_pmcount will be reduced by the scan if we encounter the
215 * port multiplier port prior to target 15.
216 */
217 if (ap->ap_ata == NULL) {
218 ap->ap_ata = kmalloc(sizeof(*ap->ap_ata) * AHCI_MAX_PMPORTS,
219 M_DEVBUF, M_INTWAIT | M_ZERO);
220 for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
221 at = &ap->ap_ata[i];
222 at->at_ahci_port = ap;
223 at->at_target = i;
3209f581 224 at->at_probe = ATA_PROBE_NEED_INIT;
831bc9e3 225 at->at_features |= ATA_PORT_F_RESCAN;
1980eff3
MD
226 ksnprintf(at->at_name, sizeof(at->at_name),
227 "%s.%d", ap->ap_name, i);
228 }
229 }
258223a3
MD
230 if (bus_space_subregion(sc->sc_iot, sc->sc_ioh,
231 AHCI_PORT_REGION(port), AHCI_PORT_SIZE, &ap->ap_ioh) != 0) {
232 device_printf(sc->sc_dev,
233 "unable to create register window for port %d\n",
234 port);
235 goto freeport;
236 }
237
238 ap->ap_sc = sc;
239 ap->ap_num = port;
3209f581 240 ap->ap_probe = ATA_PROBE_NEED_INIT;
258223a3
MD
241 TAILQ_INIT(&ap->ap_ccb_free);
242 TAILQ_INIT(&ap->ap_ccb_pending);
243 lockinit(&ap->ap_ccb_lock, "ahcipo", 0, 0);
244
245 /* Disable port interrupts */
246 ahci_pwrite(ap, AHCI_PREG_IE, 0);
831bc9e3 247 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3 248
17eab71e
MD
249 /*
250 * Sec 10.1.2 - deinitialise port if it is already running
251 */
258223a3
MD
252 cmd = ahci_pread(ap, AHCI_PREG_CMD);
253 if ((cmd & (AHCI_PREG_CMD_ST | AHCI_PREG_CMD_CR |
17eab71e 254 AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_FR)) ||
258223a3
MD
255 (ahci_pread(ap, AHCI_PREG_SCTL) & AHCI_PREG_SCTL_DET)) {
256 int r;
257
258 r = ahci_port_stop(ap, 1);
259 if (r) {
260 device_printf(sc->sc_dev,
261 "unable to disable %s, ignoring port %d\n",
262 ((r == 2) ? "CR" : "FR"), port);
263 rc = ENXIO;
264 goto freeport;
265 }
266
267 /* Write DET to zero */
cf5f3a81 268 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED);
258223a3
MD
269 }
270
271 /* Allocate RFIS */
272 ap->ap_dmamem_rfis = ahci_dmamem_alloc(sc, sc->sc_tag_rfis);
273 if (ap->ap_dmamem_rfis == NULL) {
cf5f3a81 274 kprintf("%s: NORFIS\n", PORTNAME(ap));
258223a3
MD
275 goto nomem;
276 }
277
278 /* Setup RFIS base address */
279 ap->ap_rfis = (struct ahci_rfis *) AHCI_DMA_KVA(ap->ap_dmamem_rfis);
280 dva = AHCI_DMA_DVA(ap->ap_dmamem_rfis);
281 ahci_pwrite(ap, AHCI_PREG_FBU, (u_int32_t)(dva >> 32));
282 ahci_pwrite(ap, AHCI_PREG_FB, (u_int32_t)dva);
283
831bc9e3
MD
284 /* Clear SERR before starting FIS reception or ST or anything */
285 ahci_flush_tfd(ap);
286 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
287
258223a3
MD
288 /* Enable FIS reception and activate port. */
289 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1980eff3 290 cmd &= ~(AHCI_PREG_CMD_CLO | AHCI_PREG_CMD_PMA);
258223a3
MD
291 cmd |= AHCI_PREG_CMD_FRE | AHCI_PREG_CMD_POD | AHCI_PREG_CMD_SUD;
292 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_ICC_ACTIVE);
293
294 /* Check whether port activated. Skip it if not. */
295 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
296 if ((cmd & AHCI_PREG_CMD_FRE) == 0) {
cf5f3a81 297 kprintf("%s: NOT-ACTIVATED\n", PORTNAME(ap));
258223a3
MD
298 rc = ENXIO;
299 goto freeport;
300 }
301
302 /* Allocate a CCB for each command slot */
303 ap->ap_ccbs = kmalloc(sizeof(struct ahci_ccb) * sc->sc_ncmds, M_DEVBUF,
304 M_WAITOK | M_ZERO);
305 if (ap->ap_ccbs == NULL) {
306 device_printf(sc->sc_dev,
307 "unable to allocate command list for port %d\n",
308 port);
309 goto freeport;
310 }
311
312 /* Command List Structures and Command Tables */
313 ap->ap_dmamem_cmd_list = ahci_dmamem_alloc(sc, sc->sc_tag_cmdh);
314 ap->ap_dmamem_cmd_table = ahci_dmamem_alloc(sc, sc->sc_tag_cmdt);
315 if (ap->ap_dmamem_cmd_table == NULL ||
316 ap->ap_dmamem_cmd_list == NULL) {
317nomem:
318 device_printf(sc->sc_dev,
319 "unable to allocate DMA memory for port %d\n",
320 port);
321 goto freeport;
322 }
323
324 /* Setup command list base address */
325 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_list);
326 ahci_pwrite(ap, AHCI_PREG_CLBU, (u_int32_t)(dva >> 32));
327 ahci_pwrite(ap, AHCI_PREG_CLB, (u_int32_t)dva);
328
329 /* Split CCB allocation into CCBs and assign to command header/table */
330 hdr = AHCI_DMA_KVA(ap->ap_dmamem_cmd_list);
331 table = AHCI_DMA_KVA(ap->ap_dmamem_cmd_table);
332 for (i = 0; i < sc->sc_ncmds; i++) {
333 ccb = &ap->ap_ccbs[i];
334
335 error = bus_dmamap_create(sc->sc_tag_data, BUS_DMA_ALLOCNOW,
336 &ccb->ccb_dmamap);
337 if (error) {
338 device_printf(sc->sc_dev,
339 "unable to create dmamap for port %d "
340 "ccb %d\n", port, i);
341 goto freeport;
342 }
343
344 callout_init(&ccb->ccb_timeout);
345 ccb->ccb_slot = i;
346 ccb->ccb_port = ap;
347 ccb->ccb_cmd_hdr = &hdr[i];
348 ccb->ccb_cmd_table = &table[i];
349 dva = AHCI_DMA_DVA(ap->ap_dmamem_cmd_table) +
350 ccb->ccb_slot * sizeof(struct ahci_cmd_table);
351 ccb->ccb_cmd_hdr->ctba_hi = htole32((u_int32_t)(dva >> 32));
352 ccb->ccb_cmd_hdr->ctba_lo = htole32((u_int32_t)dva);
353
354 ccb->ccb_xa.fis =
355 (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
356 ccb->ccb_xa.packetcmd = ccb->ccb_cmd_table->acmd;
357 ccb->ccb_xa.tag = i;
358
258223a3
MD
359 ccb->ccb_xa.state = ATA_S_COMPLETE;
360 ahci_put_ccb(ccb);
361 }
362
363 /* Wait for ICC change to complete */
364 ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_ICC);
365
fd8bd957 366 /*
f4553de1
MD
367 * Start the port. The helper thread will call ahci_port_init()
368 * so the ports can all be started in parallel. A failure by
369 * ahci_port_init() does not deallocate the port since we still
370 * want hot-plug events.
fd8bd957 371 */
f4553de1 372 ahci_os_start_port(ap);
fd8bd957
MD
373 return(0);
374freeport:
375 ahci_port_free(sc, port);
fd8bd957
MD
376 return (rc);
377}
378
379/*
380 * [re]initialize an idle port. No CCBs should be active.
381 *
1980eff3
MD
382 * If at is NULL we are initializing a directly connected port, otherwise
383 * we are indirectly initializing a port multiplier port.
384 *
fd8bd957
MD
385 * This function is called during the initial port allocation sequence
386 * and is also called on hot-plug insertion. We take no chances and
387 * use a portreset instead of a softreset.
388 *
22181ab7
MD
389 * This function is the only way to move a failed port back to active
390 * status.
391 *
fd8bd957
MD
392 * Returns 0 if a device is successfully detected.
393 */
394int
f4553de1 395ahci_port_init(struct ahci_port *ap, struct ata_port *atx)
fd8bd957 396{
121d8e75 397 u_int32_t data;
fd8bd957
MD
398 int rc;
399
400 /*
1980eff3 401 * Clear all notification bits
fd8bd957 402 */
121d8e75 403 if (atx == NULL && (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF))
1980eff3
MD
404 ahci_pwrite(ap, AHCI_PREG_SNTF, -1);
405
406 /*
407 * Hard-reset the port. If a device is detected but it is busy
408 * we try a second time, this time cycling the phy as well.
409 */
f4553de1
MD
410 if (atx)
411 atx->at_probe = ATA_PROBE_NEED_HARD_RESET;
412 else
413 ap->ap_probe = ATA_PROBE_NEED_HARD_RESET;
831bc9e3
MD
414 rc = ahci_port_reset(ap, atx, 2);
415#if 0
f4553de1 416 rc = ahci_port_reset(ap, atx, 1);
1980eff3 417 if (rc == EBUSY) {
f4553de1 418 rc = ahci_port_reset(ap, atx, 2);
17eab71e 419 }
831bc9e3 420#endif
fd8bd957 421
258223a3
MD
422 switch (rc) {
423 case ENODEV:
fd8bd957
MD
424 /*
425 * We had problems talking to the device on the port.
426 */
121d8e75
MD
427 if (atx) {
428 ahci_pm_read(ap, atx->at_target,
429 AHCI_PMREG_SSTS, &data);
430 } else {
431 data = ahci_pread(ap, AHCI_PREG_SSTS);
432 }
433
434 switch(data & AHCI_PREG_SSTS_DET) {
258223a3 435 case AHCI_PREG_SSTS_DET_DEV_NE:
121d8e75
MD
436 kprintf("%s: Device not communicating\n",
437 ATANAME(ap, atx));
258223a3
MD
438 break;
439 case AHCI_PREG_SSTS_DET_PHYOFFLINE:
121d8e75
MD
440 kprintf("%s: PHY offline\n",
441 ATANAME(ap, atx));
258223a3
MD
442 break;
443 default:
121d8e75
MD
444 kprintf("%s: No device detected\n",
445 ATANAME(ap, atx));
258223a3
MD
446 break;
447 }
448 break;
449
450 case EBUSY:
fd8bd957 451 /*
17eab71e
MD
452 * The device on the port is still telling us its busy,
453 * which means that it is not properly handling a SATA
454 * port COMRESET.
fd8bd957 455 *
17eab71e
MD
456 * It may be possible to softreset the device using CLO
457 * and a device reset command.
fd8bd957 458 */
121d8e75
MD
459 if (atx) {
460 kprintf("%s: Device on port is bricked, giving up\n",
461 ATANAME(ap, atx));
fd8bd957 462 } else {
121d8e75
MD
463 kprintf("%s: Device on port is bricked, "
464 "trying softreset\n", PORTNAME(ap));
465
466 rc = ahci_port_reset(ap, atx, 0);
467 if (rc) {
468 kprintf("%s: Unable unbrick device\n",
469 PORTNAME(ap));
470 } else {
471 kprintf("%s: Successfully unbricked\n",
472 PORTNAME(ap));
473 }
258223a3
MD
474 }
475 break;
476
477 default:
478 break;
479 }
480
481 /*
17eab71e
MD
482 * Command transfers can only be enabled if a device was successfully
483 * detected.
1980eff3
MD
484 *
485 * Allocate or deallocate the ap_ata array here too.
258223a3 486 */
121d8e75
MD
487 if (atx == NULL) {
488 switch(ap->ap_type) {
489 case ATA_PORT_T_NONE:
490 ap->ap_pmcount = 0;
491 break;
492 case ATA_PORT_T_PM:
493 /* already set */
494 break;
495 default:
496 ap->ap_pmcount = 1;
497 break;
498 }
1980eff3
MD
499 }
500
501 /*
502 * Start the port if we succeeded.
503 *
504 * There's nothing to start for devices behind a port multiplier.
505 */
f4553de1 506 if (rc == 0 && atx == NULL) {
17eab71e 507 if (ahci_port_start(ap)) {
fd8bd957
MD
508 kprintf("%s: failed to start command DMA on port, "
509 "disabling\n", PORTNAME(ap));
258223a3
MD
510 rc = ENXIO; /* couldn't start port */
511 }
512 }
513
17eab71e 514 /*
3209f581 515 * Flush interrupts on the port. XXX
1980eff3
MD
516 *
517 * Enable interrupts on the port whether a device is sitting on
518 * it or not, to handle hot-plug events.
17eab71e 519 */
f4553de1 520 if (atx == NULL) {
1980eff3
MD
521 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
522 ahci_write(ap->ap_sc, AHCI_REG_IS, 1 << ap->ap_num);
523
f4553de1 524 ahci_port_interrupt_enable(ap);
1980eff3 525 }
fd8bd957 526 return(rc);
258223a3
MD
527}
528
fd8bd957 529/*
f4553de1
MD
530 * Enable or re-enable interrupts on a port.
531 *
532 * This routine is called from the port initialization code or from the
533 * helper thread as the real interrupt may be forced to turn off certain
534 * interrupt sources.
535 */
536void
537ahci_port_interrupt_enable(struct ahci_port *ap)
538{
539 u_int32_t data;
540
541 data = AHCI_PREG_IE_TFEE | AHCI_PREG_IE_HBFE |
542 AHCI_PREG_IE_IFE | AHCI_PREG_IE_OFE |
543 AHCI_PREG_IE_DPE | AHCI_PREG_IE_UFE |
544 AHCI_PREG_IE_PCE | AHCI_PREG_IE_PRCE |
545 AHCI_PREG_IE_DHRE;
546 if (ap->ap_sc->sc_cap & AHCI_REG_CAP_SSNTF)
547 data |= AHCI_PREG_IE_SDBE;
548#ifdef AHCI_COALESCE
549 if (sc->sc_ccc_ports & (1 << port)
550 data &= ~(AHCI_PREG_IE_SDBE | AHCI_PREG_IE_DHRE);
551#endif
552 ahci_pwrite(ap, AHCI_PREG_IE, data);
553}
554
555/*
3209f581
MD
556 * Run the port / target state machine from a main context.
557 *
558 * The state machine for the port is always run.
559 *
560 * If atx is non-NULL run the state machine for a particular target.
561 * If atx is NULL run the state machine for all targets.
562 */
563void
831bc9e3 564ahci_port_state_machine(struct ahci_port *ap, int initial)
3209f581
MD
565{
566 struct ata_port *at;
567 u_int32_t data;
568 int target;
569 int didsleep;
831bc9e3 570 int loop;
3209f581 571
831bc9e3
MD
572 /*
573 * State machine for port. Note that CAM is not yet associated
574 * during the initial parallel probe and the port's probe state
575 * will not get past ATA_PROBE_NEED_IDENT.
576 */
3209f581
MD
577 if (ap->ap_type == ATA_PORT_T_NONE) {
578 if (ap->ap_probe == ATA_PROBE_NEED_INIT) {
579 for (target = 0; target < AHCI_MAX_PMPORTS; ++target) {
580 at = &ap->ap_ata[target];
581 at->at_probe = ATA_PROBE_NEED_INIT;
831bc9e3 582 at->at_features |= ATA_PORT_F_RESCAN;
3209f581
MD
583 }
584 ahci_port_init(ap, NULL);
585 }
586 if (ap->ap_probe == ATA_PROBE_NEED_HARD_RESET)
587 ahci_port_reset(ap, NULL, 1);
588 if (ap->ap_probe == ATA_PROBE_NEED_SOFT_RESET)
589 ahci_port_reset(ap, NULL, 0);
590 if (ap->ap_probe == ATA_PROBE_NEED_IDENT)
591 ahci_cam_probe(ap, NULL);
592 }
593 if (ap->ap_type != ATA_PORT_T_PM) {
594 if (ap->ap_probe == ATA_PROBE_FAILED) {
595 ahci_cam_changed(ap, NULL, 0);
f4553de1 596 } else if (ap->ap_probe >= ATA_PROBE_NEED_IDENT) {
3209f581
MD
597 ahci_cam_changed(ap, NULL, 1);
598 }
599 return;
600 }
601
831bc9e3
MD
602 /*
603 * Port Multiplier state machine.
604 *
605 * Get a mask of changed targets and combine with any runnable
606 * states already present.
607 */
608 for (loop = 0; ;++loop) {
3209f581
MD
609 if (ahci_pm_read(ap, 15, AHCI_PMREG_EINFO, &data)) {
610 kprintf("%s: PM unable to read hot-plug bitmap\n",
611 PORTNAME(ap));
612 break;
613 }
614 data &= (1 << ap->ap_pmcount) - 1;
615
616 /*
831bc9e3
MD
617 * Do at least one loop, then stop if no more state changes
618 * have occured. The PM might not generate a new
619 * notification until we clear the entire bitmap.
3209f581 620 */
831bc9e3 621 if (loop && data == 0)
3209f581
MD
622 break;
623
624 /*
625 * New devices showing up in the bitmap require some spin-up
626 * time before we start probing them. Reset didsleep. The
627 * first new device we detect will sleep before probing.
831bc9e3
MD
628 *
629 * This only applies to devices whos change bit is set in
630 * the data, and does not apply to the initial boot-time
631 * probe.
3209f581
MD
632 */
633 didsleep = 0;
634
635 for (target = 0; target < ap->ap_pmcount; ++target) {
636 at = &ap->ap_ata[target];
637
638 /*
639 * Check the target state for targets behind the PM
640 * which have changed state. This will adjust
641 * at_probe and set ATA_PORT_F_RESCAN
642 *
643 * We want to wait at least 4 seconds before probing
644 * a newly inserted device. If the check status
645 * indicates a device is present and in need of a
646 * hard reset, we make sure we have slept before
647 * continuing.
831bc9e3
MD
648 *
649 * NOTE: When pm_check_good finds a good port it
650 * typically starts us in probe state
651 * NEED_HARD_RESET rather than INIT.
3209f581
MD
652 */
653 if (data & (1 << target)) {
654 ahci_pm_check_good(ap, target);
831bc9e3
MD
655 if (initial == 0 && didsleep == 0 &&
656 at->at_probe <= ATA_PROBE_NEED_HARD_RESET
657 ) {
658 didsleep = 1;
121d8e75
MD
659 kprintf("%s: Waiting 10 seconds on insertion\n", PORTNAME(ap));
660 ahci_os_sleep(10000);
831bc9e3
MD
661 }
662 }
663
664 /*
665 * Report hot-plug events before the probe state
666 * really gets hot. Only actual events are reported
667 * here to reduce spew.
668 */
669 if (data & (1 << target)) {
670 kprintf("%s: HOTPLUG (PM) - ", ATANAME(ap, at));
671 switch(at->at_probe) {
672 case ATA_PROBE_NEED_INIT:
673 case ATA_PROBE_NEED_HARD_RESET:
674 kprintf("Device inserted\n");
675 break;
676 case ATA_PROBE_FAILED:
677 kprintf("Device removed\n");
678 break;
679 default:
680 kprintf("Device probe in progress\n");
681 break;
3209f581
MD
682 }
683 }
684
685 /*
831bc9e3
MD
686 * Run through the state machine as necessary if
687 * the port is not marked failed.
688 *
689 * The state machine may stop at NEED_IDENT if
690 * CAM is not yet attached.
691 *
692 * Acquire exclusive access to the port while we
693 * are doing this. This prevents command-completion
694 * from queueing commands for non-polled targets
695 * inbetween our probe steps. We need to do this
696 * because the reset probes can generate severe PHY
697 * and protocol errors and soft-brick the port.
3209f581 698 */
831bc9e3
MD
699 if (at->at_probe != ATA_PROBE_FAILED &&
700 at->at_probe != ATA_PROBE_GOOD) {
701 ahci_beg_exclusive_access(ap, at);
3209f581
MD
702 if (at->at_probe == ATA_PROBE_NEED_INIT)
703 ahci_port_init(ap, at);
704 if (at->at_probe == ATA_PROBE_NEED_HARD_RESET)
705 ahci_port_reset(ap, at, 1);
706 if (at->at_probe == ATA_PROBE_NEED_SOFT_RESET)
707 ahci_port_reset(ap, at, 0);
708 if (at->at_probe == ATA_PROBE_NEED_IDENT)
709 ahci_cam_probe(ap, at);
831bc9e3 710 ahci_end_exclusive_access(ap, at);
3209f581
MD
711 }
712
713 /*
831bc9e3 714 * Add or remove from CAM
3209f581
MD
715 */
716 if (at->at_features & ATA_PORT_F_RESCAN) {
717 at->at_features &= ~ATA_PORT_F_RESCAN;
718 if (at->at_probe == ATA_PROBE_FAILED) {
719 ahci_cam_changed(ap, at, 0);
f4553de1 720 } else if (at->at_probe >= ATA_PROBE_NEED_IDENT) {
3209f581
MD
721 ahci_cam_changed(ap, at, 1);
722 }
723 }
724 }
725 }
726}
727
728
729/*
fd8bd957
MD
730 * De-initialize and detach a port.
731 */
258223a3
MD
732void
733ahci_port_free(struct ahci_softc *sc, u_int port)
734{
735 struct ahci_port *ap = sc->sc_ports[port];
736 struct ahci_ccb *ccb;
737
17eab71e
MD
738 /*
739 * Ensure port is disabled and its interrupts are all flushed.
740 */
258223a3 741 if (ap->ap_sc) {
17eab71e 742 ahci_port_stop(ap, 1);
f4553de1 743 ahci_os_stop_port(ap);
258223a3
MD
744 ahci_pwrite(ap, AHCI_PREG_CMD, 0);
745 ahci_pwrite(ap, AHCI_PREG_IE, 0);
746 ahci_pwrite(ap, AHCI_PREG_IS, ahci_pread(ap, AHCI_PREG_IS));
747 ahci_write(sc, AHCI_REG_IS, 1 << port);
748 }
749
750 if (ap->ap_ccbs) {
751 while ((ccb = ahci_get_ccb(ap)) != NULL) {
752 if (ccb->ccb_dmamap) {
753 bus_dmamap_destroy(sc->sc_tag_data,
754 ccb->ccb_dmamap);
755 ccb->ccb_dmamap = NULL;
756 }
757 }
758 kfree(ap->ap_ccbs, M_DEVBUF);
759 ap->ap_ccbs = NULL;
760 }
761
762 if (ap->ap_dmamem_cmd_list) {
763 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_list);
764 ap->ap_dmamem_cmd_list = NULL;
765 }
766 if (ap->ap_dmamem_rfis) {
767 ahci_dmamem_free(sc, ap->ap_dmamem_rfis);
768 ap->ap_dmamem_rfis = NULL;
769 }
770 if (ap->ap_dmamem_cmd_table) {
771 ahci_dmamem_free(sc, ap->ap_dmamem_cmd_table);
772 ap->ap_dmamem_cmd_table = NULL;
773 }
1980eff3
MD
774 if (ap->ap_ata) {
775 kfree(ap->ap_ata, M_DEVBUF);
776 ap->ap_ata = NULL;
777 }
258223a3
MD
778
779 /* bus_space(9) says we dont free the subregions handle */
780
781 kfree(ap, M_DEVBUF);
782 sc->sc_ports[port] = NULL;
783}
784
fd8bd957
MD
785/*
786 * Start high-level command processing on the port
787 */
258223a3 788int
17eab71e 789ahci_port_start(struct ahci_port *ap)
258223a3 790{
8bf6a3ff 791 u_int32_t r, oldr, s, olds, is, oldis, tfd, oldtfd;
258223a3 792
17eab71e
MD
793 /*
794 * FRE must be turned on before ST. Wait for FR to go active
795 * before turning on ST. The spec doesn't seem to think this
796 * is necessary but waiting here avoids an on-off race in the
797 * ahci_port_stop() code.
798 */
cec07d75 799 /* XXX REMOVE ME */
1980eff3
MD
800 olds = ahci_pread(ap, AHCI_PREG_SERR);
801 oldis= ahci_pread(ap, AHCI_PREG_IS);
8bf6a3ff 802 oldtfd = ahci_pread(ap, AHCI_PREG_TFD);
1980eff3 803 oldr = r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
17eab71e
MD
804 if ((r & AHCI_PREG_CMD_FRE) == 0) {
805 r |= AHCI_PREG_CMD_FRE;
806 ahci_pwrite(ap, AHCI_PREG_CMD, r);
807 }
808 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0) {
809 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) {
810 kprintf("%s: Cannot start FIS reception\n",
811 PORTNAME(ap));
812 return (2);
813 }
814 }
815
816 /*
817 * Turn on ST, wait for CR to come up.
818 */
819 r |= AHCI_PREG_CMD_ST;
258223a3 820 ahci_pwrite(ap, AHCI_PREG_CMD, r);
17eab71e 821 if (ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) {
8bf6a3ff
MD
822 s = ahci_pread(ap, AHCI_PREG_SERR);
823 is = ahci_pread(ap, AHCI_PREG_IS);
824 tfd = ahci_pread(ap, AHCI_PREG_TFD);
1980eff3
MD
825 kprintf("%s: Cannot start command DMA\n"
826 "OCMD=%b OSERR=%b\n"
827 "NCMP=%b NSERR=%b\n"
8bf6a3ff
MD
828 "OLDIS=%b\nNEWIS=%b\n"
829 "OLDTFD=%b\nNEWTFD=%b\n",
1980eff3
MD
830 PORTNAME(ap),
831 oldr, AHCI_PFMT_CMD, olds, AHCI_PFMT_SERR,
832 r, AHCI_PFMT_CMD, s, AHCI_PFMT_SERR,
8bf6a3ff
MD
833 oldis, AHCI_PFMT_IS, is, AHCI_PFMT_IS,
834 oldtfd, AHCI_PFMT_TFD_STS, tfd, AHCI_PFMT_TFD_STS);
17eab71e
MD
835 return (1);
836 }
258223a3
MD
837
838#ifdef AHCI_COALESCE
17eab71e
MD
839 /*
840 * (Re-)enable coalescing on the port.
841 */
258223a3
MD
842 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
843 ap->ap_sc->sc_ccc_ports_cur |= (1 << ap->ap_num);
844 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
845 ap->ap_sc->sc_ccc_ports_cur);
846 }
847#endif
848
258223a3
MD
849 return (0);
850}
851
fd8bd957
MD
852/*
853 * Stop high-level command processing on a port
854 */
258223a3
MD
855int
856ahci_port_stop(struct ahci_port *ap, int stop_fis_rx)
857{
858 u_int32_t r;
859
860#ifdef AHCI_COALESCE
17eab71e
MD
861 /*
862 * Disable coalescing on the port while it is stopped.
863 */
258223a3
MD
864 if (ap->ap_sc->sc_ccc_ports & (1 << ap->ap_num)) {
865 ap->ap_sc->sc_ccc_ports_cur &= ~(1 << ap->ap_num);
866 ahci_write(ap->ap_sc, AHCI_REG_CCC_PORTS,
867 ap->ap_sc->sc_ccc_ports_cur);
868 }
869#endif
870
17eab71e
MD
871 /*
872 * Turn off ST, then wait for CR to go off.
873 */
258223a3
MD
874 r = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
875 r &= ~AHCI_PREG_CMD_ST;
258223a3
MD
876 ahci_pwrite(ap, AHCI_PREG_CMD, r);
877
17eab71e
MD
878 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CR)) {
879 kprintf("%s: Port bricked, unable to stop (ST)\n",
880 PORTNAME(ap));
258223a3 881 return (1);
17eab71e 882 }
258223a3 883
1980eff3 884#if 0
17eab71e
MD
885 /*
886 * Turn off FRE, then wait for FR to go off. FRE cannot
887 * be turned off until CR transitions to 0.
888 */
1980eff3
MD
889 if ((r & AHCI_PREG_CMD_FR) == 0) {
890 kprintf("%s: FR stopped, clear FRE for next start\n",
891 PORTNAME(ap));
892 stop_fis_rx = 2;
893 }
894#endif
17eab71e
MD
895 if (stop_fis_rx) {
896 r &= ~AHCI_PREG_CMD_FRE;
897 ahci_pwrite(ap, AHCI_PREG_CMD, r);
898 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR)) {
899 kprintf("%s: Port bricked, unable to stop (FRE)\n",
900 PORTNAME(ap));
901 return (2);
902 }
903 }
258223a3
MD
904
905 return (0);
906}
907
fd8bd957
MD
908/*
909 * AHCI command list override -> forcibly clear TFD.STS.{BSY,DRQ}
910 */
258223a3
MD
911int
912ahci_port_clo(struct ahci_port *ap)
913{
914 struct ahci_softc *sc = ap->ap_sc;
915 u_int32_t cmd;
916
917 /* Only attempt CLO if supported by controller */
918 if ((ahci_read(sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO) == 0)
919 return (1);
920
921 /* Issue CLO */
922 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
923#ifdef DIAGNOSTIC
924 if (cmd & AHCI_PREG_CMD_ST) {
925 kprintf("%s: CLO requested while port running\n",
926 PORTNAME(ap));
927 }
928#endif
929 ahci_pwrite(ap, AHCI_PREG_CMD, cmd | AHCI_PREG_CMD_CLO);
930
931 /* Wait for completion */
932 if (ahci_pwait_clr(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_CLO)) {
933 kprintf("%s: CLO did not complete\n", PORTNAME(ap));
934 return (1);
935 }
936
937 return (0);
938}
939
fd8bd957 940/*
1980eff3
MD
941 * Reset a port.
942 *
943 * If hard is 0 perform a softreset of the port.
944 * If hard is 1 perform a hard reset of the port.
945 * If hard is 2 perform a hard reset of the port and cycle the phy.
946 *
947 * If at is non-NULL an indirect port via a port-multiplier is being
948 * reset, otherwise a direct port is being reset.
17eab71e 949 *
1980eff3 950 * NOTE: Indirect ports can only be soft-reset.
17eab71e
MD
951 */
952int
1980eff3 953ahci_port_reset(struct ahci_port *ap, struct ata_port *at, int hard)
17eab71e
MD
954{
955 int rc;
956
957 if (hard) {
1980eff3
MD
958 if (at)
959 rc = ahci_pm_hardreset(ap, at->at_target, hard);
960 else
961 rc = ahci_port_hardreset(ap, hard);
17eab71e 962 } else {
1980eff3
MD
963 if (at)
964 rc = ahci_pm_softreset(ap, at->at_target);
965 else
966 rc = ahci_port_softreset(ap);
17eab71e
MD
967 }
968 return(rc);
969}
970
971/*
fd8bd957
MD
972 * AHCI soft reset, Section 10.4.1
973 *
1980eff3
MD
974 * (at) will be NULL when soft-resetting a directly-attached device, and
975 * non-NULL when soft-resetting a device through a port multiplier.
976 *
fd8bd957 977 * This function keeps port communications intact and attempts to generate
1980eff3 978 * a reset to the connected device using device commands.
fd8bd957 979 */
258223a3
MD
980int
981ahci_port_softreset(struct ahci_port *ap)
982{
1980eff3
MD
983 struct ahci_ccb *ccb = NULL;
984 struct ahci_cmd_hdr *cmd_slot;
985 u_int8_t *fis;
3209f581 986 int error;
1980eff3
MD
987 u_int32_t cmd;
988
3209f581 989 error = EIO;
1980eff3
MD
990
991 kprintf("%s: START SOFTRESET %b\n", PORTNAME(ap),
992 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD);
258223a3
MD
993
994 DPRINTF(AHCI_D_VERBOSE, "%s: soft reset\n", PORTNAME(ap));
995
996 crit_enter();
1980eff3
MD
997 ap->ap_flags |= AP_F_IN_RESET;
998 ap->ap_state = AP_S_NORMAL;
258223a3 999
1980eff3
MD
1000 /*
1001 * Remember port state in cmd (main to restore start/stop)
1002 *
1003 * Idle port.
1004 */
1005 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
258223a3
MD
1006 if (ahci_port_stop(ap, 0)) {
1007 kprintf("%s: failed to stop port, cannot softreset\n",
1008 PORTNAME(ap));
1009 goto err;
1010 }
cf5f3a81
MD
1011
1012 /*
1980eff3 1013 * Request CLO if device appears hung.
cf5f3a81 1014 */
258223a3 1015 if (ahci_pread(ap, AHCI_PREG_TFD) &
1980eff3 1016 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
1017 ahci_port_clo(ap);
1018 }
1019
1980eff3
MD
1020 /*
1021 * This is an attempt to clear errors so a new signature will
1022 * be latched. It isn't working properly. XXX
1023 */
cf5f3a81 1024 ahci_flush_tfd(ap);
1980eff3 1025 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3
MD
1026
1027 /* Restart port */
17eab71e 1028 if (ahci_port_start(ap)) {
258223a3
MD
1029 kprintf("%s: failed to start port, cannot softreset\n",
1030 PORTNAME(ap));
1031 goto err;
1032 }
1033
1034 /* Check whether CLO worked */
1035 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1980eff3 1036 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
1037 kprintf("%s: CLO %s, need port reset\n",
1038 PORTNAME(ap),
1039 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO)
1040 ? "failed" : "unsupported");
3209f581 1041 error = EBUSY;
258223a3
MD
1042 goto err;
1043 }
1044
cec85a37
MD
1045 /*
1046 * Prep first D2H command with SRST feature & clear busy/reset flags
1047 *
1048 * It is unclear which other fields in the FIS are used. Just zero
1049 * everything.
1050 */
258223a3 1051 ccb = ahci_get_err_ccb(ap);
1980eff3 1052 ccb->ccb_xa.at = NULL;
258223a3 1053 cmd_slot = ccb->ccb_cmd_hdr;
258223a3
MD
1054
1055 fis = ccb->ccb_cmd_table->cfis;
cec85a37 1056 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1980eff3
MD
1057 fis[0] = ATA_FIS_TYPE_H2D;
1058 fis[15] = ATA_FIS_CONTROL_SRST|ATA_FIS_CONTROL_4BIT;
258223a3
MD
1059
1060 cmd_slot->prdtl = 0;
1061 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1062 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
1063 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
258223a3
MD
1064
1065 ccb->ccb_xa.state = ATA_S_PENDING;
5f8c1efd 1066 ccb->ccb_xa.flags = 0;
831bc9e3 1067 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
5f8c1efd 1068 kprintf("%s: First FIS failed\n", PORTNAME(ap));
258223a3 1069 goto err;
cec85a37 1070 }
258223a3 1071
cec85a37 1072 /*
831bc9e3
MD
1073 * WARNING! TIME SENSITIVE SPACE! WARNING!
1074 *
1075 * The two FISes are supposed to be back to back. Don't issue other
1076 * commands or even delay if we can help it.
1980eff3 1077 */
1980eff3
MD
1078
1079 /*
cec85a37
MD
1080 * Prep second D2H command to read status and complete reset sequence
1081 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1082 * Rev 2.6 and it is unclear how the second FIS should be set up
1083 * from the AHCI document.
1084 *
b089d0bf 1085 * Give the device 3ms before sending the second FIS.
cec85a37
MD
1086 *
1087 * It is unclear which other fields in the FIS are used. Just zero
1088 * everything.
1089 */
1090 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1980eff3
MD
1091 fis[0] = ATA_FIS_TYPE_H2D;
1092 fis[15] = ATA_FIS_CONTROL_4BIT;
258223a3
MD
1093
1094 cmd_slot->prdtl = 0;
1095 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
258223a3
MD
1096
1097 ccb->ccb_xa.state = ATA_S_PENDING;
5f8c1efd 1098 ccb->ccb_xa.flags = 0;
831bc9e3 1099 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
5f8c1efd 1100 kprintf("%s: Second FIS failed\n", PORTNAME(ap));
258223a3 1101 goto err;
cec85a37 1102 }
258223a3 1103
1980eff3
MD
1104 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1105 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
258223a3
MD
1106 kprintf("%s: device didn't come ready after reset, TFD: 0x%b\n",
1107 PORTNAME(ap),
1108 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS);
3209f581 1109 error = EBUSY;
258223a3
MD
1110 goto err;
1111 }
3209f581 1112 ahci_os_sleep(10);
258223a3 1113
fd8bd957
MD
1114 /*
1115 * If the softreset is trying to clear a BSY condition after a
1116 * normal portreset we assign the port type.
1117 *
1118 * If the softreset is being run first as part of the ccb error
1119 * processing code then report if the device signature changed
1120 * unexpectedly.
1121 */
1980eff3
MD
1122 if (ap->ap_type == ATA_PORT_T_NONE) {
1123 ap->ap_type = ahci_port_signature_detect(ap, NULL);
fd8bd957 1124 } else {
1980eff3
MD
1125 if (ahci_port_signature_detect(ap, NULL) != ap->ap_type) {
1126 kprintf("%s: device signature unexpectedly "
1127 "changed\n", PORTNAME(ap));
3209f581 1128 error = EBUSY; /* XXX */
fd8bd957
MD
1129 }
1130 }
3209f581 1131 error = 0;
1980eff3 1132
3209f581 1133 ahci_os_sleep(3);
258223a3
MD
1134err:
1135 if (ccb != NULL) {
258223a3 1136 ahci_put_err_ccb(ccb);
1980eff3
MD
1137
1138 /*
1139 * If the target is busy use CLO to clear the busy
1140 * condition. The BSY should be cleared on the next
1141 * start.
1142 */
1143 if (ahci_pread(ap, AHCI_PREG_TFD) &
1144 (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1145 ahci_port_clo(ap);
1146 }
258223a3
MD
1147 }
1148
cf5f3a81
MD
1149 /*
1150 * If we failed to softreset make the port quiescent, otherwise
1151 * make sure the port's start/stop state matches what it was on
1152 * entry.
1980eff3
MD
1153 *
1154 * Don't kill the port if the softreset is on a port multiplier
1155 * target, that would kill all the targets!
cf5f3a81 1156 */
3209f581 1157 if (error) {
cf5f3a81 1158 ahci_port_hardstop(ap);
3209f581 1159 /* ap_probe set to failed */
cf5f3a81 1160 } else if (cmd & AHCI_PREG_CMD_ST) {
3209f581 1161 ap->ap_probe = ATA_PROBE_NEED_IDENT;
1980eff3 1162 kprintf("%s: STARTING PORT\n", PORTNAME(ap));
cf5f3a81
MD
1163 ahci_port_start(ap);
1164 } else {
3209f581 1165 ap->ap_probe = ATA_PROBE_NEED_IDENT;
1980eff3 1166 kprintf("%s: STOPPING PORT\n", PORTNAME(ap));
cf5f3a81
MD
1167 ahci_port_stop(ap, !(cmd & AHCI_PREG_CMD_FRE));
1168 }
3209f581 1169 ap->ap_flags &= ~AP_F_IN_RESET;
258223a3
MD
1170 crit_exit();
1171
1980eff3 1172 kprintf("%s: END SOFTRESET\n", PORTNAME(ap));
1980eff3 1173
3209f581 1174 return (error);
258223a3
MD
1175}
1176
fd8bd957
MD
1177/*
1178 * AHCI port reset, Section 10.4.2
1179 *
1180 * This function does a hard reset of the port. Note that the device
1181 * connected to the port could still end-up hung.
1182 */
258223a3 1183int
1980eff3 1184ahci_port_hardreset(struct ahci_port *ap, int hard)
258223a3 1185{
1980eff3 1186 u_int32_t cmd, r;
3209f581 1187 int error;
1980eff3
MD
1188 int loop;
1189 int type;
258223a3
MD
1190
1191 DPRINTF(AHCI_D_VERBOSE, "%s: port reset\n", PORTNAME(ap));
1192
1980eff3
MD
1193 ap->ap_flags |= AP_F_IN_RESET;
1194
1195 /*
1196 * Idle the port,
1197 */
258223a3 1198 ahci_port_stop(ap, 0);
cf5f3a81 1199 ap->ap_state = AP_S_NORMAL;
3209f581 1200 error = 0;
cf5f3a81
MD
1201
1202 /*
1980eff3
MD
1203 * The port may have been quiescent with its SUD bit cleared, so
1204 * set the SUD (spin up device).
cf5f3a81
MD
1205 */
1206 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1207 cmd |= AHCI_PREG_CMD_SUD;
1208 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
258223a3 1209
1980eff3
MD
1210 /*
1211 * Perform device detection. Cycle the PHY off, wait 10ms.
1212 * This simulates the SATA cable being physically unplugged.
1213 */
1214 ap->ap_type = ATA_PORT_T_NONE;
1215
1216 r = AHCI_PREG_SCTL_IPM_DISABLED;
1217 if (hard == 2)
1218 r |= AHCI_PREG_SCTL_DET_DISABLE;
1219 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
3209f581 1220 ahci_os_sleep(10);
258223a3 1221
1980eff3
MD
1222 /*
1223 * Start transmitting COMRESET. COMRESET must be sent for at
1224 * least 1ms.
1225 */
1226 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
258223a3
MD
1227 if (AhciForceGen1 & (1 << ap->ap_num)) {
1228 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap));
1229 r |= AHCI_PREG_SCTL_SPD_GEN1;
1230 } else {
1231 r |= AHCI_PREG_SCTL_SPD_ANY;
1232 }
1233 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
831bc9e3
MD
1234
1235 /*
1236 * Through trial and error it seems to take around 100ms
1237 * for the detect logic to settle down. If this is too
1238 * short the softreset code will fail.
1239 */
1240 ahci_os_sleep(100);
cf5f3a81
MD
1241
1242 /*
1243 * Only SERR_DIAG_X needs to be cleared for TFD updates, but
1244 * since we are hard-resetting the port we might as well clear
1245 * the whole enchillada
1246 */
1247 ahci_flush_tfd(ap);
1248 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
258223a3
MD
1249 r &= ~AHCI_PREG_SCTL_DET_INIT;
1250 r |= AHCI_PREG_SCTL_DET_NONE;
1251 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
258223a3 1252
1980eff3
MD
1253 /*
1254 * Try to determine if there is a device on the port.
1255 *
1256 * Give the device 3/10 second to at least be detected.
1257 * If we fail clear PRCS (phy detect) since we may cycled
1258 * the phy and probably caused another PRCS interrupt.
1259 */
1260 for (loop = 30; loop; --loop) {
1261 r = ahci_pread(ap, AHCI_PREG_SSTS);
1262 if (r & AHCI_PREG_SSTS_DET)
1263 break;
3209f581 1264 ahci_os_sleep(10);
1980eff3
MD
1265 }
1266 if (loop == 0) {
1267 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_PRCS);
1268 kprintf("%s: Port appears to be unplugged\n",
1269 PORTNAME(ap));
3209f581 1270 error = ENODEV;
1980eff3
MD
1271 }
1272
1273 /*
1274 * There is something on the port. Give the device 3 seconds
1275 * to fully negotiate.
1276 */
3209f581 1277 if (error == 0 &&
1980eff3
MD
1278 ahci_pwait_eq(ap, 3000, AHCI_PREG_SSTS,
1279 AHCI_PREG_SSTS_DET, AHCI_PREG_SSTS_DET_DEV)) {
1280 kprintf("%s: Device may be powered down\n",
1281 PORTNAME(ap));
3209f581 1282 error = ENODEV;
258223a3
MD
1283 }
1284
cec85a37 1285 /*
1980eff3 1286 * Wait for the device to become ready.
cec85a37 1287 *
b089d0bf
MD
1288 * This can take more then a second, give it 3 seconds. If we
1289 * succeed give the device another 3ms after that.
1980eff3 1290 *
3209f581 1291 * NOTE: Port multipliers can do two things here. First they can
1980eff3
MD
1292 * return device-ready if a device is on target 0 and also
1293 * return the signature for that device. If there is no
1294 * device on target 0 then BSY/DRQ is never cleared and
1295 * it never comes ready.
cec85a37 1296 */
3209f581 1297 if (error == 0 &&
1980eff3
MD
1298 ahci_pwait_clr_to(ap, 3000, AHCI_PREG_TFD,
1299 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1300 /*
1301 * The device is bricked or its a port multiplier and will
1302 * not unbusy until we do the pmprobe CLO softreset sequence.
1303 */
3209f581
MD
1304 error = ahci_port_pmprobe(ap);
1305 if (error) {
1980eff3
MD
1306 kprintf("%s: Device will not come ready 0x%b\n",
1307 PORTNAME(ap),
1308 ahci_pread(ap, AHCI_PREG_TFD),
1309 AHCI_PFMT_TFD_STS);
1310 } else {
1311 ap->ap_type = ATA_PORT_T_PM;
1980eff3 1312 }
3209f581 1313 } else if (error == 0) {
1980eff3
MD
1314 /*
1315 * We generally will not get a port multiplier signature in
1316 * this case even if this is a port multiplier, because of
1317 * Intel's stupidity. We almost certainly got target 0
1318 * behind the PM, if there is a PM.
1319 *
1320 * Save the signature and probe for a PM. If we do not
1321 * find a PM then use the saved signature and return
1322 * success.
1323 */
1324 type = ahci_port_signature_detect(ap, NULL);
3209f581
MD
1325 error = ahci_port_pmprobe(ap);
1326 if (error) {
1980eff3 1327 ap->ap_type = type;
3209f581 1328 error = 0;
1980eff3
MD
1329 } else {
1330 ap->ap_type = ATA_PORT_T_PM;
3209f581 1331 kprintf("%s: Port multiplier detected\n",
1980eff3
MD
1332 PORTNAME(ap));
1333 }
1334 }
1335
1336 /*
1337 * hard-stop the port if we failed. This will set ap_probe
1338 * to FAILED.
1339 */
1980eff3 1340 ap->ap_flags &= ~AP_F_IN_RESET;
3209f581
MD
1341 if (error) {
1342 ahci_port_hardstop(ap);
1343 /* ap_probe set to failed */
1344 } else {
f4553de1
MD
1345 if (ap->ap_type == ATA_PORT_T_PM)
1346 ap->ap_probe = ATA_PROBE_GOOD;
1347 else
1348 ap->ap_probe = ATA_PROBE_NEED_SOFT_RESET;
3209f581
MD
1349 }
1350 return (error);
1980eff3
MD
1351}
1352
1353/*
1354 * AHCI port multiplier probe. This routine is run by the hardreset code
1355 * if it gets past the device detect, whether or not BSY is found to be
1356 * stuck.
1357 *
1358 * We MUST use CLO to properly probe whether the port multiplier exists
1359 * or not.
1360 *
1361 * Return 0 on success, non-zero on failure.
1362 */
1363int
1364ahci_port_pmprobe(struct ahci_port *ap)
1365{
1366 struct ahci_cmd_hdr *cmd_slot;
1367 struct ahci_ccb *ccb = NULL;
1368 u_int8_t *fis = NULL;
831bc9e3 1369 int error = EIO;
1980eff3
MD
1370 u_int32_t cmd;
1371 int count;
1372
1373 /*
1374 * If we don't support port multipliers don't try to detect one.
1375 */
1376 if ((ap->ap_sc->sc_cap & AHCI_REG_CAP_SPM) == 0)
1377 return (ENODEV);
1378
1379 count = 2;
831bc9e3 1380#if 1
1980eff3
MD
1381 kprintf("%s: START PMPROBE\n", PORTNAME(ap));
1382#endif
1383retry:
1384 /*
1385 * This code is only called from hardreset, which does not
1386 * high level command processing. The port should be stopped.
1387 *
1388 * Set PMA mode while the port is stopped.
1389 *
1390 * NOTE: On retry the port might be running, stopped, or failed.
1391 */
1392 ahci_port_stop(ap, 0);
1393 ap->ap_state = AP_S_NORMAL;
1394 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
831bc9e3
MD
1395 if ((cmd & AHCI_PREG_CMD_PMA) == 0) {
1396 cmd |= AHCI_PREG_CMD_PMA;
1397 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1398 }
1980eff3
MD
1399
1400 /*
1401 * Flush any errors and request CLO unconditionally, then start
1402 * the port.
1403 */
1404 ahci_flush_tfd(ap);
1405 ahci_port_clo(ap);
1406 if (ahci_port_start(ap)) {
1407 kprintf("%s: PMPROBE failed to start port, cannot softreset\n",
1408 PORTNAME(ap));
1409 goto err;
1410 }
1411
1412 /*
1413 * Check whether CLO worked
1414 */
1415 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1416 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1417 kprintf("%s: PMPROBE CLO %s, need port reset\n",
258223a3 1418 PORTNAME(ap),
1980eff3
MD
1419 (ahci_read(ap->ap_sc, AHCI_REG_CAP) & AHCI_REG_CAP_SCLO)
1420 ? "failed" : "unsupported");
831bc9e3 1421 error = EBUSY;
258223a3
MD
1422 goto err;
1423 }
1424
1980eff3
MD
1425 /*
1426 * Prep the first H2D command with SRST feature & clear busy/reset
1427 * flags.
1428 */
1429 ccb = ahci_get_err_ccb(ap);
1430 cmd_slot = ccb->ccb_cmd_hdr;
1431
1432 fis = ccb->ccb_cmd_table->cfis;
1433 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1434 fis[0] = ATA_FIS_TYPE_H2D;
1435 fis[1] = 0x0F; /* Target 15 */
1436 fis[15] = ATA_FIS_CONTROL_SRST | ATA_FIS_CONTROL_4BIT;
1437
1438 cmd_slot->prdtl = 0;
1439 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1440 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_C); /* Clear busy on OK */
1441 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_R); /* Reset */
1442 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
1443
1444 ccb->ccb_xa.state = ATA_S_PENDING;
1445 ccb->ccb_xa.flags = 0;
1446
831bc9e3 1447 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != ATA_S_COMPLETE) {
1980eff3
MD
1448 kprintf("%s: PMPROBE First FIS failed\n", PORTNAME(ap));
1449 if (--count) {
1980eff3
MD
1450 ahci_put_err_ccb(ccb);
1451 goto retry;
1452 }
1453 goto err;
1454 }
1455 if (ahci_pwait_clr(ap, AHCI_PREG_TFD,
1456 AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1457 kprintf("%s: PMPROBE Busy after first FIS\n", PORTNAME(ap));
1458 }
1459
1460 /*
1461 * The device may have muffed up the PHY when it reset.
1462 */
831bc9e3 1463 ahci_os_sleep(100);
1980eff3
MD
1464 ahci_flush_tfd(ap);
1465 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
1466 /* ahci_pm_phy_status(ap, 15, &cmd); */
1467
1468 /*
1469 * Prep second D2H command to read status and complete reset sequence
1470 * AHCI 10.4.1 and "Serial ATA Revision 2.6". I can't find the ATA
1471 * Rev 2.6 and it is unclear how the second FIS should be set up
1472 * from the AHCI document.
1473 *
1474 * Give the device 3ms before sending the second FIS.
1475 *
1476 * It is unclear which other fields in the FIS are used. Just zero
1477 * everything.
1478 */
1479 bzero(fis, sizeof(ccb->ccb_cmd_table->cfis));
1480 fis[0] = ATA_FIS_TYPE_H2D;
1481 fis[1] = 0x0F;
1482 fis[15] = ATA_FIS_CONTROL_4BIT;
1483
1484 cmd_slot->prdtl = 0;
1485 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
1486 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_PMP); /* port 0xF */
1487
1488 ccb->ccb_xa.state = ATA_S_PENDING;
1489 ccb->ccb_xa.flags = 0;
1490
831bc9e3 1491 if (ahci_poll(ccb, 5000, ahci_quick_timeout) != ATA_S_COMPLETE) {
1980eff3
MD
1492 kprintf("%s: PMPROBE Second FIS failed\n", PORTNAME(ap));
1493 if (--count) {
1980eff3
MD
1494 ahci_put_err_ccb(ccb);
1495 goto retry;
1496 }
1497 goto err;
1498 }
1499
1500 /*
1501 * What? We succeeded? Yup, but for some reason the signature
1502 * is still latched from the original detect (that saw target 0
1503 * behind the PM), and I don't know how to clear the condition
1504 * other then by retrying the whole reset sequence.
1505 */
1506 if (--count) {
1507 fis[15] = 0;
1508 ahci_put_err_ccb(ccb);
1509 goto retry;
1510 }
1511
1512 /*
1513 * Get the signature. The caller sets the ap fields.
1514 */
1515 if (ahci_port_signature_detect(ap, NULL) == ATA_PORT_T_PM) {
1516 ap->ap_ata[15].at_probe = ATA_PROBE_GOOD;
831bc9e3 1517 error = 0;
1980eff3 1518 } else {
831bc9e3 1519 error = EBUSY;
1980eff3 1520 }
cf5f3a81
MD
1521
1522 /*
1980eff3 1523 * Fall through / clean up the CCB and perform error processing.
cf5f3a81 1524 */
258223a3 1525err:
831bc9e3 1526 if (ccb != NULL)
1980eff3 1527 ahci_put_err_ccb(ccb);
1980eff3 1528
831bc9e3 1529 if (error == 0 && ahci_pm_identify(ap)) {
3209f581
MD
1530 kprintf("%s: PM - cannot identify port multiplier\n",
1531 PORTNAME(ap));
831bc9e3 1532 error = EBUSY;
3209f581
MD
1533 }
1534#if 0
831bc9e3 1535 if (error == 0 && ahci_pm_set_feature(ap, ATA_SATAFT_ASYNCNOTIFY, 1)) {
3209f581
MD
1536 kprintf("%s: PM - Warning, cannot enable async notify\n",
1537 PORTNAME(ap));
1538 /* ignore error */
1539 }
831bc9e3 1540 if (error == 0) {
3209f581
MD
1541 u_int32_t data;
1542 if (ahci_pm_read(ap, 2, 4, &data))
1543 kprintf("Cannot read snotify\n");
1544 else
1545 kprintf("Read snotify %08x\n", data);
1546 }
1547#endif
1548
1980eff3
MD
1549 /*
1550 * If we failed turn off PMA, otherwise identify the port multiplier.
1551 * CAM will iterate the devices.
1552 */
831bc9e3 1553 if (error) {
1980eff3
MD
1554 ahci_port_stop(ap, 0);
1555 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
1556 cmd &= ~AHCI_PREG_CMD_PMA;
1557 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1980eff3
MD
1558 }
1559 ahci_port_stop(ap, 0);
1560
831bc9e3 1561 return(error);
cf5f3a81
MD
1562}
1563
1980eff3 1564
cf5f3a81
MD
1565/*
1566 * Hard-stop on hot-swap device removal. See 10.10.1
1567 *
1568 * Place the port in a mode that will allow it to detect hot-swap insertions.
1569 * This is a bit imprecise because just setting-up SCTL to DET_INIT doesn't
1570 * seem to do the job.
1571 */
1572void
1573ahci_port_hardstop(struct ahci_port *ap)
1574{
1980eff3 1575 struct ata_port *at;
cf5f3a81
MD
1576 u_int32_t r;
1577 u_int32_t cmd;
1980eff3 1578 int i;
cf5f3a81
MD
1579
1580 /*
1581 * Stop the port. We can't modify things like SUD if the port
1582 * is running.
1583 */
1584 ap->ap_state = AP_S_FATAL_ERROR;
1980eff3
MD
1585 ap->ap_probe = ATA_PROBE_FAILED;
1586 ap->ap_type = ATA_PORT_T_NONE;
cf5f3a81
MD
1587 ahci_port_stop(ap, 0);
1588 cmd = ahci_pread(ap, AHCI_PREG_CMD);
1589
1590 /*
1980eff3
MD
1591 * Clean up AT sub-ports on SATA port.
1592 */
1593 for (i = 0; ap->ap_ata && i < AHCI_MAX_PMPORTS; ++i) {
1594 at = &ap->ap_ata[i];
1595 at->at_type = ATA_PORT_T_NONE;
3209f581 1596 at->at_probe = ATA_PROBE_FAILED;
1980eff3
MD
1597 }
1598
1599 /*
1600 * Turn off port-multiplier control bit
1601 */
1602 if (cmd & AHCI_PREG_CMD_PMA) {
1603 cmd &= ~AHCI_PREG_CMD_PMA;
1604 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1605 }
1606
1607 /*
cf5f3a81
MD
1608 * Make sure FRE is active. There isn't anything we can do if it
1609 * fails so just ignore errors.
1610 */
1611 if ((cmd & AHCI_PREG_CMD_FRE) == 0) {
1612 cmd |= AHCI_PREG_CMD_FRE;
1613 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1614 if ((ap->ap_sc->sc_flags & AHCI_F_IGN_FR) == 0)
1615 ahci_pwait_set(ap, AHCI_PREG_CMD, AHCI_PREG_CMD_FR);
1616 }
1617
1618 /*
1619 * 10.10.3 DET must be set to 0 before setting SUD to 0.
1620 * 10.10.1 place us in the Listen state.
1621 *
1622 * Deactivating SUD only applies if the controller supports SUD.
1623 */
1624 ahci_pwrite(ap, AHCI_PREG_SCTL, AHCI_PREG_SCTL_IPM_DISABLED);
3209f581 1625 ahci_os_sleep(1);
cf5f3a81
MD
1626 if (cmd & AHCI_PREG_CMD_SUD) {
1627 cmd &= ~AHCI_PREG_CMD_SUD;
1628 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
1629 }
3209f581 1630 ahci_os_sleep(1);
cf5f3a81
MD
1631
1632 /*
1633 * Transition su to the spin-up state. HVA shall send COMRESET and
1634 * begin initialization sequence (whatever that means).
1635 *
1636 * This only applies if the controller supports SUD.
1637 */
1638 cmd |= AHCI_PREG_CMD_SUD;
258223a3 1639 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
3209f581 1640 ahci_os_sleep(1);
258223a3 1641
cf5f3a81
MD
1642 /*
1643 * Transition us to the Reset state. Theoretically we send a
1644 * continuous stream of COMRESETs in this state.
1645 */
1646 r = AHCI_PREG_SCTL_IPM_DISABLED | AHCI_PREG_SCTL_DET_INIT;
1647 if (AhciForceGen1 & (1 << ap->ap_num)) {
1648 kprintf("%s: Force 1.5Gbits\n", PORTNAME(ap));
1649 r |= AHCI_PREG_SCTL_SPD_GEN1;
1650 } else {
1651 r |= AHCI_PREG_SCTL_SPD_ANY;
1652 }
1653 ahci_pwrite(ap, AHCI_PREG_SCTL, r);
3209f581 1654 ahci_os_sleep(1);
cf5f3a81
MD
1655
1656 /*
1657 * Flush SERR_DIAG_X so the TFD can update.
1658 */
1659 ahci_flush_tfd(ap);
1660
1661 /*
1662 * Leave us in COMRESET (both SUD and INIT active), the HBA should
1663 * hopefully send us a DIAG_X-related interrupt if it receives
1664 * a COMINIT, and if not that then at least a Phy transition
1665 * interrupt.
1666 *
1667 * If we transition INIT from 1->0 to begin the initalization
1668 * sequence it is unclear if that sequence will remain active
1669 * until the next device insertion.
1670 *
1671 * If we go back to the listen state it is unclear if the
1672 * device will actually send us a COMINIT, since we aren't
1673 * sending any COMRESET's
1674 */
1675 /* NOP */
1676}
1677
1678/*
1679 * Multiple events may have built up in the TFD. The spec is not very
1680 * clear on this but it does seem to serialize events so clearing DIAG_X
1681 * just once might not do the job during a reset sequence.
831bc9e3
MD
1682 *
1683 * XXX this probably isn't right.
cf5f3a81
MD
1684 */
1685void
1686ahci_flush_tfd(struct ahci_port *ap)
1687{
1688 u_int32_t r;
1689
1690 r = ahci_pread(ap, AHCI_PREG_SERR);
1980eff3
MD
1691 while (r & AHCI_PREG_SERR_DIAG_X) {
1692 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_X);
3209f581 1693 ahci_os_sleep(1);
cf5f3a81
MD
1694 r = ahci_pread(ap, AHCI_PREG_SERR);
1695 }
258223a3
MD
1696}
1697
fd8bd957
MD
1698/*
1699 * Figure out what type of device is connected to the port, ATAPI or
1700 * DISK.
1701 */
1702int
1980eff3 1703ahci_port_signature_detect(struct ahci_port *ap, struct ata_port *at)
fd8bd957
MD
1704{
1705 u_int32_t sig;
1706
1707 sig = ahci_pread(ap, AHCI_PREG_SIG);
1980eff3 1708 kprintf("%s: sig %08x\n", ATANAME(ap, at), sig);
fd8bd957
MD
1709 if ((sig & 0xffff0000) == (SATA_SIGNATURE_ATAPI & 0xffff0000)) {
1710 return(ATA_PORT_T_ATAPI);
1980eff3
MD
1711 } else if ((sig & 0xffff0000) ==
1712 (SATA_SIGNATURE_PORT_MULTIPLIER & 0xffff0000)) {
1980eff3 1713 return(ATA_PORT_T_PM);
fd8bd957
MD
1714 } else {
1715 return(ATA_PORT_T_DISK);
1716 }
1717}
1718
1719/*
1720 * Load the DMA descriptor table for a CCB's buffer.
1721 */
258223a3
MD
1722int
1723ahci_load_prdt(struct ahci_ccb *ccb)
1724{
1725 struct ahci_port *ap = ccb->ccb_port;
1726 struct ahci_softc *sc = ap->ap_sc;
1727 struct ata_xfer *xa = &ccb->ccb_xa;
1728 struct ahci_prdt *prdt = ccb->ccb_cmd_table->prdt;
1729 bus_dmamap_t dmap = ccb->ccb_dmamap;
1730 struct ahci_cmd_hdr *cmd_slot = ccb->ccb_cmd_hdr;
1731 int error;
1732
1733 if (xa->datalen == 0) {
1734 ccb->ccb_cmd_hdr->prdtl = 0;
1735 return (0);
1736 }
1737
1738 error = bus_dmamap_load(sc->sc_tag_data, dmap,
1739 xa->data, xa->datalen,
1740 ahci_load_prdt_callback,
1741 &prdt,
1742 ((xa->flags & ATA_F_NOWAIT) ?
1743 BUS_DMA_NOWAIT : BUS_DMA_WAITOK));
1744 if (error != 0) {
1745 kprintf("%s: error %d loading dmamap\n", PORTNAME(ap), error);
1746 return (1);
1747 }
1748 if (xa->flags & ATA_F_PIO)
1749 prdt->flags |= htole32(AHCI_PRDT_FLAG_INTR);
1750
1751 cmd_slot->prdtl = htole16(prdt - ccb->ccb_cmd_table->prdt + 1);
1752
1753 bus_dmamap_sync(sc->sc_tag_data, dmap,
1754 (xa->flags & ATA_F_READ) ?
1755 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
1756
1757 return (0);
1758
1759#ifdef DIAGNOSTIC
1760diagerr:
1761 bus_dmamap_unload(sc->sc_tag_data, dmap);
1762 return (1);
1763#endif
1764}
1765
1766/*
1767 * Callback from BUSDMA system to load the segment list. The passed segment
1768 * list is a temporary structure.
1769 */
1770static
1771void
1772ahci_load_prdt_callback(void *info, bus_dma_segment_t *segs, int nsegs,
1773 int error)
1774{
1775 struct ahci_prdt *prd = *(void **)info;
1776 u_int64_t addr;
1777
1778 KKASSERT(nsegs <= AHCI_MAX_PRDT);
1779
1780 while (nsegs) {
1781 addr = segs->ds_addr;
1782 prd->dba_hi = htole32((u_int32_t)(addr >> 32));
1783 prd->dba_lo = htole32((u_int32_t)addr);
1784#ifdef DIAGNOSTIC
1785 KKASSERT((addr & 1) == 0);
1786 KKASSERT((segs->ds_len & 1) == 0);
1787#endif
1788 prd->flags = htole32(segs->ds_len - 1);
1789 --nsegs;
1790 if (nsegs)
1791 ++prd;
1792 ++segs;
1793 }
1794 *(void **)info = prd; /* return last valid segment */
1795}
1796
1797void
1798ahci_unload_prdt(struct ahci_ccb *ccb)
1799{
1800 struct ahci_port *ap = ccb->ccb_port;
1801 struct ahci_softc *sc = ap->ap_sc;
1802 struct ata_xfer *xa = &ccb->ccb_xa;
1803 bus_dmamap_t dmap = ccb->ccb_dmamap;
1804
1805 if (xa->datalen != 0) {
1806 bus_dmamap_sync(sc->sc_tag_data, dmap,
1807 (xa->flags & ATA_F_READ) ?
1808 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1809
1810 bus_dmamap_unload(sc->sc_tag_data, dmap);
1811
1812 if (ccb->ccb_xa.flags & ATA_F_NCQ)
1813 xa->resid = 0;
1814 else
1815 xa->resid = xa->datalen -
1816 le32toh(ccb->ccb_cmd_hdr->prdbc);
1817 }
1818}
1819
5f8c1efd
MD
1820/*
1821 * Start a command and poll for completion.
1822 *
3209f581
MD
1823 * timeout is in ms and only counts once the command gets on-chip.
1824 *
831bc9e3
MD
1825 * Returns ATA_S_* state, compare against ATA_S_COMPLETE to determine
1826 * that no error occured.
1827 *
5f8c1efd
MD
1828 * NOTE: If the caller specifies a NULL timeout function the caller is
1829 * responsible for clearing hardware state on failure, but we will
1830 * deal with removing the ccb from any pending queue.
1831 *
1832 * NOTE: NCQ should never be used with this function.
cf5f3a81
MD
1833 *
1834 * NOTE: If the port is in a failed state and stopped we do not try
1835 * to activate the ccb.
5f8c1efd 1836 */
258223a3 1837int
831bc9e3
MD
1838ahci_poll(struct ahci_ccb *ccb, int timeout,
1839 void (*timeout_fn)(struct ahci_ccb *))
258223a3 1840{
5f8c1efd 1841 struct ahci_port *ap = ccb->ccb_port;
258223a3 1842
cf5f3a81
MD
1843 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR) {
1844 ccb->ccb_xa.state = ATA_S_ERROR;
831bc9e3 1845 return(ccb->ccb_xa.state);
cf5f3a81 1846 }
258223a3
MD
1847 crit_enter();
1848 ahci_start(ccb);
1980eff3 1849
258223a3 1850 do {
f4553de1 1851 ahci_port_intr(ap, 1);
831bc9e3
MD
1852 switch(ccb->ccb_xa.state) {
1853 case ATA_S_ONCHIP:
1854 timeout -= ahci_os_softsleep();
1855 break;
1856 case ATA_S_PENDING:
1857 ahci_os_softsleep();
1858 ahci_check_active_timeouts(ap);
f4553de1 1859 break;
831bc9e3
MD
1860 default:
1861 crit_exit();
1862 return (ccb->ccb_xa.state);
f4553de1 1863 }
3209f581 1864 } while (timeout > 0);
5f8c1efd 1865
831bc9e3
MD
1866 kprintf("%s: Poll timeout slot %d CMD: %b TFD: 0x%b SERR: %b\n",
1867 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_slot,
1868 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD,
1869 ahci_pread(ap, AHCI_PREG_TFD), AHCI_PFMT_TFD_STS,
1870 ahci_pread(ap, AHCI_PREG_SERR), AHCI_PFMT_SERR);
1871
1872 timeout_fn(ccb);
5f8c1efd 1873
258223a3
MD
1874 crit_exit();
1875
831bc9e3
MD
1876 return(ccb->ccb_xa.state);
1877}
1878
1879/*
1880 * When polling we have to check if the currently active CCB(s)
1881 * have timed out as the callout will be deadlocked while we
1882 * hold the port lock.
1883 */
1884void
1885ahci_check_active_timeouts(struct ahci_port *ap)
1886{
1887 struct ahci_ccb *ccb;
1888 u_int32_t mask;
1889 int tag;
1890
1891 mask = ap->ap_active | ap->ap_sactive;
1892 while (mask) {
1893 tag = ffs(mask) - 1;
1894 mask &= ~(1 << tag);
1895 ccb = &ap->ap_ccbs[tag];
1896 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_EXPIRED) {
1897 ahci_ata_cmd_timeout(ccb);
1898 }
1899 }
258223a3
MD
1900}
1901
3209f581
MD
1902static
1903__inline
1904void
1905ahci_start_timeout(struct ahci_ccb *ccb)
1906{
1907 if (ccb->ccb_xa.flags & ATA_F_TIMEOUT_DESIRED) {
1908 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_RUNNING;
1909 callout_reset(&ccb->ccb_timeout,
1910 (ccb->ccb_xa.timeout * hz + 999) / 1000,
1911 ahci_ata_cmd_timeout_unserialized, ccb);
1912 }
1913}
1914
258223a3
MD
1915void
1916ahci_start(struct ahci_ccb *ccb)
1917{
1918 struct ahci_port *ap = ccb->ccb_port;
1919 struct ahci_softc *sc = ap->ap_sc;
1920
1921 KKASSERT(ccb->ccb_xa.state == ATA_S_PENDING);
1922
1923 /* Zero transferred byte count before transfer */
1924 ccb->ccb_cmd_hdr->prdbc = 0;
1925
1926 /* Sync command list entry and corresponding command table entry */
1927 bus_dmamap_sync(sc->sc_tag_cmdh,
1928 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
1929 BUS_DMASYNC_PREWRITE);
1930 bus_dmamap_sync(sc->sc_tag_cmdt,
1931 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
1932 BUS_DMASYNC_PREWRITE);
1933
1934 /* Prepare RFIS area for write by controller */
1935 bus_dmamap_sync(sc->sc_tag_rfis,
1936 AHCI_DMA_MAP(ap->ap_dmamem_rfis),
1937 BUS_DMASYNC_PREREAD);
1938
1939 if (ccb->ccb_xa.flags & ATA_F_NCQ) {
1980eff3
MD
1940 /*
1941 * Issue NCQ commands only when there are no outstanding
1942 * standard commands.
1943 */
1944 if (ap->ap_active || TAILQ_FIRST(&ap->ap_ccb_pending)) {
258223a3 1945 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1980eff3 1946 } else {
3209f581 1947 ahci_start_timeout(ccb);
258223a3
MD
1948 KKASSERT(ap->ap_active_cnt == 0);
1949 ap->ap_sactive |= (1 << ccb->ccb_slot);
1950 ccb->ccb_xa.state = ATA_S_ONCHIP;
1951 ahci_pwrite(ap, AHCI_PREG_SACT, 1 << ccb->ccb_slot);
1952 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1953 }
1954 } else {
5f8c1efd
MD
1955 /*
1956 * Wait for all NCQ commands to finish before issuing standard
1980eff3
MD
1957 * command. Allow up to <limit> non-NCQ commands to be active.
1958 *
1959 * XXX If ap is a port multiplier only allow 1. At least the
1960 * NVidia-MCP77 part seems to barf if more then one
1961 * command is activated, even though it isn't NCQ.
1962 *
1963 * If I set up more then one I get phy errors and the
1964 * port fails.
5f8c1efd 1965 */
1980eff3
MD
1966 int limit = (ap->ap_type == ATA_PORT_T_PM) ? 1 : 2;
1967 if (ap->ap_sactive || ap->ap_active_cnt >= limit) {
258223a3 1968 TAILQ_INSERT_TAIL(&ap->ap_ccb_pending, ccb, ccb_entry);
1980eff3 1969 } else {
3209f581 1970 ahci_start_timeout(ccb);
258223a3
MD
1971 ap->ap_active |= 1 << ccb->ccb_slot;
1972 ccb->ccb_xa.state = ATA_S_ONCHIP;
1973 ahci_pwrite(ap, AHCI_PREG_CI, 1 << ccb->ccb_slot);
1974 ap->ap_active_cnt++;
1975 }
1976 }
1977}
1978
831bc9e3
MD
1979/*
1980 * While holding the port lock acquire exclusive access to the port.
1981 *
1982 * This is used when running the state machine to initialize and identify
1983 * targets over a port multiplier. Setting exclusive access prevents
1984 * ahci_port_intr() from activating any requests sitting on the pending
1985 * queue.
1986 */
1987void
1988ahci_beg_exclusive_access(struct ahci_port *ap, struct ata_port *at)
1989{
1990 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) == 0);
1991 ap->ap_flags |= AP_F_EXCLUSIVE_ACCESS;
1992 while (ap->ap_active || ap->ap_sactive) {
1993 ahci_port_intr(ap, 1);
1994 ahci_os_softsleep();
1995 }
1996}
1997
1998void
1999ahci_end_exclusive_access(struct ahci_port *ap, struct ata_port *at)
2000{
2001 KKASSERT((ap->ap_flags & AP_F_EXCLUSIVE_ACCESS) != 0);
2002 ap->ap_flags &= ~AP_F_EXCLUSIVE_ACCESS;
2003 if (ap->ap_active == 0 && ap->ap_sactive == 0)
2004 ahci_issue_pending_commands(ap, 0);
2005}
2006
258223a3
MD
2007void
2008ahci_issue_pending_ncq_commands(struct ahci_port *ap)
2009{
2010 struct ahci_ccb *nextccb;
2011 u_int32_t sact_change = 0;
2012
2013 KKASSERT(ap->ap_active_cnt == 0);
2014
2015 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
2016 if (nextccb == NULL || !(nextccb->ccb_xa.flags & ATA_F_NCQ))
2017 return;
831bc9e3
MD
2018 if (ap->ap_flags & AP_F_EXCLUSIVE_ACCESS)
2019 return;
258223a3
MD
2020
2021 /* Start all the NCQ commands at the head of the pending list. */
2022 do {
2023 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
3209f581 2024 ahci_start_timeout(nextccb);
258223a3
MD
2025 sact_change |= 1 << nextccb->ccb_slot;
2026 nextccb->ccb_xa.state = ATA_S_ONCHIP;
2027 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
2028 } while (nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ));
2029
2030 ap->ap_sactive |= sact_change;
2031 ahci_pwrite(ap, AHCI_PREG_SACT, sact_change);
2032 ahci_pwrite(ap, AHCI_PREG_CI, sact_change);
2033
2034 return;
2035}
2036
2037void
2038ahci_issue_pending_commands(struct ahci_port *ap, int last_was_ncq)
2039{
2040 struct ahci_ccb *nextccb;
2041
2042 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
831bc9e3
MD
2043 if (nextccb == NULL)
2044 return;
2045 if (ap->ap_flags & AP_F_EXCLUSIVE_ACCESS)
2046 return;
2047
2048 if (nextccb->ccb_xa.flags & ATA_F_NCQ) {
258223a3
MD
2049 KKASSERT(last_was_ncq == 0); /* otherwise it should have
2050 * been started already. */
2051
1980eff3
MD
2052 /*
2053 * Issue NCQ commands only when there are no outstanding
2054 * standard commands.
2055 */
258223a3
MD
2056 if (ap->ap_active == 0)
2057 ahci_issue_pending_ncq_commands(ap);
2058 else
1980eff3 2059 KKASSERT(ap->ap_active_cnt > 0);
831bc9e3 2060 } else {
1980eff3 2061 if (ap->ap_sactive || last_was_ncq)
258223a3
MD
2062 KKASSERT(ap->ap_active_cnt == 0);
2063
1980eff3
MD
2064 /*
2065 * Wait for all NCQ commands to finish before issuing standard
2066 * command. Then keep up to 2 standard commands on-chip at
2067 * a time.
2068 */
2069 if (ap->ap_sactive)
258223a3
MD
2070 return;
2071
1980eff3
MD
2072 while (ap->ap_active_cnt < 2 &&
2073 nextccb && (nextccb->ccb_xa.flags & ATA_F_NCQ) == 0) {
258223a3 2074 TAILQ_REMOVE(&ap->ap_ccb_pending, nextccb, ccb_entry);
3209f581 2075 ahci_start_timeout(nextccb);
258223a3
MD
2076 ap->ap_active |= 1 << nextccb->ccb_slot;
2077 nextccb->ccb_xa.state = ATA_S_ONCHIP;
2078 ahci_pwrite(ap, AHCI_PREG_CI, 1 << nextccb->ccb_slot);
1980eff3 2079 ap->ap_active_cnt++;
258223a3 2080 nextccb = TAILQ_FIRST(&ap->ap_ccb_pending);
1980eff3 2081 }
258223a3
MD
2082 }
2083}
2084
2085void
2086ahci_intr(void *arg)
2087{
f4553de1
MD
2088 struct ahci_softc *sc = arg;
2089 struct ahci_port *ap;
2090 u_int32_t is, ack = 0;
2091 int port;
258223a3 2092
f4553de1
MD
2093 /*
2094 * Check if the master enable is up, and whether any interrupts are
2095 * pending.
2096 */
2097 if ((sc->sc_flags & AHCI_F_INT_GOOD) == 0)
2098 return;
258223a3
MD
2099 is = ahci_read(sc, AHCI_REG_IS);
2100 if (is == 0 || is == 0xffffffff)
2101 return;
2102 ack = is;
2103
2104#ifdef AHCI_COALESCE
2105 /* Check coalescing interrupt first */
2106 if (is & sc->sc_ccc_mask) {
2107 DPRINTF(AHCI_D_INTR, "%s: command coalescing interrupt\n",
2108 DEVNAME(sc));
2109 is &= ~sc->sc_ccc_mask;
2110 is |= sc->sc_ccc_ports_cur;
2111 }
2112#endif
2113
f4553de1
MD
2114 /*
2115 * Process interrupts for each port in a non-blocking fashion.
2116 */
258223a3
MD
2117 while (is) {
2118 port = ffs(is) - 1;
f4553de1
MD
2119 ap = sc->sc_ports[port];
2120 if (ap) {
2121 if (ahci_os_lock_port_nb(ap) == 0) {
2122 ahci_port_intr(ap, 0);
2123 ahci_os_unlock_port(ap);
2124 } else {
2125 ahci_pwrite(ap, AHCI_PREG_IE, 0);
2126 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2127 }
2128 }
258223a3
MD
2129 is &= ~(1 << port);
2130 }
2131
2132 /* Finally, acknowledge global interrupt */
2133 ahci_write(sc, AHCI_REG_IS, ack);
2134}
2135
f4553de1
MD
2136/*
2137 * Core called from helper thread.
2138 */
3209f581 2139void
f4553de1 2140ahci_port_thread_core(struct ahci_port *ap, int mask)
258223a3 2141{
f4553de1
MD
2142 /*
2143 * Process any expired timedouts.
2144 */
2145 ahci_os_lock_port(ap);
2146 if (mask & AP_SIGF_TIMEOUT) {
831bc9e3 2147 ahci_check_active_timeouts(ap);
f4553de1
MD
2148 }
2149
2150 /*
2151 * Process port interrupts which require a higher level of
2152 * intervention.
2153 */
2154 if (mask & AP_SIGF_PORTINT) {
2155 ahci_port_intr(ap, 1);
f4553de1 2156 ahci_port_interrupt_enable(ap);
831bc9e3 2157 ahci_os_unlock_port(ap);
f4553de1
MD
2158 } else {
2159 ahci_os_unlock_port(ap);
2160 }
2161}
2162
2163/*
2164 * Core per-port interrupt handler.
2165 *
2166 * If blockable is 0 we cannot call ahci_os_sleep() at all and we can only
2167 * deal with normal command completions which do not require blocking.
2168 */
2169void
2170ahci_port_intr(struct ahci_port *ap, int blockable)
2171{
2172 struct ahci_softc *sc = ap->ap_sc;
2173 u_int32_t is, ci_saved, ci_masked;
2174 int slot;
2175 struct ahci_ccb *ccb = NULL;
2176 struct ata_port *ccb_at = NULL;
2177 volatile u_int32_t *active;
258223a3 2178#ifdef DIAGNOSTIC
f4553de1 2179 u_int32_t tmp;
258223a3 2180#endif
f4553de1
MD
2181 const u_int32_t blockable_mask = AHCI_PREG_IS_TFES |
2182 AHCI_PREG_IS_IFS |
2183 AHCI_PREG_IS_PCS |
2184 AHCI_PREG_IS_PRCS |
2185 AHCI_PREG_IS_HBFS |
2186 AHCI_PREG_IS_OFS |
2187 AHCI_PREG_IS_UFS;
2188
22181ab7
MD
2189 enum { NEED_NOTHING, NEED_RESTART, NEED_HOTPLUG_INSERT,
2190 NEED_HOTPLUG_REMOVE } need = NEED_NOTHING;
258223a3
MD
2191
2192 is = ahci_pread(ap, AHCI_PREG_IS);
f4553de1
MD
2193
2194 /*
2195 * All basic command completions are always processed.
2196 */
cec07d75
MD
2197 if (is & AHCI_PREG_IS_DPS)
2198 ahci_pwrite(ap, AHCI_PREG_IS, is & AHCI_PREG_IS_DPS);
258223a3 2199
f4553de1
MD
2200 /*
2201 * If we can't block then we can't handle these here. Disable
2202 * the interrupts in question so we don't live-lock, the helper
2203 * thread will re-enable them.
2204 *
2205 * If the port is in a completely failed state we do not want
dbef6246 2206 * to drop through to failed-command-processing if blockable is 0,
f4553de1 2207 * just let the thread deal with it all.
dbef6246
MD
2208 *
2209 * Otherwise we fall through and still handle DHRS and any commands
2210 * which completed normally. Even if we are errored we haven't
2211 * stopped the port yet so CI/SACT are still good.
f4553de1
MD
2212 */
2213 if (blockable == 0) {
2214 if (ap->ap_state == AP_S_FATAL_ERROR) {
2215 ahci_pwrite(ap, AHCI_PREG_IE,
2216 ahci_pread(ap, AHCI_PREG_IE) & ~is);
2217 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2218 return;
2219 }
2220 if (is & blockable_mask) {
f4553de1 2221 ahci_pwrite(ap, AHCI_PREG_IE,
dbef6246
MD
2222 ahci_pread(ap, AHCI_PREG_IE) & ~blockable_mask);
2223 is &= ~blockable_mask;
f4553de1
MD
2224 ahci_os_signal_port_thread(ap, AP_SIGF_PORTINT);
2225 }
2226 }
2227
1980eff3
MD
2228#if 0
2229 kprintf("%s: INTERRUPT %b\n", PORTNAME(ap),
2230 is, AHCI_PFMT_IS);
2231#endif
2232
3209f581 2233 /*
f4553de1 2234 * Either NCQ or non-NCQ commands will be active, never both.
3209f581 2235 */
258223a3 2236 if (ap->ap_sactive) {
258223a3
MD
2237 KKASSERT(ap->ap_active == 0);
2238 KKASSERT(ap->ap_active_cnt == 0);
2239 ci_saved = ahci_pread(ap, AHCI_PREG_SACT);
2240 active = &ap->ap_sactive;
2241 } else {
258223a3
MD
2242 ci_saved = ahci_pread(ap, AHCI_PREG_CI);
2243 active = &ap->ap_active;
2244 }
2245
258223a3 2246 if (is & AHCI_PREG_IS_TFES) {
1980eff3 2247 /*
f4553de1
MD
2248 * Command failed (blockable).
2249 *
2250 * See AHCI 1.1 spec 6.2.2.1 and 6.2.2.2.
1980eff3
MD
2251 *
2252 * This stops command processing.
2253 */
2254 u_int32_t tfd, serr;
2255 int err_slot;
258223a3
MD
2256
2257 tfd = ahci_pread(ap, AHCI_PREG_TFD);
2258 serr = ahci_pread(ap, AHCI_PREG_SERR);
2259
cf5f3a81
MD
2260 /*
2261 * If no NCQ commands are active the error slot is easily
2262 * determined, otherwise we have to extract the error
2263 * from the log page.
2264 */
258223a3 2265 if (ap->ap_sactive == 0) {
cf5f3a81
MD
2266 err_slot = AHCI_PREG_CMD_CCS(
2267 ahci_pread(ap, AHCI_PREG_CMD));
258223a3 2268 ccb = &ap->ap_ccbs[err_slot];
1980eff3 2269 ccb_at = ccb->ccb_xa.at; /* can be NULL */
258223a3
MD
2270
2271 /* Preserve received taskfile data from the RFIS. */
2272 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
cf5f3a81
MD
2273 sizeof(struct ata_fis_d2h));
2274 } else {
2275 err_slot = -1;
2276 }
258223a3 2277
1980eff3
MD
2278 DPRINTF(AHCI_D_VERBOSE, "%s: errd slot %d, TFD: %b, SERR: %b\n",
2279 PORTNAME(ap), err_slot,
2280 tfd, AHCI_PFMT_TFD_STS,
2281 serr, AHCI_PFMT_SERR);
258223a3 2282
cf5f3a81 2283 /* Stopping the port clears CI and SACT */
258223a3 2284 ahci_port_stop(ap, 0);
22181ab7 2285 need = NEED_RESTART;
258223a3 2286
cf5f3a81
MD
2287 /*
2288 * Clear SERR (primarily DIAG_X) to enable capturing of the
2289 * next error.
2290 */
258223a3
MD
2291 ahci_pwrite(ap, AHCI_PREG_SERR, serr);
2292
2293 /* Acknowledge the interrupts we can recover from. */
cf5f3a81 2294 ahci_pwrite(ap, AHCI_PREG_IS,
cec07d75 2295 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_IFS));
1980eff3 2296 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_IFS);
258223a3
MD
2297
2298 /* If device hasn't cleared its busy status, try to idle it. */
2299 if (tfd & (AHCI_PREG_TFD_STS_BSY | AHCI_PREG_TFD_STS_DRQ)) {
1980eff3
MD
2300 kprintf("%s: Attempting to idle device ccb=%p\n",
2301 PORTNAME(ap), ccb_at);
2302 if (ap->ap_flags & AP_F_IN_RESET)
2303 goto fatal;
2304 /*
2305 * XXX how do we unbrick a PM target (ccb_at != NULL).
2306 *
2307 * For now fail the target and use CLO to clear the
2308 * busy condition and make the ahci port usable for
2309 * the remaining devices.
2310 */
2311 if (ccb_at) {
2312 ccb_at->at_probe = ATA_PROBE_FAILED;
2313 ahci_port_clo(ap);
2314 } else if (ahci_port_reset(ap, ccb_at, 0)) {
17eab71e
MD
2315 kprintf("%s: Unable to idle device, port "
2316 "bricked on us\n",
258223a3 2317 PORTNAME(ap));
17eab71e 2318 goto fatal;
258223a3
MD
2319 }
2320
2321 /* Had to reset device, can't gather extended info. */
2322 } else if (ap->ap_sactive) {
1980eff3
MD
2323 /*
2324 * Recover the NCQ error from log page 10h.
2325 *
2326 * XXX NCQ currently not supported with port
2327 * multiplier.
2328 */
258223a3 2329 ahci_port_read_ncq_error(ap, &err_slot);
cec07d75
MD
2330 kprintf("recover from NCQ error err_slot %d\n",
2331 err_slot);
258223a3
MD
2332 if (err_slot < 0)
2333 goto failall;
2334
2335 DPRINTF(AHCI_D_VERBOSE, "%s: NCQ errored slot %d\n",
2336 PORTNAME(ap), err_slot);
2337
2338 ccb = &ap->ap_ccbs[err_slot];
2339 } else {
2340 /* Didn't reset, could gather extended info from log. */
1980eff3
MD
2341 kprintf("%s: didn't reset err_slot %d "
2342 "sact=%08x act=%08x\n",
2343 PORTNAME(ap),
cf5f3a81 2344 err_slot, ap->ap_sactive, ap->ap_active);
258223a3
MD
2345 }
2346
2347 /*
2348 * If we couldn't determine the errored slot, reset the port
2349 * and fail all the active slots.
2350 */
2351 if (err_slot == -1) {
1980eff3
MD
2352 if (ap->ap_flags & AP_F_IN_RESET)
2353 goto fatal;
2354 /*
2355 * XXX how do we unbrick a PM target (ccb_at != NULL).
2356 *
2357 * For now fail the target and use CLO to clear the
2358 * busy condition and make the ahci port usable for
2359 * the remaining devices.
2360 */
2361 if (ccb_at) {
2362 ccb_at->at_probe = ATA_PROBE_FAILED;
2363 ahci_port_clo(ap);
2364 } else if (ahci_port_reset(ap, ccb_at, 0)) {
17eab71e
MD
2365 kprintf("%s: Unable to idle device after "
2366 "NCQ error, port bricked on us\n",
258223a3
MD
2367 PORTNAME(ap));
2368 goto fatal;
2369 }
2370 kprintf("%s: couldn't recover NCQ error, failing "
2371 "all outstanding commands.\n",
2372 PORTNAME(ap));
2373 goto failall;
2374 }
2375
2376 /* Clear the failed command in saved CI so completion runs. */
2377 ci_saved &= ~(1 << err_slot);
2378
2379 /* Note the error in the ata_xfer. */
2380 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2381 ccb->ccb_xa.state = ATA_S_ERROR;
2382
2383#ifdef DIAGNOSTIC
2384 /* There may only be one outstanding standard command now. */
2385 if (ap->ap_sactive == 0) {
2386 tmp = ci_saved;
2387 if (tmp) {
2388 slot = ffs(tmp) - 1;
2389 tmp &= ~(1 << slot);
2390 KKASSERT(tmp == 0);
2391 }
2392 }
2393#endif
1980eff3
MD
2394 } else if (is & AHCI_PREG_IS_DHRS) {
2395 /*
f4553de1
MD
2396 * Command posted D2H register FIS to the rfis (non-blocking).
2397 *
1980eff3 2398 * Command posted D2H register FIS to the rfis. This
8bf6a3ff
MD
2399 * does NOT stop command processing and it is unclear
2400 * how we are supposed to deal with it other then using
2401 * only a queue of 1.
2402 *
2403 * We must copy the port rfis to the ccb and restart
2404 * command processing. ahci_pm_read() does not function
2405 * without this support.
1980eff3
MD
2406 */
2407 int err_slot;
2408
2409 if (ap->ap_sactive == 0) {
2410 err_slot = AHCI_PREG_CMD_CCS(
2411 ahci_pread(ap, AHCI_PREG_CMD));
2412 ccb = &ap->ap_ccbs[err_slot];
2413 ccb_at = ccb->ccb_xa.at; /* can be NULL */
2414
2415 memcpy(&ccb->ccb_xa.rfis, ap->ap_rfis->rfis,
2416 sizeof(struct ata_fis_d2h));
2417 } else {
2418 kprintf("%s: Unexpected DHRS posted while "
2419 "NCQ running\n", PORTNAME(ap));
2420 err_slot = -1;
2421 }
cec07d75
MD
2422 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_DHRS);
2423 is &= ~AHCI_PREG_IS_DHRS;
1980eff3
MD
2424 }
2425
2426 /*
f4553de1 2427 * Device notification to us (non-blocking)
1980eff3 2428 *
cec07d75
MD
2429 * NOTE! On some parts notification bits can get set without
2430 * generating an interrupt. It is unclear whether this is
2431 * a bug in the PM (sending a DTOH device setbits with 'N' set
2432 * and 'I' not set), or a bug in the host controller.
2433 *
2434 * It only seems to occur under load.
1980eff3 2435 */
cec07d75 2436 if (/*(is & AHCI_PREG_IS_SDBS) &&*/ (sc->sc_cap & AHCI_REG_CAP_SSNTF)) {
1980eff3 2437 u_int32_t data;
cec07d75 2438 const char *xstr;
1980eff3
MD
2439
2440 data = ahci_pread(ap, AHCI_PREG_SNTF);
cec07d75
MD
2441 if (is & AHCI_PREG_IS_SDBS) {
2442 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_SDBS);
2443 is &= ~AHCI_PREG_IS_SDBS;
2444 xstr = " (no SDBS!)";
2445 } else {
2446 xstr = "";
2447 }
1980eff3 2448 if (data) {
1980eff3 2449 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_SDBS);
cec07d75
MD
2450
2451 kprintf("%s: NOTIFY %08x%s\n",
2452 PORTNAME(ap), data, xstr);
2453 ahci_pwrite(ap, AHCI_PREG_SERR, AHCI_PREG_SERR_DIAG_N);
3209f581
MD
2454 ahci_pwrite(ap, AHCI_PREG_SNTF, data);
2455 ahci_cam_changed(ap, NULL, -1);
1980eff3
MD
2456 }
2457 }
3209f581
MD
2458
2459 /*
f4553de1
MD
2460 * Spurious IFS errors (blockable).
2461 *
3209f581
MD
2462 * Spurious IFS errors can occur while we are doing a reset
2463 * sequence through a PM. Try to recover if we are being asked
2464 * to ignore IFS errors during these periods.
2465 */
2466 if ((is & AHCI_PREG_IS_IFS) && (ap->ap_flags & AP_F_IGNORE_IFS)) {
1980eff3 2467 u_int32_t serr = ahci_pread(ap, AHCI_PREG_SERR);
3209f581
MD
2468 if ((ap->ap_flags & AP_F_IFS_IGNORED) == 0) {
2469 kprintf("%s: Ignoring IFS (XXX) (IS: %b, SERR: %b)\n",
2470 PORTNAME(ap),
2471 is, AHCI_PFMT_IS,
2472 serr, AHCI_PFMT_SERR);
2473 ap->ap_flags |= AP_F_IFS_IGNORED;
2474 }
2475 ap->ap_flags |= AP_F_IFS_OCCURED;
1980eff3
MD
2476 ahci_pwrite(ap, AHCI_PREG_SERR, -1);
2477 ahci_pwrite(ap, AHCI_PREG_IS, AHCI_PREG_IS_IFS);
2478 is &= ~AHCI_PREG_IS_IFS;
2479 ahci_port_stop(ap, 0);
2480 ahci_port_start(ap);
2481 need = NEED_RESTART;
258223a3
MD
2482 }
2483
2484 /*
f4553de1 2485 * Port change (hot-plug) (blockable).
258223a3
MD
2486 *
2487 * A PCS interrupt will occur on hot-plug once communication is
2488 * established.
2489 *
2490 * A PRCS interrupt will occur on hot-unplug (and possibly also
2491 * on hot-plug).
2492 *
22181ab7
MD
2493 * XXX We can then check the CPS (Cold Presence State) bit, if
2494 * supported, to determine if a device is plugged in or not and do
2495 * the right thing.
2496 *
2497 * WARNING: A PCS interrupt is cleared by clearing DIAG_X, and
2498 * can also occur if an unsolicited COMINIT is received.
2499 * If this occurs command processing is automatically
2500 * stopped (CR goes inactive) and the port must be stopped
2501 * and restarted.
258223a3
MD
2502 */
2503 if (is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS)) {
cec07d75
MD
2504 ahci_pwrite(ap, AHCI_PREG_IS,
2505 is & (AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS));
2506 is &= ~(AHCI_PREG_IS_PCS | AHCI_PREG_IS_PRCS);
258223a3 2507 ahci_pwrite(ap, AHCI_PREG_SERR,
1980eff3 2508 (AHCI_PREG_SERR_DIAG_N | AHCI_PREG_SERR_DIAG_X));
22181ab7 2509 ahci_port_stop(ap, 0);
258223a3
MD
2510 switch (ahci_pread(ap, AHCI_PREG_SSTS) & AHCI_PREG_SSTS_DET) {
2511 case AHCI_PREG_SSTS_DET_DEV:
1980eff3 2512 if (ap->ap_type == ATA_PORT_T_NONE) {
22181ab7
MD
2513 need = NEED_HOTPLUG_INSERT;
2514 goto fatal;
258223a3 2515 }
22181ab7 2516 need = NEED_RESTART;
258223a3
MD
2517 break;
2518 default:
1980eff3 2519 if (ap->ap_type != ATA_PORT_T_NONE) {
22181ab7
MD
2520 need = NEED_HOTPLUG_REMOVE;
2521 goto fatal;
258223a3 2522 }
22181ab7 2523 need = NEED_RESTART;
258223a3
MD
2524 break;
2525 }
2526 }
2527
22181ab7 2528 /*
f4553de1 2529 * Check for remaining errors - they are fatal. (blockable)
22181ab7 2530 */
258223a3
MD
2531 if (is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS | AHCI_PREG_IS_IFS |
2532 AHCI_PREG_IS_OFS | AHCI_PREG_IS_UFS)) {
cec07d75
MD
2533 u_int32_t serr;
2534
2535 ahci_pwrite(ap, AHCI_PREG_IS,
2536 is & (AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2537 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2538 AHCI_PREG_IS_UFS));
cec07d75 2539 serr = ahci_pread(ap, AHCI_PREG_SERR);
831bc9e3 2540 kprintf("%s: Unrecoverable errors (IS: %b, SERR: %b), "
4444122d
MD
2541 "disabling port.\n",
2542 PORTNAME(ap),
2543 is, AHCI_PFMT_IS,
1980eff3 2544 serr, AHCI_PFMT_SERR
4444122d 2545 );
831bc9e3
MD
2546 is &= ~(AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2547 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2548 AHCI_PREG_IS_UFS);
258223a3
MD
2549 /* XXX try recovery first */
2550 goto fatal;
2551 }
2552
22181ab7
MD
2553 /*
2554 * Fail all outstanding commands if we know the port won't recover.
1980eff3
MD
2555 *
2556 * We may have a ccb_at if the failed command is known and was
2557 * being sent to a device over a port multiplier (PM). In this
2558 * case if the port itself has not completely failed we fail just
2559 * the commands related to that target.
22181ab7 2560 */
258223a3
MD
2561 if (ap->ap_state == AP_S_FATAL_ERROR) {
2562fatal:
2563 ap->ap_state = AP_S_FATAL_ERROR;
2564failall:
2565
cf5f3a81
MD
2566 /* Stopping the port clears CI/SACT */
2567 ahci_port_stop(ap, 0);
258223a3 2568
1980eff3
MD
2569 /*
2570 * Error all the active slots. If running across a PM
2571 * try to error out just the slots related to the target.
2572 */
258223a3
MD
2573 ci_masked = ci_saved & *active;
2574 while (ci_masked) {
2575 slot = ffs(ci_masked) - 1;
2576 ccb = &ap->ap_ccbs[slot];
1980eff3
MD
2577 if (ccb_at == ccb->ccb_xa.at ||
2578 ap->ap_state == AP_S_FATAL_ERROR) {
2579 ci_masked &= ~(1 << slot);
2580 ccb->ccb_xa.state = ATA_S_ERROR;
2581 }
258223a3
MD
2582 }
2583
2584 /* Run completion for all active slots. */
2585 ci_saved &= ~*active;
2586
2587 /*
2588 * Don't restart the port if our problems were deemed fatal.
2589 *
2590 * Also acknowlege all fatal interrupt sources to prevent
2591 * a livelock.
2592 */
2593 if (ap->ap_state == AP_S_FATAL_ERROR) {
22181ab7
MD
2594 if (need == NEED_RESTART)
2595 need = NEED_NOTHING;
258223a3
MD
2596 ahci_pwrite(ap, AHCI_PREG_IS,
2597 AHCI_PREG_IS_TFES | AHCI_PREG_IS_HBFS |
2598 AHCI_PREG_IS_IFS | AHCI_PREG_IS_OFS |
2599 AHCI_PREG_IS_UFS);
2600 }
2601 }
2602
2603 /*
f4553de1
MD
2604 * CCB completion (non blocking).
2605 *
258223a3
MD
2606 * CCB completion is detected by noticing its slot's bit in CI has
2607 * changed to zero some time after we activated it.
2608 * If we are polling, we may only be interested in particular slot(s).
cf5f3a81
MD
2609 *
2610 * Any active bits not saved are completed within the restrictions
2611 * imposed by the caller.
258223a3 2612 */
3209f581 2613 ci_masked = ~ci_saved & *active;
258223a3
MD
2614 while (ci_masked) {
2615 slot = ffs(ci_masked) - 1;
2616 ccb = &ap->ap_ccbs[slot];
2617 ci_masked &= ~(1 << slot);
2618
2619 DPRINTF(AHCI_D_INTR, "%s: slot %d is complete%s\n",
2620 PORTNAME(ap), slot, ccb->ccb_xa.state == ATA_S_ERROR ?
2621 " (error)" : "");
2622
2623 bus_dmamap_sync(sc->sc_tag_cmdh,
2624 AHCI_DMA_MAP(ap->ap_dmamem_cmd_list),
2625 BUS_DMASYNC_POSTWRITE);
2626
2627 bus_dmamap_sync(sc->sc_tag_cmdt,
2628 AHCI_DMA_MAP(ap->ap_dmamem_cmd_table),
2629 BUS_DMASYNC_POSTWRITE);
2630
2631 bus_dmamap_sync(sc->sc_tag_rfis,
2632 AHCI_DMA_MAP(ap->ap_dmamem_rfis),
2633 BUS_DMASYNC_POSTREAD);
2634
2635 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
2636 if (active == &ap->ap_active) {
2637 KKASSERT(ap->ap_active_cnt > 0);
2638 --ap->ap_active_cnt;
2639 }
258223a3 2640 ccb->ccb_done(ccb);
258223a3
MD
2641 }
2642
f4553de1
MD
2643 /*
2644 * Cleanup. Will not be set if non-blocking.
2645 */
22181ab7
MD
2646 switch(need) {
2647 case NEED_RESTART:
2648 /*
2649 * A recoverable error occured and we can restart outstanding
2650 * commands on the port.
2651 */
17eab71e 2652 ahci_port_start(ap);
258223a3 2653
258223a3
MD
2654 if (ci_saved) {
2655#ifdef DIAGNOSTIC
2656 tmp = ci_saved;
2657 while (tmp) {
2658 slot = ffs(tmp) - 1;
2659 tmp &= ~(1 << slot);
2660 ccb = &ap->ap_ccbs[slot];
2661 KKASSERT(ccb->ccb_xa.state == ATA_S_ONCHIP);
2662 KKASSERT((!!(ccb->ccb_xa.flags & ATA_F_NCQ)) ==
2663 (!!ap->ap_sactive));
2664 }
2665#endif
2666 DPRINTF(AHCI_D_VERBOSE, "%s: ahci_port_intr "
2667 "re-enabling%s slots %08x\n", PORTNAME(ap),
2668 ap->ap_sactive ? " NCQ" : "", ci_saved);
2669
2670 if (ap->ap_sactive)
2671 ahci_pwrite(ap, AHCI_PREG_SACT, ci_saved);
2672 ahci_pwrite(ap, AHCI_PREG_CI, ci_saved);
2673 }
22181ab7
MD
2674 break;
2675 case NEED_HOTPLUG_INSERT:
2676 /*
cf5f3a81
MD
2677 * A hot-plug insertion event has occured and all
2678 * outstanding commands have already been revoked.
1980eff3
MD
2679 *
2680 * Don't recurse if this occurs while we are
2681 * resetting the port.
22181ab7 2682 */
1980eff3
MD
2683 if ((ap->ap_flags & AP_F_IN_RESET) == 0) {
2684 kprintf("%s: HOTPLUG - Device inserted\n",
2685 PORTNAME(ap));
3209f581
MD
2686 ap->ap_probe = ATA_PROBE_NEED_INIT;
2687 ahci_cam_changed(ap, NULL, -1);
1980eff3 2688 }
22181ab7
MD
2689 break;
2690 case NEED_HOTPLUG_REMOVE:
cf5f3a81
MD
2691 /*
2692 * A hot-plug removal event has occured and all
2693 * outstanding commands have already been revoked.
1980eff3
MD
2694 *
2695 * Don't recurse if this occurs while we are
2696 * resetting the port.
cf5f3a81 2697 */
1980eff3
MD
2698 if ((ap->ap_flags & AP_F_IN_RESET) == 0) {
2699 kprintf("%s: HOTPLUG - Device removed\n",
2700 PORTNAME(ap));
2701 ahci_port_hardstop(ap);
3209f581
MD
2702 /* ap_probe set to failed */
2703 ahci_cam_changed(ap, NULL, -1);
1980eff3 2704 }
22181ab7
MD
2705 break;
2706 default:
2707 break;
258223a3 2708 }
258223a3
MD
2709}
2710
2711struct ahci_ccb *
2712ahci_get_ccb(struct ahci_port *ap)
2713{
2714 struct ahci_ccb *ccb;
2715
2716 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE);
2717 ccb = TAILQ_FIRST(&ap->ap_ccb_free);
2718 if (ccb != NULL) {
2719 KKASSERT(ccb->ccb_xa.state == ATA_S_PUT);
2720 TAILQ_REMOVE(&ap->ap_ccb_free, ccb, ccb_entry);
2721 ccb->ccb_xa.state = ATA_S_SETUP;
1980eff3 2722 ccb->ccb_xa.at = NULL;
258223a3
MD
2723 }
2724 lockmgr(&ap->ap_ccb_lock, LK_RELEASE);
2725
2726 return (ccb);
2727}
2728
2729void
2730ahci_put_ccb(struct ahci_ccb *ccb)
2731{
2732 struct ahci_port *ap = ccb->ccb_port;
2733
2734#ifdef DIAGNOSTIC
2735 if (ccb->ccb_xa.state != ATA_S_COMPLETE &&
2736 ccb->ccb_xa.state != ATA_S_TIMEOUT &&
2737 ccb->ccb_xa.state != ATA_S_ERROR) {
2738 kprintf("%s: invalid ata_xfer state %02x in ahci_put_ccb, "
2739 "slot %d\n",
2740 PORTNAME(ccb->ccb_port), ccb->ccb_xa.state,
2741 ccb->ccb_slot);
2742 }
2743#endif
2744
2745 ccb->ccb_xa.state = ATA_S_PUT;
2746 lockmgr(&ap->ap_ccb_lock, LK_EXCLUSIVE);
2747 TAILQ_INSERT_TAIL(&ap->ap_ccb_free, ccb, ccb_entry);
2748 lockmgr(&ap->ap_ccb_lock, LK_RELEASE);
2749}
2750
2751struct ahci_ccb *
2752ahci_get_err_ccb(struct ahci_port *ap)
2753{
2754 struct ahci_ccb *err_ccb;
2755 u_int32_t sact;
2756
2757 /* No commands may be active on the chip. */
2758 sact = ahci_pread(ap, AHCI_PREG_SACT);
2759 if (sact != 0)
2760 kprintf("ahci_get_err_ccb but SACT %08x != 0?\n", sact);
2761 KKASSERT(ahci_pread(ap, AHCI_PREG_CI) == 0);
2762
2763#ifdef DIAGNOSTIC
2764 KKASSERT(ap->ap_err_busy == 0);
2765 ap->ap_err_busy = 1;
2766#endif
2767 /* Save outstanding command state. */
2768 ap->ap_err_saved_active = ap->ap_active;
2769 ap->ap_err_saved_active_cnt = ap->ap_active_cnt;
2770 ap->ap_err_saved_sactive = ap->ap_sactive;
2771
2772 /*
2773 * Pretend we have no commands outstanding, so that completions won't
2774 * run prematurely.
2775 */
2776 ap->ap_active = ap->ap_active_cnt = ap->ap_sactive = 0;
2777
2778 /*
2779 * Grab a CCB to use for error recovery. This should never fail, as
2780 * we ask atascsi to reserve one for us at init time.
2781 */
2782 err_ccb = ahci_get_ccb(ap);
2783 KKASSERT(err_ccb != NULL);
2784 err_ccb->ccb_xa.flags = 0;
2785 err_ccb->ccb_done = ahci_empty_done;
2786
2787 return err_ccb;
2788}
2789
2790void
2791ahci_put_err_ccb(struct ahci_ccb *ccb)
2792{
2793 struct ahci_port *ap = ccb->ccb_port;
2794 u_int32_t sact;
5f8c1efd 2795 u_int32_t ci;
258223a3
MD
2796
2797#ifdef DIAGNOSTIC
2798 KKASSERT(ap->ap_err_busy);
2799#endif
5f8c1efd
MD
2800 /*
2801 * No commands may be active on the chip
2802 */
258223a3 2803 sact = ahci_pread(ap, AHCI_PREG_SACT);
5f8c1efd
MD
2804 if (sact) {
2805 panic("ahci_port_err_ccb(%d) but SACT %08x != 0\n",
2806 ccb->ccb_slot, sact);
2807 }
2808 ci = ahci_pread(ap, AHCI_PREG_CI);
2809 if (ci) {
cf5f3a81
MD
2810 panic("ahci_put_err_ccb(%d) but CI %08x != 0 "
2811 "(act=%08x sact=%08x)\n",
2812 ccb->ccb_slot, ci,
2813 ap->ap_active, ap->ap_sactive);
258223a3 2814 }
258223a3
MD
2815
2816 /* Done with the CCB */
2817 ahci_put_ccb(ccb);
2818
2819 /* Restore outstanding command state */
2820 ap->ap_sactive = ap->ap_err_saved_sactive;
2821 ap->ap_active_cnt = ap->ap_err_saved_active_cnt;
2822 ap->ap_active = ap->ap_err_saved_active;
2823
2824#ifdef DIAGNOSTIC
2825 ap->ap_err_busy = 0;
2826#endif
2827}
2828
1980eff3
MD
2829/*
2830 * Read log page to get NCQ error.
2831 *
2832 * NOTE: NCQ not currently supported on port multipliers. XXX
2833 */
258223a3
MD
2834int
2835ahci_port_read_ncq_error(struct ahci_port *ap, int *err_slotp)
2836{
2837 struct ahci_ccb *ccb;
2838 struct ahci_cmd_hdr *cmd_slot;
2839 u_int32_t cmd;
2840 struct ata_fis_h2d *fis;
2841 int rc = EIO;
2842
2843 DPRINTF(AHCI_D_VERBOSE, "%s: read log page\n", PORTNAME(ap));
2844
2845 /* Save command register state. */
2846 cmd = ahci_pread(ap, AHCI_PREG_CMD) & ~AHCI_PREG_CMD_ICC;
2847
2848 /* Port should have been idled already. Start it. */
2849 KKASSERT((cmd & AHCI_PREG_CMD_CR) == 0);
17eab71e 2850 ahci_port_start(ap);
258223a3
MD
2851
2852 /* Prep error CCB for READ LOG EXT, page 10h, 1 sector. */
2853 ccb = ahci_get_err_ccb(ap);
2854 ccb->ccb_xa.flags = ATA_F_NOWAIT | ATA_F_READ | ATA_F_POLL;
2855 ccb->ccb_xa.data = ap->ap_err_scratch;
2856 ccb->ccb_xa.datalen = 512;
2857 cmd_slot = ccb->ccb_cmd_hdr;
2858 bzero(ccb->ccb_cmd_table, sizeof(struct ahci_cmd_table));
2859
2860 fis = (struct ata_fis_h2d *)ccb->ccb_cmd_table->cfis;
2861 fis->type = ATA_FIS_TYPE_H2D;
2862 fis->flags = ATA_H2D_FLAGS_CMD;
2863 fis->command = ATA_C_READ_LOG_EXT;
2864 fis->lba_low = 0x10; /* queued error log page (10h) */
2865 fis->sector_count = 1; /* number of sectors (1) */
2866 fis->sector_count_exp = 0;
2867 fis->lba_mid = 0; /* starting offset */
2868 fis->lba_mid_exp = 0;
2869 fis->device = 0;
2870
2871 cmd_slot->flags = htole16(5); /* FIS length: 5 DWORDS */
2872
2873 if (ahci_load_prdt(ccb) != 0) {
2874 rc = ENOMEM; /* XXX caller must abort all commands */
2875 goto err;
2876 }
2877
2878 ccb->ccb_xa.state = ATA_S_PENDING;
831bc9e3 2879 if (ahci_poll(ccb, 1000, ahci_quick_timeout) != 0)
258223a3
MD
2880 goto err;
2881
2882 rc = 0;
2883err:
2884 /* Abort our command, if it failed, by stopping command DMA. */
831bc9e3 2885 if (rc) {
258223a3
MD
2886 kprintf("%s: log page read failed, slot %d was still active.\n",
2887 PORTNAME(ap), ccb->ccb_slot);
258223a3
MD
2888 }
2889
2890 /* Done with the error CCB now. */
2891 ahci_unload_prdt(ccb);
2892 ahci_put_err_ccb(ccb);
2893
2894 /* Extract failed register set and tags from the scratch space. */
2895 if (rc == 0) {
2896 struct ata_log_page_10h *log;
2897 int err_slot;
2898
2899 log = (struct ata_log_page_10h *)ap->ap_err_scratch;
2900 if (log->err_regs.type & ATA_LOG_10H_TYPE_NOTQUEUED) {
2901 /* Not queued bit was set - wasn't an NCQ error? */
2902 kprintf("%s: read NCQ error page, but not an NCQ "
2903 "error?\n",
2904 PORTNAME(ap));
2905 rc = ESRCH;
2906 } else {
2907 /* Copy back the log record as a D2H register FIS. */
2908 *err_slotp = err_slot = log->err_regs.type &
2909 ATA_LOG_10H_TYPE_TAG_MASK;
2910
2911 ccb = &ap->ap_ccbs[err_slot];
2912 memcpy(&ccb->ccb_xa.rfis, &log->err_regs,
2913 sizeof(struct ata_fis_d2h));
2914 ccb->ccb_xa.rfis.type = ATA_FIS_TYPE_D2H;
2915 ccb->ccb_xa.rfis.flags = 0;
2916 }
2917 }
2918
2919 /* Restore saved CMD register state */
2920 ahci_pwrite(ap, AHCI_PREG_CMD, cmd);
2921
2922 return (rc);
2923}
2924
2925/*
2926 * Allocate memory for various structures DMAd by hardware. The maximum
2927 * number of segments for these tags is 1 so the DMA memory will have a
2928 * single physical base address.
2929 */
2930struct ahci_dmamem *
2931ahci_dmamem_alloc(struct ahci_softc *sc, bus_dma_tag_t tag)
2932{
2933 struct ahci_dmamem *adm;
2934 int error;
2935
2936 adm = kmalloc(sizeof(*adm), M_DEVBUF, M_INTWAIT | M_ZERO);
2937
2938 error = bus_dmamem_alloc(tag, (void **)&adm->adm_kva,
2939 BUS_DMA_ZERO, &adm->adm_map);
2940 if (error == 0) {
2941 adm->adm_tag = tag;
2942 error = bus_dmamap_load(tag, adm->adm_map,
2943 adm->adm_kva,
2944 bus_dma_tag_getmaxsize(tag),
2945 ahci_dmamem_saveseg, &adm->adm_busaddr,
2946 0);
2947 }
2948 if (error) {
2949 if (adm->adm_map) {
2950 bus_dmamap_destroy(tag, adm->adm_map);
2951 adm->adm_map = NULL;
2952 adm->adm_tag = NULL;
2953 adm->adm_kva = NULL;
2954 }
2955 kfree(adm, M_DEVBUF);
2956 adm = NULL;
2957 }
2958 return (adm);
2959}
2960
2961static
2962void
2963ahci_dmamem_saveseg(void *info, bus_dma_segment_t *segs, int nsegs, int error)
2964{
2965 KKASSERT(error == 0);
2966 KKASSERT(nsegs == 1);
2967 *(bus_addr_t *)info = segs->ds_addr;
2968}
2969
2970
2971void
2972ahci_dmamem_free(struct ahci_softc *sc, struct ahci_dmamem *adm)
2973{
2974 if (adm->adm_map) {
2975 bus_dmamap_unload(adm->adm_tag, adm->adm_map);
2976 bus_dmamap_destroy(adm->adm_tag, adm->adm_map);
2977 adm->adm_map = NULL;
2978 adm->adm_tag = NULL;
2979 adm->adm_kva = NULL;
2980 }
2981 kfree(adm, M_DEVBUF);
2982}
2983
2984u_int32_t
2985ahci_read(struct ahci_softc *sc, bus_size_t r)
2986{
2987 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2988 BUS_SPACE_BARRIER_READ);
2989 return (bus_space_read_4(sc->sc_iot, sc->sc_ioh, r));
2990}
2991
2992void
2993ahci_write(struct ahci_softc *sc, bus_size_t r, u_int32_t v)
2994{
2995 bus_space_write_4(sc->sc_iot, sc->sc_ioh, r, v);
2996 bus_space_barrier(sc->sc_iot, sc->sc_ioh, r, 4,
2997 BUS_SPACE_BARRIER_WRITE);
2998}
2999
258223a3
MD
3000u_int32_t
3001ahci_pread(struct ahci_port *ap, bus_size_t r)
3002{
3003 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
3004 BUS_SPACE_BARRIER_READ);
3005 return (bus_space_read_4(ap->ap_sc->sc_iot, ap->ap_ioh, r));
3006}
3007
3008void
3009ahci_pwrite(struct ahci_port *ap, bus_size_t r, u_int32_t v)
3010{
3011 bus_space_write_4(ap->ap_sc->sc_iot, ap->ap_ioh, r, v);
3012 bus_space_barrier(ap->ap_sc->sc_iot, ap->ap_ioh, r, 4,
3013 BUS_SPACE_BARRIER_WRITE);
3014}
3015
831bc9e3
MD
3016/*
3017 * Wait up to (timeout) milliseconds for the masked port register to
3018 * match the target.
3019 *
3020 * Timeout is in milliseconds.
3021 */
258223a3 3022int
cec85a37
MD
3023ahci_pwait_eq(struct ahci_port *ap, int timeout,
3024 bus_size_t r, u_int32_t mask, u_int32_t target)
258223a3 3025{
831bc9e3 3026 int t;
258223a3 3027
831bc9e3
MD
3028 /*
3029 * Loop hard up to 100uS
3030 */
3031 for (t = 0; t < 100; ++t) {
258223a3
MD
3032 if ((ahci_pread(ap, r) & mask) == target)
3033 return (0);
831bc9e3 3034 ahci_os_hardsleep(1); /* us */
258223a3
MD
3035 }
3036
831bc9e3
MD
3037 do {
3038 timeout -= ahci_os_softsleep();
3039 if ((ahci_pread(ap, r) & mask) == target)
3040 return (0);
3041 } while (timeout > 0);
258223a3
MD
3042 return (1);
3043}
3044
831bc9e3
MD
3045int
3046ahci_wait_ne(struct ahci_softc *sc, bus_size_t r, u_int32_t mask,
3047 u_int32_t target)
3048{
3049 int t;
3050
3051 /*
3052 * Loop hard up to 100uS
3053 */
3054 for (t = 0; t < 100; ++t) {
3055 if ((ahci_read(sc, r) & mask) != target)
3056 return (0);
3057 ahci_os_hardsleep(1); /* us */
3058 }
3059
3060 /*
3061 * And one millisecond the slow way
3062 */
3063 t = 1000;
3064 do {
3065 t -= ahci_os_softsleep();
3066 if ((ahci_read(sc, r) & mask) != target)
3067 return (0);
3068 } while (t > 0);
3069
3070 return (1);
3071}
3072
3073
1980eff3
MD
3074/*
3075 * Acquire an ata transfer.
3076 *
3077 * Pass a NULL at for direct-attached transfers, and a non-NULL at for
3078 * targets that go through the port multiplier.
3079 */
258223a3 3080struct ata_xfer *
1980eff3 3081ahci_ata_get_xfer(struct ahci_port *ap, struct ata_port *at)
258223a3 3082{
258223a3
MD
3083 struct ahci_ccb *ccb;
3084
3085 ccb = ahci_get_ccb(ap);
3086 if (ccb == NULL) {
3087 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer: NULL ccb\n",
3088 PORTNAME(ap));
3089 return (NULL);
3090 }
3091
3092 DPRINTF(AHCI_D_XFER, "%s: ahci_ata_get_xfer got slot %d\n",
3093 PORTNAME(ap), ccb->ccb_slot);
3094
1980eff3 3095 ccb->ccb_xa.at = at;
258223a3
MD
3096 ccb->ccb_xa.fis->type = ATA_FIS_TYPE_H2D;
3097
3098 return (&ccb->ccb_xa);
3099}
3100
3101void
3102ahci_ata_put_xfer(struct ata_xfer *xa)
3103{
3104 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
3105
3106 DPRINTF(AHCI_D_XFER, "ahci_ata_put_xfer slot %d\n", ccb->ccb_slot);
3107
3108 ahci_put_ccb(ccb);
3109}
3110
3111int
3112ahci_ata_cmd(struct ata_xfer *xa)
3113{
3114 struct ahci_ccb *ccb = (struct ahci_ccb *)xa;
3115 struct ahci_cmd_hdr *cmd_slot;
3116
3117 KKASSERT(xa->state == ATA_S_SETUP);
3118
3119 if (ccb->ccb_port->ap_state == AP_S_FATAL_ERROR)
3120 goto failcmd;
1980eff3
MD
3121#if 0
3122 kprintf("%s: started std command %b ccb %d ccb_at %p %d\n",
3123 ATANAME(ccb->ccb_port, ccb->ccb_xa.at),
3124 ahci_pread(ccb->ccb_port, AHCI_PREG_CMD), AHCI_PFMT_CMD,
3125 ccb->ccb_slot,
3126 ccb->ccb_xa.at,
3127 ccb->ccb_xa.at ? ccb->ccb_xa.at->at_target : -1);
3128#endif
258223a3
MD
3129
3130 ccb->ccb_done = ahci_ata_cmd_done;
3131
3132 cmd_slot = ccb->ccb_cmd_hdr;
3133 cmd_slot->flags = htole16(5); /* FIS length (in DWORDs) */
1980eff3
MD
3134 if (ccb->ccb_xa.at) {
3135 cmd_slot->flags |= htole16(ccb->ccb_xa.at->at_target <<
3136 AHCI_CMD_LIST_FLAG_PMP_SHIFT);
3137 }
258223a3
MD
3138
3139 if (xa->flags & ATA_F_WRITE)
3140 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_W);
3141
3142 if (xa->flags & ATA_F_PACKET)
3143 cmd_slot->flags |= htole16(AHCI_CMD_LIST_FLAG_A);
3144
3145 if (ahci_load_prdt(ccb) != 0)
3146 goto failcmd;
3147
3148 xa->state = ATA_S_PENDING;
3149
831bc9e3
MD
3150 if (xa->flags & ATA_F_POLL)
3151 return (ahci_poll(ccb, xa->timeout, ahci_ata_cmd_timeout));
258223a3
MD
3152
3153 crit_enter();
f4553de1 3154 KKASSERT((xa->flags & ATA_F_TIMEOUT_EXPIRED) == 0);
3209f581 3155 xa->flags |= ATA_F_TIMEOUT_DESIRED;
258223a3
MD
3156 ahci_start(ccb);
3157 crit_exit();
831bc9e3 3158 return (xa->state);
258223a3
MD
3159
3160failcmd:
3161 crit_enter();
3162 xa->state = ATA_S_ERROR;
3163 xa->complete(xa);
3164 crit_exit();
831bc9e3 3165 return (ATA_S_ERROR);
258223a3
MD
3166}
3167
3168void
3169ahci_ata_cmd_done(struct ahci_ccb *ccb)
3170{
3171 struct ata_xfer *xa = &ccb->ccb_xa;
3172
831bc9e3
MD
3173 /*
3174 * NOTE: callout does not lock port and may race us modifying
3175 * the flags, so make sure its stopped.
3176 */
258223a3 3177 if (xa->flags & ATA_F_TIMEOUT_RUNNING) {
258223a3 3178 callout_stop(&ccb->ccb_timeout);
831bc9e3 3179 xa->flags &= ~ATA_F_TIMEOUT_RUNNING;
258223a3 3180 }
f4553de1 3181 xa->flags &= ~(ATA_F_TIMEOUT_DESIRED | ATA_F_TIMEOUT_EXPIRED);
258223a3 3182
f4553de1 3183 if (xa->state == ATA_S_ONCHIP || xa->state == ATA_S_ERROR) {
258223a3 3184 ahci_issue_pending_commands(ccb->ccb_port,
f4553de1
MD
3185 xa->flags & ATA_F_NCQ);
3186 }
258223a3
MD
3187
3188 ahci_unload_prdt(ccb);
3189
3190 if (xa->state == ATA_S_ONCHIP)
3191 xa->state = ATA_S_COMPLETE;
3192#ifdef DIAGNOSTIC
3193 else if (xa->state != ATA_S_ERROR && xa->state != ATA_S_TIMEOUT)
3194 kprintf("%s: invalid ata_xfer state %02x in ahci_ata_cmd_done, "
3195 "slot %d\n",
3196 PORTNAME(ccb->ccb_port), xa->state, ccb->ccb_slot);
3197#endif
3198 if (xa->state != ATA_S_TIMEOUT)
3199 xa->complete(xa);
3200}
3201
f4553de1
MD
3202/*
3203 * Timeout from callout, MPSAFE - nothing can mess with the CCB's flags
3204 * while the callout is runing.
3205 *
3206 * We can't safely get the port lock here or delay, we could block
3207 * the callout thread.
3208 */
258223a3
MD
3209static void
3210ahci_ata_cmd_timeout_unserialized(void *arg)
3211{
3212 struct ahci_ccb *ccb = arg;
3213 struct ahci_port *ap = ccb->ccb_port;
3214
f4553de1
MD
3215 ccb->ccb_xa.flags &= ~ATA_F_TIMEOUT_RUNNING;
3216 ccb->ccb_xa.flags |= ATA_F_TIMEOUT_EXPIRED;
3217 ahci_os_signal_port_thread(ap, AP_SIGF_TIMEOUT);
258223a3
MD
3218}
3219
1980eff3 3220void
831bc9e3 3221ahci_ata_cmd_timeout(struct ahci_ccb *ccb)
258223a3 3222{
258223a3
MD
3223 struct ata_xfer *xa = &ccb->ccb_xa;
3224 struct ahci_port *ap = ccb->ccb_port;
3225 volatile u_int32_t *active;
3226 int ccb_was_started, ncq_cmd;
131be210 3227 int status;
258223a3
MD
3228
3229 crit_enter();
831bc9e3 3230 kprintf("%s: CMD TIMEOUT state=%d cmd-reg 0x%b\n"
cf5f3a81
MD
3231 "\tsactive=%08x active=%08x\n"
3232 "\t sact=%08x ci=%08x\n",
1980eff3 3233 ATANAME(ap, ccb->ccb_xa.at),
831bc9e3 3234 ccb->ccb_xa.state,
258223a3 3235 ahci_pread(ap, AHCI_PREG_CMD), AHCI_PFMT_CMD,
cf5f3a81 3236 ap->ap_sactive, ap->ap_active,
258223a3
MD
3237 ahci_pread(ap, AHCI_PREG_SACT),
3238 ahci_pread(ap, AHCI_PREG_CI));
3239
9e145b23
MD
3240 /*
3241 * NOTE: Timeout will not be running if the command was polled.
3209f581 3242 * If we got here at least one of these flags should be set.
9e145b23 3243 */
3209f581
MD
3244 KKASSERT(xa->flags & (ATA_F_POLL | ATA_F_TIMEOUT_DESIRED |
3245 ATA_F_TIMEOUT_RUNNING));
f4553de1 3246 xa->flags &= ~(ATA_F_TIMEOUT_RUNNING | ATA_F_TIMEOUT_EXPIRED);
258223a3
MD
3247 ncq_cmd = (xa->flags & ATA_F_NCQ);
3248 active = ncq_cmd ? &ap->ap_sactive : &ap->ap_active;
3249
3250 if (ccb->ccb_xa.state == ATA_S_PENDING) {
3251 DPRINTF(AHCI_D_TIMEOUT, "%s: command for slot %d timed out "
3252 "before it got on chip\n", PORTNAME(ap), ccb->ccb_slot);
3253 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
3254 ccb_was_started = 0;
258223a3
MD
3255 } else if (ccb->ccb_xa.state != ATA_S_ONCHIP) {
3256 DPRINTF(AHCI_D_TIMEOUT, "%s: command slot %d already "
3257 "handled%s\n", PORTNAME(ap), ccb->ccb_slot,
3258 (*active & (1 << ccb->ccb_slot)) ?
3259 " but slot is still active?" : ".");
3260 goto ret;
3261 } else if ((ahci_pread(ap, ncq_cmd ? AHCI_PREG_SACT : AHCI_PREG_CI) &
3262 (1 << ccb->ccb_slot)) == 0 &&
3263 (*active & (1 << ccb->ccb_slot))) {
831bc9e3
MD
3264 kprintf("%s: ahci_port_intr() failed to detect "
3265 "completed slot\n", ATANAME(ap, ccb->ccb_xa.at));
258223a3 3266 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
3267 if (ncq_cmd == 0) {
3268 KKASSERT(ap->ap_active_cnt > 0);
3269 --ap->ap_active_cnt;
3270 }
258223a3
MD
3271 ccb->ccb_done(ccb);
3272 goto ret;
3273 } else {
258223a3
MD
3274 ccb_was_started = 1;
3275 }
3276
3277 /* Complete the slot with a timeout error. */
3278 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3279 *active &= ~(1 << ccb->ccb_slot);
1980eff3
MD
3280 if (ncq_cmd == 0) {
3281 KKASSERT(ap->ap_active_cnt > 0);
3282 --ap->ap_active_cnt;
3283 }
258223a3
MD
3284 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (1)\n", PORTNAME(ap));
3285 ccb->ccb_done(ccb); /* This won't issue pending commands or run the
3286 atascsi completion. */
3287
3288 /* Reset port to abort running command. */
3289 if (ccb_was_started) {
3290 DPRINTF(AHCI_D_TIMEOUT, "%s: resetting port to abort%s command "
3291 "in slot %d, active %08x\n", PORTNAME(ap), ncq_cmd ? " NCQ"
3292 : "", ccb->ccb_slot, *active);
1980eff3 3293 /* XXX */
3209f581 3294 if (ccb->ccb_xa.at && ap->ap_type == ATA_PORT_T_PM) {
1980eff3 3295 /* XXX how do we unbrick a PM target? */
131be210
MD
3296 kprintf("%s: PM target bricked and timed-out, "
3297 "disabling PM target but trying to "
3298 "leave the port intact\n",
3299 ATANAME(ap, ccb->ccb_xa.at));
3300 ccb->ccb_xa.at->at_probe = ATA_PROBE_FAILED;
f4553de1 3301 ahci_port_intr(ap, 1);
131be210
MD
3302 ahci_port_stop(ap, 0);
3303 ahci_port_clo(ap);
3304 ahci_port_start(ap);
3305 status = 0;
1980eff3 3306 } else if (ahci_port_reset(ap, ccb->ccb_xa.at, 0)) {
cf5f3a81
MD
3307 /*
3308 * If the softreset failed place the port in a
3309 * failed state and use ahci_port_intr() to cancel
3310 * any remaining commands.
3311 */
17eab71e
MD
3312 kprintf("%s: Unable to reset during timeout, port "
3313 "bricked on us\n",
258223a3
MD
3314 PORTNAME(ap));
3315 ap->ap_state = AP_S_FATAL_ERROR;
f4553de1 3316 ahci_port_intr(ap, 1);
131be210 3317 status = 1;
cf5f3a81 3318 } else {
131be210
MD
3319 status = 0;
3320 }
3321 if (status == 0) {
cf5f3a81
MD
3322 /*
3323 * Restart any other commands that were aborted
3324 * by the reset.
3325 */
3326 if (*active) {
3327 DPRINTF(AHCI_D_TIMEOUT, "%s: re-enabling%s slots "
3328 "%08x\n", PORTNAME(ap), ncq_cmd ? " NCQ" : "",
3329 *active);
3330 if (ncq_cmd)
3331 ahci_pwrite(ap, AHCI_PREG_SACT, *active);
3332 ahci_pwrite(ap, AHCI_PREG_CI, *active);
3333 }
258223a3
MD
3334 }
3335 }
3336
3337 /* Issue any pending commands now. */
3338 DPRINTF(AHCI_D_TIMEOUT, "%s: issue pending\n", PORTNAME(ap));
3339 if (ccb_was_started)
3340 ahci_issue_pending_commands(ap, ncq_cmd);
3341 else if (ap->ap_active == 0)
3342 ahci_issue_pending_ncq_commands(ap);
3343
3344 /* Complete the timed out ata_xfer I/O (may generate new I/O). */
3345 DPRINTF(AHCI_D_TIMEOUT, "%s: run completion (2)\n", PORTNAME(ap));
3346 xa->complete(xa);
3347
3348 DPRINTF(AHCI_D_TIMEOUT, "%s: splx\n", PORTNAME(ap));
3349ret:
3350 crit_exit();
3351}
3352
831bc9e3
MD
3353/*
3354 * Used by the softreset, pmprobe, and read_ncq_error only, in very
3355 * specialized, controlled circumstances.
3356 *
3357 * Only one command may be pending.
3358 */
3359void
3360ahci_quick_timeout(struct ahci_ccb *ccb)
3361{
3362 struct ahci_port *ap = ccb->ccb_port;
3363
3364 switch (ccb->ccb_xa.state) {
3365 case ATA_S_PENDING:
3366 TAILQ_REMOVE(&ap->ap_ccb_pending, ccb, ccb_entry);
3367 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3368 break;
3369 case ATA_S_ONCHIP:
3370 KKASSERT(ap->ap_active == (1 << ccb->ccb_slot) &&
3371 ap->ap_sactive == 0);
3372 ahci_port_stop(ap, 0);
3373 ahci_port_start(ap);
3374
3375 ccb->ccb_xa.state = ATA_S_TIMEOUT;
3376 ap->ap_active &= ~(1 << ccb->ccb_slot);
3377 KKASSERT(ap->ap_active_cnt > 0);
3378 --ap->ap_active_cnt;
3379 break;
3380 default:
3381 panic("%s: ahci_quick_timeout: ccb in bad state %d",
3382 ATANAME(ap, ccb->ccb_xa.at), ccb->ccb_xa.state);
3383 }
3384}
3385
258223a3
MD
3386void
3387ahci_empty_done(struct ahci_ccb *ccb)
3388{
831bc9e3
MD
3389 if (ccb->ccb_xa.state == ATA_S_ONCHIP)
3390 ccb->ccb_xa.state = ATA_S_COMPLETE;
258223a3 3391}