TRIM support
[dragonfly.git] / sys / dev / disk / ahci / ahci_cam.c
index 0225764..8e61fbf 100644 (file)
@@ -1,4 +1,6 @@
 /*
+ * (MPSAFE)
+ *
  * Copyright (c) 2009 The DragonFly Project.  All rights reserved.
  *
  * This code is derived from software contributed to The DragonFly Project
 
 #include "ahci.h"
 
-const char *ScsiTypeArray[32] = {
-       "DIRECT",
-       "SEQUENTIAL",
-       "PRINTER",
-       "PROCESSOR",
-       "WORM",
-       "CDROM",
-       "SCANNER",
-       "OPTICAL",
-       "CHANGER",
-       "COMM",
-       "ASC0",
-       "ASC1",
-       "STORARRAY",
-       "ENCLOSURE",
-       "RBC",
-       "OCRW",
-       "0x10",
-       "OSD",
-       "ADC",
-       "0x13",
-       "0x14",
-       "0x15",
-       "0x16",
-       "0x17",
-       "0x18",
-       "0x19",
-       "0x1A",
-       "0x1B",
-       "0x1C",
-       "0x1D",
-       "0x1E",
-       "NODEVICE"
-};
-
 static void ahci_xpt_action(struct cam_sim *sim, union ccb *ccb);
 static void ahci_xpt_poll(struct cam_sim *sim);
 static void ahci_xpt_scsi_disk_io(struct ahci_port *ap,
                        struct ata_port *at, union ccb *ccb);
 static void ahci_xpt_scsi_atapi_io(struct ahci_port *ap,
                        struct ata_port *at, union ccb *ccb);
+static void ahci_xpt_page_inquiry(struct ahci_port *ap,
+                       struct ata_port *at, union ccb *ccb);
 
 static void ahci_ata_complete_disk_rw(struct ata_xfer *xa);
 static void ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa);
@@ -109,12 +78,13 @@ static void ahci_ata_dummy_sense(struct scsi_sense_data *sense_data);
 static void ahci_ata_atapi_sense(struct ata_fis_d2h *rfis,
                     struct scsi_sense_data *sense_data);
 
-static int ahci_cam_probe(struct ahci_port *ap, struct ata_port *at);
 static int ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *at);
 static int ahci_cam_probe_atapi(struct ahci_port *ap, struct ata_port *at);
+static int ahci_set_xfer(struct ahci_port *ap, struct ata_port *atx);
 static void ahci_ata_dummy_done(struct ata_xfer *xa);
 static void ata_fix_identify(struct ata_identify *id);
 static void ahci_cam_rescan(struct ahci_port *ap);
+static void ahci_strip_string(const char **basep, int *lenp);
 
 int
 ahci_cam_attach(struct ahci_port *ap)
@@ -136,61 +106,89 @@ ahci_cam_attach(struct ahci_port *ap)
        if (devq == NULL) {
                return (ENOMEM);
        }
+
+       /*
+        * Give the devq enough room to run with 32 max_dev_transactions,
+        * but set the overall max tags to 1 until NCQ is negotiated.
+        */
        sim = cam_sim_alloc(ahci_xpt_action, ahci_xpt_poll, "ahci",
-                          (void *)ap, unit, &sim_mplock, 1, 1, devq);
+                          (void *)ap, unit, &ap->ap_sim_lock,
+                          32, 1, devq);
        cam_simq_release(devq);
        if (sim == NULL) {
                return (ENOMEM);
        }
        ap->ap_sim = sim;
+       ahci_os_unlock_port(ap);
+       lockmgr(&ap->ap_sim_lock, LK_EXCLUSIVE);
        error = xpt_bus_register(ap->ap_sim, ap->ap_num);
+       lockmgr(&ap->ap_sim_lock, LK_RELEASE);
+       ahci_os_lock_port(ap);
        if (error != CAM_SUCCESS) {
                ahci_cam_detach(ap);
                return (EINVAL);
        }
        ap->ap_flags |= AP_F_BUS_REGISTERED;
-       error = xpt_create_path(&ap->ap_path, NULL, cam_sim_path(sim),
-                               CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
-       if (error != CAM_REQ_CMP) {
-               ahci_cam_detach(ap);
-               return (ENOMEM);
-       }
 
-       error = ahci_cam_probe(ap, NULL);
+       if (ap->ap_probe == ATA_PROBE_NEED_IDENT)
+               error = ahci_cam_probe(ap, NULL);
+       else
+               error = 0;
        if (error) {
                ahci_cam_detach(ap);
                return (EIO);
        }
        ap->ap_flags |= AP_F_CAM_ATTACHED;
 
-       ahci_cam_rescan(ap);
-
        return(0);
 }
 
 /*
- * Use a wildcard path to indicate that all the devices on the bus
- * were found, or lost.
+ * The state of the port has changed.
+ *
+ * If at is NULL the physical port has changed state.
+ * If at is non-NULL a particular target behind a PM has changed state.
+ *
+ * If found is -1 the target state must be queued to a non-interrupt context.
+ * (only works with at == NULL).
+ *
+ * If found is 0 the target was removed.
+ * If found is 1 the target was inserted.
  */
 void
-ahci_cam_changed(struct ahci_port *ap, int found)
+ahci_cam_changed(struct ahci_port *ap, struct ata_port *atx, int found)
 {
        struct cam_path *tmppath;
+       int status;
+       int target;
+
+       target = atx ? atx->at_target : CAM_TARGET_WILDCARD;
 
        if (ap->ap_sim == NULL)
                return;
-       if (xpt_create_path(&tmppath, NULL, cam_sim_path(ap->ap_sim),
-                           CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
-               return;
-       }
-
-       if (found) {
-               kprintf("%s: CAM RESCAN\n", PORTNAME(ap));
-               ahci_cam_probe(ap, NULL);
-               /*xpt_async(AC_FOUND_DEVICE, tmppath, NULL);*/
+       if (found == CAM_TARGET_WILDCARD) {
+               status = xpt_create_path(&tmppath, NULL,
+                                        cam_sim_path(ap->ap_sim),
+                                        target, CAM_LUN_WILDCARD);
+               if (status != CAM_REQ_CMP)
+                       return;
                ahci_cam_rescan(ap);
        } else {
-               xpt_async(AC_LOST_DEVICE, tmppath, NULL);
+               status = xpt_create_path(&tmppath, NULL,
+                                        cam_sim_path(ap->ap_sim),
+                                        target,
+                                        CAM_LUN_WILDCARD);
+               if (status != CAM_REQ_CMP)
+                       return;
+#if 0
+               /*
+                * This confuses CAM
+                */
+               if (found)
+                       xpt_async(AC_FOUND_DEVICE, tmppath, NULL);
+               else
+                       xpt_async(AC_LOST_DEVICE, tmppath, NULL);
+#endif
        }
        xpt_free_path(tmppath);
 }
@@ -202,14 +200,10 @@ ahci_cam_detach(struct ahci_port *ap)
 
        if ((ap->ap_flags & AP_F_CAM_ATTACHED) == 0)
                return;
-       get_mplock();
+       lockmgr(&ap->ap_sim_lock, LK_EXCLUSIVE);
        if (ap->ap_sim) {
                xpt_freeze_simq(ap->ap_sim, 1);
        }
-       if (ap->ap_path) {
-               xpt_free_path(ap->ap_path);
-               ap->ap_path = NULL;
-       }
        if (ap->ap_flags & AP_F_BUS_REGISTERED) {
                error = xpt_bus_deregister(cam_sim_path(ap->ap_sim));
                KKASSERT(error == CAM_REQ_CMP);
@@ -219,7 +213,7 @@ ahci_cam_detach(struct ahci_port *ap)
                cam_sim_free(ap->ap_sim);
                ap->ap_sim = NULL;
        }
-       rel_mplock();
+       lockmgr(&ap->ap_sim_lock, LK_RELEASE);
        ap->ap_flags &= ~AP_F_CAM_ATTACHED;
 }
 
@@ -230,7 +224,7 @@ ahci_cam_detach(struct ahci_port *ap)
  * If at is NULL we are probing the direct-attached device on the port,
  * which may or may not be a port multiplier.
  */
-static int
+int
 ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
 {
        struct ata_port *at;
@@ -238,21 +232,26 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
        u_int64_t       capacity;
        u_int64_t       capacity_bytes;
        int             model_len;
-       int             status;
+       int             firmware_len;
+       int             serial_len;
        int             error;
        int             devncqdepth;
        int             i;
+       const char      *model_id;
+       const char      *firmware_id;
+       const char      *serial_id;
        const char      *wcstr;
        const char      *rastr;
        const char      *scstr;
        const char      *type;
 
+       error = EIO;
+
        /*
-        * If nothing is probed on the port then there is nothing for us
-        * to do.  In this state ap->ap_ata will also probably be NULL.
+        * Delayed CAM attachment for initial probe, sim may be NULL
         */
-       if (ap->ap_type == ATA_PORT_T_NONE)
-               return (EIO);
+       if (ap->ap_sim == NULL)
+               return(0);
 
        /*
         * A NULL atx indicates a probe of the directly connected device.
@@ -263,26 +262,27 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
         * an (at) pointing to target 0.
         */
        if (atx == NULL) {
+               at = ap->ap_ata[0];     /* direct attached - device 0 */
                if (ap->ap_type == ATA_PORT_T_PM) {
                        kprintf("%s: Found Port Multiplier\n",
                                ATANAME(ap, atx));
-                       ap->ap_probe = ATA_PROBE_GOOD;
                        return (0);
                }
-               at = ap->ap_ata;        /* direct attached - device 0 */
                at->at_type = ap->ap_type;
        } else {
+               at = atx;
                if (atx->at_type == ATA_PORT_T_PM) {
                        kprintf("%s: Bogus device, reducing port count to %d\n",
                                ATANAME(ap, atx), atx->at_target);
                        if (ap->ap_pmcount > atx->at_target)
                                ap->ap_pmcount = atx->at_target;
-                       return (EIO);
+                       goto err;
                }
-               at = atx;
        }
+       if (ap->ap_type == ATA_PORT_T_NONE)
+               goto err;
        if (at->at_type == ATA_PORT_T_NONE)
-               return (EIO);
+               goto err;
 
        /*
         * Issue identify, saving the result
@@ -291,6 +291,7 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
        xa->complete = ahci_ata_dummy_done;
        xa->data = &at->at_identify;
        xa->datalen = sizeof(at->at_identify);
+       xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
        xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
 
        switch(at->at_type) {
@@ -300,6 +301,7 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
                break;
        case ATA_PORT_T_ATAPI:
                xa->fis->command = ATA_C_ATAPI_IDENTIFY;
+               xa->flags |= ATA_F_AUTOSENSE;
                type = "ATAPI";
                break;
        default:
@@ -309,22 +311,13 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
        }
        xa->fis->features = 0;
        xa->fis->device = 0;
-       xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
-       xa->timeout = hz;
+       xa->timeout = 1000;
 
-       status = ahci_ata_cmd(xa);
-       if (status != ATA_COMPLETE) {
+       if (ahci_ata_cmd(xa) != ATA_S_COMPLETE) {
                kprintf("%s: Detected %s device but unable to IDENTIFY\n",
                        ATANAME(ap, atx), type);
                ahci_ata_put_xfer(xa);
-               return(EIO);
-       }
-       if (xa->state != ATA_S_COMPLETE) {
-               kprintf("%s: Detected %s device but unable to IDENTIFY "
-                       " xa->state=%d\n",
-                       ATANAME(ap, atx), type, xa->state);
-               ahci_ata_put_xfer(xa);
-               return(EIO);
+               goto err;
        }
        ahci_ata_put_xfer(xa);
 
@@ -346,10 +339,11 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
                capacity <<= 16;
                capacity += le16toh(at->at_identify.addrsec[0]);
        }
+       if (capacity == 0)
+               capacity = 1024 * 1024 / 512;
        at->at_capacity = capacity;
        if (atx == NULL)
                ap->ap_probe = ATA_PROBE_GOOD;
-       at->at_probe = ATA_PROBE_GOOD;
 
        capacity_bytes = capacity * 512;
 
@@ -384,24 +378,25 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
                                }
                        }
                        if (at->at_ncqdepth >= ap->ap_sc->sc_ncmds) {
-                               cam_devq_resize(ap->ap_sim->devq,
-                                               at->at_ncqdepth - 1);
+                               cam_sim_set_max_tags(ap->ap_sim,
+                                                    at->at_ncqdepth - 1);
                        }
                }
        } else {
                devncqdepth = 0;
        }
 
-       /*
-        * Make the model string a bit more presentable
-        */
-       for (model_len = 40; model_len; --model_len) {
-               if (at->at_identify.model[model_len-1] == ' ')
-                       continue;
-               if (at->at_identify.model[model_len-1] == 0)
-                       continue;
-               break;
-       }
+       model_len = sizeof(at->at_identify.model);
+       model_id = at->at_identify.model;
+       ahci_strip_string(&model_id, &model_len);
+
+       firmware_len = sizeof(at->at_identify.firmware);
+       firmware_id = at->at_identify.firmware;
+       ahci_strip_string(&firmware_id, &firmware_len);
+
+       serial_len = sizeof(at->at_identify.serial);
+       serial_id = at->at_identify.serial;
+       ahci_strip_string(&serial_id, &serial_len);
 
        /*
         * Generate informatiive strings.
@@ -436,30 +431,33 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
                        scstr = "frozen";
                else if (at->at_type == ATA_PORT_T_ATAPI)
                        scstr = "unfrozen";
+               else if (AhciNoFeatures & (1 << ap->ap_num))
+                       scstr = "<disabled>";
                else
                        scstr = "freezing";
        } else {
                    scstr = "notsupp";
        }
 
-       kprintf("%s: Found %s \"%*.*s %8.8s\" serial=\"%20.20s\"\n"
-               "%s: tags=%d/%d satacaps=%04x satafeat=%04x "
-               "capacity=%lld.%02dMB\n"
-               "%s: f85=%04x f86=%04x f87=%04x WC=%s RA=%s SEC=%s\n",
+       kprintf("%s: Found %s \"%*.*s %*.*s\" serial=\"%*.*s\"\n"
+               "%s: tags=%d/%d satacap=%04x satafea=%04x NCQ=%s "
+               "capacity=%lld.%02dMB\n",
+
                ATANAME(ap, atx),
                type,
-               model_len, model_len,
-               at->at_identify.model,
-               at->at_identify.firmware,
-               at->at_identify.serial,
+               model_len, model_len, model_id,
+               firmware_len, firmware_len, firmware_id,
+               serial_len, serial_len, serial_id,
 
                ATANAME(ap, atx),
                devncqdepth, ap->ap_sc->sc_ncmds,
                at->at_identify.satacap,
                at->at_identify.satafsup,
+               (at->at_ncqdepth > 1 ? "YES" : "NO"),
                (long long)capacity_bytes / (1024 * 1024),
-               (int)(capacity_bytes % (1024 * 1024)) * 100 / (1024 * 1024),
-
+               (int)(capacity_bytes % (1024 * 1024)) * 100 / (1024 * 1024)
+       );
+       kprintf("%s: f85=%04x f86=%04x f87=%04x WC=%s RA=%s SEC=%s\n",
                ATANAME(ap, atx),
                at->at_identify.features85,
                at->at_identify.features86,
@@ -483,7 +481,17 @@ ahci_cam_probe(struct ahci_port *ap, struct ata_port *atx)
                error = EIO;
                break;
        }
-       return (0);
+err:
+       if (error) {
+               at->at_probe = ATA_PROBE_FAILED;
+               if (atx == NULL)
+                       ap->ap_probe = at->at_probe;
+       } else {
+               at->at_probe = ATA_PROBE_GOOD;
+               if (atx == NULL)
+                       ap->ap_probe = at->at_probe;
+       }
+       return (error);
 }
 
 /*
@@ -494,9 +502,13 @@ ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
 {
        struct ata_port *at;
        struct ata_xfer *xa;
-       int status;
 
-       at = atx ? atx : ap->ap_ata;
+       at = atx ? atx : ap->ap_ata[0];
+
+       /*
+        * Set dummy xfer mode
+        */
+       ahci_set_xfer(ap, atx);
 
        /*
         * Enable write cache if supported
@@ -513,16 +525,18 @@ ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
                xa = ahci_ata_get_xfer(ap, atx);
                xa->complete = ahci_ata_dummy_done;
                xa->fis->command = ATA_C_SET_FEATURES;
-               /*xa->fis->features = ATA_SF_WRITECACHE_EN;*/
-               xa->fis->features = ATA_SF_LOOKAHEAD_EN;
+               xa->fis->features = ATA_SF_WRITECACHE_EN;
+               /* xa->fis->features = ATA_SF_LOOKAHEAD_EN; */
                xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
                xa->fis->device = 0;
-               xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
-               xa->timeout = hz;
+               xa->flags = ATA_F_PIO | ATA_F_POLL;
+               xa->timeout = 1000;
                xa->datalen = 0;
-               status = ahci_ata_cmd(xa);
-               if (status == ATA_COMPLETE)
+               if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
                        at->at_features |= ATA_PORT_F_WCACHE;
+               else
+                       kprintf("%s: Unable to enable write-caching\n",
+                               ATANAME(ap, atx));
                ahci_ata_put_xfer(xa);
        }
 
@@ -537,12 +551,14 @@ ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
                xa->fis->features = ATA_SF_LOOKAHEAD_EN;
                xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
                xa->fis->device = 0;
-               xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
-               xa->timeout = hz;
+               xa->flags = ATA_F_PIO | ATA_F_POLL;
+               xa->timeout = 1000;
                xa->datalen = 0;
-               status = ahci_ata_cmd(xa);
-               if (status == ATA_COMPLETE)
+               if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
                        at->at_features |= ATA_PORT_F_RAHEAD;
+               else
+                       kprintf("%s: Unable to enable read-ahead\n",
+                               ATANAME(ap, atx));
                ahci_ata_put_xfer(xa);
        }
 
@@ -554,17 +570,20 @@ ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
         * support it
         */
        if ((at->at_identify.cmdset82 & ATA_IDENTIFY_SECURITY) &&
-           (at->at_identify.securestatus & ATA_SECURE_FROZEN) == 0) {
+           (at->at_identify.securestatus & ATA_SECURE_FROZEN) == 0 &&
+           (AhciNoFeatures & (1 << ap->ap_num)) == 0) {
                xa = ahci_ata_get_xfer(ap, atx);
                xa->complete = ahci_ata_dummy_done;
                xa->fis->command = ATA_C_SEC_FREEZE_LOCK;
                xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
-               xa->flags = ATA_F_READ | ATA_F_PIO | ATA_F_POLL;
-               xa->timeout = hz;
+               xa->flags = ATA_F_PIO | ATA_F_POLL;
+               xa->timeout = 1000;
                xa->datalen = 0;
-               status = ahci_ata_cmd(xa);
-               if (status == ATA_COMPLETE)
+               if (ahci_ata_cmd(xa) == ATA_S_COMPLETE)
                        at->at_features |= ATA_PORT_F_FRZLCK;
+               else
+                       kprintf("%s: Unable to set security freeze\n",
+                               ATANAME(ap, atx));
                ahci_ata_put_xfer(xa);
        }
 
@@ -577,6 +596,68 @@ ahci_cam_probe_disk(struct ahci_port *ap, struct ata_port *atx)
 static int
 ahci_cam_probe_atapi(struct ahci_port *ap, struct ata_port *atx)
 {
+       ahci_set_xfer(ap, atx);
+       return(0);
+}
+
+/*
+ * Setting the transfer mode is irrelevant for the SATA transport
+ * but some (atapi) devices seem to need it anyway.  In addition
+ * if we are running through a SATA->PATA converter for some reason
+ * beyond my comprehension we might have to set the mode.
+ *
+ * We only support DMA modes for SATA attached devices, so don't bother
+ * with legacy modes.
+ */
+static int
+ahci_set_xfer(struct ahci_port *ap, struct ata_port *atx)
+{
+       struct ata_port *at;
+       struct ata_xfer *xa;
+       u_int16_t mode;
+       u_int16_t mask;
+
+       at = atx ? atx : ap->ap_ata[0];
+
+       /*
+        * Figure out the supported UDMA mode.  Ignore other legacy modes.
+        */
+       mask = le16toh(at->at_identify.ultradma);
+       if ((mask & 0xFF) == 0 || mask == 0xFFFF)
+               return(0);
+       mask &= 0xFF;
+       mode = 0x4F;
+       while ((mask & 0x8000) == 0) {
+               mask <<= 1;
+               --mode;
+       }
+
+       /*
+        * SATA atapi devices often still report a dma mode, even though
+        * it is irrelevant for SATA transport.  It is also possible that
+        * we are running through a SATA->PATA converter and seeing the
+        * PATA dma mode.
+        *
+        * In this case the device may require a (dummy) SETXFER to be
+        * sent before it will work properly.
+        */
+       xa = ahci_ata_get_xfer(ap, atx);
+       xa->complete = ahci_ata_dummy_done;
+       xa->fis->command = ATA_C_SET_FEATURES;
+       xa->fis->features = ATA_SF_SETXFER;
+       xa->fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
+       xa->fis->sector_count = mode;
+       xa->flags = ATA_F_PIO | ATA_F_POLL;
+       xa->timeout = 1000;
+       xa->datalen = 0;
+       if (ahci_ata_cmd(xa) != ATA_S_COMPLETE) {
+               kprintf("%s: Unable to set dummy xfer mode \n",
+                       ATANAME(ap, atx));
+       } else if (bootverbose) {
+               kprintf("%s: Set dummy xfer mode to %02x\n",
+                       ATANAME(ap, atx), mode);
+       }
+       ahci_ata_put_xfer(xa);
        return(0);
 }
 
@@ -612,14 +693,26 @@ ahci_ata_dummy_done(struct ata_xfer *xa)
 }
 
 /*
- * Initiate a bus scan.
+ * Use an engineering request to initiate a target scan for devices
+ * behind a port multiplier.
  *
- * An asynchronous bus scan is used to avoid reentrancy issues
+ * An asynchronous bus scan is used to avoid reentrancy issues.
  */
 static void
 ahci_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb)
 {
-       kfree(ccb, M_TEMP);
+       struct ahci_port *ap = ccb->ccb_h.sim_priv.entries[0].ptr;
+
+       if (ccb->ccb_h.func_code == XPT_SCAN_BUS) {
+               ap->ap_flags &= ~AP_F_SCAN_RUNNING;
+               if (ap->ap_flags & AP_F_SCAN_REQUESTED) {
+                       ap->ap_flags &= ~AP_F_SCAN_REQUESTED;
+                       ahci_cam_rescan(ap);
+               }
+               ap->ap_flags |= AP_F_SCAN_COMPLETED;
+               wakeup(&ap->ap_flags);
+       }
+       xpt_free_ccb(ccb);
 }
 
 static void
@@ -628,20 +721,50 @@ ahci_cam_rescan(struct ahci_port *ap)
        struct cam_path *path;
        union ccb *ccb;
        int status;
+       int i;
+
+       if (ap->ap_flags & AP_F_SCAN_RUNNING) {
+               ap->ap_flags |= AP_F_SCAN_REQUESTED;
+               return;
+       }
+       ap->ap_flags |= AP_F_SCAN_RUNNING;
+       for (i = 0; i < AHCI_MAX_PMPORTS; ++i) {
+               ap->ap_ata[i]->at_features |= ATA_PORT_F_RESCAN;
+       }
 
-       ccb = kmalloc(sizeof(*ccb), M_TEMP, M_WAITOK | M_ZERO);
        status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim),
                                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
        if (status != CAM_REQ_CMP)
                return;
 
+       ccb = xpt_alloc_ccb();
        xpt_setup_ccb(&ccb->ccb_h, path, 5);    /* 5 = low priority */
-       ccb->ccb_h.func_code = XPT_SCAN_BUS | XPT_FC_QUEUED;
+       ccb->ccb_h.func_code = XPT_ENG_EXEC;
        ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback;
+       ccb->ccb_h.sim_priv.entries[0].ptr = ap;
        ccb->crcn.flags = CAM_FLAG_NONE;
-       xpt_action(ccb);
+       xpt_action_async(ccb);
+}
+
+static void
+ahci_xpt_rescan(struct ahci_port *ap)
+{
+       struct cam_path *path;
+       union ccb *ccb;
+       int status;
 
-       /* scan is now underway */
+       status = xpt_create_path(&path, xpt_periph, cam_sim_path(ap->ap_sim),
+                                CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
+       if (status != CAM_REQ_CMP)
+               return;
+
+       ccb = xpt_alloc_ccb();
+       xpt_setup_ccb(&ccb->ccb_h, path, 5);    /* 5 = low priority */
+       ccb->ccb_h.func_code = XPT_SCAN_BUS;
+       ccb->ccb_h.cbfcnp = ahci_cam_rescan_callback;
+       ccb->ccb_h.sim_priv.entries[0].ptr = ap;
+       ccb->crcn.flags = CAM_FLAG_NONE;
+       xpt_action_async(ccb);
 }
 
 /*
@@ -658,13 +781,16 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
 
        /* XXX lock */
        ap = cam_sim_softc(sim);
-       at = ap->ap_ata;
        atx = NULL;
        KKASSERT(ap != NULL);
        ccbh = &ccb->ccb_h;
        unit = cam_sim_unit(sim);
 
        /*
+        * Early failure checks.  These checks do not apply to XPT_PATH_INQ,
+        * otherwise the bus rescan will not remove the dead devices when
+        * unplugging a PM.
+        *
         * For non-wildcards we have one target (0) and one lun (0),
         * unless we have a port multiplier.
         *
@@ -677,9 +803,10 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
         *
         * XXX What do we do with a LUN wildcard?
         */
-       if (ccbh->target_id != CAM_TARGET_WILDCARD) {
+       if (ccbh->target_id != CAM_TARGET_WILDCARD &&
+           ccbh->func_code != XPT_PATH_INQ) {
                if (ap->ap_type == ATA_PORT_T_NONE) {
-                       ccbh->status = CAM_REQ_INVALID;
+                       ccbh->status = CAM_DEV_NOT_THERE;
                        xpt_done(ccb);
                        return;
                }
@@ -688,7 +815,7 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                        xpt_done(ccb);
                        return;
                }
-               at += ccbh->target_id;
+               at = ap->ap_ata[ccbh->target_id];
                if (ap->ap_type == ATA_PORT_T_PM)
                        atx = at;
 
@@ -697,20 +824,38 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                        xpt_done(ccb);
                        return;
                }
+       } else {
+               at = ap->ap_ata[0];
        }
 
        /*
         * Switch on the meta XPT command
         */
        switch(ccbh->func_code) {
+       case XPT_ENG_EXEC:
+               /*
+                * This routine is called after a port multiplier has been
+                * probed.
+                */
+               ccbh->status = CAM_REQ_CMP;
+               ahci_os_lock_port(ap);
+               ahci_port_state_machine(ap, 0);
+               ahci_os_unlock_port(ap);
+               xpt_done(ccb);
+               ahci_xpt_rescan(ap);
+               break;
        case XPT_PATH_INQ:
+               /*
+                * This command always succeeds, otherwise the bus scan
+                * will not detach dead devices.
+                */
                ccb->cpi.version_num = 1;
                ccb->cpi.hba_inquiry = 0;
                ccb->cpi.target_sprt = 0;
                ccb->cpi.hba_misc = PIM_SEQSCAN;
                ccb->cpi.hba_eng_cnt = 0;
                bzero(ccb->cpi.vuhba_flags, sizeof(ccb->cpi.vuhba_flags));
-               ccb->cpi.max_target = AHCI_MAX_PMPORTS;
+               ccb->cpi.max_target = AHCI_MAX_PMPORTS - 1;
                ccb->cpi.max_lun = 0;
                ccb->cpi.async_flags = 0;
                ccb->cpi.hpath_id = 0;
@@ -718,19 +863,17 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                ccb->cpi.unit_number = cam_sim_unit(sim);
                ccb->cpi.bus_id = cam_sim_bus(sim);
                ccb->cpi.base_transfer_speed = 150000;
-               ccb->cpi.transport = XPORT_AHCI;
+               ccb->cpi.transport = XPORT_SATA;
                ccb->cpi.transport_version = 1;
                ccb->cpi.protocol = PROTO_SCSI;
                ccb->cpi.protocol_version = SCSI_REV_2;
 
-               /*
-                * Non-zero target and lun ids will be used for future
-                * port multiplication(?).  A target wildcard indicates only
-                * the general bus is being probed.
-                *
-                * XXX What do we do with a LUN wildcard?
-                */
-               if (ccbh->target_id != CAM_TARGET_WILDCARD) {
+               ccbh->status = CAM_REQ_CMP;
+               if (ccbh->target_id == CAM_TARGET_WILDCARD) {
+                       ahci_os_lock_port(ap);
+                       ahci_port_state_machine(ap, 0);
+                       ahci_os_unlock_port(ap);
+               } else {
                        switch(ahci_pread(ap, AHCI_PREG_SSTS) &
                               AHCI_PREG_SSTS_SPD) {
                        case AHCI_PREG_SSTS_SPD_GEN1:
@@ -739,65 +882,36 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                        case AHCI_PREG_SSTS_SPD_GEN2:
                                ccb->cpi.base_transfer_speed = 300000;
                                break;
+                       case AHCI_PREG_SSTS_SPD_GEN3:
+                               ccb->cpi.base_transfer_speed = 600000;
+                               break;
                        default:
                                /* unknown */
                                ccb->cpi.base_transfer_speed = 1000;
                                break;
                        }
-                       ccbh->status = CAM_REQ_CMP;
-
-                       /*
-                        * Initialize the port as needed
-                        */
-                       if (ap->ap_type == ATA_PORT_T_NONE) {
-                               if (ap->ap_probe == ATA_PROBE_NEED_HARD_RESET)
-                                       ahci_port_reset(ap, NULL, 1);
-                               if (ap->ap_probe == ATA_PROBE_NEED_SOFT_RESET)
-                                       ahci_port_reset(ap, NULL, 0);
-                               if (ap->ap_probe == ATA_PROBE_NEED_IDENT)
-                                       ahci_cam_probe(ap, atx);
-                               if (ap->ap_type == ATA_PORT_T_NONE)
-                                       ccbh->status = CAM_DEV_NOT_THERE;
-                       }
-
-                       /*
-                        * Initialize the target (if behind a port multiplier)
-                        * as needed.
-                        */
-                       if (ap->ap_type != ATA_PORT_T_NONE &&
-                           atx && atx->at_type == ATA_PORT_T_NONE) {
-                               if (atx->at_probe == ATA_PROBE_NEED_HARD_RESET)
-                                       ahci_port_reset(ap, atx, 1);
-                               if (atx->at_probe == ATA_PROBE_NEED_SOFT_RESET)
-                                       ahci_port_reset(ap, atx, 0);
-                               if (atx->at_probe == ATA_PROBE_NEED_IDENT)
-                                       ahci_cam_probe(ap, atx);
-                               if (atx->at_type == ATA_PORT_T_NONE)
-                                       ccbh->status = CAM_DEV_NOT_THERE;
-                       }
-               } else {
-                       ccbh->status = CAM_REQ_CMP;
+#if 0
+                       if (ap->ap_type == ATA_PORT_T_NONE)
+                               ccbh->status = CAM_DEV_NOT_THERE;
+#endif
                }
                xpt_done(ccb);
                break;
        case XPT_RESET_DEV:
-               lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
+               ahci_os_lock_port(ap);
                if (ap->ap_type == ATA_PORT_T_NONE) {
-                       ccbh->status = CAM_REQ_INVALID;
+                       ccbh->status = CAM_DEV_NOT_THERE;
                } else {
                        ahci_port_reset(ap, atx, 0);
                        ccbh->status = CAM_REQ_CMP;
                }
-               lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+               ahci_os_unlock_port(ap);
                xpt_done(ccb);
                break;
        case XPT_RESET_BUS:
-               lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
+               ahci_os_lock_port(ap);
                ahci_port_reset(ap, NULL, 1);
-               lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
-
-               xpt_async(AC_BUS_RESET, ap->ap_path, NULL);
-
+               ahci_os_unlock_port(ap);
                ccbh->status = CAM_REQ_CMP;
                xpt_done(ccb);
                break;
@@ -808,7 +922,7 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
        case XPT_GET_TRAN_SETTINGS:
                ccb->cts.protocol = PROTO_SCSI;
                ccb->cts.protocol_version = SCSI_REV_2;
-               ccb->cts.transport = XPORT_AHCI;
+               ccb->cts.transport = XPORT_SATA;
                ccb->cts.transport_version = XPORT_VERSION_UNSPECIFIED;
                ccb->cts.proto_specific.valid = 0;
                ccb->cts.xport_specific.valid = 0;
@@ -820,6 +934,17 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                xpt_done(ccb);
                break;
        case XPT_SCSI_IO:
+               /*
+                * Our parallel startup code might have only probed through
+                * to the IDENT, so do the last step if necessary.
+                */
+               if (at->at_probe == ATA_PROBE_NEED_IDENT)
+                       ahci_cam_probe(ap, atx);
+               if (at->at_probe != ATA_PROBE_GOOD) {
+                       ccbh->status = CAM_DEV_NOT_THERE;
+                       xpt_done(ccb);
+                       break;
+               }
                switch(at->at_type) {
                case ATA_PORT_T_DISK:
                        ahci_xpt_scsi_disk_io(ap, atx, ccb);
@@ -833,6 +958,17 @@ ahci_xpt_action(struct cam_sim *sim, union ccb *ccb)
                        break;
                }
                break;
+       case XPT_TRIM:
+       {
+               scsi_cdb_t cdb;
+               struct ccb_scsiio *csio;
+               csio = &ccb->csio;
+               cdb = (void *)((ccbh->flags & CAM_CDB_POINTER) ?
+                   csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
+               cdb->generic.opcode = TRIM;
+               ahci_xpt_scsi_disk_io(ap, atx, ccb);
+               break;
+       }
        default:
                ccbh->status = CAM_REQ_INVALID;
                xpt_done(ccb);
@@ -854,9 +990,9 @@ ahci_xpt_poll(struct cam_sim *sim)
 
        ap = cam_sim_softc(sim);
        crit_enter();
-       lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
-       ahci_port_intr(ap, AHCI_PREG_CI_ALL_SLOTS);
-       lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+       ahci_os_lock_port(ap);
+       ahci_port_intr(ap, 1);
+       ahci_os_unlock_port(ap);
        crit_exit();
 }
 
@@ -879,6 +1015,8 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
        struct ata_xfer *xa;
        struct ata_port *at;
        struct ata_fis_h2d *fis;
+       struct ata_pass_12 *atp12;
+       struct ata_pass_16 *atp16;
        scsi_cdb_t cdb;
        union scsi_data *rdata;
        int rdata_len;
@@ -888,7 +1026,7 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
 
        ccbh = &ccb->csio.ccb_h;
        csio = &ccb->csio;
-       at = atx ? atx : &ap->ap_ata[0];
+       at = atx ? atx : ap->ap_ata[0];
 
        /*
         * XXX not passing NULL at for direct attach!
@@ -918,17 +1056,7 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                 * [opcode, byte2, page_code, length, control]
                 */
                if (cdb->inquiry.byte2 & SI_EVPD) {
-                       switch(cdb->inquiry.page_code) {
-                       case SVPD_SUPPORTED_PAGE_LIST:
-                               /* XXX atascsi_disk_vpd_supported */
-                       case SVPD_UNIT_SERIAL_NUMBER:
-                               /* XXX atascsi_disk_vpd_serial */
-                       case SVPD_UNIT_DEVID:
-                               /* XXX atascsi_disk_vpd_ident */
-                       default:
-                               ccbh->status = CAM_FUNC_NOTAVAIL;
-                               break;
-                       }
+                       ahci_xpt_page_inquiry(ap, at, ccb);
                } else {
                        bzero(rdata, rdata_len);
                        if (rdata_len < SHORT_INQUIRY_LENGTH) {
@@ -950,6 +1078,17 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                              sizeof(rdata->inquiry_data.revision));
                        ccbh->status = CAM_REQ_CMP;
                }
+               
+               /*
+                * Use the vendor specific area to set the TRIM status
+                * for scsi_da
+                */
+               if (at->at_identify.support_dsm) {
+                       rdata->inquiry_data.vendor_specific1[0] =
+                           at->at_identify.support_dsm &ATA_SUPPORT_DSM_TRIM;
+                       rdata->inquiry_data.vendor_specific1[1] = 
+                           at->at_identify.max_dsm_blocks;
+               }
                break;
        case READ_CAPACITY_16:
                if (cdb->read_capacity_16.service_action != SRC16_SERVICE_ACTION) {
@@ -996,12 +1135,42 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                fis->flags = ATA_H2D_FLAGS_CMD;
                fis->command = ATA_C_FLUSH_CACHE;
                fis->device = 0;
-               if (xa->timeout < 45 * hz)
-                       xa->timeout = 45 * hz;
+               if (xa->timeout < 45000)
+                       xa->timeout = 45000;
                xa->datalen = 0;
-               xa->flags = ATA_F_READ;
+               xa->flags = 0;
                xa->complete = ahci_ata_complete_disk_synchronize_cache;
                break;
+       case TRIM:
+               fis = xa->fis;
+               fis->command = ATA_C_DATA_SET_MANAGEMENT;
+               fis->features = (u_int8_t)ATA_SF_DSM_TRIM;
+               fis->features_exp = (u_int8_t)(ATA_SF_DSM_TRIM>> 8);
+
+               xa->flags = ATA_F_WRITE;
+               fis->flags = ATA_H2D_FLAGS_CMD;
+
+               xa->data = csio->data_ptr;
+               xa->datalen = csio->dxfer_len;
+               xa->timeout = ccbh->timeout*50; /* milliseconds */
+
+               fis->sector_count =(u_int8_t)(xa->datalen/512);
+               fis->sector_count_exp =(u_int8_t)((xa->datalen/512)>>8);
+
+               lba = 0;
+               fis->lba_low = (u_int8_t)lba;
+               fis->lba_mid = (u_int8_t)(lba >> 8);
+               fis->lba_high = (u_int8_t)(lba >> 16);
+               fis->lba_low_exp = (u_int8_t)(lba >> 24);
+               fis->lba_mid_exp = (u_int8_t)(lba >> 32);
+               fis->lba_high_exp = (u_int8_t)(lba >> 40);
+                  
+               fis->device = ATA_H2D_DEVICE_LBA;
+               xa->data = csio->data_ptr;
+
+               xa->complete = ahci_ata_complete_disk_rw;
+               ccbh->status = CAM_REQ_INPROG;
+               break;
        case TEST_UNIT_READY:
        case START_STOP_UNIT:
        case PREVENT_ALLOW:
@@ -1012,11 +1181,92 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                rdata_len = 0;
                break;
        case ATA_PASS_12:
+               atp12 = &cdb->ata_pass_12;
+               fis = xa->fis;
+               /*
+                * Figure out the flags to be used, depending on the direction of the
+                * CAM request.
+                */
+               switch (ccbh->flags & CAM_DIR_MASK) {
+               case CAM_DIR_IN:
+                       xa->flags = ATA_F_READ;
+                       break;
+               case CAM_DIR_OUT:
+                       xa->flags = ATA_F_WRITE;
+                       break;
+               default:
+                       xa->flags = 0;
+               }
+               xa->flags |= ATA_F_POLL | ATA_F_EXCLUSIVE;
+               xa->data = csio->data_ptr;
+               xa->datalen = csio->dxfer_len;
+               xa->complete = ahci_ata_complete_disk_rw;
+               xa->timeout = ccbh->timeout;
+
+               /*
+                * Populate the fis from the information we received through CAM
+                * ATA passthrough.
+                */
+               fis->flags = ATA_H2D_FLAGS_CMD; /* maybe also atp12->flags ? */
+               fis->features = atp12->features;
+               fis->sector_count = atp12->sector_count;
+               fis->lba_low = atp12->lba_low;
+               fis->lba_mid = atp12->lba_mid;
+               fis->lba_high = atp12->lba_high;
+               fis->device = atp12->device;    /* maybe always 0? */
+               fis->command = atp12->command;
+               fis->control = atp12->control;
+
+               /*
+                * Mark as in progress so it is sent to the device.
+                */
+               ccbh->status = CAM_REQ_INPROG;
+               break;
        case ATA_PASS_16:
+               atp16 = &cdb->ata_pass_16;
+               fis = xa->fis;
                /*
-                * XXX implement pass-through
+                * Figure out the flags to be used, depending on the direction of the
+                * CAM request.
                 */
-               ccbh->status = CAM_FUNC_NOTAVAIL;
+               switch (ccbh->flags & CAM_DIR_MASK) {
+               case CAM_DIR_IN:
+                       xa->flags = ATA_F_READ;
+                       break;
+               case CAM_DIR_OUT:
+                       xa->flags = ATA_F_WRITE;
+                       break;
+               default:
+                       xa->flags = 0;
+               }
+               xa->flags |= ATA_F_POLL | ATA_F_EXCLUSIVE;
+               xa->data = csio->data_ptr;
+               xa->datalen = csio->dxfer_len;
+               xa->complete = ahci_ata_complete_disk_rw;
+               xa->timeout = ccbh->timeout;
+
+               /*
+                * Populate the fis from the information we received through CAM
+                * ATA passthrough.
+                */
+               fis->flags = ATA_H2D_FLAGS_CMD; /* maybe also atp16->flags ? */
+               fis->features = atp16->features;
+               fis->features_exp = atp16->features_ext;
+               fis->sector_count = atp16->sector_count;
+               fis->sector_count_exp = atp16->sector_count_ext;
+               fis->lba_low = atp16->lba_low;
+               fis->lba_low_exp = atp16->lba_low_ext;
+               fis->lba_mid = atp16->lba_mid;
+               fis->lba_mid_exp = atp16->lba_mid_ext;
+               fis->lba_high = atp16->lba_high;
+               fis->lba_mid_exp = atp16->lba_mid_ext;
+               fis->device = atp16->device;    /* maybe always 0? */
+               fis->command = atp16->command;
+
+               /*
+                * Mark as in progress so it is sent to the device.
+                */
+               ccbh->status = CAM_REQ_INPROG;
                break;
        default:
                switch(cdb->generic.opcode) {
@@ -1094,7 +1344,7 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                        fis->sector_count = xa->tag << 3;
                        fis->features = (u_int8_t)count;
                        fis->features_exp = (u_int8_t)(count >> 8);
-               } else if (count > 0x100 || lba > 0xFFFFFFFFU) {
+               } else if (count > 0x100 || lba > 0x0FFFFFFFU) {
                        /*
                         * Use LBA48
                         */
@@ -1120,7 +1370,11 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
                xa->data = csio->data_ptr;
                xa->datalen = csio->dxfer_len;
                xa->complete = ahci_ata_complete_disk_rw;
-               xa->timeout = ccbh->timeout * hz / 1000;
+               xa->timeout = ccbh->timeout;    /* milliseconds */
+#if 0
+               if (xa->timeout > 10000)        /* XXX - debug */
+                       xa->timeout = 10000;
+#endif
                if (ccbh->flags & CAM_POLLED)
                        xa->flags |= ATA_F_POLL;
                break;
@@ -1128,17 +1382,17 @@ ahci_xpt_scsi_disk_io(struct ahci_port *ap, struct ata_port *atx,
 
        /*
         * If the request is still in progress the xa and FIS have
-        * been set up and must be dispatched.  Otherwise the request
-        * is complete.
+        * been set up (except for the PM target), and must be dispatched.
+        * Otherwise the request was completed.
         */
        if (ccbh->status == CAM_REQ_INPROG) {
                KKASSERT(xa->complete != NULL);
                xa->atascsi_private = ccb;
                ccb->ccb_h.sim_priv.entries[0].ptr = ap;
-               lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
-               fis->flags |= at->at_target;
+               ahci_os_lock_port(ap);
+               xa->fis->flags |= at->at_target;
                ahci_ata_cmd(xa);
-               lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+               ahci_os_unlock_port(ap);
        } else {
                ahci_ata_put_xfer(xa);
                xpt_done(ccb);
@@ -1167,7 +1421,7 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
 
        ccbh = &ccb->csio.ccb_h;
        csio = &ccb->csio;
-       at = atx ? atx : &ap->ap_ata[0];
+       at = atx ? atx : ap->ap_ata[0];
 
        switch (ccbh->flags & CAM_DIR_MASK) {
        case CAM_DIR_IN:
@@ -1186,6 +1440,13 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
                /* NOT REACHED */
        }
 
+       /*
+        * Special handling to get the rfis back into host memory while
+        * still allowing the chip to run commands in parallel to
+        * ATAPI devices behind a PM.
+        */
+       flags |= ATA_F_AUTOSENSE;
+
        /*
         * The command has to fit in the packet command buffer.
         */
@@ -1196,7 +1457,8 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
        }
 
        /*
-        * Initialize the XA and FIS.
+        * Initialize the XA and FIS.  It is unclear how much of
+        * this has to mimic the equivalent ATA command.
         *
         * XXX not passing NULL at for direct attach!
         */
@@ -1205,18 +1467,26 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
 
        fis->flags = ATA_H2D_FLAGS_CMD | at->at_target;
        fis->command = ATA_C_PACKET;
-       fis->device = 0;
+       fis->device = ATA_H2D_DEVICE_LBA;
        fis->sector_count = xa->tag << 3;
-       fis->features = ATA_H2D_FEATURES_DMA |
-                   ((flags & ATA_F_WRITE) ?
-                   ATA_H2D_FEATURES_DIR_WRITE : ATA_H2D_FEATURES_DIR_READ);
-       fis->lba_mid = 0x00;
-       fis->lba_high = 0x20;
+       if (flags & (ATA_F_READ | ATA_F_WRITE)) {
+               if (flags & ATA_F_WRITE) {
+                       fis->features = ATA_H2D_FEATURES_DMA |
+                                      ATA_H2D_FEATURES_DIR_WRITE;
+               } else {
+                       fis->features = ATA_H2D_FEATURES_DMA |
+                                      ATA_H2D_FEATURES_DIR_READ;
+               }
+       } else {
+               fis->lba_mid = 0;
+               fis->lba_high = 0;
+       }
+       fis->control = ATA_FIS_CONTROL_4BIT;
 
        xa->flags = flags;
        xa->data = csio->data_ptr;
        xa->datalen = csio->dxfer_len;
-       xa->timeout = ccbh->timeout * hz / 1000;
+       xa->timeout = ccbh->timeout;    /* milliseconds */
 
        if (ccbh->flags & CAM_POLLED)
                xa->flags |= ATA_F_POLL;
@@ -1224,10 +1494,14 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
        /*
         * Copy the cdb to the packetcmd buffer in the FIS using a
         * convenient pointer in the xa.
+        *
+        * Zero-out any trailing bytes in case the ATAPI device cares.
         */
        cdbs = (void *)((ccbh->flags & CAM_CDB_POINTER) ?
                        csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
        bcopy(cdbs, xa->packetcmd, csio->cdb_len);
+       if (csio->cdb_len < 16)
+               bzero(xa->packetcmd + csio->cdb_len, 16 - csio->cdb_len);
 
 #if 0
        kprintf("opcode %d cdb_len %d dxfer_len %d\n",
@@ -1241,18 +1515,26 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
        cdbd = (void *)xa->packetcmd;
 
        switch(cdbd->generic.opcode) {
-       case INQUIRY:
+       case REQUEST_SENSE:
                /*
-                * Some ATAPI devices can't handle SI_EVPD being set
-                * for a basic inquiry (page_code == 0).
+                * Force SENSE requests to the ATAPI sense length.
                 *
+                * It is unclear if this is needed or not.
+                */
+               if (cdbd->sense.length == SSD_FULL_SIZE) {
+                       if (bootverbose) {
+                               kprintf("%s: Shortening sense request\n",
+                                       PORTNAME(ap));
+                       }
+                       cdbd->sense.length = offsetof(struct scsi_sense_data,
+                                                     extra_bytes[0]);
+               }
+               break;
+       case INQUIRY:
+               /*
                 * Some ATAPI devices can't handle long inquiry lengths,
                 * don't ask me why.  Truncate the inquiry length.
                 */
-               if ((cdbd->inquiry.byte2 & SI_EVPD) &&
-                   cdbd->inquiry.page_code == 0) {
-                       cdbd->inquiry.byte2 &= ~SI_EVPD;
-               }
                if (cdbd->inquiry.page_code == 0 &&
                    cdbd->inquiry.length > SHORT_INQUIRY_LENGTH) {
                        cdbd->inquiry.length = SHORT_INQUIRY_LENGTH;
@@ -1285,7 +1567,78 @@ ahci_xpt_scsi_atapi_io(struct ahci_port *ap, struct ata_port *atx,
        xa->complete = ahci_atapi_complete_cmd;
        xa->atascsi_private = ccb;
        ccb->ccb_h.sim_priv.entries[0].ptr = ap;
+       ahci_os_lock_port(ap);
        ahci_ata_cmd(xa);
+       ahci_os_unlock_port(ap);
+}
+
+/*
+ * Simulate page inquiries for disk attachments.
+ */
+static
+void
+ahci_xpt_page_inquiry(struct ahci_port *ap, struct ata_port *at, union ccb *ccb)
+{
+       union {
+               struct scsi_vpd_supported_page_list     list;
+               struct scsi_vpd_unit_serial_number      serno;
+               struct scsi_vpd_unit_devid              devid;
+               char                                    buf[256];
+       } *page;
+       scsi_cdb_t cdb;
+       int i;
+       int j;
+       int len;
+
+       page = kmalloc(sizeof(*page), M_DEVBUF, M_WAITOK | M_ZERO);
+
+       cdb = (void *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
+                       ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes);
+
+       switch(cdb->inquiry.page_code) {
+       case SVPD_SUPPORTED_PAGE_LIST:
+               i = 0;
+               page->list.device = T_DIRECT;
+               page->list.page_code = SVPD_SUPPORTED_PAGE_LIST;
+               page->list.list[i++] = SVPD_SUPPORTED_PAGE_LIST;
+               page->list.list[i++] = SVPD_UNIT_SERIAL_NUMBER;
+               page->list.list[i++] = SVPD_UNIT_DEVID;
+               page->list.length = i;
+               len = offsetof(struct scsi_vpd_supported_page_list, list[3]);
+               break;
+       case SVPD_UNIT_SERIAL_NUMBER:
+               i = 0;
+               j = sizeof(at->at_identify.serial);
+               for (i = 0; i < j && at->at_identify.serial[i] == ' '; ++i)
+                       ;
+               while (j > i && at->at_identify.serial[j-1] == ' ')
+                       --j;
+               page->serno.device = T_DIRECT;
+               page->serno.page_code = SVPD_UNIT_SERIAL_NUMBER;
+               page->serno.length = j - i;
+               bcopy(at->at_identify.serial + i,
+                     page->serno.serial_num, j - i);
+               len = offsetof(struct scsi_vpd_unit_serial_number,
+                              serial_num[j-i]);
+               break;
+       case SVPD_UNIT_DEVID:
+               /* fall through for now */
+       default:
+               ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
+               len = 0;
+               break;
+       }
+       if (ccb->ccb_h.status == CAM_REQ_INPROG) {
+               if (len <= ccb->csio.dxfer_len) {
+                       ccb->ccb_h.status = CAM_REQ_CMP;
+                       bzero(ccb->csio.data_ptr, ccb->csio.dxfer_len);
+                       bcopy(page, ccb->csio.data_ptr, len);
+                       ccb->csio.resid = ccb->csio.dxfer_len - len;
+               } else {
+                       ccb->ccb_h.status = CAM_CCB_LEN_ERR;
+               }
+       }
+       kfree(page, M_DEVBUF);
 }
 
 /*
@@ -1323,9 +1676,9 @@ ahci_ata_complete_disk_synchronize_cache(struct ata_xfer *xa)
                break;
        }
        ahci_ata_put_xfer(xa);
-       lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+       ahci_os_unlock_port(ap);
        xpt_done(ccb);
-       lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
+       ahci_os_lock_port(ap);
 }
 
 /*
@@ -1353,6 +1706,8 @@ ahci_ata_complete_disk_rw(struct ata_xfer *xa)
        case ATA_S_TIMEOUT:
                kprintf("%s: disk_rw: timeout\n", ATANAME(ap, xa->at));
                ccbh->status = CAM_CMD_TIMEOUT;
+               ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+               ahci_ata_dummy_sense(&ccb->csio.sense_data);
                break;
        default:
                kprintf("%s: disk_rw: unknown state %d\n",
@@ -1362,9 +1717,9 @@ ahci_ata_complete_disk_rw(struct ata_xfer *xa)
        }
        ccb->csio.resid = xa->resid;
        ahci_ata_put_xfer(xa);
-       lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+       ahci_os_unlock_port(ap);
        xpt_done(ccb);
-       lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
+       ahci_os_lock_port(ap);
 }
 
 /*
@@ -1390,8 +1745,6 @@ ahci_atapi_complete_cmd(struct ata_xfer *xa)
                ccb->csio.scsi_status = SCSI_STATUS_OK;
                break;
        case ATA_S_ERROR:
-               kprintf("%s: cmd %d: error\n",
-                       PORTNAME(ap), cdb->generic.opcode);
                ccbh->status = CAM_SCSI_STATUS_ERROR;
                ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
                ahci_ata_atapi_sense(&xa->rfis, &ccb->csio.sense_data);
@@ -1400,6 +1753,8 @@ ahci_atapi_complete_cmd(struct ata_xfer *xa)
                kprintf("%s: cmd %d: timeout\n",
                        PORTNAME(ap), cdb->generic.opcode);
                ccbh->status = CAM_CMD_TIMEOUT;
+               ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+               ahci_ata_dummy_sense(&ccb->csio.sense_data);
                break;
        default:
                kprintf("%s: cmd %d: unknown state %d\n",
@@ -1409,9 +1764,9 @@ ahci_atapi_complete_cmd(struct ata_xfer *xa)
        }
        ccb->csio.resid = xa->resid;
        ahci_ata_put_xfer(xa);
-       lwkt_serialize_exit(&ap->ap_sc->sc_serializer);
+       ahci_os_unlock_port(ap);
        xpt_done(ccb);
-       lwkt_serialize_enter(&ap->ap_sc->sc_serializer);
+       ahci_os_lock_port(ap);
 }
 
 /*
@@ -1457,3 +1812,20 @@ ahci_ata_atapi_sense(struct ata_fis_d2h *rfis,
        sense_data->info[3] = 0;
        sense_data->extra_len = 0;
 }
+
+static
+void
+ahci_strip_string(const char **basep, int *lenp)
+{
+       const char *base = *basep;
+       int len = *lenp;
+
+       while (len && (*base == 0 || *base == ' ')) {
+               --len;
+               ++base;
+       }
+       while (len && (base[len-1] == 0 || base[len-1] == ' '))
+               --len;
+       *basep = base;
+       *lenp = len;
+}