sys/dev/virtual/virtio/scsi/virtio_scsi.c

   1 /*-
   2  * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * $FreeBSD: head/sys/dev/virtio/scsi/virtio_scsi.c 311305 2017-01-04 20:26:42Z asomers $
  27  */
  28
  29 /* Driver for VirtIO SCSI devices. */
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/kthread.h>
  35 #include <sys/malloc.h>
  36 #include <sys/module.h>
  37 #include <sys/sglist.h>
  38 #include <sys/sysctl.h>
  39 #include <sys/lock.h>
  40 #include <sys/callout.h>
  41 #include <sys/queue.h>
  42 #include <sys/sbuf.h>
  43
  44 #include <machine/stdarg.h>
  45
  46 #include <sys/bus.h>
  47 #include <sys/rman.h>
  48
  49 #include <bus/cam/cam.h>
  50 #include <bus/cam/cam_ccb.h>
  51 #include <bus/cam/cam_sim.h>
  52 #include <bus/cam/cam_periph.h>
  53 #include <bus/cam/cam_xpt_periph.h>
  54 #include <bus/cam/cam_xpt_sim.h>
  55 #include <bus/cam/cam_debug.h>
  56 #include <bus/cam/scsi/scsi_all.h>
  57 #include <bus/cam/scsi/scsi_message.h>
  58
  59 #include <dev/virtual/virtio/virtio/virtio.h>
  60 #include <dev/virtual/virtio/virtio/virtqueue.h>
  61 #include <dev/virtual/virtio/scsi/virtio_scsi.h>
  62 #include <dev/virtual/virtio/scsi/virtio_scsivar.h>
  63
  64 static int      vtscsi_modevent(module_t, int, void *);
  65
  66 static int      vtscsi_probe(device_t);
  67 static int      vtscsi_attach(device_t);
  68 static int      vtscsi_detach(device_t);
  69 static int      vtscsi_suspend(device_t);
  70 static int      vtscsi_resume(device_t);
  71
  72 static void     vtscsi_negotiate_features(struct vtscsi_softc *);
  73 static void     vtscsi_read_config(struct vtscsi_softc *,
  74                     struct virtio_scsi_config *);
  75 static int      vtscsi_maximum_segments(struct vtscsi_softc *, int);
  76 static int      vtscsi_alloc_intrs(struct vtscsi_softc *);
  77 static int      vtscsi_alloc_virtqueues(struct vtscsi_softc *);
  78 static void     vtscsi_write_device_config(struct vtscsi_softc *);
  79 static int      vtscsi_reinit(struct vtscsi_softc *);
  80
  81 static int      vtscsi_alloc_cam(struct vtscsi_softc *);
  82 static int      vtscsi_register_cam(struct vtscsi_softc *);
  83 static void     vtscsi_free_cam(struct vtscsi_softc *);
  84 static void     vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *);
  85 static int      vtscsi_register_async(struct vtscsi_softc *);
  86 static void     vtscsi_deregister_async(struct vtscsi_softc *);
  87 static void     vtscsi_cam_action(struct cam_sim *, union ccb *);
  88 static void     vtscsi_cam_poll(struct cam_sim *);
  89
  90 static void     vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *,
  91                     union ccb *);
  92 static void     vtscsi_cam_get_tran_settings(struct vtscsi_softc *,
  93                     union ccb *);
  94 static void     vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *);
  95 static void     vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *);
  96 static void     vtscsi_cam_abort(struct vtscsi_softc *, union ccb *);
  97 static void     vtscsi_cam_path_inquiry(struct vtscsi_softc *,
  98                     struct cam_sim *, union ccb *);
  99
 100 static int      vtscsi_sg_append_scsi_buf(struct vtscsi_softc *,
 101                     struct sglist *, struct ccb_scsiio *);
 102 static int      vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *,
 103                     struct vtscsi_request *, int *, int *);
 104 static int      vtscsi_execute_scsi_cmd(struct vtscsi_softc *,
 105                     struct vtscsi_request *);
 106 static int      vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *);
 107 static void     vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *,
 108                     struct vtscsi_request *);
 109 static int      vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *,
 110                     struct vtscsi_request *);
 111 static void     vtscsi_timedout_scsi_cmd(void *);
 112 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *);
 113 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *,
 114                     struct ccb_scsiio *, struct virtio_scsi_cmd_resp *);
 115 static void     vtscsi_complete_scsi_cmd(struct vtscsi_softc *,
 116                     struct vtscsi_request *);
 117
 118 static void     vtscsi_poll_ctrl_req(struct vtscsi_softc *,
 119                     struct vtscsi_request *);
 120 static int      vtscsi_execute_ctrl_req(struct vtscsi_softc *,
 121                     struct vtscsi_request *, struct sglist *, int, int, int);
 122 static void     vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c,
 123                     struct vtscsi_request *);
 124 static int      vtscsi_execute_abort_task_cmd(struct vtscsi_softc *,
 125                     struct vtscsi_request *);
 126 static int      vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *,
 127                     struct vtscsi_request *);
 128
 129 static void     vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
 130 static void     vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
 131 static void     vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
 132                     struct virtio_scsi_cmd_req *);
 133 static void     vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
 134                     uintptr_t, struct virtio_scsi_ctrl_tmf_req *);
 135
 136 static void     vtscsi_freeze_simq(struct vtscsi_softc *, int);
 137 static int      vtscsi_thaw_simq(struct vtscsi_softc *, int);
 138
 139 static void     vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t,
 140                     lun_id_t);
 141 static void     vtscsi_cam_rescan_callback(struct cam_periph *periph,
 142                     union ccb *ccb);
 143 static void     vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t,
 144                     lun_id_t);
 145 static void     vtscsi_execute_rescan_bus(struct vtscsi_softc *);
 146
 147 static void     vtscsi_handle_event(struct vtscsi_softc *,
 148                     struct virtio_scsi_event *);
 149 static int      vtscsi_enqueue_event_buf(struct vtscsi_softc *,
 150                     struct virtio_scsi_event *);
 151 static int      vtscsi_init_event_vq(struct vtscsi_softc *);
 152 static void     vtscsi_reinit_event_vq(struct vtscsi_softc *);
 153 static void     vtscsi_drain_event_vq(struct vtscsi_softc *);
 154
 155 static void     vtscsi_complete_vqs_locked(struct vtscsi_softc *);
 156 static void     vtscsi_complete_vqs(struct vtscsi_softc *);
 157 static void     vtscsi_drain_vqs(struct vtscsi_softc *);
 158 static void     vtscsi_cancel_request(struct vtscsi_softc *,
 159                     struct vtscsi_request *);
 160 static void     vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *);
 161 static void     vtscsi_stop(struct vtscsi_softc *);
 162 static int      vtscsi_reset_bus(struct vtscsi_softc *);
 163
 164 static void     vtscsi_init_request(struct vtscsi_softc *,
 165                     struct vtscsi_request *);
 166 static int      vtscsi_alloc_requests(struct vtscsi_softc *);
 167 static void     vtscsi_free_requests(struct vtscsi_softc *);
 168 static void     vtscsi_enqueue_request(struct vtscsi_softc *,
 169                     struct vtscsi_request *);
 170 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *);
 171
 172 static void     vtscsi_complete_request(struct vtscsi_request *);
 173 static void     vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);
 174
 175 static void     vtscsi_control_vq_intr(void *);
 176 static void     vtscsi_event_vq_intr(void *);
 177 static void     vtscsi_request_vq_intr(void *);
 178 static void     vtscsi_disable_vqs_intr(struct vtscsi_softc *);
 179 static void     vtscsi_enable_vqs_intr(struct vtscsi_softc *);
 180
 181 static void     vtscsi_get_tunables(struct vtscsi_softc *);
 182 static void     vtscsi_add_sysctl(struct vtscsi_softc *);
 183
 184 static void     vtscsi_printf_req(struct vtscsi_request *, const char *,
 185                     const char *, ...) __printflike(3, 4);
 186
 187 /* Global tunables. */
 188 /*
 189  * The current QEMU VirtIO SCSI implementation does not cancel in-flight
 190  * IO during virtio_stop(). So in-flight requests still complete after the
 191  * device reset. We would have to wait for all the in-flight IO to complete,
 192  * which defeats the typical purpose of a bus reset. We could simulate the
 193  * bus reset with either I_T_NEXUS_RESET of all the targets, or with
 194  * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the
 195  * control virtqueue). But this isn't very useful if things really go off
 196  * the rails, so default to disabled for now.
 197  */
 198 static int vtscsi_bus_reset_disable = 1;
 199 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable);
 200
 201 static struct virtio_feature_desc vtscsi_feature_desc[] = {
 202         { VIRTIO_SCSI_F_INOUT,          "InOut"         },
 203         { VIRTIO_SCSI_F_HOTPLUG,        "Hotplug"       },
 204
 205         { 0, NULL }
 206 };
 207
 208 static device_method_t vtscsi_methods[] = {
 209         /* Device methods. */
 210         DEVMETHOD(device_probe,         vtscsi_probe),
 211         DEVMETHOD(device_attach,        vtscsi_attach),
 212         DEVMETHOD(device_detach,        vtscsi_detach),
 213         DEVMETHOD(device_suspend,       vtscsi_suspend),
 214         DEVMETHOD(device_resume,        vtscsi_resume),
 215
 216         DEVMETHOD_END
 217 };
 218
 219 static driver_t vtscsi_driver = {
 220         "vtscsi",
 221         vtscsi_methods,
 222         sizeof(struct vtscsi_softc)
 223 };
 224 static devclass_t vtscsi_devclass;
 225
 226 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass,
 227     vtscsi_modevent, NULL);
 228 MODULE_VERSION(virtio_scsi, 1);
 229 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);
 230 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);
 231
 232 static int
 233 vtscsi_modevent(module_t mod, int type, void *unused)
 234 {
 235         int error;
 236
 237         switch (type) {
 238         case MOD_LOAD:
 239         case MOD_UNLOAD:
 240         case MOD_SHUTDOWN:
 241                 error = 0;
 242                 break;
 243         default:
 244                 error = EOPNOTSUPP;
 245                 break;
 246         }
 247
 248         return (error);
 249 }
 250
 251 static int
 252 vtscsi_probe(device_t dev)
 253 {
 254
 255         if (virtio_get_device_type(dev) != VIRTIO_ID_SCSI)
 256                 return (ENXIO);
 257
 258         device_set_desc(dev, "VirtIO SCSI Adapter");
 259
 260         return (BUS_PROBE_DEFAULT);
 261 }
 262
 263 struct irqmap {
 264         int irq;
 265         driver_intr_t *handler;
 266 };
 267
 268 static int
 269 vtscsi_attach(device_t dev)
 270 {
 271         struct vtscsi_softc *sc;
 272         struct virtio_scsi_config scsicfg;
 273         int i, error;
 274
 275         sc = device_get_softc(dev);
 276         sc->vtscsi_dev = dev;
 277
 278         VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));
 279         TAILQ_INIT(&sc->vtscsi_req_free);
 280
 281         vtscsi_get_tunables(sc);
 282         vtscsi_add_sysctl(sc);
 283
 284         virtio_set_feature_desc(dev, vtscsi_feature_desc);
 285         vtscsi_negotiate_features(sc);
 286
 287         if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
 288                 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT;
 289 #ifndef __DragonFly__ /* XXX swildner */
 290         if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
 291                 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL;
 292 #endif
 293         if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
 294                 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;
 295
 296         vtscsi_read_config(sc, &scsicfg);
 297
 298         sc->vtscsi_max_channel = scsicfg.max_channel;
 299         sc->vtscsi_max_target = scsicfg.max_target;
 300         sc->vtscsi_max_lun = scsicfg.max_lun;
 301         sc->vtscsi_event_buf_size = scsicfg.event_info_size;
 302
 303         vtscsi_write_device_config(sc);
 304
 305         sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max);
 306         sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT);
 307         if (sc->vtscsi_sglist == NULL) {
 308                 error = ENOMEM;
 309                 device_printf(dev, "cannot allocate sglist\n");
 310                 goto fail;
 311         }
 312
 313         error = vtscsi_alloc_intrs(sc);
 314         if (error) {
 315                 device_printf(dev, "cannot allocate interrupts\n");
 316                 goto fail;
 317         }
 318
 319         error = vtscsi_alloc_virtqueues(sc);
 320         if (error) {
 321                 device_printf(dev, "cannot allocate virtqueues\n");
 322                 goto fail;
 323         }
 324
 325         /* XXX Separate function */
 326         struct irqmap info[3];
 327
 328         /* Possible "Virtqueue <-> IRQ" configurations */
 329         switch (sc->vtscsi_nintr) {
 330         case 1:
 331                 info[0] = (struct irqmap){0, vtscsi_control_vq_intr};
 332                 info[1] = (struct irqmap){0, vtscsi_event_vq_intr};
 333                 info[2] = (struct irqmap){0, vtscsi_request_vq_intr};
 334                 break;
 335         case 2:
 336                 info[0] = (struct irqmap){0, vtscsi_control_vq_intr};
 337                 info[1] = (struct irqmap){0, vtscsi_event_vq_intr};
 338                 info[2] = (struct irqmap){1, vtscsi_request_vq_intr};
 339                 break;
 340         case 3:
 341                 info[0] = (struct irqmap){0, vtscsi_control_vq_intr};
 342                 info[1] = (struct irqmap){1, vtscsi_event_vq_intr};
 343                 info[2] = (struct irqmap){2, vtscsi_request_vq_intr};
 344                 break;
 345         default:
 346                 device_printf(dev, "Invalid interrupt vector count: %d\n",
 347                     sc->vtscsi_nintr);
 348                 goto fail;
 349         }
 350         for (i = 0; i < 3; i++) {
 351                 error = virtio_bind_intr(sc->vtscsi_dev, info[i].irq, i,
 352                     info[i].handler, sc);
 353                 if (error) {
 354                         device_printf(dev,
 355                             "cannot bind virtqueue IRQs\n");
 356                         goto fail;
 357                 }
 358         }
 359
 360         error = vtscsi_init_event_vq(sc);
 361         if (error) {
 362                 device_printf(dev, "cannot populate the eventvq\n");
 363                 goto fail;
 364         }
 365
 366         error = vtscsi_alloc_requests(sc);
 367         if (error) {
 368                 device_printf(dev, "cannot allocate requests\n");
 369                 goto fail;
 370         }
 371
 372         error = vtscsi_alloc_cam(sc);
 373         if (error) {
 374                 device_printf(dev, "cannot allocate CAM structures\n");
 375                 goto fail;
 376         }
 377
 378         for (i = 0; i < sc->vtscsi_nintr; i++) {
 379                 error = virtio_setup_intr(dev, i, NULL);
 380                 if (error) {
 381                         device_printf(dev, "cannot setup virtqueue "
 382                             "interrupts\n");
 383                         goto fail;
 384                 }
 385         }
 386
 387         vtscsi_enable_vqs_intr(sc);
 388
 389         /*
 390          * Register with CAM after interrupts are enabled so we will get
 391          * notified of the probe responses.
 392          */
 393         error = vtscsi_register_cam(sc);
 394         if (error) {
 395                 device_printf(dev, "cannot register with CAM\n");
 396                 goto fail;
 397         }
 398
 399 fail:
 400         if (error)
 401                 vtscsi_detach(dev);
 402
 403         return (error);
 404 }
 405
 406 static int
 407 vtscsi_detach(device_t dev)
 408 {
 409         struct vtscsi_softc *sc;
 410         int i;
 411
 412         sc = device_get_softc(dev);
 413
 414         for (i = 0; i < sc->vtscsi_nintr; i++)
 415                 virtio_teardown_intr(dev, i);
 416
 417         VTSCSI_LOCK(sc);
 418         sc->vtscsi_flags |= VTSCSI_FLAG_DETACH;
 419         if (device_is_attached(dev))
 420                 vtscsi_stop(sc);
 421         VTSCSI_UNLOCK(sc);
 422
 423         vtscsi_complete_vqs(sc);
 424         vtscsi_drain_vqs(sc);
 425
 426         vtscsi_free_cam(sc);
 427         vtscsi_free_requests(sc);
 428
 429         if (sc->vtscsi_sglist != NULL) {
 430                 sglist_free(sc->vtscsi_sglist);
 431                 sc->vtscsi_sglist = NULL;
 432         }
 433
 434         VTSCSI_LOCK_DESTROY(sc);
 435
 436         return (0);
 437 }
 438
 439 static int
 440 vtscsi_suspend(device_t dev)
 441 {
 442
 443         return (0);
 444 }
 445
 446 static int
 447 vtscsi_resume(device_t dev)
 448 {
 449
 450         return (0);
 451 }
 452
 453 static void
 454 vtscsi_negotiate_features(struct vtscsi_softc *sc)
 455 {
 456         device_t dev;
 457         uint64_t features;
 458
 459         dev = sc->vtscsi_dev;
 460         features = virtio_negotiate_features(dev, VTSCSI_FEATURES);
 461         sc->vtscsi_features = features;
 462 }
 463
 464 #define VTSCSI_GET_CONFIG(_dev, _field, _cfg)                   \
 465         virtio_read_device_config(_dev,                         \
 466             offsetof(struct virtio_scsi_config, _field),        \
 467             &(_cfg)->_field, sizeof((_cfg)->_field))            \
 468
 469 static void
 470 vtscsi_read_config(struct vtscsi_softc *sc,
 471     struct virtio_scsi_config *scsicfg)
 472 {
 473         device_t dev;
 474
 475         dev = sc->vtscsi_dev;
 476
 477         bzero(scsicfg, sizeof(struct virtio_scsi_config));
 478
 479         VTSCSI_GET_CONFIG(dev, num_queues, scsicfg);
 480         VTSCSI_GET_CONFIG(dev, seg_max, scsicfg);
 481         VTSCSI_GET_CONFIG(dev, max_sectors, scsicfg);
 482         VTSCSI_GET_CONFIG(dev, cmd_per_lun, scsicfg);
 483         VTSCSI_GET_CONFIG(dev, event_info_size, scsicfg);
 484         VTSCSI_GET_CONFIG(dev, sense_size, scsicfg);
 485         VTSCSI_GET_CONFIG(dev, cdb_size, scsicfg);
 486         VTSCSI_GET_CONFIG(dev, max_channel, scsicfg);
 487         VTSCSI_GET_CONFIG(dev, max_target, scsicfg);
 488         VTSCSI_GET_CONFIG(dev, max_lun, scsicfg);
 489 }
 490
 491 #undef VTSCSI_GET_CONFIG
 492
 493 static int
 494 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max)
 495 {
 496         int nsegs;
 497
 498         nsegs = VTSCSI_MIN_SEGMENTS;
 499
 500         if (seg_max > 0) {
 501                 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1);
 502                 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT)
 503                         nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
 504         } else
 505                 nsegs += 1;
 506
 507         return (nsegs);
 508 }
 509
 510
 511 static int
 512 vtscsi_alloc_intrs(struct vtscsi_softc *sc)
 513 {
 514         int intrcount = virtio_intr_count(sc->vtscsi_dev);
 515         int cnt, i, error;
 516
 517         for (i = 0; i < NELEM(sc->vtscsi_cpus); i++)
 518                 sc->vtscsi_cpus[i] = -1;
 519
 520         intrcount = imin(intrcount, 3);
 521         if (intrcount < 1)
 522                 return (ENXIO);
 523
 524         cnt = intrcount;
 525         error = virtio_intr_alloc(sc->vtscsi_dev, &cnt, 0, sc->vtscsi_cpus);
 526         if (error != 0) {
 527                 virtio_intr_release(sc->vtscsi_dev);
 528                 return (error);
 529         }
 530         sc->vtscsi_nintr = cnt;
 531         return (0);
 532 }
 533
 534 static int
 535 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc)
 536 {
 537         device_t dev = sc->vtscsi_dev;
 538         struct vq_alloc_info vq_info[3];
 539         int nvqs = 3;
 540
 541         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, &sc->vtscsi_control_vq,
 542             "%s control", device_get_nameunit(dev));
 543
 544         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, &sc->vtscsi_event_vq,
 545             "%s event", device_get_nameunit(dev));
 546
 547         VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs,
 548             &sc->vtscsi_request_vq, "%s request", device_get_nameunit(dev));
 549
 550         return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
 551 }
 552
 553 static void
 554 vtscsi_write_device_config(struct vtscsi_softc *sc)
 555 {
 556
 557         virtio_write_dev_config_4(sc->vtscsi_dev,
 558             offsetof(struct virtio_scsi_config, sense_size),
 559             VIRTIO_SCSI_SENSE_SIZE);
 560
 561         /*
 562          * This is the size in the virtio_scsi_cmd_req structure. Note
 563          * this value (32) is larger than the maximum CAM CDB size (16).
 564          */
 565         virtio_write_dev_config_4(sc->vtscsi_dev,
 566             offsetof(struct virtio_scsi_config, cdb_size),
 567             VIRTIO_SCSI_CDB_SIZE);
 568 }
 569
 570 static int
 571 vtscsi_reinit(struct vtscsi_softc *sc)
 572 {
 573         device_t dev;
 574         int error;
 575
 576         dev = sc->vtscsi_dev;
 577
 578         error = virtio_reinit(dev, sc->vtscsi_features);
 579         if (error == 0) {
 580                 vtscsi_write_device_config(sc);
 581                 vtscsi_reinit_event_vq(sc);
 582                 virtio_reinit_complete(dev);
 583
 584                 vtscsi_enable_vqs_intr(sc);
 585         }
 586
 587         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error);
 588
 589         return (error);
 590 }
 591
 592 static int
 593 vtscsi_alloc_cam(struct vtscsi_softc *sc)
 594 {
 595         device_t dev;
 596         struct cam_devq *devq;
 597         int openings;
 598
 599         dev = sc->vtscsi_dev;
 600         openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS;
 601
 602         devq = cam_simq_alloc(openings);
 603         if (devq == NULL) {
 604                 device_printf(dev, "cannot allocate SIM queue\n");
 605                 return (ENOMEM);
 606         }
 607
 608         sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll,
 609             "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1,
 610             openings, devq);
 611         cam_simq_release(devq);
 612         if (sc->vtscsi_sim == NULL) {
 613                 device_printf(dev, "cannot allocate SIM\n");
 614                 return (ENOMEM);
 615         }
 616
 617         return (0);
 618 }
 619
 620 static int
 621 vtscsi_register_cam(struct vtscsi_softc *sc)
 622 {
 623         device_t dev;
 624         int registered, error;
 625
 626         dev = sc->vtscsi_dev;
 627         registered = 0;
 628
 629         VTSCSI_LOCK(sc);
 630
 631         if (xpt_bus_register(sc->vtscsi_sim, 0) != CAM_SUCCESS) {
 632                 error = ENOMEM;
 633                 device_printf(dev, "cannot register XPT bus\n");
 634                 goto fail;
 635         }
 636
 637         registered = 1;
 638
 639         if (xpt_create_path(&sc->vtscsi_path, NULL,
 640             cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD,
 641             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 642                 error = ENOMEM;
 643                 device_printf(dev, "cannot create bus path\n");
 644                 goto fail;
 645         }
 646
 647         if (vtscsi_register_async(sc) != CAM_REQ_CMP) {
 648                 error = EIO;
 649                 device_printf(dev, "cannot register async callback\n");
 650                 goto fail;
 651         }
 652
 653         VTSCSI_UNLOCK(sc);
 654
 655         return (0);
 656
 657 fail:
 658         if (sc->vtscsi_path != NULL) {
 659                 xpt_free_path(sc->vtscsi_path);
 660                 sc->vtscsi_path = NULL;
 661         }
 662
 663         if (registered != 0)
 664                 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));
 665
 666         VTSCSI_UNLOCK(sc);
 667
 668         return (error);
 669 }
 670
 671 static void
 672 vtscsi_free_cam(struct vtscsi_softc *sc)
 673 {
 674
 675         VTSCSI_LOCK(sc);
 676
 677         if (sc->vtscsi_path != NULL) {
 678                 vtscsi_deregister_async(sc);
 679
 680                 xpt_free_path(sc->vtscsi_path);
 681                 sc->vtscsi_path = NULL;
 682
 683                 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));
 684         }
 685
 686         if (sc->vtscsi_sim != NULL) {
 687                 cam_sim_free(sc->vtscsi_sim);
 688                 sc->vtscsi_sim = NULL;
 689         }
 690
 691         VTSCSI_UNLOCK(sc);
 692 }
 693
 694 static void
 695 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg)
 696 {
 697         struct cam_sim *sim;
 698         struct vtscsi_softc *sc;
 699
 700         sim = cb_arg;
 701         sc = cam_sim_softc(sim);
 702
 703         vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code);
 704
 705         /*
 706          * TODO Once QEMU supports event reporting, we should
 707          *      (un)subscribe to events here.
 708          */
 709         switch (code) {
 710         case AC_FOUND_DEVICE:
 711                 break;
 712         case AC_LOST_DEVICE:
 713                 break;
 714         }
 715 }
 716
 717 static int
 718 vtscsi_register_async(struct vtscsi_softc *sc)
 719 {
 720         struct ccb_setasync *csa;
 721         u_int32_t status;
 722
 723         csa = &xpt_alloc_ccb()->csa;
 724
 725         xpt_setup_ccb(&csa->ccb_h, sc->vtscsi_path, 5);
 726         csa->ccb_h.func_code = XPT_SASYNC_CB;
 727         csa->event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE;
 728         csa->callback = vtscsi_cam_async;
 729         csa->callback_arg = sc->vtscsi_sim;
 730
 731         xpt_action((union ccb *)csa);
 732         status = csa->ccb_h.status;
 733         xpt_free_ccb(&csa->ccb_h);
 734
 735         return status;
 736 }
 737
 738 static void
 739 vtscsi_deregister_async(struct vtscsi_softc *sc)
 740 {
 741         struct ccb_setasync *csa;
 742
 743         csa = &xpt_alloc_ccb()->csa;
 744         xpt_setup_ccb(&csa->ccb_h, sc->vtscsi_path, 5);
 745         csa->ccb_h.func_code = XPT_SASYNC_CB;
 746         csa->event_enable = 0;
 747         csa->callback = vtscsi_cam_async;
 748         csa->callback_arg = sc->vtscsi_sim;
 749         xpt_action((union ccb *)csa);
 750         xpt_free_ccb(&csa->ccb_h);
 751 }
 752
 753 static void
 754 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
 755 {
 756         struct vtscsi_softc *sc;
 757         struct ccb_hdr *ccbh;
 758
 759         sc = cam_sim_softc(sim);
 760         ccbh = &ccb->ccb_h;
 761
 762         VTSCSI_LOCK_OWNED(sc);
 763
 764         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) {
 765                 /*
 766                  * The VTSCSI_MTX is briefly dropped between setting
 767                  * VTSCSI_FLAG_DETACH and deregistering with CAM, so
 768                  * drop any CCBs that come in during that window.
 769                  */
 770                 ccbh->status = CAM_NO_HBA;
 771                 xpt_done(ccb);
 772                 return;
 773         }
 774
 775         switch (ccbh->func_code) {
 776         case XPT_SCSI_IO:
 777                 vtscsi_cam_scsi_io(sc, sim, ccb);
 778                 break;
 779
 780         case XPT_SET_TRAN_SETTINGS:
 781                 ccbh->status = CAM_FUNC_NOTAVAIL;
 782                 xpt_done(ccb);
 783                 break;
 784
 785         case XPT_GET_TRAN_SETTINGS:
 786                 vtscsi_cam_get_tran_settings(sc, ccb);
 787                 break;
 788
 789         case XPT_RESET_BUS:
 790                 vtscsi_cam_reset_bus(sc, ccb);
 791                 break;
 792
 793         case XPT_RESET_DEV:
 794                 vtscsi_cam_reset_dev(sc, ccb);
 795                 break;
 796
 797         case XPT_ABORT:
 798                 vtscsi_cam_abort(sc, ccb);
 799                 break;
 800
 801         case XPT_CALC_GEOMETRY:
 802                 cam_calc_geometry(&ccb->ccg, 1);
 803                 xpt_done(ccb);
 804                 break;
 805
 806         case XPT_PATH_INQ:
 807                 vtscsi_cam_path_inquiry(sc, sim, ccb);
 808                 break;
 809
 810         default:
 811                 vtscsi_dprintf(sc, VTSCSI_ERROR,
 812                     "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code);
 813
 814                 ccbh->status = CAM_REQ_INVALID;
 815                 xpt_done(ccb);
 816                 break;
 817         }
 818 }
 819
 820 static void
 821 vtscsi_cam_poll(struct cam_sim *sim)
 822 {
 823         struct vtscsi_softc *sc;
 824
 825         sc = cam_sim_softc(sim);
 826
 827         vtscsi_complete_vqs_locked(sc);
 828 }
 829
 830 static void
 831 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim,
 832     union ccb *ccb)
 833 {
 834         struct ccb_hdr *ccbh;
 835         struct ccb_scsiio *csio;
 836         int error;
 837
 838         ccbh = &ccb->ccb_h;
 839         csio = &ccb->csio;
 840
 841         if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) {
 842                 error = EINVAL;
 843                 ccbh->status = CAM_REQ_INVALID;
 844                 goto done;
 845         }
 846
 847 #ifndef __DragonFly__ /* XXX swildner */
 848         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH &&
 849             (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) {
 850                 error = EINVAL;
 851                 ccbh->status = CAM_REQ_INVALID;
 852                 goto done;
 853         }
 854 #endif
 855
 856         error = vtscsi_start_scsi_cmd(sc, ccb);
 857
 858 done:
 859         if (error) {
 860                 vtscsi_dprintf(sc, VTSCSI_ERROR,
 861                     "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status);
 862                 xpt_done(ccb);
 863         }
 864 }
 865
 866 static void
 867 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb)
 868 {
 869         struct ccb_trans_settings *cts;
 870         struct ccb_trans_settings_scsi *scsi;
 871
 872         cts = &ccb->cts;
 873         scsi = &cts->proto_specific.scsi;
 874
 875         cts->protocol = PROTO_SCSI;
 876         cts->protocol_version = SCSI_REV_SPC3;
 877         cts->transport = XPORT_SAS;
 878         cts->transport_version = 0;
 879
 880         scsi->valid = CTS_SCSI_VALID_TQ;
 881         scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
 882
 883         ccb->ccb_h.status = CAM_REQ_CMP;
 884         xpt_done(ccb);
 885 }
 886
 887 static void
 888 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb)
 889 {
 890         int error;
 891
 892         error = vtscsi_reset_bus(sc);
 893         if (error == 0)
 894                 ccb->ccb_h.status = CAM_REQ_CMP;
 895         else
 896                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 897
 898         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n",
 899             error, ccb, ccb->ccb_h.status);
 900
 901         xpt_done(ccb);
 902 }
 903
 904 static void
 905 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb)
 906 {
 907         struct ccb_hdr *ccbh;
 908         struct vtscsi_request *req;
 909         int error;
 910
 911         ccbh = &ccb->ccb_h;
 912
 913         req = vtscsi_dequeue_request(sc);
 914         if (req == NULL) {
 915                 error = EAGAIN;
 916                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
 917                 goto fail;
 918         }
 919
 920         req->vsr_ccb = ccb;
 921
 922         error = vtscsi_execute_reset_dev_cmd(sc, req);
 923         if (error == 0)
 924                 return;
 925
 926         vtscsi_enqueue_request(sc, req);
 927
 928 fail:
 929         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
 930             error, req, ccb);
 931
 932         if (error == EAGAIN)
 933                 ccbh->status = CAM_RESRC_UNAVAIL;
 934         else
 935                 ccbh->status = CAM_REQ_CMP_ERR;
 936
 937         xpt_done(ccb);
 938 }
 939
 940 static void
 941 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb)
 942 {
 943         struct vtscsi_request *req;
 944         struct ccb_hdr *ccbh;
 945         int error;
 946
 947         ccbh = &ccb->ccb_h;
 948
 949         req = vtscsi_dequeue_request(sc);
 950         if (req == NULL) {
 951                 error = EAGAIN;
 952                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
 953                 goto fail;
 954         }
 955
 956         req->vsr_ccb = ccb;
 957
 958         error = vtscsi_execute_abort_task_cmd(sc, req);
 959         if (error == 0)
 960                 return;
 961
 962         vtscsi_enqueue_request(sc, req);
 963
 964 fail:
 965         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
 966             error, req, ccb);
 967
 968         if (error == EAGAIN)
 969                 ccbh->status = CAM_RESRC_UNAVAIL;
 970         else
 971                 ccbh->status = CAM_REQ_CMP_ERR;
 972
 973         xpt_done(ccb);
 974 }
 975
 976 static void
 977 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim,
 978     union ccb *ccb)
 979 {
 980         device_t dev;
 981         struct ccb_pathinq *cpi;
 982
 983         dev = sc->vtscsi_dev;
 984         cpi = &ccb->cpi;
 985
 986         vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb);
 987
 988         cpi->version_num = 1;
 989         cpi->hba_inquiry = PI_TAG_ABLE;
 990         cpi->target_sprt = 0;
 991         cpi->hba_misc = PIM_SEQSCAN;
 992         if (vtscsi_bus_reset_disable != 0)
 993                 cpi->hba_misc |= PIM_NOBUSRESET;
 994         cpi->hba_eng_cnt = 0;
 995
 996         cpi->max_target = sc->vtscsi_max_target;
 997         cpi->max_lun = sc->vtscsi_max_lun;
 998         cpi->initiator_id = VTSCSI_INITIATOR_ID;
 999
1000         strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1001         strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
1002         strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1003
1004         cpi->unit_number = cam_sim_unit(sim);
1005         cpi->bus_id = cam_sim_bus(sim);
1006
1007         cpi->base_transfer_speed = 300000;
1008
1009         cpi->protocol = PROTO_SCSI;
1010         cpi->protocol_version = SCSI_REV_SPC3;
1011         cpi->transport = XPORT_SAS;
1012         cpi->transport_version = 0;
1013
1014         cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) *
1015             PAGE_SIZE;
1016
1017 #if 0
1018         cpi->hba_vendor = virtio_get_vendor(dev);
1019         cpi->hba_device = virtio_get_device(dev);
1020         cpi->hba_subvendor = virtio_get_subvendor(dev);
1021         cpi->hba_subdevice = virtio_get_subdevice(dev);
1022 #endif
1023
1024         ccb->ccb_h.status = CAM_REQ_CMP;
1025         xpt_done(ccb);
1026 }
1027
1028 static int
1029 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg,
1030     struct ccb_scsiio *csio)
1031 {
1032         struct ccb_hdr *ccbh;
1033         struct bus_dma_segment *dseg;
1034         int i, error;
1035
1036         ccbh = &csio->ccb_h;
1037         error = 0;
1038
1039         if ((ccbh->flags & CAM_SCATTER_VALID) == 0) {
1040
1041                 if ((ccbh->flags & CAM_DATA_PHYS) == 0)
1042                         error = sglist_append(sg,
1043                             csio->data_ptr, csio->dxfer_len);
1044                 else
1045                         error = sglist_append_phys(sg,
1046                             (vm_paddr_t)(vm_offset_t) csio->data_ptr,
1047                             csio->dxfer_len);
1048         } else {
1049
1050                 for (i = 0; i < csio->sglist_cnt && error == 0; i++) {
1051                         dseg = &((struct bus_dma_segment *)csio->data_ptr)[i];
1052
1053                         if ((ccbh->flags & CAM_SG_LIST_PHYS) == 0)
1054                                 error = sglist_append(sg,
1055                                     (void *)(vm_offset_t) dseg->ds_addr,
1056                                     dseg->ds_len);
1057                         else
1058                                 error = sglist_append_phys(sg,
1059                                     (vm_paddr_t) dseg->ds_addr, dseg->ds_len);
1060                 }
1061         }
1062
1063         return (error);
1064 }
1065
1066 static int
1067 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req,
1068     int *readable, int *writable)
1069 {
1070         struct sglist *sg;
1071         struct ccb_hdr *ccbh;
1072         struct ccb_scsiio *csio;
1073         struct virtio_scsi_cmd_req *cmd_req;
1074         struct virtio_scsi_cmd_resp *cmd_resp;
1075         int error;
1076
1077         sg = sc->vtscsi_sglist;
1078         csio = &req->vsr_ccb->csio;
1079         ccbh = &csio->ccb_h;
1080         cmd_req = &req->vsr_cmd_req;
1081         cmd_resp = &req->vsr_cmd_resp;
1082
1083         sglist_reset(sg);
1084
1085         sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req));
1086         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1087                 error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
1088                 /* At least one segment must be left for the response. */
1089                 if (error || sg->sg_nseg == sg->sg_maxseg)
1090                         goto fail;
1091         }
1092
1093         *readable = sg->sg_nseg;
1094
1095         sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp));
1096         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) {
1097                 error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
1098                 if (error)
1099                         goto fail;
1100         }
1101
1102         *writable = sg->sg_nseg - *readable;
1103
1104         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d "
1105             "writable=%d\n", req, ccbh, *readable, *writable);
1106
1107         return (0);
1108
1109 fail:
1110         /*
1111          * This should never happen unless maxio was incorrectly set.
1112          */
1113         vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0);
1114
1115         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p "
1116             "nseg=%d maxseg=%d\n",
1117             error, req, ccbh, sg->sg_nseg, sg->sg_maxseg);
1118
1119         return (EFBIG);
1120 }
1121
1122 static int
1123 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
1124 {
1125         struct sglist *sg;
1126         struct virtqueue *vq;
1127         struct ccb_scsiio *csio;
1128         struct ccb_hdr *ccbh;
1129         struct virtio_scsi_cmd_req *cmd_req;
1130         struct virtio_scsi_cmd_resp *cmd_resp;
1131         int readable, writable, error;
1132
1133         sg = sc->vtscsi_sglist;
1134         vq = sc->vtscsi_request_vq;
1135         csio = &req->vsr_ccb->csio;
1136         ccbh = &csio->ccb_h;
1137         cmd_req = &req->vsr_cmd_req;
1138         cmd_resp = &req->vsr_cmd_resp;
1139
1140         vtscsi_init_scsi_cmd_req(csio, cmd_req);
1141
1142         error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);
1143         if (error)
1144                 return (error);
1145
1146         req->vsr_complete = vtscsi_complete_scsi_cmd;
1147         cmd_resp->response = -1;
1148
1149         error = virtqueue_enqueue(vq, req, sg, readable, writable);
1150         if (error) {
1151                 vtscsi_dprintf(sc, VTSCSI_ERROR,
1152                     "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh);
1153
1154                 ccbh->status = CAM_REQUEUE_REQ;
1155                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ);
1156                 return (error);
1157         }
1158
1159         ccbh->status |= CAM_SIM_QUEUED;
1160         ccbh->ccbh_vtscsi_req = req;
1161
1162         virtqueue_notify(vq, NULL);
1163
1164         if (ccbh->timeout != CAM_TIME_INFINITY) {
1165                 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET;
1166                 callout_reset(&req->vsr_callout, ccbh->timeout * hz / 1000,
1167                     vtscsi_timedout_scsi_cmd, req);
1168         }
1169
1170         vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n",
1171             req, ccbh);
1172
1173         return (0);
1174 }
1175
1176 static int
1177 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb)
1178 {
1179         struct vtscsi_request *req;
1180         int error;
1181
1182         req = vtscsi_dequeue_request(sc);
1183         if (req == NULL) {
1184                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1185                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
1186                 return (ENOBUFS);
1187         }
1188
1189         req->vsr_ccb = ccb;
1190
1191         error = vtscsi_execute_scsi_cmd(sc, req);
1192         if (error)
1193                 vtscsi_enqueue_request(sc, req);
1194
1195         return (error);
1196 }
1197
1198 static void
1199 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
1200     struct vtscsi_request *req)
1201 {
1202         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1203         struct vtscsi_request *to_req;
1204         uint8_t response;
1205
1206         tmf_resp = &req->vsr_tmf_resp;
1207         response = tmf_resp->response;
1208         to_req = req->vsr_timedout_req;
1209
1210         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n",
1211             req, to_req, response);
1212
1213         vtscsi_enqueue_request(sc, req);
1214
1215         /*
1216          * The timedout request could have completed between when the
1217          * abort task was sent and when the host processed it.
1218          */
1219         if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT)
1220                 return;
1221
1222         /* The timedout request was successfully aborted. */
1223         if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE)
1224                 return;
1225
1226         /* Don't bother if the device is going away. */
1227         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
1228                 return;
1229
1230         /* The timedout request will be aborted by the reset. */
1231         if (sc->vtscsi_flags & VTSCSI_FLAG_RESET)
1232                 return;
1233
1234         vtscsi_reset_bus(sc);
1235 }
1236
1237 static int
1238 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
1239     struct vtscsi_request *to_req)
1240 {
1241         struct sglist *sg;
1242         struct ccb_hdr *to_ccbh;
1243         struct vtscsi_request *req;
1244         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1245         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1246         int error;
1247
1248         sg = sc->vtscsi_sglist;
1249         to_ccbh = &to_req->vsr_ccb->ccb_h;
1250
1251         req = vtscsi_dequeue_request(sc);
1252         if (req == NULL) {
1253                 error = ENOBUFS;
1254                 goto fail;
1255         }
1256
1257         tmf_req = &req->vsr_tmf_req;
1258         tmf_resp = &req->vsr_tmf_resp;
1259
1260         vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1261             (uintptr_t) to_ccbh, tmf_req);
1262
1263         sglist_reset(sg);
1264         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1265         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1266
1267         req->vsr_timedout_req = to_req;
1268         req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd;
1269         tmf_resp->response = -1;
1270
1271         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1272             VTSCSI_EXECUTE_ASYNC);
1273         if (error == 0)
1274                 return (0);
1275
1276         vtscsi_enqueue_request(sc, req);
1277
1278 fail:
1279         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p "
1280             "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh);
1281
1282         return (error);
1283 }
1284
1285 static void
1286 vtscsi_timedout_scsi_cmd(void *xreq)
1287 {
1288         struct vtscsi_softc *sc;
1289         struct vtscsi_request *to_req;
1290
1291         to_req = xreq;
1292         sc = to_req->vsr_softc;
1293
1294         vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n",
1295             to_req, to_req->vsr_ccb, to_req->vsr_state);
1296
1297         /* Don't bother if the device is going away. */
1298         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
1299                 return;
1300
1301         /*
1302          * Bail if the request is not in use. We likely raced when
1303          * stopping the callout handler or it has already been aborted.
1304          */
1305         if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE ||
1306             (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0)
1307                 return;
1308
1309         /*
1310          * Complete the request queue in case the timedout request is
1311          * actually just pending.
1312          */
1313         vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
1314         if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE)
1315                 return;
1316
1317         sc->vtscsi_stats.scsi_cmd_timeouts++;
1318         to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT;
1319
1320         if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0)
1321                 return;
1322
1323         vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n");
1324         vtscsi_reset_bus(sc);
1325 }
1326
1327 static cam_status
1328 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp)
1329 {
1330         cam_status status;
1331
1332         switch (cmd_resp->response) {
1333         case VIRTIO_SCSI_S_OK:
1334                 status = CAM_REQ_CMP;
1335                 break;
1336         case VIRTIO_SCSI_S_OVERRUN:
1337                 status = CAM_DATA_RUN_ERR;
1338                 break;
1339         case VIRTIO_SCSI_S_ABORTED:
1340                 status = CAM_REQ_ABORTED;
1341                 break;
1342         case VIRTIO_SCSI_S_BAD_TARGET:
1343                 /*
1344                  * A CAM_SEL_TIMEOUT here will cause the entire device to
1345                  * be lost, which is not desirable when scanning LUNs.
1346                  * Use CAM_DEV_NOT_THERE instead.
1347                  */
1348                 status = CAM_DEV_NOT_THERE;
1349                 break;
1350         case VIRTIO_SCSI_S_RESET:
1351                 status = CAM_SCSI_BUS_RESET;
1352                 break;
1353         case VIRTIO_SCSI_S_BUSY:
1354                 status = CAM_SCSI_BUSY;
1355                 break;
1356         case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
1357         case VIRTIO_SCSI_S_TARGET_FAILURE:
1358         case VIRTIO_SCSI_S_NEXUS_FAILURE:
1359                 status = CAM_SCSI_IT_NEXUS_LOST;
1360                 break;
1361         default: /* VIRTIO_SCSI_S_FAILURE */
1362                 status = CAM_REQ_CMP_ERR;
1363                 break;
1364         }
1365
1366         return (status);
1367 }
1368
1369 static cam_status
1370 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,
1371     struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp)
1372 {
1373         cam_status status;
1374
1375         csio->scsi_status = cmd_resp->status;
1376         csio->resid = cmd_resp->resid;
1377
1378         if (csio->scsi_status == SCSI_STATUS_OK)
1379                 status = CAM_REQ_CMP;
1380         else
1381                 status = CAM_SCSI_STATUS_ERROR;
1382
1383         if (cmd_resp->sense_len > 0) {
1384                 status |= CAM_AUTOSNS_VALID;
1385
1386                 if (cmd_resp->sense_len < csio->sense_len)
1387                         csio->sense_resid = csio->sense_len -
1388                             cmd_resp->sense_len;
1389                 else
1390                         csio->sense_resid = 0;
1391
1392                 bzero(&csio->sense_data, sizeof(csio->sense_data));
1393                 memcpy(cmd_resp->sense, &csio->sense_data,
1394                     csio->sense_len - csio->sense_resid);
1395         }
1396
1397         vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR,
1398             "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n",
1399             csio, csio->scsi_status, csio->resid, csio->sense_resid);
1400
1401         return (status);
1402 }
1403
1404 static void
1405 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
1406 {
1407         struct ccb_hdr *ccbh;
1408         struct ccb_scsiio *csio;
1409         struct virtio_scsi_cmd_resp *cmd_resp;
1410         cam_status status;
1411
1412         csio = &req->vsr_ccb->csio;
1413         ccbh = &csio->ccb_h;
1414         cmd_resp = &req->vsr_cmd_resp;
1415
1416         KASSERT(ccbh->ccbh_vtscsi_req == req,
1417             ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req));
1418
1419         if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
1420                 callout_stop(&req->vsr_callout);
1421
1422         status = vtscsi_scsi_cmd_cam_status(cmd_resp);
1423         if (status == CAM_REQ_ABORTED) {
1424                 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT)
1425                         status = CAM_CMD_TIMEOUT;
1426         } else if (status == CAM_REQ_CMP)
1427                 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp);
1428
1429         if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1430                 status |= CAM_DEV_QFRZN;
1431                 xpt_freeze_devq(ccbh->path, 1);
1432         }
1433
1434         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
1435                 status |= CAM_RELEASE_SIMQ;
1436
1437         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n",
1438             req, ccbh, status);
1439
1440         ccbh->status = status;
1441         xpt_done(req->vsr_ccb);
1442         vtscsi_enqueue_request(sc, req);
1443 }
1444
1445 static void
1446 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req)
1447 {
1448
1449         /* XXX We probably shouldn't poll forever. */
1450         req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED;
1451         do
1452                 vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
1453         while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0);
1454
1455         req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED;
1456 }
1457
1458 static int
1459 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req,
1460     struct sglist *sg, int readable, int writable, int flag)
1461 {
1462         struct virtqueue *vq;
1463         int error;
1464
1465         vq = sc->vtscsi_control_vq;
1466
1467         KKASSERT(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL);
1468
1469         error = virtqueue_enqueue(vq, req, sg, readable, writable);
1470         if (error) {
1471                 /*
1472                  * Return EAGAIN when the virtqueue does not have enough
1473                  * descriptors available.
1474                  */
1475                 if (error == ENOSPC || error == EMSGSIZE)
1476                         error = EAGAIN;
1477
1478                 return (error);
1479         }
1480
1481         virtqueue_notify(vq, NULL);
1482         if (flag == VTSCSI_EXECUTE_POLL)
1483                 vtscsi_poll_ctrl_req(sc, req);
1484
1485         return (0);
1486 }
1487
1488 static void
1489 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc,
1490     struct vtscsi_request *req)
1491 {
1492         union ccb *ccb;
1493         struct ccb_hdr *ccbh;
1494         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1495
1496         ccb = req->vsr_ccb;
1497         ccbh = &ccb->ccb_h;
1498         tmf_resp = &req->vsr_tmf_resp;
1499
1500         switch (tmf_resp->response) {
1501         case VIRTIO_SCSI_S_FUNCTION_COMPLETE:
1502                 ccbh->status = CAM_REQ_CMP;
1503                 break;
1504         case VIRTIO_SCSI_S_FUNCTION_REJECTED:
1505                 ccbh->status = CAM_UA_ABORT;
1506                 break;
1507         default:
1508                 ccbh->status = CAM_REQ_CMP_ERR;
1509                 break;
1510         }
1511
1512         xpt_done(ccb);
1513         vtscsi_enqueue_request(sc, req);
1514 }
1515
1516 static int
1517 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc,
1518     struct vtscsi_request *req)
1519 {
1520         struct sglist *sg;
1521         struct ccb_abort *cab;
1522         struct ccb_hdr *ccbh;
1523         struct ccb_hdr *abort_ccbh;
1524         struct vtscsi_request *abort_req;
1525         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1526         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1527         int error;
1528
1529         sg = sc->vtscsi_sglist;
1530         cab = &req->vsr_ccb->cab;
1531         ccbh = &cab->ccb_h;
1532         tmf_req = &req->vsr_tmf_req;
1533         tmf_resp = &req->vsr_tmf_resp;
1534
1535         /* CCB header and request that's to be aborted. */
1536         abort_ccbh = &cab->abort_ccb->ccb_h;
1537         abort_req = abort_ccbh->ccbh_vtscsi_req;
1538
1539         if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) {
1540                 error = EINVAL;
1541                 goto fail;
1542         }
1543
1544         /* Only attempt to abort requests that could be in-flight. */
1545         if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) {
1546                 error = EALREADY;
1547                 goto fail;
1548         }
1549
1550         abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED;
1551         if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
1552                 callout_stop(&abort_req->vsr_callout);
1553
1554         vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1555             (uintptr_t) abort_ccbh, tmf_req);
1556
1557         sglist_reset(sg);
1558         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1559         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1560
1561         req->vsr_complete = vtscsi_complete_abort_task_cmd;
1562         tmf_resp->response = -1;
1563
1564         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1565             VTSCSI_EXECUTE_ASYNC);
1566
1567 fail:
1568         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p "
1569             "abort_req=%p\n", error, req, abort_ccbh, abort_req);
1570
1571         return (error);
1572 }
1573
1574 static void
1575 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc,
1576     struct vtscsi_request *req)
1577 {
1578         union ccb *ccb;
1579         struct ccb_hdr *ccbh;
1580         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1581
1582         ccb = req->vsr_ccb;
1583         ccbh = &ccb->ccb_h;
1584         tmf_resp = &req->vsr_tmf_resp;
1585
1586         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n",
1587             req, ccb, tmf_resp->response);
1588
1589         if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) {
1590                 ccbh->status = CAM_REQ_CMP;
1591                 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id,
1592                     ccbh->target_lun);
1593         } else
1594                 ccbh->status = CAM_REQ_CMP_ERR;
1595
1596         xpt_done(ccb);
1597         vtscsi_enqueue_request(sc, req);
1598 }
1599
1600 static int
1601 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc,
1602     struct vtscsi_request *req)
1603 {
1604         struct sglist *sg;
1605         struct ccb_resetdev *crd;
1606         struct ccb_hdr *ccbh;
1607         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1608         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1609         uint32_t subtype;
1610         int error;
1611
1612         sg = sc->vtscsi_sglist;
1613         crd = &req->vsr_ccb->crd;
1614         ccbh = &crd->ccb_h;
1615         tmf_req = &req->vsr_tmf_req;
1616         tmf_resp = &req->vsr_tmf_resp;
1617
1618         if (ccbh->target_lun == CAM_LUN_WILDCARD)
1619                 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET;
1620         else
1621                 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;
1622
1623         vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req);
1624
1625         sglist_reset(sg);
1626         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1627         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1628
1629         req->vsr_complete = vtscsi_complete_reset_dev_cmd;
1630         tmf_resp->response = -1;
1631
1632         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1633             VTSCSI_EXECUTE_ASYNC);
1634
1635         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n",
1636             error, req, ccbh);
1637
1638         return (error);
1639 }
1640
1641 static void
1642 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id)
1643 {
1644
1645         *target_id = lun[1];
1646         *lun_id = (lun[2] << 8) | lun[3];
1647 }
1648
1649 static void
1650 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[])
1651 {
1652
1653         lun[0] = 1;
1654         lun[1] = ccbh->target_id;
1655         lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F);
1656         lun[3] = ccbh->target_lun & 0xFF;
1657 }
1658
1659 static void
1660 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio,
1661     struct virtio_scsi_cmd_req *cmd_req)
1662 {
1663         uint8_t attr;
1664
1665         switch (csio->tag_action) {
1666         case MSG_HEAD_OF_Q_TAG:
1667                 attr = VIRTIO_SCSI_S_HEAD;
1668                 break;
1669         case MSG_ORDERED_Q_TAG:
1670                 attr = VIRTIO_SCSI_S_ORDERED;
1671                 break;
1672         case MSG_ACA_TASK:
1673                 attr = VIRTIO_SCSI_S_ACA;
1674                 break;
1675         default: /* MSG_SIMPLE_Q_TAG */
1676                 attr = VIRTIO_SCSI_S_SIMPLE;
1677                 break;
1678         }
1679
1680         vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);
1681         cmd_req->tag = (uintptr_t) csio;
1682         cmd_req->task_attr = attr;
1683
1684         memcpy(cmd_req->cdb,
1685             csio->ccb_h.flags & CAM_CDB_POINTER ?
1686                 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes,
1687             csio->cdb_len);
1688 }
1689
1690 static void
1691 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype,
1692     uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
1693 {
1694
1695         vtscsi_set_request_lun(ccbh, tmf_req->lun);
1696
1697         tmf_req->type = VIRTIO_SCSI_T_TMF;
1698         tmf_req->subtype = subtype;
1699         tmf_req->tag = tag;
1700 }
1701
1702 static void
1703 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason)
1704 {
1705         int frozen;
1706
1707         frozen = sc->vtscsi_frozen;
1708
1709         if (reason & VTSCSI_REQUEST &&
1710             (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0)
1711                 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS;
1712
1713         if (reason & VTSCSI_REQUEST_VQ &&
1714             (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0)
1715                 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL;
1716
1717         /* Freeze the SIMQ if transitioned to frozen. */
1718         if (frozen == 0 && sc->vtscsi_frozen != 0) {
1719                 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n");
1720                 xpt_freeze_simq(sc->vtscsi_sim, 1);
1721         }
1722 }
1723
1724 static int
1725 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason)
1726 {
1727         int thawed;
1728
1729         if (sc->vtscsi_frozen == 0 || reason == 0)
1730                 return (0);
1731
1732         if (reason & VTSCSI_REQUEST &&
1733             sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS)
1734                 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS;
1735
1736         if (reason & VTSCSI_REQUEST_VQ &&
1737             sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL)
1738                 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL;
1739
1740         thawed = sc->vtscsi_frozen == 0;
1741         if (thawed != 0)
1742                 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n");
1743
1744         return (thawed);
1745 }
1746
1747 static void
1748 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code,
1749     target_id_t target_id, lun_id_t lun_id)
1750 {
1751         struct cam_path *path;
1752
1753         /* Use the wildcard path from our softc for bus announcements. */
1754         if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) {
1755                 xpt_async(ac_code, sc->vtscsi_path, NULL);
1756                 return;
1757         }
1758
1759         if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim),
1760             target_id, lun_id) != CAM_REQ_CMP) {
1761                 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n");
1762                 return;
1763         }
1764
1765         xpt_async(ac_code, path, NULL);
1766         xpt_free_path(path);
1767 }
1768
1769 static void
1770 vtscsi_cam_rescan_callback(struct cam_periph *periph, union ccb *ccb)
1771 {
1772         xpt_free_path(ccb->ccb_h.path);
1773         xpt_free_ccb(&ccb->ccb_h);
1774 }
1775
1776 static void
1777 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id,
1778     lun_id_t lun_id)
1779 {
1780         union ccb *ccb;
1781         cam_status status;
1782
1783         ccb = xpt_alloc_ccb();
1784         if (ccb == NULL) {
1785                 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n");
1786                 return;
1787         }
1788
1789         status = xpt_create_path(&ccb->ccb_h.path, NULL,
1790             cam_sim_path(sc->vtscsi_sim), target_id, lun_id);
1791         if (status != CAM_REQ_CMP) {
1792                 xpt_free_ccb(&ccb->ccb_h);
1793                 return;
1794         }
1795
1796         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/);
1797         ccb->ccb_h.func_code = XPT_SCAN_LUN;
1798         ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback;
1799         ccb->crcn.flags = CAM_FLAG_NONE;
1800         xpt_action(ccb);
1801 }
1802
1803 static void
1804 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc)
1805 {
1806         union ccb *ccb;
1807         cam_status status;
1808
1809         ccb = xpt_alloc_ccb();
1810         if (ccb == NULL) {
1811                 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n");
1812                 return;
1813         }
1814
1815         status = xpt_create_path(&ccb->ccb_h.path, NULL,
1816             cam_sim_path(sc->vtscsi_sim),
1817             CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
1818         if (status != CAM_REQ_CMP) {
1819                 xpt_free_ccb(&ccb->ccb_h);
1820                 return;
1821         }
1822
1823         xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5/*priority (low)*/);
1824         ccb->ccb_h.func_code = XPT_SCAN_BUS;
1825         ccb->ccb_h.cbfcnp = vtscsi_cam_rescan_callback;
1826         ccb->crcn.flags = CAM_FLAG_NONE;
1827         xpt_action(ccb);
1828 }
1829
1830 static void
1831 vtscsi_transport_reset_event(struct vtscsi_softc *sc,
1832     struct virtio_scsi_event *event)
1833 {
1834         target_id_t target_id;
1835         lun_id_t lun_id;
1836
1837         vtscsi_get_request_lun(event->lun, &target_id, &lun_id);
1838
1839         switch (event->reason) {
1840         case VIRTIO_SCSI_EVT_RESET_RESCAN:
1841         case VIRTIO_SCSI_EVT_RESET_REMOVED:
1842                 vtscsi_execute_rescan(sc, target_id, lun_id);
1843                 break;
1844         default:
1845                 device_printf(sc->vtscsi_dev,
1846                     "unhandled transport event reason: %d\n", event->reason);
1847                 break;
1848         }
1849 }
1850
1851 static void
1852 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event)
1853 {
1854         int error;
1855
1856         if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) {
1857                 switch (event->event) {
1858                 case VIRTIO_SCSI_T_TRANSPORT_RESET:
1859                         vtscsi_transport_reset_event(sc, event);
1860                         break;
1861                 default:
1862                         device_printf(sc->vtscsi_dev,
1863                             "unhandled event: %d\n", event->event);
1864                         break;
1865                 }
1866         } else
1867                 vtscsi_execute_rescan_bus(sc);
1868
1869         /*
1870          * This should always be successful since the buffer
1871          * was just dequeued.
1872          */
1873         error = vtscsi_enqueue_event_buf(sc, event);
1874         KASSERT(error == 0,
1875             ("cannot requeue event buffer: %d", error));
1876 }
1877
1878 static int
1879 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc,
1880     struct virtio_scsi_event *event)
1881 {
1882         struct sglist *sg;
1883         struct virtqueue *vq;
1884         int size, error;
1885
1886         sg = sc->vtscsi_sglist;
1887         vq = sc->vtscsi_event_vq;
1888         size = sc->vtscsi_event_buf_size;
1889
1890         bzero(event, size);
1891
1892         sglist_reset(sg);
1893         error = sglist_append(sg, event, size);
1894         if (error)
1895                 return (error);
1896
1897         error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg);
1898         if (error)
1899                 return (error);
1900
1901         virtqueue_notify(vq, NULL);
1902
1903         return (0);
1904 }
1905
1906 static int
1907 vtscsi_init_event_vq(struct vtscsi_softc *sc)
1908 {
1909         struct virtio_scsi_event *event;
1910         int i, size, error;
1911
1912         /*
1913          * The first release of QEMU with VirtIO SCSI support would crash
1914          * when attempting to notify the event virtqueue. This was fixed
1915          * when hotplug support was added.
1916          */
1917         if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG)
1918                 size = sc->vtscsi_event_buf_size;
1919         else
1920                 size = 0;
1921
1922         if (size < sizeof(struct virtio_scsi_event))
1923                 return (0);
1924
1925         for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
1926                 event = &sc->vtscsi_event_bufs[i];
1927
1928                 error = vtscsi_enqueue_event_buf(sc, event);
1929                 if (error)
1930                         break;
1931         }
1932
1933         /*
1934          * Even just one buffer is enough. Missed events are
1935          * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag.
1936          */
1937         if (i > 0)
1938                 error = 0;
1939
1940         return (error);
1941 }
1942
1943 static void
1944 vtscsi_reinit_event_vq(struct vtscsi_softc *sc)
1945 {
1946         struct virtio_scsi_event *event;
1947         int i, error;
1948
1949         if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 ||
1950             sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event))
1951                 return;
1952
1953         for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
1954                 event = &sc->vtscsi_event_bufs[i];
1955
1956                 error = vtscsi_enqueue_event_buf(sc, event);
1957                 if (error)
1958                         break;
1959         }
1960
1961         KASSERT(i > 0, ("cannot reinit event vq: %d", error));
1962 }
1963
1964 static void
1965 vtscsi_drain_event_vq(struct vtscsi_softc *sc)
1966 {
1967         struct virtqueue *vq;
1968         int last;
1969
1970         vq = sc->vtscsi_event_vq;
1971         last = 0;
1972
1973         while (virtqueue_drain(vq, &last) != NULL)
1974                 ;
1975
1976         KASSERT(virtqueue_empty(vq), ("eventvq not empty"));
1977 }
1978
1979 static void
1980 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc)
1981 {
1982
1983         VTSCSI_LOCK_OWNED(sc);
1984
1985         if (sc->vtscsi_request_vq != NULL)
1986                 vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
1987         if (sc->vtscsi_control_vq != NULL)
1988                 vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
1989 }
1990
1991 static void
1992 vtscsi_complete_vqs(struct vtscsi_softc *sc)
1993 {
1994
1995         VTSCSI_LOCK(sc);
1996         vtscsi_complete_vqs_locked(sc);
1997         VTSCSI_UNLOCK(sc);
1998 }
1999
2000 static void
2001 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
2002 {
2003         union ccb *ccb;
2004         int detach;
2005
2006         ccb = req->vsr_ccb;
2007
2008         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb);
2009
2010         /*
2011          * The callout must be drained when detaching since the request is
2012          * about to be freed. The VTSCSI_MTX must not be held for this in
2013          * case the callout is pending because there is a deadlock potential.
2014          * Otherwise, the virtqueue is being drained because of a bus reset
2015          * so we only need to attempt to stop the callouts.
2016          */
2017         detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0;
2018         if (detach != 0)
2019                 VTSCSI_LOCK_NOTOWNED(sc);
2020         else
2021                 VTSCSI_LOCK_OWNED(sc);
2022
2023         if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) {
2024                 if (detach != 0)
2025                         callout_drain(&req->vsr_callout);
2026                 else
2027                         callout_stop(&req->vsr_callout);
2028         }
2029
2030         if (ccb != NULL) {
2031                 if (detach != 0) {
2032                         VTSCSI_LOCK(sc);
2033                         ccb->ccb_h.status = CAM_NO_HBA;
2034                 } else
2035                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
2036                 xpt_done(ccb);
2037                 if (detach != 0)
2038                         VTSCSI_UNLOCK(sc);
2039         }
2040
2041         vtscsi_enqueue_request(sc, req);
2042 }
2043
2044 static void
2045 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
2046 {
2047         struct vtscsi_request *req;
2048         int last;
2049
2050         last = 0;
2051
2052         vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq);
2053
2054         while ((req = virtqueue_drain(vq, &last)) != NULL)
2055                 vtscsi_cancel_request(sc, req);
2056
2057         KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
2058 }
2059
2060 static void
2061 vtscsi_drain_vqs(struct vtscsi_softc *sc)
2062 {
2063
2064         if (sc->vtscsi_control_vq != NULL)
2065                 vtscsi_drain_vq(sc, sc->vtscsi_control_vq);
2066         if (sc->vtscsi_request_vq != NULL)
2067                 vtscsi_drain_vq(sc, sc->vtscsi_request_vq);
2068         if (sc->vtscsi_event_vq != NULL)
2069                 vtscsi_drain_event_vq(sc);
2070 }
2071
2072 static void
2073 vtscsi_stop(struct vtscsi_softc *sc)
2074 {
2075
2076         vtscsi_disable_vqs_intr(sc);
2077         virtio_stop(sc->vtscsi_dev);
2078 }
2079
2080 static int
2081 vtscsi_reset_bus(struct vtscsi_softc *sc)
2082 {
2083         int error;
2084
2085         VTSCSI_LOCK_OWNED(sc);
2086
2087         if (vtscsi_bus_reset_disable != 0) {
2088                 device_printf(sc->vtscsi_dev, "bus reset disabled\n");
2089                 return (0);
2090         }
2091
2092         sc->vtscsi_flags |= VTSCSI_FLAG_RESET;
2093
2094         /*
2095          * vtscsi_stop() will cause the in-flight requests to be canceled.
2096          * Those requests are then completed here so CAM will retry them
2097          * after the reset is complete.
2098          */
2099         vtscsi_stop(sc);
2100         vtscsi_complete_vqs_locked(sc);
2101
2102         /* Rid the virtqueues of any remaining requests. */
2103         vtscsi_drain_vqs(sc);
2104
2105         /*
2106          * Any resource shortage that froze the SIMQ cannot persist across
2107          * a bus reset so ensure it gets thawed here.
2108          */
2109         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
2110                 xpt_release_simq(sc->vtscsi_sim, 0);
2111
2112         error = vtscsi_reinit(sc);
2113         if (error) {
2114                 device_printf(sc->vtscsi_dev,
2115                     "reinitialization failed, stopping device...\n");
2116                 vtscsi_stop(sc);
2117         } else
2118                 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
2119                     CAM_LUN_WILDCARD);
2120
2121         sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET;
2122
2123         return (error);
2124 }
2125
2126 static void
2127 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
2128 {
2129
2130 #ifdef INVARIANTS
2131         int req_nsegs, resp_nsegs;
2132
2133         req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq));
2134         resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp));
2135
2136         KASSERT(req_nsegs == 1, ("request crossed page boundary"));
2137         KASSERT(resp_nsegs == 1, ("response crossed page boundary"));
2138 #endif
2139
2140         req->vsr_softc = sc;
2141         callout_init_lk(&req->vsr_callout, VTSCSI_MTX(sc));
2142 }
2143
2144 static int
2145 vtscsi_alloc_requests(struct vtscsi_softc *sc)
2146 {
2147         struct vtscsi_request *req;
2148         int i, nreqs;
2149
2150         /*
2151          * Commands destined for either the request or control queues come
2152          * from the same SIM queue. Use the size of the request virtqueue
2153          * as it (should) be much more frequently used. Some additional
2154          * requests are allocated for internal (TMF) use.
2155          */
2156         nreqs = virtqueue_size(sc->vtscsi_request_vq);
2157         if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0)
2158                 nreqs /= VTSCSI_MIN_SEGMENTS;
2159         nreqs += VTSCSI_RESERVED_REQUESTS;
2160
2161         for (i = 0; i < nreqs; i++) {
2162                 req = contigmalloc(sizeof(struct vtscsi_request), M_DEVBUF,
2163                     M_WAITOK, 0, BUS_SPACE_MAXADDR, 16, 0);
2164                 if (req == NULL)
2165                         return (ENOMEM);
2166
2167                 vtscsi_init_request(sc, req);
2168
2169                 sc->vtscsi_nrequests++;
2170                 vtscsi_enqueue_request(sc, req);
2171         }
2172
2173         return (0);
2174 }
2175
2176 static void
2177 vtscsi_free_requests(struct vtscsi_softc *sc)
2178 {
2179         struct vtscsi_request *req;
2180
2181         while ((req = vtscsi_dequeue_request(sc)) != NULL) {
2182                 KASSERT(callout_active(&req->vsr_callout) == 0,
2183                     ("request callout still active"));
2184
2185                 sc->vtscsi_nrequests--;
2186                 contigfree(req, sizeof(struct vtscsi_request), M_DEVBUF);
2187         }
2188
2189         KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d",
2190             sc->vtscsi_nrequests));
2191 }
2192
2193 static void
2194 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
2195 {
2196
2197         KASSERT(req->vsr_softc == sc,
2198             ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc));
2199
2200         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);
2201
2202         /* A request is available so the SIMQ could be released. */
2203         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0)
2204                 xpt_release_simq(sc->vtscsi_sim, 1);
2205
2206         req->vsr_ccb = NULL;
2207         req->vsr_complete = NULL;
2208         req->vsr_ptr0 = NULL;
2209         req->vsr_state = VTSCSI_REQ_STATE_FREE;
2210         req->vsr_flags = 0;
2211
2212         bzero(&req->vsr_ureq, sizeof(req->vsr_ureq));
2213         bzero(&req->vsr_uresp, sizeof(req->vsr_uresp));
2214
2215         /*
2216          * We insert at the tail of the queue in order to make it
2217          * very unlikely a request will be reused if we race with
2218          * stopping its callout handler.
2219          */
2220         TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link);
2221 }
2222
2223 static struct vtscsi_request *
2224 vtscsi_dequeue_request(struct vtscsi_softc *sc)
2225 {
2226         struct vtscsi_request *req;
2227
2228         req = TAILQ_FIRST(&sc->vtscsi_req_free);
2229         if (req != NULL) {
2230                 req->vsr_state = VTSCSI_REQ_STATE_INUSE;
2231                 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link);
2232         } else
2233                 sc->vtscsi_stats.dequeue_no_requests++;
2234
2235         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);
2236
2237         return (req);
2238 }
2239
2240 static void
2241 vtscsi_complete_request(struct vtscsi_request *req)
2242 {
2243
2244         if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED)
2245                 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE;
2246
2247         if (req->vsr_complete != NULL)
2248                 req->vsr_complete(req->vsr_softc, req);
2249 }
2250
2251 static void
2252 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
2253 {
2254         struct vtscsi_request *req;
2255
2256         VTSCSI_LOCK_OWNED(sc);
2257
2258         while ((req = virtqueue_dequeue(vq, NULL)) != NULL)
2259                 vtscsi_complete_request(req);
2260 }
2261
2262 static void
2263 vtscsi_control_vq_intr(void *xsc)
2264 {
2265         struct vtscsi_softc *sc;
2266         struct virtqueue *vq;
2267
2268         sc = xsc;
2269         vq = sc->vtscsi_control_vq;
2270
2271 again:
2272         VTSCSI_LOCK(sc);
2273         if (!virtqueue_pending(vq))
2274                 goto done;
2275
2276         vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
2277
2278         if (virtqueue_enable_intr(vq) != 0) {
2279                 virtqueue_disable_intr(vq);
2280                 VTSCSI_UNLOCK(sc);
2281                 goto again;
2282         }
2283
2284 done:
2285         VTSCSI_UNLOCK(sc);
2286 }
2287
2288 static void
2289 vtscsi_event_vq_intr(void *xsc)
2290 {
2291         struct vtscsi_softc *sc;
2292         struct virtqueue *vq;
2293         struct virtio_scsi_event *event;
2294
2295         sc = xsc;
2296         vq = sc->vtscsi_event_vq;
2297
2298 again:
2299         VTSCSI_LOCK(sc);
2300         if (!virtqueue_pending(vq))
2301                 goto done;
2302
2303         while ((event = virtqueue_dequeue(vq, NULL)) != NULL)
2304                 vtscsi_handle_event(sc, event);
2305
2306         if (virtqueue_enable_intr(vq) != 0) {
2307                 virtqueue_disable_intr(vq);
2308                 VTSCSI_UNLOCK(sc);
2309                 goto again;
2310         }
2311
2312 done:
2313         VTSCSI_UNLOCK(sc);
2314 }
2315
2316 static void
2317 vtscsi_request_vq_intr(void *xsc)
2318 {
2319         struct vtscsi_softc *sc;
2320         struct virtqueue *vq;
2321
2322         sc = xsc;
2323         vq = sc->vtscsi_request_vq;
2324
2325 again:
2326         VTSCSI_LOCK(sc);
2327         if (!virtqueue_pending(vq))
2328                 goto done;
2329
2330         vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
2331
2332         if (virtqueue_enable_intr(vq) != 0) {
2333                 virtqueue_disable_intr(vq);
2334                 VTSCSI_UNLOCK(sc);
2335                 goto again;
2336         }
2337
2338 done:
2339         VTSCSI_UNLOCK(sc);
2340 }
2341
2342 static void
2343 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc)
2344 {
2345
2346         virtqueue_disable_intr(sc->vtscsi_control_vq);
2347         virtqueue_disable_intr(sc->vtscsi_event_vq);
2348         virtqueue_disable_intr(sc->vtscsi_request_vq);
2349 }
2350
2351 static void
2352 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc)
2353 {
2354
2355         virtqueue_enable_intr(sc->vtscsi_control_vq);
2356         virtqueue_enable_intr(sc->vtscsi_event_vq);
2357         virtqueue_enable_intr(sc->vtscsi_request_vq);
2358 }
2359
2360 static void
2361 vtscsi_get_tunables(struct vtscsi_softc *sc)
2362 {
2363         char tmpstr[64];
2364
2365         TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug);
2366
2367         ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level",
2368             device_get_unit(sc->vtscsi_dev));
2369         TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug);
2370 }
2371
2372 static void
2373 vtscsi_add_sysctl(struct vtscsi_softc *sc)
2374 {
2375         device_t dev;
2376         struct vtscsi_statistics *stats;
2377         struct sysctl_ctx_list *ctx;
2378         struct sysctl_oid *tree;
2379         struct sysctl_oid_list *child;
2380
2381         dev = sc->vtscsi_dev;
2382         stats = &sc->vtscsi_stats;
2383         ctx = device_get_sysctl_ctx(dev);
2384         tree = device_get_sysctl_tree(dev);
2385         child = SYSCTL_CHILDREN(tree);
2386
2387         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level",
2388             CTLFLAG_RW, &sc->vtscsi_debug, 0,
2389             "Debug level");
2390
2391         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts",
2392             CTLFLAG_RD, &stats->scsi_cmd_timeouts,
2393             "SCSI command timeouts");
2394         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests",
2395             CTLFLAG_RD, &stats->dequeue_no_requests,
2396             "No available requests to dequeue");
2397 }
2398
2399 static void
2400 vtscsi_printf_req(struct vtscsi_request *req, const char *func,
2401     const char *fmt, ...)
2402 {
2403         struct vtscsi_softc *sc;
2404         union ccb *ccb;
2405         struct sbuf sb;
2406         __va_list ap;
2407         char str[192];
2408         char path_str[64];
2409
2410         if (req == NULL)
2411                 return;
2412
2413         sc = req->vsr_softc;
2414         ccb = req->vsr_ccb;
2415
2416         __va_start(ap, fmt);
2417         sbuf_new(&sb, str, sizeof(str), 0);
2418
2419         if (ccb == NULL) {
2420                 sbuf_printf(&sb, "(noperiph:%s%d:%u): ",
2421                     cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim),
2422                     cam_sim_bus(sc->vtscsi_sim));
2423         } else {
2424                 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str));
2425                 sbuf_cat(&sb, path_str);
2426                 if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
2427                         scsi_command_string(&ccb->csio, &sb);
2428                         sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len);
2429                 }
2430         }
2431
2432         sbuf_vprintf(&sb, fmt, ap);
2433         __va_end(ap);
2434
2435         sbuf_finish(&sb);
2436         kprintf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func,
2437             sbuf_data(&sb));
2438 }