sys/dev/virtual/virtio/block/virtio_blk.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * $FreeBSD: src/sys/dev/virtio/block/virtio_blk.c,v 1.4 2012/04/16 18:29:12 grehan Exp $
  27  */
  28
  29 /* Driver for VirtIO block devices. */
  30
  31 #include <sys/param.h>
  32 #include <sys/systm.h>
  33 #include <sys/kernel.h>
  34 #include <sys/bio.h>
  35 #include <sys/malloc.h>
  36 #include <sys/module.h>
  37 #include <sys/sglist.h>
  38 #include <sys/lock.h>
  39 #include <sys/queue.h>
  40 #include <sys/serialize.h>
  41 #include <sys/buf2.h>
  42 #include <sys/rman.h>
  43 #include <sys/disk.h>
  44 #include <sys/devicestat.h>
  45
  46 #include <dev/virtual/virtio/virtio/virtio.h>
  47 #include <dev/virtual/virtio/virtio/virtqueue.h>
  48 #include "virtio_blk.h"
  49
  50 struct vtblk_request {
  51         struct virtio_blk_outhdr        vbr_hdr;
  52         struct bio                      *vbr_bp;
  53         uint8_t                         vbr_ack;
  54
  55         TAILQ_ENTRY(vtblk_request)      vbr_link;
  56 };
  57
  58 struct vtblk_softc {
  59         device_t                        vtblk_dev;
  60         struct lwkt_serialize           vtblk_slz;
  61         uint64_t                        vtblk_features;
  62
  63 #define VTBLK_FLAG_READONLY             0x0002
  64 #define VTBLK_FLAG_DETACH               0x0004
  65 #define VTBLK_FLAG_SUSPEND              0x0008
  66         uint32_t                        vtblk_flags;
  67
  68         struct virtqueue                *vtblk_vq;
  69         struct sglist                   *vtblk_sglist;
  70         struct disk                     vtblk_disk;
  71         cdev_t                          cdev;
  72         struct devstat                  stats;
  73
  74         struct bio_queue_head           vtblk_bioq;
  75         TAILQ_HEAD(, vtblk_request)     vtblk_req_free;
  76         TAILQ_HEAD(, vtblk_request)     vtblk_req_ready;
  77
  78         int                             vtblk_sector_size;
  79         int                             vtblk_max_nsegs;
  80         int                             vtblk_unit;
  81         int                             vtblk_request_count;
  82
  83         struct vtblk_request            vtblk_dump_request;
  84 };
  85
  86 static struct virtio_feature_desc vtblk_feature_desc[] = {
  87         { VIRTIO_BLK_F_BARRIER,         "HostBarrier"   },
  88         { VIRTIO_BLK_F_SIZE_MAX,        "MaxSegSize"    },
  89         { VIRTIO_BLK_F_SEG_MAX,         "MaxNumSegs"    },
  90         { VIRTIO_BLK_F_GEOMETRY,        "DiskGeometry"  },
  91         { VIRTIO_BLK_F_RO,              "ReadOnly"      },
  92         { VIRTIO_BLK_F_BLK_SIZE,        "BlockSize"     },
  93         { VIRTIO_BLK_F_SCSI,            "SCSICmds"      },
  94         { VIRTIO_BLK_F_FLUSH,           "FlushCmd"      },
  95         { VIRTIO_BLK_F_TOPOLOGY,        "Topology"      },
  96
  97         { 0, NULL }
  98 };
  99
 100 static int      vtblk_modevent(module_t, int, void *);
 101
 102 static int      vtblk_probe(device_t);
 103 static int      vtblk_attach(device_t);
 104 static int      vtblk_detach(device_t);
 105 static int      vtblk_suspend(device_t);
 106 static int      vtblk_resume(device_t);
 107 static int      vtblk_shutdown(device_t);
 108
 109 static void     vtblk_negotiate_features(struct vtblk_softc *);
 110 static int      vtblk_maximum_segments(struct vtblk_softc *,
 111                                        struct virtio_blk_config *);
 112 static int      vtblk_alloc_virtqueue(struct vtblk_softc *);
 113 static void     vtblk_alloc_disk(struct vtblk_softc *,
 114                                  struct virtio_blk_config *);
 115 /*
 116  * Interface to the device switch.
 117  */
 118 static d_open_t         vtblk_open;
 119 static d_strategy_t     vtblk_strategy;
 120 static d_dump_t         vtblk_dump;
 121
 122 static struct dev_ops vbd_disk_ops = {
 123         { "vbd", 200, D_DISK | D_MPSAFE },
 124         .d_open         = vtblk_open,
 125         .d_close        = nullclose,
 126         .d_read         = physread,
 127         .d_write        = physwrite,
 128         .d_strategy     = vtblk_strategy,
 129         .d_dump         = vtblk_dump,
 130 };
 131
 132 static void vtblk_startio(struct vtblk_softc *);
 133 static struct vtblk_request *vtblk_bio_request(struct vtblk_softc *);
 134 static int vtblk_execute_request(struct vtblk_softc *, struct vtblk_request *);
 135
 136 static int              vtblk_vq_intr(void *);
 137 static void             vtblk_complete(void *);
 138
 139 static void             vtblk_stop(struct vtblk_softc *);
 140
 141 static void             vtblk_drain_vq(struct vtblk_softc *, int);
 142 static void             vtblk_drain(struct vtblk_softc *);
 143
 144 static int              vtblk_alloc_requests(struct vtblk_softc *);
 145 static void             vtblk_free_requests(struct vtblk_softc *);
 146 static struct vtblk_request *vtblk_dequeue_request(struct vtblk_softc *);
 147 static void             vtblk_enqueue_request(struct vtblk_softc *,
 148                                               struct vtblk_request *);
 149
 150 static struct vtblk_request *vtblk_dequeue_ready(struct vtblk_softc *);
 151 static void             vtblk_enqueue_ready(struct vtblk_softc *,
 152                                             struct vtblk_request *);
 153
 154 static void             vtblk_bio_error(struct bio *, int);
 155
 156 /* Tunables. */
 157 static int vtblk_no_ident = 0;
 158 TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
 159
 160 /* Features desired/implemented by this driver. */
 161 #define VTBLK_FEATURES \
 162     (VIRTIO_BLK_F_BARRIER               | \
 163      VIRTIO_BLK_F_SIZE_MAX              | \
 164      VIRTIO_BLK_F_SEG_MAX               | \
 165      VIRTIO_BLK_F_GEOMETRY              | \
 166      VIRTIO_BLK_F_RO                    | \
 167      VIRTIO_BLK_F_BLK_SIZE              | \
 168      VIRTIO_BLK_F_FLUSH)
 169
 170 /*
 171  * Each block request uses at least two segments - one for the header
 172  * and one for the status.
 173  */
 174 #define VTBLK_MIN_SEGMENTS      2
 175
 176 static device_method_t vtblk_methods[] = {
 177         /* Device methods. */
 178         DEVMETHOD(device_probe,         vtblk_probe),
 179         DEVMETHOD(device_attach,        vtblk_attach),
 180         DEVMETHOD(device_detach,        vtblk_detach),
 181         DEVMETHOD(device_suspend,       vtblk_suspend),
 182         DEVMETHOD(device_resume,        vtblk_resume),
 183         DEVMETHOD(device_shutdown,      vtblk_shutdown),
 184
 185         DEVMETHOD_END
 186 };
 187
 188 static driver_t vtblk_driver = {
 189         "vtblk",
 190         vtblk_methods,
 191         sizeof(struct vtblk_softc)
 192 };
 193 static devclass_t vtblk_devclass;
 194
 195 DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
 196               vtblk_modevent, NULL);
 197 MODULE_VERSION(virtio_blk, 1);
 198 MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
 199
 200 static int
 201 vtblk_modevent(module_t mod, int type, void *unused)
 202 {
 203         int error;
 204
 205         error = 0;
 206
 207         switch (type) {
 208         case MOD_LOAD:
 209                 break;
 210         case MOD_UNLOAD:
 211                 break;
 212         case MOD_SHUTDOWN:
 213                 break;
 214         default:
 215                 error = EOPNOTSUPP;
 216                 break;
 217         }
 218
 219         return (error);
 220 }
 221
 222 static int
 223 vtblk_probe(device_t dev)
 224 {
 225
 226         if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK)
 227                 return (ENXIO);
 228
 229         device_set_desc(dev, "VirtIO Block Adapter");
 230
 231         return (BUS_PROBE_DEFAULT);
 232 }
 233
 234 static int
 235 vtblk_attach(device_t dev)
 236 {
 237         struct vtblk_softc *sc;
 238         struct virtio_blk_config blkcfg;
 239         int error;
 240
 241         sc = device_get_softc(dev);
 242         sc->vtblk_dev = dev;
 243         sc->vtblk_unit = device_get_unit(dev);
 244
 245         lwkt_serialize_init(&sc->vtblk_slz);
 246
 247         bioq_init(&sc->vtblk_bioq);
 248         TAILQ_INIT(&sc->vtblk_req_free);
 249         TAILQ_INIT(&sc->vtblk_req_ready);
 250
 251         virtio_set_feature_desc(dev, vtblk_feature_desc);
 252         vtblk_negotiate_features(sc);
 253
 254         if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
 255                 sc->vtblk_flags |= VTBLK_FLAG_READONLY;
 256
 257         /* Get local copy of config. */
 258         virtio_read_device_config(dev, 0, &blkcfg,
 259                                   sizeof(struct virtio_blk_config));
 260
 261         /*
 262          * With the current sglist(9) implementation, it is not easy
 263          * for us to support a maximum segment size as adjacent
 264          * segments are coalesced. For now, just make sure it's larger
 265          * than the maximum supported transfer size.
 266          */
 267         if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
 268                 if (blkcfg.size_max < MAXPHYS) {
 269                         error = ENOTSUP;
 270                         device_printf(dev, "host requires unsupported "
 271                             "maximum segment size feature\n");
 272                         goto fail;
 273                 }
 274         }
 275
 276         sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
 277         if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
 278                 error = EINVAL;
 279                 device_printf(dev, "fewer than minimum number of segments "
 280                     "allowed: %d\n", sc->vtblk_max_nsegs);
 281                 goto fail;
 282         }
 283
 284         /*
 285          * Allocate working sglist. The number of segments may be too
 286          * large to safely store on the stack.
 287          */
 288         sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
 289         if (sc->vtblk_sglist == NULL) {
 290                 error = ENOMEM;
 291                 device_printf(dev, "cannot allocate sglist\n");
 292                 goto fail;
 293         }
 294
 295         error = vtblk_alloc_virtqueue(sc);
 296         if (error) {
 297                 device_printf(dev, "cannot allocate virtqueue\n");
 298                 goto fail;
 299         }
 300
 301         error = vtblk_alloc_requests(sc);
 302         if (error) {
 303                 device_printf(dev, "cannot preallocate requests\n");
 304                 goto fail;
 305         }
 306
 307         vtblk_alloc_disk(sc, &blkcfg);
 308
 309         error = virtio_setup_intr(dev, &sc->vtblk_slz);
 310         if (error) {
 311                 device_printf(dev, "cannot setup virtqueue interrupt\n");
 312                 goto fail;
 313         }
 314
 315         virtqueue_enable_intr(sc->vtblk_vq);
 316
 317 fail:
 318         if (error)
 319                 vtblk_detach(dev);
 320
 321         return (error);
 322 }
 323
 324 static int
 325 vtblk_detach(device_t dev)
 326 {
 327         struct vtblk_softc *sc;
 328
 329         sc = device_get_softc(dev);
 330
 331         lwkt_serialize_enter(&sc->vtblk_slz);
 332         sc->vtblk_flags |= VTBLK_FLAG_DETACH;
 333         if (device_is_attached(dev))
 334                 vtblk_stop(sc);
 335         lwkt_serialize_exit(&sc->vtblk_slz);
 336
 337         vtblk_drain(sc);
 338
 339         if (sc->vtblk_sglist != NULL) {
 340                 sglist_free(sc->vtblk_sglist);
 341                 sc->vtblk_sglist = NULL;
 342         }
 343
 344         return (0);
 345 }
 346
 347 static int
 348 vtblk_suspend(device_t dev)
 349 {
 350         struct vtblk_softc *sc;
 351
 352         sc = device_get_softc(dev);
 353
 354         lwkt_serialize_enter(&sc->vtblk_slz);
 355         sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
 356         /* TODO Wait for any inflight IO to complete? */
 357         lwkt_serialize_exit(&sc->vtblk_slz);
 358
 359         return (0);
 360 }
 361
 362 static int
 363 vtblk_resume(device_t dev)
 364 {
 365         struct vtblk_softc *sc;
 366
 367         sc = device_get_softc(dev);
 368
 369         lwkt_serialize_enter(&sc->vtblk_slz);
 370         sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
 371         /* TODO Resume IO? */
 372         lwkt_serialize_exit(&sc->vtblk_slz);
 373
 374         return (0);
 375 }
 376
 377 static int
 378 vtblk_shutdown(device_t dev)
 379 {
 380         return (0);
 381 }
 382
 383 static int
 384 vtblk_open(struct dev_open_args *ap)
 385 {
 386         struct vtblk_softc *sc;
 387         cdev_t dev = ap->a_head.a_dev;
 388         sc = dev->si_drv1;
 389         if (sc == NULL)
 390                 return (ENXIO);
 391
 392         return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
 393 }
 394
 395 static int
 396 vtblk_dump(struct dev_dump_args *ap)
 397 {
 398         /* XXX */
 399         return (ENXIO);
 400 }
 401
 402 static int
 403 vtblk_strategy(struct dev_strategy_args *ap)
 404 {
 405         struct vtblk_softc *sc;
 406         cdev_t dev = ap->a_head.a_dev;
 407         sc = dev->si_drv1;
 408         struct bio *bio = ap->a_bio;
 409         struct buf *bp = bio->bio_buf;
 410
 411         if (sc == NULL) {
 412                 vtblk_bio_error(bio, EINVAL);
 413                 return EINVAL;
 414         }
 415
 416         /*
 417          * Fail any write if RO. Unfortunately, there does not seem to
 418          * be a better way to report our readonly'ness to GEOM above.
 419          *
 420          * XXX: Is that true in DFly?
 421          */
 422         if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
 423             (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_FLUSH)) {
 424                 vtblk_bio_error(bio, EROFS);
 425                 return (EINVAL);
 426         }
 427
 428         lwkt_serialize_enter(&sc->vtblk_slz);
 429         if ((sc->vtblk_flags & VTBLK_FLAG_DETACH) == 0) {
 430                 devstat_start_transaction(&sc->stats);
 431                 bioqdisksort(&sc->vtblk_bioq, bio);
 432                 vtblk_startio(sc);
 433         } else {
 434                 vtblk_bio_error(bio, ENXIO);
 435         }
 436         lwkt_serialize_exit(&sc->vtblk_slz);
 437         return 0;
 438 }
 439
 440 static void
 441 vtblk_negotiate_features(struct vtblk_softc *sc)
 442 {
 443         device_t dev;
 444         uint64_t features;
 445
 446         dev = sc->vtblk_dev;
 447         features = VTBLK_FEATURES;
 448
 449         sc->vtblk_features = virtio_negotiate_features(dev, features);
 450 }
 451
 452 static int
 453 vtblk_maximum_segments(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 454 {
 455         device_t dev;
 456         int nsegs;
 457
 458         dev = sc->vtblk_dev;
 459         nsegs = VTBLK_MIN_SEGMENTS;
 460
 461         if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
 462                 nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1);
 463         } else {
 464                 nsegs += 1;
 465         }
 466
 467         return (nsegs);
 468 }
 469
 470 static int
 471 vtblk_alloc_virtqueue(struct vtblk_softc *sc)
 472 {
 473         device_t dev;
 474         struct vq_alloc_info vq_info;
 475
 476         dev = sc->vtblk_dev;
 477
 478         VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
 479                            vtblk_vq_intr, sc, &sc->vtblk_vq,
 480                            "%s request", device_get_nameunit(dev));
 481
 482         return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
 483 }
 484
 485 static void
 486 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 487 {
 488
 489         struct disk_info info;
 490
 491         /* construct the disk_info */
 492         bzero(&info, sizeof(info));
 493
 494         if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_BLK_SIZE))
 495                 sc->vtblk_sector_size = blkcfg->blk_size;
 496         else
 497                 sc->vtblk_sector_size = DEV_BSIZE;
 498
 499         info.d_media_blksize = sc->vtblk_sector_size;
 500         info.d_media_blocks = blkcfg->capacity;
 501
 502         info.d_ncylinders = blkcfg->geometry.cylinders;
 503         info.d_nheads = blkcfg->geometry.heads;
 504         info.d_secpertrack = blkcfg->geometry.sectors;
 505
 506         info.d_secpercyl = info.d_secpertrack * info.d_nheads;
 507
 508         devstat_add_entry(&sc->stats, "vbd", device_get_unit(sc->vtblk_dev),
 509                           DEV_BSIZE, DEVSTAT_ALL_SUPPORTED,
 510                           DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
 511                           DEVSTAT_PRIORITY_DISK);
 512
 513         /* attach a generic disk device to ourselves */
 514         sc->cdev = disk_create(device_get_unit(sc->vtblk_dev), &sc->vtblk_disk,
 515                                &vbd_disk_ops);
 516
 517         sc->cdev->si_drv1 = sc;
 518         disk_setdiskinfo(&sc->vtblk_disk, &info);
 519 }
 520
 521 static void
 522 vtblk_startio(struct vtblk_softc *sc)
 523 {
 524         struct virtqueue *vq;
 525         struct vtblk_request *req;
 526         int enq;
 527
 528         vq = sc->vtblk_vq;
 529         enq = 0;
 530
 531         ASSERT_SERIALIZED(&sc->vtblk_slz);
 532
 533         if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
 534                 return;
 535
 536         while (!virtqueue_full(vq)) {
 537                 if ((req = vtblk_dequeue_ready(sc)) == NULL)
 538                         req = vtblk_bio_request(sc);
 539                 if (req == NULL)
 540                         break;
 541
 542                 if (vtblk_execute_request(sc, req) != 0) {
 543                         vtblk_enqueue_ready(sc, req);
 544                         break;
 545                 }
 546
 547                 enq++;
 548         }
 549
 550         if (enq > 0)
 551                 virtqueue_notify(vq, &sc->vtblk_slz);
 552 }
 553
 554 static struct vtblk_request *
 555 vtblk_bio_request(struct vtblk_softc *sc)
 556 {
 557         struct bio_queue_head *bioq;
 558         struct vtblk_request *req;
 559         struct bio *bio;
 560         struct buf *bp;
 561
 562         bioq = &sc->vtblk_bioq;
 563
 564         if (bioq_first(bioq) == NULL)
 565                 return (NULL);
 566
 567         req = vtblk_dequeue_request(sc);
 568         if (req == NULL)
 569                 return (NULL);
 570
 571         bio = bioq_takefirst(bioq);
 572         req->vbr_bp = bio;
 573         req->vbr_ack = -1;
 574         req->vbr_hdr.ioprio = 1;
 575         bp = bio->bio_buf;
 576
 577         switch (bp->b_cmd) {
 578         case BUF_CMD_FLUSH:
 579                 req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
 580                 break;
 581         case BUF_CMD_READ:
 582                 req->vbr_hdr.type = VIRTIO_BLK_T_IN;
 583                 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
 584                 break;
 585         case BUF_CMD_WRITE:
 586                 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
 587                 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
 588                 break;
 589         default:
 590                 KASSERT(0, ("bio with unhandled cmd: %d", bp->b_cmd));
 591                 req->vbr_hdr.type = -1;
 592                 break;
 593         }
 594
 595         if (bp->b_flags & B_ORDERED)
 596                 req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
 597
 598         return (req);
 599 }
 600
 601 static int
 602 vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 603 {
 604         struct sglist *sg;
 605         struct bio *bio;
 606         struct buf *bp;
 607         int writable, error;
 608
 609         sg = sc->vtblk_sglist;
 610         bio = req->vbr_bp;
 611         bp = bio->bio_buf;
 612         writable = 0;
 613
 614         /*
 615          * sglist is live throughout this subroutine.
 616          */
 617         sglist_reset(sg);
 618
 619         error = sglist_append(sg, &req->vbr_hdr,
 620                               sizeof(struct virtio_blk_outhdr));
 621         KASSERT(error == 0, ("error adding header to sglist"));
 622         KASSERT(sg->sg_nseg == 1,
 623             ("header spanned multiple segments: %d", sg->sg_nseg));
 624
 625         if (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_WRITE) {
 626                 error = sglist_append(sg, bp->b_data, bp->b_bcount);
 627                 KASSERT(error == 0, ("error adding buffer to sglist"));
 628
 629                 /* BUF_CMD_READ means the host writes into our buffer. */
 630                 if (bp->b_cmd == BUF_CMD_READ)
 631                         writable += sg->sg_nseg - 1;
 632         }
 633
 634         error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
 635         KASSERT(error == 0, ("error adding ack to sglist"));
 636         writable++;
 637
 638         KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
 639             ("fewer than min segments: %d", sg->sg_nseg));
 640
 641         error = virtqueue_enqueue(sc->vtblk_vq, req, sg,
 642                                   sg->sg_nseg - writable, writable);
 643
 644         sglist_reset(sg);
 645
 646         return (error);
 647 }
 648
 649 static int
 650 vtblk_vq_intr(void *xsc)
 651 {
 652         vtblk_complete(xsc);
 653
 654         return (1);
 655 }
 656
 657 static void
 658 vtblk_complete(void *arg)
 659 {
 660         struct vtblk_softc *sc;
 661         struct vtblk_request *req;
 662         struct virtqueue *vq;
 663         struct bio *bio;
 664         struct buf *bp;
 665
 666         sc = arg;
 667         vq = sc->vtblk_vq;
 668
 669         lwkt_serialize_handler_disable(&sc->vtblk_slz);
 670         virtqueue_disable_intr(sc->vtblk_vq);
 671         ASSERT_SERIALIZED(&sc->vtblk_slz);
 672
 673 retry:
 674         if (sc->vtblk_flags & VTBLK_FLAG_DETACH)
 675                 return;
 676
 677         while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
 678                 bio = req->vbr_bp;
 679                 bp = bio->bio_buf;
 680
 681                 if (req->vbr_ack == VIRTIO_BLK_S_OK)
 682                         bp->b_resid = 0;
 683                 else {
 684                         bp->b_flags |= B_ERROR;
 685                         if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP) {
 686                                 bp->b_error = ENOTSUP;
 687                         } else {
 688                                 bp->b_error = EIO;
 689                         }
 690                 }
 691
 692                 devstat_end_transaction_buf(&sc->stats, bio->bio_buf);
 693
 694                 lwkt_serialize_exit(&sc->vtblk_slz);
 695                 /*
 696                  * Unlocking the controller around biodone() does not allow
 697                  * processing further device interrupts; when we queued
 698                  * vtblk_complete, we disabled interrupts. It will allow
 699                  * concurrent vtblk_strategy/_startio command dispatches.
 700                  */
 701                 biodone(bio);
 702                 lwkt_serialize_enter(&sc->vtblk_slz);
 703
 704                 vtblk_enqueue_request(sc, req);
 705         }
 706
 707         vtblk_startio(sc);
 708
 709         if (virtqueue_enable_intr(vq) != 0) {
 710                 /*
 711                  * If new virtqueue entries appeared immediately after
 712                  * enabling interrupts, process them now. Release and
 713                  * retake softcontroller lock to try to avoid blocking
 714                  * I/O dispatch for too long.
 715                  */
 716                 virtqueue_disable_intr(vq);
 717                 goto retry;
 718         }
 719         lwkt_serialize_handler_enable(&sc->vtblk_slz);
 720 }
 721
 722 static void
 723 vtblk_stop(struct vtblk_softc *sc)
 724 {
 725         virtqueue_disable_intr(sc->vtblk_vq);
 726         virtio_stop(sc->vtblk_dev);
 727 }
 728
 729 static void
 730 vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
 731 {
 732         struct virtqueue *vq;
 733         struct vtblk_request *req;
 734         int last;
 735
 736         vq = sc->vtblk_vq;
 737         last = 0;
 738
 739         while ((req = virtqueue_drain(vq, &last)) != NULL) {
 740                 if (!skip_done)
 741                         vtblk_bio_error(req->vbr_bp, ENXIO);
 742
 743                 vtblk_enqueue_request(sc, req);
 744         }
 745
 746         KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
 747 }
 748
 749 static void
 750 vtblk_drain(struct vtblk_softc *sc)
 751 {
 752         struct bio_queue_head *bioq;
 753         struct vtblk_request *req;
 754         struct bio *bp;
 755
 756         bioq = &sc->vtblk_bioq;
 757
 758         if (sc->vtblk_vq != NULL)
 759                 vtblk_drain_vq(sc, 0);
 760
 761         while ((req = vtblk_dequeue_ready(sc)) != NULL) {
 762                 vtblk_bio_error(req->vbr_bp, ENXIO);
 763                 vtblk_enqueue_request(sc, req);
 764         }
 765
 766         while (bioq_first(bioq) != NULL) {
 767                 bp = bioq_takefirst(bioq);
 768                 vtblk_bio_error(bp, ENXIO);
 769         }
 770
 771         vtblk_free_requests(sc);
 772 }
 773
 774 static int
 775 vtblk_alloc_requests(struct vtblk_softc *sc)
 776 {
 777         struct vtblk_request *req;
 778         int i, nreqs;
 779
 780         nreqs = virtqueue_size(sc->vtblk_vq);
 781
 782         /*
 783          * Preallocate sufficient requests to keep the virtqueue full. Each
 784          * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
 785          * the number allocated when indirect descriptors are not available.
 786          */
 787         nreqs /= VTBLK_MIN_SEGMENTS;
 788
 789         for (i = 0; i < nreqs; i++) {
 790                 req = kmalloc(sizeof(struct vtblk_request), M_DEVBUF, M_WAITOK);
 791
 792                 sc->vtblk_request_count++;
 793                 vtblk_enqueue_request(sc, req);
 794         }
 795
 796         return (0);
 797 }
 798
 799 static void
 800 vtblk_free_requests(struct vtblk_softc *sc)
 801 {
 802         struct vtblk_request *req;
 803
 804         while ((req = vtblk_dequeue_request(sc)) != NULL) {
 805                 sc->vtblk_request_count--;
 806                 kfree(req, M_DEVBUF);
 807         }
 808
 809         KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
 810 }
 811
 812 static struct vtblk_request *
 813 vtblk_dequeue_request(struct vtblk_softc *sc)
 814 {
 815         struct vtblk_request *req;
 816
 817         req = TAILQ_FIRST(&sc->vtblk_req_free);
 818         if (req != NULL)
 819                 TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
 820
 821         return (req);
 822 }
 823
 824 static void
 825 vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
 826 {
 827         bzero(req, sizeof(struct vtblk_request));
 828         TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
 829 }
 830
 831 static struct vtblk_request *
 832 vtblk_dequeue_ready(struct vtblk_softc *sc)
 833 {
 834         struct vtblk_request *req;
 835
 836         req = TAILQ_FIRST(&sc->vtblk_req_ready);
 837         if (req != NULL)
 838                 TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
 839
 840         return (req);
 841 }
 842
 843 static void
 844 vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
 845 {
 846         TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
 847 }
 848
 849 static void
 850 vtblk_bio_error(struct bio *bp, int error)
 851 {
 852         biodone(bp);
 853 }