68680dc90bf29dc9cc6842dc5659605f49bb44e9
[dragonfly.git] / sys / dev / virtual / virtio / block / virtio_blk.c
1 /*-
2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * $FreeBSD: src/sys/dev/virtio/block/virtio_blk.c,v 1.4 2012/04/16 18:29:12 grehan Exp $
27  */
28
29 /* Driver for VirtIO block devices. */
30
31 #include <sys/cdefs.h>
32
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/bio.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/sglist.h>
40 #include <sys/lock.h>
41 #include <sys/queue.h>
42 #include <sys/taskqueue.h>
43
44 #include <sys/buf2.h>
45 #include <sys/rman.h>
46 #include <sys/disk.h>
47 #include <sys/spinlock.h>
48 #include <sys/spinlock2.h>
49 #include <sys/devicestat.h>
50
51 #include <dev/virtual/virtio/virtio/virtio.h>
52 #include <dev/virtual/virtio/virtio/virtqueue.h>
53 #include "virtio_blk.h"
54
55 struct vtblk_request {
56         struct virtio_blk_outhdr        vbr_hdr;
57         struct bio                      *vbr_bp;
58         uint8_t                         vbr_ack;
59
60         TAILQ_ENTRY(vtblk_request)      vbr_link;
61 };
62
63 struct vtblk_softc {
64         device_t                        vtblk_dev;
65         struct spinlock                 vtblk_mtx;
66         uint64_t                        vtblk_features;
67
68 #define VTBLK_FLAG_READONLY             0x0002
69 #define VTBLK_FLAG_DETACH               0x0004
70 #define VTBLK_FLAG_SUSPEND              0x0008
71 #define VTBLK_FLAG_DUMPING              0x0010
72         uint32_t                        vtblk_flags;
73
74         struct virtqueue                *vtblk_vq;
75         struct sglist                   *vtblk_sglist;
76         struct disk                     vtblk_disk;
77         cdev_t                          cdev;
78         struct devstat                  stats;
79
80         struct bio_queue_head           vtblk_bioq;
81         TAILQ_HEAD(, vtblk_request)     vtblk_req_free;
82         TAILQ_HEAD(, vtblk_request)     vtblk_req_ready;
83
84         struct task                     vtblk_intr_task;
85
86         int                             vtblk_sector_size;
87         int                             vtblk_max_nsegs;
88         int                             vtblk_unit;
89         int                             vtblk_request_count;
90
91         struct vtblk_request            vtblk_dump_request;
92 };
93
94 static struct virtio_feature_desc vtblk_feature_desc[] = {
95         { VIRTIO_BLK_F_BARRIER,         "HostBarrier"   },
96         { VIRTIO_BLK_F_SIZE_MAX,        "MaxSegSize"    },
97         { VIRTIO_BLK_F_SEG_MAX,         "MaxNumSegs"    },
98         { VIRTIO_BLK_F_GEOMETRY,        "DiskGeometry"  },
99         { VIRTIO_BLK_F_RO,              "ReadOnly"      },
100         { VIRTIO_BLK_F_BLK_SIZE,        "BlockSize"     },
101         { VIRTIO_BLK_F_SCSI,            "SCSICmds"      },
102         { VIRTIO_BLK_F_FLUSH,           "FlushCmd"      },
103         { VIRTIO_BLK_F_TOPOLOGY,        "Topology"      },
104
105         { 0, NULL }
106 };
107
108 static int      vtblk_modevent(module_t, int, void *);
109
110 static int      vtblk_probe(device_t);
111 static int      vtblk_attach(device_t);
112 static int      vtblk_detach(device_t);
113 static int      vtblk_suspend(device_t);
114 static int      vtblk_resume(device_t);
115 static int      vtblk_shutdown(device_t);
116
117 static void     vtblk_negotiate_features(struct vtblk_softc *);
118 static int      vtblk_maximum_segments(struct vtblk_softc *,
119                                        struct virtio_blk_config *);
120 static int      vtblk_alloc_virtqueue(struct vtblk_softc *);
121 static void     vtblk_alloc_disk(struct vtblk_softc *,
122                                  struct virtio_blk_config *);
123 /*
124  * Interface to the device switch.
125  */
126 static d_open_t         vtblk_open;
127 static d_strategy_t     vtblk_strategy;
128 static d_dump_t         vtblk_dump;
129
130 static struct dev_ops vbd_disk_ops = {
131         { "vbd", 200, D_DISK | D_MPSAFE },
132         .d_open         = vtblk_open,
133         .d_close        = nullclose,
134         .d_read         = physread,
135         .d_write        = physwrite,
136         .d_strategy     = vtblk_strategy,
137         .d_dump         = vtblk_dump,
138 };
139
140 static void             vtblk_startio(struct vtblk_softc *);
141 static struct vtblk_request *vtblk_bio_request(struct vtblk_softc *);
142 static int              vtblk_execute_request(struct vtblk_softc *,
143                                               struct vtblk_request *);
144
145 static int              vtblk_vq_intr(void *);
146 static void             vtblk_complete(void *, int);
147
148 static void             vtblk_stop(struct vtblk_softc *);
149
150 static void             vtblk_prepare_dump(struct vtblk_softc *);
151 static int              vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
152 static int              vtblk_flush_dump(struct vtblk_softc *);
153 static int              vtblk_poll_request(struct vtblk_softc *,
154                                            struct vtblk_request *);
155
156 static void             vtblk_drain_vq(struct vtblk_softc *, int);
157 static void             vtblk_drain(struct vtblk_softc *);
158
159 static int              vtblk_alloc_requests(struct vtblk_softc *);
160 static void             vtblk_free_requests(struct vtblk_softc *);
161 static struct vtblk_request *vtblk_dequeue_request(struct vtblk_softc *);
162 static void             vtblk_enqueue_request(struct vtblk_softc *,
163                                               struct vtblk_request *);
164
165 static struct vtblk_request *vtblk_dequeue_ready(struct vtblk_softc *);
166 static void             vtblk_enqueue_ready(struct vtblk_softc *,
167                                             struct vtblk_request *);
168
169 static void             vtblk_bio_error(struct bio *, int);
170
171 /* Tunables. */
172 static int vtblk_no_ident = 0;
173 TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
174
175 /* Features desired/implemented by this driver. */
176 #define VTBLK_FEATURES \
177     (VIRTIO_BLK_F_BARRIER               | \
178      VIRTIO_BLK_F_SIZE_MAX              | \
179      VIRTIO_BLK_F_SEG_MAX               | \
180      VIRTIO_BLK_F_GEOMETRY              | \
181      VIRTIO_BLK_F_RO                    | \
182      VIRTIO_BLK_F_BLK_SIZE              | \
183      VIRTIO_BLK_F_FLUSH)
184
185 #define VTBLK_MTX(_sc)          &(_sc)->vtblk_mtx
186 #define VTBLK_LOCK_INIT(_sc)    spin_init(&(_sc)->vtblk_mtx)
187 #define VTBLK_LOCK(_sc)         spin_lock(VTBLK_MTX((_sc)))
188 #define VTBLK_TRYLOCK(_sc)      spin_trylock(VTBLK_MTX((_sc)))
189 #define VTBLK_UNLOCK(_sc)       spin_unlock(VTBLK_MTX((_sc)))
190 #define VTBLK_LOCK_DESTROY(_sc) spin_uninit(VTBLK_MTX((_sc)))
191
192 #define VTBLK_LOCK_ASSERT(_sc)
193 #define VTBLK_LOCK_ASSERT_NOTOWNED(_sc)
194
195 /*
196  * Each block request uses at least two segments - one for the header
197  * and one for the status.
198  */
199 #define VTBLK_MIN_SEGMENTS      2
200
201 static device_method_t vtblk_methods[] = {
202         /* Device methods. */
203         DEVMETHOD(device_probe,         vtblk_probe),
204         DEVMETHOD(device_attach,        vtblk_attach),
205         DEVMETHOD(device_detach,        vtblk_detach),
206         DEVMETHOD(device_suspend,       vtblk_suspend),
207         DEVMETHOD(device_resume,        vtblk_resume),
208         DEVMETHOD(device_shutdown,      vtblk_shutdown),
209
210         { 0, 0 }
211 };
212
213 static driver_t vtblk_driver = {
214         "vtblk",
215         vtblk_methods,
216         sizeof(struct vtblk_softc)
217 };
218 static devclass_t vtblk_devclass;
219
220 DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
221               vtblk_modevent, NULL);
222 MODULE_VERSION(virtio_blk, 1);
223 MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
224
225 static int
226 vtblk_modevent(module_t mod, int type, void *unused)
227 {
228         int error;
229
230         error = 0;
231
232         switch (type) {
233         case MOD_LOAD:
234                 break;
235         case MOD_UNLOAD:
236                 break;
237         case MOD_SHUTDOWN:
238                 break;
239         default:
240                 error = EOPNOTSUPP;
241                 break;
242         }
243
244         return (error);
245 }
246
247 static int
248 vtblk_probe(device_t dev)
249 {
250
251         if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK)
252                 return (ENXIO);
253
254         device_set_desc(dev, "VirtIO Block Adapter");
255
256         return (BUS_PROBE_DEFAULT);
257 }
258
259 static int
260 vtblk_attach(device_t dev)
261 {
262         struct vtblk_softc *sc;
263         struct virtio_blk_config blkcfg;
264         int error;
265
266         sc = device_get_softc(dev);
267         sc->vtblk_dev = dev;
268         sc->vtblk_unit = device_get_unit(dev);
269
270         VTBLK_LOCK_INIT(sc);
271
272         bioq_init(&sc->vtblk_bioq);
273         TAILQ_INIT(&sc->vtblk_req_free);
274         TAILQ_INIT(&sc->vtblk_req_ready);
275
276         virtio_set_feature_desc(dev, vtblk_feature_desc);
277         vtblk_negotiate_features(sc);
278
279         if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
280                 sc->vtblk_flags |= VTBLK_FLAG_READONLY;
281
282         /* Get local copy of config. */
283         virtio_read_device_config(dev, 0, &blkcfg,
284                                   sizeof(struct virtio_blk_config));
285
286         /*
287          * With the current sglist(9) implementation, it is not easy
288          * for us to support a maximum segment size as adjacent
289          * segments are coalesced. For now, just make sure it's larger
290          * than the maximum supported transfer size.
291          */
292         if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
293                 if (blkcfg.size_max < MAXPHYS) {
294                         error = ENOTSUP;
295                         device_printf(dev, "host requires unsupported "
296                             "maximum segment size feature\n");
297                         goto fail;
298                 }
299         }
300
301         sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
302         if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
303                 error = EINVAL;
304                 device_printf(dev, "fewer than minimum number of segments "
305                     "allowed: %d\n", sc->vtblk_max_nsegs);
306                 goto fail;
307         }
308
309         /*
310          * Allocate working sglist. The number of segments may be too
311          * large to safely store on the stack.
312          */
313         sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
314         if (sc->vtblk_sglist == NULL) {
315                 error = ENOMEM;
316                 device_printf(dev, "cannot allocate sglist\n");
317                 goto fail;
318         }
319
320         error = vtblk_alloc_virtqueue(sc);
321         if (error) {
322                 device_printf(dev, "cannot allocate virtqueue\n");
323                 goto fail;
324         }
325
326         error = vtblk_alloc_requests(sc);
327         if (error) {
328                 device_printf(dev, "cannot preallocate requests\n");
329                 goto fail;
330         }
331
332         vtblk_alloc_disk(sc, &blkcfg);
333
334         TASK_INIT(&sc->vtblk_intr_task, 0, vtblk_complete, sc);
335
336         error = virtio_setup_intr(dev);
337         if (error) {
338                 device_printf(dev, "cannot setup virtqueue interrupt\n");
339                 goto fail;
340         }
341
342         virtqueue_enable_intr(sc->vtblk_vq);
343
344 fail:
345         if (error)
346                 vtblk_detach(dev);
347
348         return (error);
349 }
350
351 static int
352 vtblk_detach(device_t dev)
353 {
354         struct vtblk_softc *sc;
355
356         sc = device_get_softc(dev);
357
358         VTBLK_LOCK(sc);
359         sc->vtblk_flags |= VTBLK_FLAG_DETACH;
360         if (device_is_attached(dev))
361                 vtblk_stop(sc);
362         VTBLK_UNLOCK(sc);
363
364         taskqueue_drain(taskqueue_thread[mycpuid], &sc->vtblk_intr_task);
365
366         vtblk_drain(sc);
367
368         if (sc->vtblk_sglist != NULL) {
369                 sglist_free(sc->vtblk_sglist);
370                 sc->vtblk_sglist = NULL;
371         }
372
373         VTBLK_LOCK_DESTROY(sc);
374
375         return (0);
376 }
377
378 static int
379 vtblk_suspend(device_t dev)
380 {
381         struct vtblk_softc *sc;
382
383         sc = device_get_softc(dev);
384
385         VTBLK_LOCK(sc);
386         sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
387         /* TODO Wait for any inflight IO to complete? */
388         VTBLK_UNLOCK(sc);
389
390         return (0);
391 }
392
393 static int
394 vtblk_resume(device_t dev)
395 {
396         struct vtblk_softc *sc;
397
398         sc = device_get_softc(dev);
399
400         VTBLK_LOCK(sc);
401         sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
402         /* TODO Resume IO? */
403         VTBLK_UNLOCK(sc);
404
405         return (0);
406 }
407
408 static int
409 vtblk_shutdown(device_t dev)
410 {
411         return (0);
412 }
413
414 static int
415 vtblk_open(struct dev_open_args *ap)
416 {
417         struct vtblk_softc *sc;
418         cdev_t dev = ap->a_head.a_dev;
419         sc = dev->si_drv1;
420         if (sc == NULL)
421                 return (ENXIO);
422
423         return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
424 }
425
426 static int
427 vtblk_dump(struct dev_dump_args *ap)
428 {
429         struct vtblk_softc *sc;
430         int error;
431
432         error = 0;
433
434         cdev_t dev = ap->a_head.a_dev;
435         sc = dev->si_drv1;
436
437         if (sc == NULL)
438                 return (ENXIO);
439
440         if (VTBLK_TRYLOCK(sc) == 0) {
441                 device_printf(sc->vtblk_dev,
442                     "softc already locked, cannot dump...\n");
443                 return (EBUSY);
444         }
445
446         if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
447                 vtblk_prepare_dump(sc);
448                 sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
449         }
450
451         if (ap->a_length > 0) {
452                 error = vtblk_write_dump(sc, ap->a_virtual, ap->a_offset,
453                                          ap->a_length);
454         } else if (ap->a_virtual == NULL && ap->a_offset == 0) {
455                 error = vtblk_flush_dump(sc);
456         }
457
458         VTBLK_UNLOCK(sc);
459
460         return (error);
461 }
462
463 static int
464 vtblk_strategy(struct dev_strategy_args *ap)
465 {
466         struct vtblk_softc *sc;
467         cdev_t dev = ap->a_head.a_dev;
468         sc = dev->si_drv1;
469         struct bio *bio = ap->a_bio;
470         struct buf *bp = bio->bio_buf;
471
472         if (sc == NULL) {
473                 vtblk_bio_error(bio, EINVAL);
474                 return EINVAL;
475         }
476
477         /*
478          * Fail any write if RO. Unfortunately, there does not seem to
479          * be a better way to report our readonly'ness to GEOM above.
480          *
481          * XXX: Is that true in DFly?
482          */
483         if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
484             (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_FLUSH)) {
485                 vtblk_bio_error(bio, EROFS);
486                 return (EINVAL);
487         }
488
489         VTBLK_LOCK(sc);
490         if ((sc->vtblk_flags & VTBLK_FLAG_DETACH) == 0) {
491                 devstat_start_transaction(&sc->stats);
492                 bioqdisksort(&sc->vtblk_bioq, bio);
493                 vtblk_startio(sc);
494         } else {
495                 vtblk_bio_error(bio, ENXIO);
496         }
497         VTBLK_UNLOCK(sc);
498         return 0;
499 }
500
501 static void
502 vtblk_negotiate_features(struct vtblk_softc *sc)
503 {
504         device_t dev;
505         uint64_t features;
506
507         dev = sc->vtblk_dev;
508         features = VTBLK_FEATURES;
509
510         sc->vtblk_features = virtio_negotiate_features(dev, features);
511 }
512
513 static int
514 vtblk_maximum_segments(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
515 {
516         device_t dev;
517         int nsegs;
518
519         dev = sc->vtblk_dev;
520         nsegs = VTBLK_MIN_SEGMENTS;
521
522         if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
523                 nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1);
524         } else {
525                 nsegs += 1;
526         }
527
528         return (nsegs);
529 }
530
531 static int
532 vtblk_alloc_virtqueue(struct vtblk_softc *sc)
533 {
534         device_t dev;
535         struct vq_alloc_info vq_info;
536
537         dev = sc->vtblk_dev;
538
539         VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
540                            vtblk_vq_intr, sc, &sc->vtblk_vq,
541                            "%s request", device_get_nameunit(dev));
542
543         return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
544 }
545
546 static void
547 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
548 {
549
550         struct disk_info info;
551
552         /* construct the disk_info */
553         bzero(&info, sizeof(info));
554
555         if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_BLK_SIZE))
556                 sc->vtblk_sector_size = blkcfg->blk_size;
557         else
558                 sc->vtblk_sector_size = DEV_BSIZE;
559
560         info.d_media_blksize = sc->vtblk_sector_size;
561         info.d_media_blocks = blkcfg->capacity;
562
563         info.d_ncylinders = blkcfg->geometry.cylinders;
564         info.d_nheads = blkcfg->geometry.heads;
565         info.d_secpertrack = blkcfg->geometry.sectors;
566
567         info.d_secpercyl = info.d_secpertrack * info.d_nheads;
568
569         devstat_add_entry(&sc->stats, "vbd", device_get_unit(sc->vtblk_dev),
570                           DEV_BSIZE, DEVSTAT_ALL_SUPPORTED,
571                           DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
572                           DEVSTAT_PRIORITY_DISK);
573
574         /* attach a generic disk device to ourselves */
575         sc->cdev = disk_create(device_get_unit(sc->vtblk_dev), &sc->vtblk_disk,
576                                &vbd_disk_ops);
577
578         sc->cdev->si_drv1 = sc;
579         disk_setdiskinfo(&sc->vtblk_disk, &info);
580 }
581
582 static void
583 vtblk_startio(struct vtblk_softc *sc)
584 {
585         struct virtqueue *vq;
586         struct vtblk_request *req;
587         int enq;
588
589         vq = sc->vtblk_vq;
590         enq = 0;
591
592         VTBLK_LOCK_ASSERT(sc);
593
594         if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
595                 return;
596
597         while (!virtqueue_full(vq)) {
598                 if ((req = vtblk_dequeue_ready(sc)) == NULL)
599                         req = vtblk_bio_request(sc);
600                 if (req == NULL)
601                         break;
602
603                 if (vtblk_execute_request(sc, req) != 0) {
604                         vtblk_enqueue_ready(sc, req);
605                         break;
606                 }
607
608                 enq++;
609         }
610
611         if (enq > 0)
612                 virtqueue_notify(vq, &sc->vtblk_mtx);
613 }
614
615 static struct vtblk_request *
616 vtblk_bio_request(struct vtblk_softc *sc)
617 {
618         struct bio_queue_head *bioq;
619         struct vtblk_request *req;
620         struct bio *bio;
621         struct buf *bp;
622
623         bioq = &sc->vtblk_bioq;
624
625         if (bioq_first(bioq) == NULL)
626                 return (NULL);
627
628         req = vtblk_dequeue_request(sc);
629         if (req == NULL)
630                 return (NULL);
631
632         bio = bioq_takefirst(bioq);
633         req->vbr_bp = bio;
634         req->vbr_ack = -1;
635         req->vbr_hdr.ioprio = 1;
636         bp = bio->bio_buf;
637
638         switch (bp->b_cmd) {
639         case BUF_CMD_FLUSH:
640                 req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
641                 break;
642         case BUF_CMD_READ:
643                 req->vbr_hdr.type = VIRTIO_BLK_T_IN;
644                 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
645                 break;
646         case BUF_CMD_WRITE:
647                 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
648                 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
649                 break;
650         default:
651                 KASSERT(0, ("bio with unhandled cmd: %d", bp->b_cmd));
652                 req->vbr_hdr.type = -1;
653                 break;
654         }
655
656         if (bp->b_flags & B_ORDERED)
657                 req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
658
659         return (req);
660 }
661
662 static int
663 vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
664 {
665         struct sglist *sg;
666         struct bio *bio;
667         struct buf *bp;
668         int writable, error;
669
670         sg = sc->vtblk_sglist;
671         bio = req->vbr_bp;
672         bp = bio->bio_buf;
673         writable = 0;
674
675         /*
676          * sglist is live throughout this subroutine.
677          */
678         sglist_reset(sg);
679         
680         error = sglist_append(sg, &req->vbr_hdr,
681                               sizeof(struct virtio_blk_outhdr));
682         KASSERT(error == 0, ("error adding header to sglist"));
683         KASSERT(sg->sg_nseg == 1,
684             ("header spanned multiple segments: %d", sg->sg_nseg));
685
686         if (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_WRITE) {
687                 error = sglist_append(sg, bp->b_data, bp->b_bcount);
688                 KASSERT(error == 0, ("error adding buffer to sglist"));
689
690                 /* BUF_CMD_READ means the host writes into our buffer. */
691                 if (bp->b_cmd == BUF_CMD_READ)
692                         writable += sg->sg_nseg - 1;
693         }
694
695         error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
696         KASSERT(error == 0, ("error adding ack to sglist"));
697         writable++;
698
699         KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
700             ("fewer than min segments: %d", sg->sg_nseg));
701
702         error = virtqueue_enqueue(sc->vtblk_vq, req, sg,
703                                   sg->sg_nseg - writable, writable);
704
705         sglist_reset(sg);
706
707         return (error);
708 }
709
710 static int
711 vtblk_vq_intr(void *xsc)
712 {
713         struct vtblk_softc *sc;
714
715         sc = xsc;
716
717         virtqueue_disable_intr(sc->vtblk_vq);
718         taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtblk_intr_task);
719
720         return (1);
721 }
722
723 static void
724 vtblk_complete(void *arg, int pending)
725 {
726         struct vtblk_softc *sc;
727         struct vtblk_request *req;
728         struct virtqueue *vq;
729         struct bio *bio;
730         struct buf *bp;
731         
732         sc = arg;
733         vq = sc->vtblk_vq;
734
735 retry:
736         VTBLK_LOCK(sc);
737         if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
738                 VTBLK_UNLOCK(sc);
739                 return;
740         }
741
742         while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
743                 bio = req->vbr_bp;
744                 bp = bio->bio_buf;
745
746                 if (req->vbr_ack == VIRTIO_BLK_S_OK)
747                         bp->b_resid = 0;
748                 else {
749                         bp->b_flags |= B_ERROR;
750                         if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP) {
751                                 bp->b_error = ENOTSUP;
752                         } else {
753                                 bp->b_error = EIO;
754                         }
755                 }
756
757                 devstat_end_transaction_buf(&sc->stats, bio->bio_buf);
758
759                 VTBLK_UNLOCK(sc);
760                 /*
761                  * Unlocking the controller around biodone() does not allow
762                  * processing further device interrupts; when we queued
763                  * vtblk_intr_task, we disabled interrupts. It will allow
764                  * concurrent vtblk_strategy/_startio command dispatches.
765                  */
766                 biodone(bio);
767                 VTBLK_LOCK(sc);
768
769                 vtblk_enqueue_request(sc, req);
770         }
771
772         vtblk_startio(sc);
773
774         if (virtqueue_enable_intr(vq) != 0) {
775                 /* 
776                  * If new virtqueue entries appeared immediately after
777                  * enabling interrupts, process them now. Release and
778                  * retake softcontroller lock to try to avoid blocking
779                  * I/O dispatch for too long.
780                  */
781                 virtqueue_disable_intr(vq);
782                 VTBLK_UNLOCK(sc);
783                 goto retry;
784         }
785
786         VTBLK_UNLOCK(sc);
787 }
788
789 static void
790 vtblk_stop(struct vtblk_softc *sc)
791 {
792         virtqueue_disable_intr(sc->vtblk_vq);
793         virtio_stop(sc->vtblk_dev);
794 }
795
796 static void
797 vtblk_prepare_dump(struct vtblk_softc *sc)
798 {
799         device_t dev;
800         struct virtqueue *vq;
801
802         dev = sc->vtblk_dev;
803         vq = sc->vtblk_vq;
804
805         vtblk_stop(sc);
806
807         /*
808          * Drain all requests caught in-flight in the virtqueue,
809          * skipping biodone(). When dumping, only one request is
810          * outstanding at a time, and we just poll the virtqueue
811          * for the response.
812          */
813         vtblk_drain_vq(sc, 1);
814
815         if (virtio_reinit(dev, sc->vtblk_features) != 0)
816                 panic("cannot reinit VirtIO block device during dump");
817
818         virtqueue_disable_intr(vq);
819         virtio_reinit_complete(dev);
820 }
821
822 static int
823 vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
824                  size_t length)
825 {
826         struct bio bio;
827         struct vtblk_request *req;
828         struct buf *bp;
829
830         req = &sc->vtblk_dump_request;
831         req->vbr_ack = -1;
832         req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
833         req->vbr_hdr.ioprio = 1;
834         req->vbr_hdr.sector = offset / DEV_BSIZE;
835
836         req->vbr_bp = &bio;
837         bzero(&buf, sizeof(struct bio));
838         bp = bio.bio_buf;
839
840         bp->b_cmd = BUF_CMD_WRITE;
841         bp->b_data = virtual;
842         bp->b_bcount = length;
843
844         return (vtblk_poll_request(sc, req));
845 }
846
847 static int
848 vtblk_flush_dump(struct vtblk_softc *sc)
849 {
850         struct bio bio;
851         struct vtblk_request *req;
852         struct buf *bp;
853
854         req = &sc->vtblk_dump_request;
855         req->vbr_ack = -1;
856         req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
857         req->vbr_hdr.ioprio = 1;
858         req->vbr_hdr.sector = 0;
859
860         req->vbr_bp = &bio;
861         bzero(&buf, sizeof(struct bio));
862         bp = bio.bio_buf;
863         
864         bp->b_cmd = BUF_CMD_FLUSH;
865
866         return (vtblk_poll_request(sc, req));
867 }
868
869 static int
870 vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
871 {
872         device_t dev;
873         struct virtqueue *vq;
874         struct vtblk_request *r __debugvar;
875         int error;
876
877         dev = sc->vtblk_dev;
878         vq = sc->vtblk_vq;
879
880         if (!virtqueue_empty(vq))
881                 return (EBUSY);
882
883         error = vtblk_execute_request(sc, req);
884         if (error)
885                 return (error);
886
887         virtqueue_notify(vq, &sc->vtblk_mtx);
888
889         r = virtqueue_poll(vq, NULL);
890         KASSERT(r == req, ("unexpected request response"));
891
892         if (req->vbr_ack != VIRTIO_BLK_S_OK) {
893                 error = req->vbr_ack == VIRTIO_BLK_S_UNSUPP ? ENOTSUP : EIO;
894                 if (bootverbose)
895                         device_printf(dev,
896                             "vtblk_poll_request: IO error: %d\n", error);
897         }
898
899         return (error);
900 }
901
902 static void
903 vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
904 {
905         struct virtqueue *vq;
906         struct vtblk_request *req;
907         int last;
908
909         vq = sc->vtblk_vq;
910         last = 0;
911
912         while ((req = virtqueue_drain(vq, &last)) != NULL) {
913                 if (!skip_done)
914                         vtblk_bio_error(req->vbr_bp, ENXIO);
915
916                 vtblk_enqueue_request(sc, req);
917         }
918
919         KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
920 }
921
922 static void
923 vtblk_drain(struct vtblk_softc *sc)
924 {
925         struct bio_queue_head *bioq;
926         struct vtblk_request *req;
927         struct bio *bp;
928
929         bioq = &sc->vtblk_bioq;
930
931         if (sc->vtblk_vq != NULL)
932                 vtblk_drain_vq(sc, 0);
933
934         while ((req = vtblk_dequeue_ready(sc)) != NULL) {
935                 vtblk_bio_error(req->vbr_bp, ENXIO);
936                 vtblk_enqueue_request(sc, req);
937         }
938
939         while (bioq_first(bioq) != NULL) {
940                 bp = bioq_takefirst(bioq);
941                 vtblk_bio_error(bp, ENXIO);
942         }
943
944         vtblk_free_requests(sc);
945 }
946
947 static int
948 vtblk_alloc_requests(struct vtblk_softc *sc)
949 {
950         struct vtblk_request *req;
951         int i, nreqs;
952
953         nreqs = virtqueue_size(sc->vtblk_vq);
954
955         /*
956          * Preallocate sufficient requests to keep the virtqueue full. Each
957          * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
958          * the number allocated when indirect descriptors are not available.
959          */
960         nreqs /= VTBLK_MIN_SEGMENTS;
961
962         for (i = 0; i < nreqs; i++) {
963                 req = kmalloc(sizeof(struct vtblk_request), M_DEVBUF, M_WAITOK);
964
965                 sc->vtblk_request_count++;
966                 vtblk_enqueue_request(sc, req);
967         }
968
969         return (0);
970 }
971
972 static void
973 vtblk_free_requests(struct vtblk_softc *sc)
974 {
975         struct vtblk_request *req;
976
977         while ((req = vtblk_dequeue_request(sc)) != NULL) {
978                 sc->vtblk_request_count--;
979                 kfree(req, M_DEVBUF);
980         }
981
982         KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
983 }
984
985 static struct vtblk_request *
986 vtblk_dequeue_request(struct vtblk_softc *sc)
987 {
988         struct vtblk_request *req;
989
990         req = TAILQ_FIRST(&sc->vtblk_req_free);
991         if (req != NULL)
992                 TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
993
994         return (req);
995 }
996
997 static void
998 vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
999 {
1000         bzero(req, sizeof(struct vtblk_request));
1001         TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
1002 }
1003
1004 static struct vtblk_request *
1005 vtblk_dequeue_ready(struct vtblk_softc *sc)
1006 {
1007         struct vtblk_request *req;
1008
1009         req = TAILQ_FIRST(&sc->vtblk_req_ready);
1010         if (req != NULL)
1011                 TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
1012
1013         return (req);
1014 }
1015
1016 static void
1017 vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
1018 {
1019         TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
1020 }
1021
1022 static void
1023 vtblk_bio_error(struct bio *bp, int error)
1024 {
1025         biodone(bp);
1026 }