Build fixes.
[dragonfly.git] / sys / dev / virtual / virtio / block / virtio_blk.c
CommitLineData
11447b59
VS
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: src/sys/dev/virtio/block/virtio_blk.c,v 1.4 2012/04/16 18:29:12 grehan Exp $
27 */
28
29/* Driver for VirtIO block devices. */
30
31#include <sys/cdefs.h>
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/kernel.h>
36#include <sys/bio.h>
37#include <sys/malloc.h>
38#include <sys/module.h>
39#include <sys/sglist.h>
40#include <sys/lock.h>
41#include <sys/queue.h>
42#include <sys/taskqueue.h>
43
44#include <sys/buf2.h>
45#include <sys/rman.h>
46#include <sys/disk.h>
47#include <sys/spinlock.h>
48#include <sys/spinlock2.h>
49#include <sys/devicestat.h>
50
dcbcbc86
VS
51#include <dev/virtual/virtio/virtio/virtio.h>
52#include <dev/virtual/virtio/virtio/virtqueue.h>
11447b59
VS
53#include "virtio_blk.h"
54
55struct vtblk_request {
56 struct virtio_blk_outhdr vbr_hdr;
57 struct bio *vbr_bp;
58 uint8_t vbr_ack;
59
60 TAILQ_ENTRY(vtblk_request) vbr_link;
61};
62
63struct vtblk_softc {
64 device_t vtblk_dev;
65 struct spinlock vtblk_mtx;
66 uint64_t vtblk_features;
67
68#define VTBLK_FLAG_READONLY 0x0002
69#define VTBLK_FLAG_DETACH 0x0004
70#define VTBLK_FLAG_SUSPEND 0x0008
71#define VTBLK_FLAG_DUMPING 0x0010
72 uint32_t vtblk_flags;
73
74 struct virtqueue *vtblk_vq;
75 struct sglist *vtblk_sglist;
76 struct disk vtblk_disk;
77 cdev_t cdev;
78 struct devstat stats;
79
80 struct bio_queue_head vtblk_bioq;
81 TAILQ_HEAD(, vtblk_request) vtblk_req_free;
82 TAILQ_HEAD(, vtblk_request) vtblk_req_ready;
83
84 struct task vtblk_intr_task;
85
86 int vtblk_sector_size;
87 int vtblk_max_nsegs;
88 int vtblk_unit;
89 int vtblk_request_count;
90
91 struct vtblk_request vtblk_dump_request;
92};
93
94static struct virtio_feature_desc vtblk_feature_desc[] = {
95 { VIRTIO_BLK_F_BARRIER, "HostBarrier" },
96 { VIRTIO_BLK_F_SIZE_MAX, "MaxSegSize" },
97 { VIRTIO_BLK_F_SEG_MAX, "MaxNumSegs" },
98 { VIRTIO_BLK_F_GEOMETRY, "DiskGeometry" },
99 { VIRTIO_BLK_F_RO, "ReadOnly" },
100 { VIRTIO_BLK_F_BLK_SIZE, "BlockSize" },
101 { VIRTIO_BLK_F_SCSI, "SCSICmds" },
102 { VIRTIO_BLK_F_FLUSH, "FlushCmd" },
103 { VIRTIO_BLK_F_TOPOLOGY, "Topology" },
104
105 { 0, NULL }
106};
107
108static int vtblk_modevent(module_t, int, void *);
109
110static int vtblk_probe(device_t);
111static int vtblk_attach(device_t);
112static int vtblk_detach(device_t);
113static int vtblk_suspend(device_t);
114static int vtblk_resume(device_t);
115static int vtblk_shutdown(device_t);
116
117static void vtblk_negotiate_features(struct vtblk_softc *);
118static int vtblk_maximum_segments(struct vtblk_softc *,
119 struct virtio_blk_config *);
120static int vtblk_alloc_virtqueue(struct vtblk_softc *);
121static void vtblk_alloc_disk(struct vtblk_softc *,
122 struct virtio_blk_config *);
123/*
124 * Interface to the device switch.
125 */
126static d_open_t vtblk_open;
127static d_strategy_t vtblk_strategy;
128static d_dump_t vtblk_dump;
129
130static struct dev_ops vbd_disk_ops = {
131 { "vbd", 200, D_DISK | D_MPSAFE },
132 .d_open = vtblk_open,
133 .d_close = nullclose,
134 .d_read = physread,
135 .d_write = physwrite,
136 .d_strategy = vtblk_strategy,
137 .d_dump = vtblk_dump,
138};
139
140static void vtblk_startio(struct vtblk_softc *);
141static struct vtblk_request *vtblk_bio_request(struct vtblk_softc *);
142static int vtblk_execute_request(struct vtblk_softc *,
143 struct vtblk_request *);
144
145static int vtblk_vq_intr(void *);
595bcf84 146static void vtblk_complete(void *, int);
11447b59
VS
147
148static void vtblk_stop(struct vtblk_softc *);
149
150static void vtblk_prepare_dump(struct vtblk_softc *);
151static int vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
152static int vtblk_flush_dump(struct vtblk_softc *);
153static int vtblk_poll_request(struct vtblk_softc *,
154 struct vtblk_request *);
155
156static void vtblk_drain_vq(struct vtblk_softc *, int);
157static void vtblk_drain(struct vtblk_softc *);
158
159static int vtblk_alloc_requests(struct vtblk_softc *);
160static void vtblk_free_requests(struct vtblk_softc *);
161static struct vtblk_request *vtblk_dequeue_request(struct vtblk_softc *);
162static void vtblk_enqueue_request(struct vtblk_softc *,
163 struct vtblk_request *);
164
165static struct vtblk_request *vtblk_dequeue_ready(struct vtblk_softc *);
166static void vtblk_enqueue_ready(struct vtblk_softc *,
167 struct vtblk_request *);
168
169static void vtblk_bio_error(struct bio *, int);
170
171/* Tunables. */
172static int vtblk_no_ident = 0;
173TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
174
175/* Features desired/implemented by this driver. */
176#define VTBLK_FEATURES \
177 (VIRTIO_BLK_F_BARRIER | \
178 VIRTIO_BLK_F_SIZE_MAX | \
179 VIRTIO_BLK_F_SEG_MAX | \
180 VIRTIO_BLK_F_GEOMETRY | \
181 VIRTIO_BLK_F_RO | \
182 VIRTIO_BLK_F_BLK_SIZE | \
183 VIRTIO_BLK_F_FLUSH)
184
185#define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx
186#define VTBLK_LOCK_INIT(_sc) spin_init(&(_sc)->vtblk_mtx)
187#define VTBLK_LOCK(_sc) spin_lock(VTBLK_MTX((_sc)))
188#define VTBLK_TRYLOCK(_sc) spin_trylock(VTBLK_MTX((_sc)))
189#define VTBLK_UNLOCK(_sc) spin_unlock(VTBLK_MTX((_sc)))
190#define VTBLK_LOCK_DESTROY(_sc) spin_uninit(VTBLK_MTX((_sc)))
191
192#define VTBLK_LOCK_ASSERT(_sc)
193#define VTBLK_LOCK_ASSERT_NOTOWNED(_sc)
194
195/*
196 * Each block request uses at least two segments - one for the header
197 * and one for the status.
198 */
199#define VTBLK_MIN_SEGMENTS 2
200
201static device_method_t vtblk_methods[] = {
202 /* Device methods. */
203 DEVMETHOD(device_probe, vtblk_probe),
204 DEVMETHOD(device_attach, vtblk_attach),
205 DEVMETHOD(device_detach, vtblk_detach),
206 DEVMETHOD(device_suspend, vtblk_suspend),
207 DEVMETHOD(device_resume, vtblk_resume),
208 DEVMETHOD(device_shutdown, vtblk_shutdown),
209
210 { 0, 0 }
211};
212
213static driver_t vtblk_driver = {
214 "vtblk",
215 vtblk_methods,
216 sizeof(struct vtblk_softc)
217};
218static devclass_t vtblk_devclass;
219
220DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
221 vtblk_modevent, NULL);
222MODULE_VERSION(virtio_blk, 1);
223MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
224
225static int
226vtblk_modevent(module_t mod, int type, void *unused)
227{
228 int error;
229
230 error = 0;
231
232 switch (type) {
233 case MOD_LOAD:
234 break;
235 case MOD_UNLOAD:
236 break;
237 case MOD_SHUTDOWN:
238 break;
239 default:
240 error = EOPNOTSUPP;
241 break;
242 }
243
244 return (error);
245}
246
247static int
248vtblk_probe(device_t dev)
249{
250
251 if (virtio_get_device_type(dev) != VIRTIO_ID_BLOCK)
252 return (ENXIO);
253
254 device_set_desc(dev, "VirtIO Block Adapter");
255
256 return (BUS_PROBE_DEFAULT);
257}
258
259static int
260vtblk_attach(device_t dev)
261{
262 struct vtblk_softc *sc;
263 struct virtio_blk_config blkcfg;
264 int error;
265
266 sc = device_get_softc(dev);
267 sc->vtblk_dev = dev;
268 sc->vtblk_unit = device_get_unit(dev);
269
270 VTBLK_LOCK_INIT(sc);
271
272 bioq_init(&sc->vtblk_bioq);
273 TAILQ_INIT(&sc->vtblk_req_free);
274 TAILQ_INIT(&sc->vtblk_req_ready);
275
276 virtio_set_feature_desc(dev, vtblk_feature_desc);
277 vtblk_negotiate_features(sc);
278
279 if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
280 sc->vtblk_flags |= VTBLK_FLAG_READONLY;
281
282 /* Get local copy of config. */
283 virtio_read_device_config(dev, 0, &blkcfg,
284 sizeof(struct virtio_blk_config));
285
286 /*
287 * With the current sglist(9) implementation, it is not easy
288 * for us to support a maximum segment size as adjacent
289 * segments are coalesced. For now, just make sure it's larger
290 * than the maximum supported transfer size.
291 */
292 if (virtio_with_feature(dev, VIRTIO_BLK_F_SIZE_MAX)) {
293 if (blkcfg.size_max < MAXPHYS) {
294 error = ENOTSUP;
295 device_printf(dev, "host requires unsupported "
296 "maximum segment size feature\n");
297 goto fail;
298 }
299 }
300
301 sc->vtblk_max_nsegs = vtblk_maximum_segments(sc, &blkcfg);
302 if (sc->vtblk_max_nsegs <= VTBLK_MIN_SEGMENTS) {
303 error = EINVAL;
304 device_printf(dev, "fewer than minimum number of segments "
305 "allowed: %d\n", sc->vtblk_max_nsegs);
306 goto fail;
307 }
308
309 /*
310 * Allocate working sglist. The number of segments may be too
311 * large to safely store on the stack.
312 */
313 sc->vtblk_sglist = sglist_alloc(sc->vtblk_max_nsegs, M_NOWAIT);
314 if (sc->vtblk_sglist == NULL) {
315 error = ENOMEM;
316 device_printf(dev, "cannot allocate sglist\n");
317 goto fail;
318 }
319
320 error = vtblk_alloc_virtqueue(sc);
321 if (error) {
322 device_printf(dev, "cannot allocate virtqueue\n");
323 goto fail;
324 }
325
326 error = vtblk_alloc_requests(sc);
327 if (error) {
328 device_printf(dev, "cannot preallocate requests\n");
329 goto fail;
330 }
331
332 vtblk_alloc_disk(sc, &blkcfg);
333
595bcf84 334 TASK_INIT(&sc->vtblk_intr_task, 0, vtblk_complete, sc);
11447b59
VS
335
336 error = virtio_setup_intr(dev);
337 if (error) {
338 device_printf(dev, "cannot setup virtqueue interrupt\n");
339 goto fail;
340 }
341
342 virtqueue_enable_intr(sc->vtblk_vq);
343
344fail:
345 if (error)
346 vtblk_detach(dev);
347
348 return (error);
349}
350
351static int
352vtblk_detach(device_t dev)
353{
354 struct vtblk_softc *sc;
355
356 sc = device_get_softc(dev);
357
358 VTBLK_LOCK(sc);
359 sc->vtblk_flags |= VTBLK_FLAG_DETACH;
360 if (device_is_attached(dev))
361 vtblk_stop(sc);
362 VTBLK_UNLOCK(sc);
363
364 taskqueue_drain(taskqueue_thread[mycpuid], &sc->vtblk_intr_task);
365
366 vtblk_drain(sc);
367
368 if (sc->vtblk_sglist != NULL) {
369 sglist_free(sc->vtblk_sglist);
370 sc->vtblk_sglist = NULL;
371 }
372
373 VTBLK_LOCK_DESTROY(sc);
374
375 return (0);
376}
377
378static int
379vtblk_suspend(device_t dev)
380{
381 struct vtblk_softc *sc;
382
383 sc = device_get_softc(dev);
384
385 VTBLK_LOCK(sc);
386 sc->vtblk_flags |= VTBLK_FLAG_SUSPEND;
387 /* TODO Wait for any inflight IO to complete? */
388 VTBLK_UNLOCK(sc);
389
390 return (0);
391}
392
393static int
394vtblk_resume(device_t dev)
395{
396 struct vtblk_softc *sc;
397
398 sc = device_get_softc(dev);
399
400 VTBLK_LOCK(sc);
401 sc->vtblk_flags &= ~VTBLK_FLAG_SUSPEND;
402 /* TODO Resume IO? */
403 VTBLK_UNLOCK(sc);
404
405 return (0);
406}
407
408static int
409vtblk_shutdown(device_t dev)
410{
411 return (0);
412}
413
414static int
415vtblk_open(struct dev_open_args *ap)
416{
417 struct vtblk_softc *sc;
418 cdev_t dev = ap->a_head.a_dev;
419 sc = dev->si_drv1;
420 if (sc == NULL)
421 return (ENXIO);
422
423 return (sc->vtblk_flags & VTBLK_FLAG_DETACH ? ENXIO : 0);
424}
425
426static int
427vtblk_dump(struct dev_dump_args *ap)
428{
429 struct vtblk_softc *sc;
430 int error;
431
432 error = 0;
433
434 cdev_t dev = ap->a_head.a_dev;
435 sc = dev->si_drv1;
436
437 if (sc == NULL)
438 return (ENXIO);
439
440 if (VTBLK_TRYLOCK(sc) == 0) {
441 device_printf(sc->vtblk_dev,
442 "softc already locked, cannot dump...\n");
443 return (EBUSY);
444 }
445
446 if ((sc->vtblk_flags & VTBLK_FLAG_DUMPING) == 0) {
447 vtblk_prepare_dump(sc);
448 sc->vtblk_flags |= VTBLK_FLAG_DUMPING;
449 }
450
451 if (ap->a_length > 0) {
452 error = vtblk_write_dump(sc, ap->a_virtual, ap->a_offset,
453 ap->a_length);
454 } else if (ap->a_virtual == NULL && ap->a_offset == 0) {
455 error = vtblk_flush_dump(sc);
456 }
457
458 VTBLK_UNLOCK(sc);
459
460 return (error);
461}
462
463static int
464vtblk_strategy(struct dev_strategy_args *ap)
465{
466 struct vtblk_softc *sc;
467 cdev_t dev = ap->a_head.a_dev;
468 sc = dev->si_drv1;
469 struct bio *bio = ap->a_bio;
470 struct buf *bp = bio->bio_buf;
471
11447b59
VS
472 if (sc == NULL) {
473 vtblk_bio_error(bio, EINVAL);
474 return EINVAL;
475 }
476
477 /*
478 * Fail any write if RO. Unfortunately, there does not seem to
479 * be a better way to report our readonly'ness to GEOM above.
480 *
481 * XXX: Is that true in DFly?
482 */
483 if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
484 (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_FLUSH)) {
485 vtblk_bio_error(bio, EROFS);
486 return (EINVAL);
487 }
488
489 VTBLK_LOCK(sc);
490 if ((sc->vtblk_flags & VTBLK_FLAG_DETACH) == 0) {
491 devstat_start_transaction(&sc->stats);
492 bioqdisksort(&sc->vtblk_bioq, bio);
493 vtblk_startio(sc);
494 } else {
495 vtblk_bio_error(bio, ENXIO);
496 }
497 VTBLK_UNLOCK(sc);
498 return 0;
499}
500
501static void
502vtblk_negotiate_features(struct vtblk_softc *sc)
503{
504 device_t dev;
505 uint64_t features;
506
507 dev = sc->vtblk_dev;
508 features = VTBLK_FEATURES;
509
510 sc->vtblk_features = virtio_negotiate_features(dev, features);
511}
512
513static int
514vtblk_maximum_segments(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
515{
516 device_t dev;
517 int nsegs;
518
519 dev = sc->vtblk_dev;
520 nsegs = VTBLK_MIN_SEGMENTS;
521
522 if (virtio_with_feature(dev, VIRTIO_BLK_F_SEG_MAX)) {
523 nsegs += MIN(blkcfg->seg_max, MAXPHYS / PAGE_SIZE + 1);
524 } else {
525 nsegs += 1;
526 }
527
528 return (nsegs);
529}
530
531static int
532vtblk_alloc_virtqueue(struct vtblk_softc *sc)
533{
534 device_t dev;
535 struct vq_alloc_info vq_info;
536
537 dev = sc->vtblk_dev;
538
539 VQ_ALLOC_INFO_INIT(&vq_info, sc->vtblk_max_nsegs,
540 vtblk_vq_intr, sc, &sc->vtblk_vq,
541 "%s request", device_get_nameunit(dev));
542
543 return (virtio_alloc_virtqueues(dev, 0, 1, &vq_info));
544}
545
546static void
547vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
548{
549
550 struct disk_info info;
551
552 /* construct the disk_info */
553 bzero(&info, sizeof(info));
554
555 if (virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_BLK_SIZE))
556 sc->vtblk_sector_size = blkcfg->blk_size;
557 else
558 sc->vtblk_sector_size = DEV_BSIZE;
559
560 info.d_media_blksize = sc->vtblk_sector_size;
561 info.d_media_blocks = blkcfg->capacity;
562
563 info.d_ncylinders = blkcfg->geometry.cylinders;
564 info.d_nheads = blkcfg->geometry.heads;
565 info.d_secpertrack = blkcfg->geometry.sectors;
566
567 info.d_secpercyl = info.d_secpertrack * info.d_nheads;
568
569 devstat_add_entry(&sc->stats, "vbd", device_get_unit(sc->vtblk_dev),
570 DEV_BSIZE, DEVSTAT_ALL_SUPPORTED,
571 DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER,
572 DEVSTAT_PRIORITY_DISK);
573
574 /* attach a generic disk device to ourselves */
575 sc->cdev = disk_create(device_get_unit(sc->vtblk_dev), &sc->vtblk_disk,
576 &vbd_disk_ops);
577
578 sc->cdev->si_drv1 = sc;
579 disk_setdiskinfo(&sc->vtblk_disk, &info);
580}
581
582static void
583vtblk_startio(struct vtblk_softc *sc)
584{
585 struct virtqueue *vq;
586 struct vtblk_request *req;
587 int enq;
588
589 vq = sc->vtblk_vq;
590 enq = 0;
591
592 VTBLK_LOCK_ASSERT(sc);
593
594 if (sc->vtblk_flags & VTBLK_FLAG_SUSPEND)
595 return;
596
597 while (!virtqueue_full(vq)) {
598 if ((req = vtblk_dequeue_ready(sc)) == NULL)
599 req = vtblk_bio_request(sc);
600 if (req == NULL)
601 break;
602
603 if (vtblk_execute_request(sc, req) != 0) {
604 vtblk_enqueue_ready(sc, req);
605 break;
606 }
607
608 enq++;
609 }
610
611 if (enq > 0)
612 virtqueue_notify(vq, &sc->vtblk_mtx);
613}
614
615static struct vtblk_request *
616vtblk_bio_request(struct vtblk_softc *sc)
617{
618 struct bio_queue_head *bioq;
619 struct vtblk_request *req;
620 struct bio *bio;
621 struct buf *bp;
622
623 bioq = &sc->vtblk_bioq;
624
625 if (bioq_first(bioq) == NULL)
626 return (NULL);
627
628 req = vtblk_dequeue_request(sc);
629 if (req == NULL)
630 return (NULL);
631
632 bio = bioq_takefirst(bioq);
633 req->vbr_bp = bio;
634 req->vbr_ack = -1;
635 req->vbr_hdr.ioprio = 1;
636 bp = bio->bio_buf;
637
638 switch (bp->b_cmd) {
639 case BUF_CMD_FLUSH:
640 req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
641 break;
642 case BUF_CMD_READ:
643 req->vbr_hdr.type = VIRTIO_BLK_T_IN;
644 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
645 break;
646 case BUF_CMD_WRITE:
647 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
648 req->vbr_hdr.sector = bio->bio_offset / DEV_BSIZE;
649 break;
650 default:
651 KASSERT(0, ("bio with unhandled cmd: %d", bp->b_cmd));
652 req->vbr_hdr.type = -1;
653 break;
654 }
655
656 if (bp->b_flags & B_ORDERED)
657 req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;
658
659 return (req);
660}
661
662static int
663vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
664{
665 struct sglist *sg;
666 struct bio *bio;
667 struct buf *bp;
668 int writable, error;
669
670 sg = sc->vtblk_sglist;
671 bio = req->vbr_bp;
672 bp = bio->bio_buf;
673 writable = 0;
674
675 /*
676 * sglist is live throughout this subroutine.
677 */
678 sglist_reset(sg);
679
680 error = sglist_append(sg, &req->vbr_hdr,
681 sizeof(struct virtio_blk_outhdr));
682 KASSERT(error == 0, ("error adding header to sglist"));
683 KASSERT(sg->sg_nseg == 1,
684 ("header spanned multiple segments: %d", sg->sg_nseg));
685
686 if (bp->b_cmd == BUF_CMD_READ || bp->b_cmd == BUF_CMD_WRITE) {
687 error = sglist_append(sg, bp->b_data, bp->b_bcount);
688 KASSERT(error == 0, ("error adding buffer to sglist"));
689
690 /* BUF_CMD_READ means the host writes into our buffer. */
691 if (bp->b_cmd == BUF_CMD_READ)
692 writable += sg->sg_nseg - 1;
693 }
694
695 error = sglist_append(sg, &req->vbr_ack, sizeof(uint8_t));
696 KASSERT(error == 0, ("error adding ack to sglist"));
697 writable++;
698
699 KASSERT(sg->sg_nseg >= VTBLK_MIN_SEGMENTS,
700 ("fewer than min segments: %d", sg->sg_nseg));
701
702 error = virtqueue_enqueue(sc->vtblk_vq, req, sg,
703 sg->sg_nseg - writable, writable);
704
705 sglist_reset(sg);
706
707 return (error);
708}
709
710static int
711vtblk_vq_intr(void *xsc)
712{
713 struct vtblk_softc *sc;
714
715 sc = xsc;
716
717 virtqueue_disable_intr(sc->vtblk_vq);
718 taskqueue_enqueue(taskqueue_thread[mycpuid], &sc->vtblk_intr_task);
719
720 return (1);
721}
722
723static void
595bcf84 724vtblk_complete(void *arg, int pending)
11447b59
VS
725{
726 struct vtblk_softc *sc;
727 struct vtblk_request *req;
728 struct virtqueue *vq;
729 struct bio *bio;
730 struct buf *bp;
731
732 sc = arg;
733 vq = sc->vtblk_vq;
734
735retry:
736 VTBLK_LOCK(sc);
737 if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
738 VTBLK_UNLOCK(sc);
739 return;
740 }
741
742 while ((req = virtqueue_dequeue(vq, NULL)) != NULL) {
743 bio = req->vbr_bp;
744 bp = bio->bio_buf;
745
746 if (req->vbr_ack == VIRTIO_BLK_S_OK)
747 bp->b_resid = 0;
748 else {
749 bp->b_flags |= B_ERROR;
750 if (req->vbr_ack == VIRTIO_BLK_S_UNSUPP) {
751 bp->b_error = ENOTSUP;
752 } else {
753 bp->b_error = EIO;
754 }
755 }
756
757 devstat_end_transaction_buf(&sc->stats, bio->bio_buf);
758
759 VTBLK_UNLOCK(sc);
760 /*
761 * Unlocking the controller around biodone() does not allow
762 * processing further device interrupts; when we queued
763 * vtblk_intr_task, we disabled interrupts. It will allow
764 * concurrent vtblk_strategy/_startio command dispatches.
765 */
766 biodone(bio);
767 VTBLK_LOCK(sc);
768
769 vtblk_enqueue_request(sc, req);
770 }
771
772 vtblk_startio(sc);
773
774 if (virtqueue_enable_intr(vq) != 0) {
775 /*
776 * If new virtqueue entries appeared immediately after
777 * enabling interrupts, process them now. Release and
778 * retake softcontroller lock to try to avoid blocking
779 * I/O dispatch for too long.
780 */
781 virtqueue_disable_intr(vq);
782 VTBLK_UNLOCK(sc);
783 goto retry;
784 }
785
786 VTBLK_UNLOCK(sc);
787}
788
789static void
790vtblk_stop(struct vtblk_softc *sc)
791{
792 virtqueue_disable_intr(sc->vtblk_vq);
793 virtio_stop(sc->vtblk_dev);
794}
795
796static void
797vtblk_prepare_dump(struct vtblk_softc *sc)
798{
799 device_t dev;
800 struct virtqueue *vq;
801
802 dev = sc->vtblk_dev;
803 vq = sc->vtblk_vq;
804
805 vtblk_stop(sc);
806
807 /*
808 * Drain all requests caught in-flight in the virtqueue,
809 * skipping biodone(). When dumping, only one request is
810 * outstanding at a time, and we just poll the virtqueue
811 * for the response.
812 */
813 vtblk_drain_vq(sc, 1);
814
815 if (virtio_reinit(dev, sc->vtblk_features) != 0)
816 panic("cannot reinit VirtIO block device during dump");
817
818 virtqueue_disable_intr(vq);
819 virtio_reinit_complete(dev);
820}
821
822static int
823vtblk_write_dump(struct vtblk_softc *sc, void *virtual, off_t offset,
824 size_t length)
825{
826 struct bio bio;
827 struct vtblk_request *req;
828 struct buf *bp;
829
830 req = &sc->vtblk_dump_request;
831 req->vbr_ack = -1;
832 req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
833 req->vbr_hdr.ioprio = 1;
834 req->vbr_hdr.sector = offset / DEV_BSIZE;
835
836 req->vbr_bp = &bio;
837 bzero(&buf, sizeof(struct bio));
838 bp = bio.bio_buf;
839
840 bp->b_cmd = BUF_CMD_WRITE;
841 bp->b_data = virtual;
842 bp->b_bcount = length;
843
844 return (vtblk_poll_request(sc, req));
845}
846
847static int
848vtblk_flush_dump(struct vtblk_softc *sc)
849{
850 struct bio bio;
851 struct vtblk_request *req;
852 struct buf *bp;
853
854 req = &sc->vtblk_dump_request;
855 req->vbr_ack = -1;
856 req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
857 req->vbr_hdr.ioprio = 1;
858 req->vbr_hdr.sector = 0;
859
860 req->vbr_bp = &bio;
861 bzero(&buf, sizeof(struct bio));
862 bp = bio.bio_buf;
863
864 bp->b_cmd = BUF_CMD_FLUSH;
865
866 return (vtblk_poll_request(sc, req));
867}
868
869static int
870vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
871{
872 device_t dev;
873 struct virtqueue *vq;
9aa0ab36 874 struct vtblk_request *r __debugvar;
11447b59
VS
875 int error;
876
877 dev = sc->vtblk_dev;
878 vq = sc->vtblk_vq;
879
880 if (!virtqueue_empty(vq))
881 return (EBUSY);
882
883 error = vtblk_execute_request(sc, req);
884 if (error)
885 return (error);
886
887 virtqueue_notify(vq, &sc->vtblk_mtx);
888
889 r = virtqueue_poll(vq, NULL);
890 KASSERT(r == req, ("unexpected request response"));
891
892 if (req->vbr_ack != VIRTIO_BLK_S_OK) {
893 error = req->vbr_ack == VIRTIO_BLK_S_UNSUPP ? ENOTSUP : EIO;
894 if (bootverbose)
895 device_printf(dev,
896 "vtblk_poll_request: IO error: %d\n", error);
897 }
898
899 return (error);
900}
901
902static void
903vtblk_drain_vq(struct vtblk_softc *sc, int skip_done)
904{
905 struct virtqueue *vq;
906 struct vtblk_request *req;
907 int last;
908
909 vq = sc->vtblk_vq;
910 last = 0;
911
912 while ((req = virtqueue_drain(vq, &last)) != NULL) {
913 if (!skip_done)
914 vtblk_bio_error(req->vbr_bp, ENXIO);
915
916 vtblk_enqueue_request(sc, req);
917 }
918
919 KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
920}
921
922static void
923vtblk_drain(struct vtblk_softc *sc)
924{
925 struct bio_queue_head *bioq;
926 struct vtblk_request *req;
927 struct bio *bp;
928
929 bioq = &sc->vtblk_bioq;
930
931 if (sc->vtblk_vq != NULL)
932 vtblk_drain_vq(sc, 0);
933
934 while ((req = vtblk_dequeue_ready(sc)) != NULL) {
935 vtblk_bio_error(req->vbr_bp, ENXIO);
936 vtblk_enqueue_request(sc, req);
937 }
938
939 while (bioq_first(bioq) != NULL) {
940 bp = bioq_takefirst(bioq);
941 vtblk_bio_error(bp, ENXIO);
942 }
943
944 vtblk_free_requests(sc);
945}
946
947static int
948vtblk_alloc_requests(struct vtblk_softc *sc)
949{
950 struct vtblk_request *req;
951 int i, nreqs;
952
953 nreqs = virtqueue_size(sc->vtblk_vq);
954
955 /*
956 * Preallocate sufficient requests to keep the virtqueue full. Each
957 * request consumes VTBLK_MIN_SEGMENTS or more descriptors so reduce
958 * the number allocated when indirect descriptors are not available.
959 */
960 nreqs /= VTBLK_MIN_SEGMENTS;
961
962 for (i = 0; i < nreqs; i++) {
963 req = kmalloc(sizeof(struct vtblk_request), M_DEVBUF, M_WAITOK);
964
965 sc->vtblk_request_count++;
966 vtblk_enqueue_request(sc, req);
967 }
968
969 return (0);
970}
971
972static void
973vtblk_free_requests(struct vtblk_softc *sc)
974{
975 struct vtblk_request *req;
976
977 while ((req = vtblk_dequeue_request(sc)) != NULL) {
978 sc->vtblk_request_count--;
979 kfree(req, M_DEVBUF);
980 }
981
982 KASSERT(sc->vtblk_request_count == 0, ("leaked requests"));
983}
984
985static struct vtblk_request *
986vtblk_dequeue_request(struct vtblk_softc *sc)
987{
988 struct vtblk_request *req;
989
990 req = TAILQ_FIRST(&sc->vtblk_req_free);
991 if (req != NULL)
992 TAILQ_REMOVE(&sc->vtblk_req_free, req, vbr_link);
993
994 return (req);
995}
996
997static void
998vtblk_enqueue_request(struct vtblk_softc *sc, struct vtblk_request *req)
999{
1000 bzero(req, sizeof(struct vtblk_request));
1001 TAILQ_INSERT_HEAD(&sc->vtblk_req_free, req, vbr_link);
1002}
1003
1004static struct vtblk_request *
1005vtblk_dequeue_ready(struct vtblk_softc *sc)
1006{
1007 struct vtblk_request *req;
1008
1009 req = TAILQ_FIRST(&sc->vtblk_req_ready);
1010 if (req != NULL)
1011 TAILQ_REMOVE(&sc->vtblk_req_ready, req, vbr_link);
1012
1013 return (req);
1014}
1015
1016static void
1017vtblk_enqueue_ready(struct vtblk_softc *sc, struct vtblk_request *req)
1018{
1019 TAILQ_INSERT_HEAD(&sc->vtblk_req_ready, req, vbr_link);
1020}
1021
1022static void
1023vtblk_bio_error(struct bio *bp, int error)
1024{
1025 biodone(bp);
1026}