Build fixes.
[dragonfly.git] / sys / dev / virtual / virtio / pci / virtio_pci.c
CommitLineData
11447b59
VS
1/*-
2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
10 * disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: src/sys/dev/virtio/pci/virtio_pci.c,v 1.3 2012/04/14 05:48:04 grehan Exp $
27 */
28
29/* Driver for the VirtIO PCI interface. */
30
31#include <sys/cdefs.h>
32
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/bus.h>
36#include <sys/kernel.h>
37#include <sys/module.h>
38#include <sys/malloc.h>
39
40#include <bus/pci/pcivar.h>
41#include <bus/pci/pcireg.h>
42
43#include <sys/bus.h>
44#include <sys/param.h>
45#include <sys/rman.h>
46
dcbcbc86
VS
47#include <dev/virtual/virtio/virtio/virtio.h>
48#include <dev/virtual/virtio/virtio/virtqueue.h>
11447b59 49#include "virtio_pci.h"
11447b59 50#include "virtio_if.h"
dcbcbc86 51#include "virtio_bus_if.h"
11447b59
VS
52
53struct vtpci_softc {
54 device_t vtpci_dev;
55 struct resource *vtpci_res;
56 struct resource *vtpci_msix_res;
57 uint64_t vtpci_features;
58 uint32_t vtpci_flags;
59 int vtpci_irq_type;
60 int vtpci_irq_rid;
61#define VIRTIO_PCI_FLAG_NO_MSI 0x0001
62#define VIRTIO_PCI_FLAG_MSI 0x0002
63#define VIRTIO_PCI_FLAG_NO_MSIX 0x0010
64#define VIRTIO_PCI_FLAG_MSIX 0x0020
65#define VIRTIO_PCI_FLAG_SHARED_MSIX 0x0040
66
67 device_t vtpci_child_dev;
68 struct virtio_feature_desc *vtpci_child_feat_desc;
69
70 /*
71 * Ideally, each virtqueue that the driver provides a callback for
72 * will receive its own MSIX vector. If there are not sufficient
73 * vectors available, we will then attempt to have all the VQs
74 * share one vector. Note that when using MSIX, the configuration
75 * changed notifications must be on their own vector.
76 *
77 * If MSIX is not available, we will attempt to have the whole
78 * device share one MSI vector, and then, finally, one legacy
79 * interrupt.
80 */
81 int vtpci_nvqs;
82 struct vtpci_virtqueue {
83 struct virtqueue *vq;
84
85 /* Index into vtpci_intr_res[] below. Unused, then -1. */
86 int ires_idx;
87 } vtpci_vqx[VIRTIO_MAX_VIRTQUEUES];
88
89 /*
90 * When using MSIX interrupts, the first element of vtpci_intr_res[]
91 * is always the configuration changed notifications. The remaining
92 * element(s) are used for the virtqueues.
93 *
94 * With MSI and legacy interrupts, only the first element of
95 * vtpci_intr_res[] is used.
96 */
97 int vtpci_nintr_res;
98 struct vtpci_intr_resource {
99 struct resource *irq;
100 int rid;
101 void *intrhand;
102 } vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES];
103};
104
105static int vtpci_probe(device_t);
106static int vtpci_attach(device_t);
107static int vtpci_detach(device_t);
108static int vtpci_suspend(device_t);
109static int vtpci_resume(device_t);
110static int vtpci_shutdown(device_t);
111static void vtpci_driver_added(device_t, driver_t *);
112static void vtpci_child_detached(device_t, device_t);
113static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
114static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
115
116static uint64_t vtpci_negotiate_features(device_t, uint64_t);
117static int vtpci_with_feature(device_t, uint64_t);
118static int vtpci_alloc_virtqueues(device_t, int, int,
119 struct vq_alloc_info *);
120static int vtpci_setup_intr(device_t);
121static void vtpci_stop(device_t);
122static int vtpci_reinit(device_t, uint64_t);
123static void vtpci_reinit_complete(device_t);
124static void vtpci_notify_virtqueue(device_t, uint16_t);
125static uint8_t vtpci_get_status(device_t);
126static void vtpci_set_status(device_t, uint8_t);
127static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
128static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
129
130static void vtpci_describe_features(struct vtpci_softc *, const char *,
131 uint64_t);
132static void vtpci_probe_and_attach_child(struct vtpci_softc *);
133
134static int vtpci_alloc_interrupts(struct vtpci_softc *, int, int,
135 struct vq_alloc_info *);
136static int vtpci_alloc_intr_resources(struct vtpci_softc *, int,
137 struct vq_alloc_info *);
138static int vtpci_alloc_msi(struct vtpci_softc *);
139static int vtpci_alloc_msix(struct vtpci_softc *, int);
140static int vtpci_register_msix_vector(struct vtpci_softc *, int, int);
141
142static void vtpci_free_interrupts(struct vtpci_softc *);
143static void vtpci_free_virtqueues(struct vtpci_softc *);
144static void vtpci_release_child_resources(struct vtpci_softc *);
145static void vtpci_reset(struct vtpci_softc *);
146
147static int vtpci_legacy_intr(void *);
148static int vtpci_vq_shared_intr(void *);
149static int vtpci_vq_intr(void *);
150static int vtpci_config_intr(void *);
151
152/*
153 * I/O port read/write wrappers.
154 */
155#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
156#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
157#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
158#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
159#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
160#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
161
162/* Tunables. */
163static int vtpci_disable_msix = 0;
164TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
165
166static device_method_t vtpci_methods[] = {
167 /* Device interface. */
168 DEVMETHOD(device_probe, vtpci_probe),
169 DEVMETHOD(device_attach, vtpci_attach),
170 DEVMETHOD(device_detach, vtpci_detach),
171 DEVMETHOD(device_suspend, vtpci_suspend),
172 DEVMETHOD(device_resume, vtpci_resume),
173 DEVMETHOD(device_shutdown, vtpci_shutdown),
174
175 /* Bus interface. */
176 DEVMETHOD(bus_driver_added, vtpci_driver_added),
177 DEVMETHOD(bus_child_detached, vtpci_child_detached),
178 DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
179 DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
180
181 /* VirtIO bus interface. */
182 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
183 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
184 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
185 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
186 DEVMETHOD(virtio_bus_stop, vtpci_stop),
187 DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
188 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
189 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
190 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
191 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
192
193 { 0, 0 }
194};
195
196static driver_t vtpci_driver = {
197 "virtio_pci",
198 vtpci_methods,
199 sizeof(struct vtpci_softc)
200};
201
202devclass_t vtpci_devclass;
203
204DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
205MODULE_VERSION(virtio_pci, 1);
206MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
207MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
208
209static int
210vtpci_probe(device_t dev)
211{
212 char desc[36];
213 const char *name;
214
215 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
216 return (ENXIO);
217
218 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
219 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
220 return (ENXIO);
221
222 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
223 return (ENXIO);
224
225 name = virtio_device_name(pci_get_subdevice(dev));
226 if (name == NULL)
227 name = "Unknown";
228
229 ksnprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
230 device_set_desc_copy(dev, desc);
231
232 return (BUS_PROBE_DEFAULT);
233}
234
235static int
236vtpci_attach(device_t dev)
237{
238 struct vtpci_softc *sc;
239 device_t child;
240 int rid;
241
242 sc = device_get_softc(dev);
243 sc->vtpci_dev = dev;
244
245 pci_enable_busmaster(dev);
246
247 rid = PCIR_BAR(0);
248 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
249 RF_ACTIVE);
250 if (sc->vtpci_res == NULL) {
251 device_printf(dev, "cannot map I/O space\n");
252 return (ENXIO);
253 }
254
255 if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0)
256 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSI;
257 /* XXX(vsrinivas): Check out how to get MSI-X */
258#if OLD_MSI
259 if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) {
260 rid = PCIR_BAR(1);
261 sc->vtpci_msix_res = bus_alloc_resource_any(dev,
262 SYS_RES_MEMORY, &rid, RF_ACTIVE);
263 }
264#endif
265 if (sc->vtpci_msix_res == NULL)
266 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSIX;
267
268 vtpci_reset(sc);
269
270 /* Tell the host we've noticed this device. */
271 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
272
273 if ((child = device_add_child(dev, NULL, -1)) == NULL) {
274 device_printf(dev, "cannot create child device\n");
275 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
276 vtpci_detach(dev);
277 return (ENOMEM);
278 }
279
280 sc->vtpci_child_dev = child;
281 vtpci_probe_and_attach_child(sc);
282
283 return (0);
284}
285
286static int
287vtpci_detach(device_t dev)
288{
289 struct vtpci_softc *sc;
290 device_t child;
291 int error;
292
293 sc = device_get_softc(dev);
294
295 if ((child = sc->vtpci_child_dev) != NULL) {
296 error = device_delete_child(dev, child);
297 if (error)
298 return (error);
299 sc->vtpci_child_dev = NULL;
300 }
301
302 vtpci_reset(sc);
303
304 if (sc->vtpci_msix_res != NULL) {
305 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
306 sc->vtpci_msix_res);
307 sc->vtpci_msix_res = NULL;
308 }
309
310 if (sc->vtpci_res != NULL) {
311 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
312 sc->vtpci_res);
313 sc->vtpci_res = NULL;
314 }
315
316 return (0);
317}
318
319static int
320vtpci_suspend(device_t dev)
321{
322
323 return (bus_generic_suspend(dev));
324}
325
326static int
327vtpci_resume(device_t dev)
328{
329
330 return (bus_generic_resume(dev));
331}
332
333static int
334vtpci_shutdown(device_t dev)
335{
336
337 (void) bus_generic_shutdown(dev);
338 /* Forcibly stop the host device. */
339 vtpci_stop(dev);
340
341 return (0);
342}
343
344static void
345vtpci_driver_added(device_t dev, driver_t *driver)
346{
347 struct vtpci_softc *sc;
348
349 sc = device_get_softc(dev);
350
351 vtpci_probe_and_attach_child(sc);
352}
353
354static void
355vtpci_child_detached(device_t dev, device_t child)
356{
357 struct vtpci_softc *sc;
358
359 sc = device_get_softc(dev);
360
361 vtpci_reset(sc);
362 vtpci_release_child_resources(sc);
363}
364
365static int
366vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
367{
368 struct vtpci_softc *sc;
369
370 sc = device_get_softc(dev);
371
372 if (sc->vtpci_child_dev != child)
373 return (ENOENT);
374
375 switch (index) {
376 case VIRTIO_IVAR_DEVTYPE:
377 *result = pci_get_subdevice(dev);
378 break;
379 default:
380 return (ENOENT);
381 }
382
383 return (0);
384}
385
386static int
387vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
388{
389 struct vtpci_softc *sc;
390
391 sc = device_get_softc(dev);
392
393 if (sc->vtpci_child_dev != child)
394 return (ENOENT);
395
396 switch (index) {
397 case VIRTIO_IVAR_FEATURE_DESC:
398 sc->vtpci_child_feat_desc = (void *) value;
399 break;
400 default:
401 return (ENOENT);
402 }
403
404 return (0);
405}
406
407static uint64_t
408vtpci_negotiate_features(device_t dev, uint64_t child_features)
409{
410 struct vtpci_softc *sc;
411 uint64_t host_features, features;
412
413 sc = device_get_softc(dev);
414
415 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
416 vtpci_describe_features(sc, "host", host_features);
417
418 /*
419 * Limit negotiated features to what the driver, virtqueue, and
420 * host all support.
421 */
422 features = host_features & child_features;
423 features = virtqueue_filter_features(features);
424 sc->vtpci_features = features;
425
426 vtpci_describe_features(sc, "negotiated", features);
427 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
428
429 return (features);
430}
431
432static int
433vtpci_with_feature(device_t dev, uint64_t feature)
434{
435 struct vtpci_softc *sc;
436
437 sc = device_get_softc(dev);
438
439 return ((sc->vtpci_features & feature) != 0);
440}
441
442static int
443vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
444 struct vq_alloc_info *vq_info)
445{
446 struct vtpci_softc *sc;
447 struct vtpci_virtqueue *vqx;
448 struct vq_alloc_info *info;
449 int queue, error;
450 uint16_t vq_size;
451
452 sc = device_get_softc(dev);
453
454 if (sc->vtpci_nvqs != 0 || nvqs <= 0 ||
455 nvqs > VIRTIO_MAX_VIRTQUEUES)
456 return (EINVAL);
457
458 error = vtpci_alloc_interrupts(sc, flags, nvqs, vq_info);
459 if (error) {
460 device_printf(dev, "cannot allocate interrupts\n");
461 return (error);
462 }
463
464 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
465 error = vtpci_register_msix_vector(sc,
466 VIRTIO_MSI_CONFIG_VECTOR, 0);
467 if (error)
468 return (error);
469 }
470
471 for (queue = 0; queue < nvqs; queue++) {
472 vqx = &sc->vtpci_vqx[queue];
473 info = &vq_info[queue];
474
475 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
476
477 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
478 error = virtqueue_alloc(dev, queue, vq_size,
479 VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vqx->vq);
480 if (error)
481 return (error);
482
483 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
484 error = vtpci_register_msix_vector(sc,
485 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx);
486 if (error)
487 return (error);
488 }
489
490 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
491 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
492
493 *info->vqai_vq = vqx->vq;
494 sc->vtpci_nvqs++;
495 }
496
497 return (0);
498}
499
500static int
501vtpci_setup_intr(device_t dev)
502{
503 struct vtpci_softc *sc;
504 struct vtpci_intr_resource *ires;
505 struct vtpci_virtqueue *vqx;
506 int i, flags, error;
507
508 sc = device_get_softc(dev);
509 flags = INTR_MPSAFE;
510 ires = &sc->vtpci_intr_res[0];
511
512 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) == 0) {
513 error = bus_setup_intr(dev, ires->irq, flags,
514 (driver_intr_t *) vtpci_legacy_intr, sc, &ires->intrhand, NULL);
515
516 return (error);
517 }
518
519 error = bus_setup_intr(dev, ires->irq, flags,(driver_intr_t *) vtpci_config_intr,
520 sc, &ires->intrhand, NULL);
521 if (error)
522 return (error);
523
524 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) {
525 ires = &sc->vtpci_intr_res[1];
526 error = bus_setup_intr(dev, ires->irq, flags,
527 (driver_intr_t *) vtpci_vq_shared_intr, sc, &ires->intrhand, NULL);
528
529 return (error);
530 }
531
532 /* Setup an interrupt handler for each virtqueue. */
533 for (i = 0; i < sc->vtpci_nvqs; i++) {
534 vqx = &sc->vtpci_vqx[i];
535 if (vqx->ires_idx < 1)
536 continue;
537
538 ires = &sc->vtpci_intr_res[vqx->ires_idx];
539 error = bus_setup_intr(dev, ires->irq, flags,
540 (driver_intr_t *) vtpci_vq_intr, vqx->vq, &ires->intrhand, NULL);
541 if (error)
542 return (error);
543 }
544
545 return (0);
546}
547
548static void
549vtpci_stop(device_t dev)
550{
551
552 vtpci_reset(device_get_softc(dev));
553}
554
555static int
556vtpci_reinit(device_t dev, uint64_t features)
557{
558 struct vtpci_softc *sc;
559 struct vtpci_virtqueue *vqx;
560 struct virtqueue *vq;
561 int queue, error;
562 uint16_t vq_size;
563
564 sc = device_get_softc(dev);
565
566 /*
567 * Redrive the device initialization. This is a bit of an abuse
568 * of the specification, but both VirtualBox and QEMU/KVM seem
569 * to play nice. We do not allow the host device to change from
570 * what was originally negotiated beyond what the guest driver
571 * changed (MSIX state should not change, number of virtqueues
572 * and their size remain the same, etc).
573 */
574
575 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
576 vtpci_stop(dev);
577
578 /*
579 * Quickly drive the status through ACK and DRIVER. The device
580 * does not become usable again until vtpci_reinit_complete().
581 */
582 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
583 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
584
585 vtpci_negotiate_features(dev, features);
586
587 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
588 error = vtpci_register_msix_vector(sc,
589 VIRTIO_MSI_CONFIG_VECTOR, 0);
590 if (error)
591 return (error);
592 }
593
594 for (queue = 0; queue < sc->vtpci_nvqs; queue++) {
595 vqx = &sc->vtpci_vqx[queue];
596 vq = vqx->vq;
597
598 KASSERT(vq != NULL, ("vq %d not allocated", queue));
599 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue);
600
601 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
602 error = virtqueue_reinit(vq, vq_size);
603 if (error)
604 return (error);
605
606 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
607 error = vtpci_register_msix_vector(sc,
608 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx);
609 if (error)
610 return (error);
611 }
612
613 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
614 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
615 }
616
617 return (0);
618}
619
620static void
621vtpci_reinit_complete(device_t dev)
622{
623
624 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
625}
626
627static void
628vtpci_notify_virtqueue(device_t dev, uint16_t queue)
629{
630 struct vtpci_softc *sc;
631
632 sc = device_get_softc(dev);
633
634 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
635}
636
637static uint8_t
638vtpci_get_status(device_t dev)
639{
640 struct vtpci_softc *sc;
641
642 sc = device_get_softc(dev);
643
644 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
645}
646
647static void
648vtpci_set_status(device_t dev, uint8_t status)
649{
650 struct vtpci_softc *sc;
651
652 sc = device_get_softc(dev);
653
654 if (status != VIRTIO_CONFIG_STATUS_RESET)
655 status |= vtpci_get_status(dev);
656
657 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
658}
659
660static void
661vtpci_read_dev_config(device_t dev, bus_size_t offset,
662 void *dst, int length)
663{
664 struct vtpci_softc *sc;
665 bus_size_t off;
666 uint8_t *d;
667 int size;
668
669 sc = device_get_softc(dev);
670 off = VIRTIO_PCI_CONFIG(sc) + offset;
671
672 for (d = dst; length > 0; d += size, off += size, length -= size) {
673 if (length >= 4) {
674 size = 4;
675 *(uint32_t *)d = vtpci_read_config_4(sc, off);
676 } else if (length >= 2) {
677 size = 2;
678 *(uint16_t *)d = vtpci_read_config_2(sc, off);
679 } else {
680 size = 1;
681 *d = vtpci_read_config_1(sc, off);
682 }
683 }
684}
685
686static void
687vtpci_write_dev_config(device_t dev, bus_size_t offset,
688 void *src, int length)
689{
690 struct vtpci_softc *sc;
691 bus_size_t off;
692 uint8_t *s;
693 int size;
694
695 sc = device_get_softc(dev);
696 off = VIRTIO_PCI_CONFIG(sc) + offset;
697
698 for (s = src; length > 0; s += size, off += size, length -= size) {
699 if (length >= 4) {
700 size = 4;
701 vtpci_write_config_4(sc, off, *(uint32_t *)s);
702 } else if (length >= 2) {
703 size = 2;
704 vtpci_write_config_2(sc, off, *(uint16_t *)s);
705 } else {
706 size = 1;
707 vtpci_write_config_1(sc, off, *s);
708 }
709 }
710}
711
712static void
713vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
714 uint64_t features)
715{
716 device_t dev, child;
717
718 dev = sc->vtpci_dev;
719 child = sc->vtpci_child_dev;
720
721 if (device_is_attached(child) && bootverbose == 0)
722 return;
723
724 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
725}
726
727static void
728vtpci_probe_and_attach_child(struct vtpci_softc *sc)
729{
730 device_t dev, child;
731
732 dev = sc->vtpci_dev;
733 child = sc->vtpci_child_dev;
734
735 if (child == NULL)
736 return;
737
738 if (device_get_state(child) != DS_NOTPRESENT)
739 return;
740
741 if (device_probe_child(dev, child) != 0)
742 return;
743
744 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
745 if (DEVICE_ATTACH(child) != 0) {
746 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
747 vtpci_reset(sc);
748 vtpci_release_child_resources(sc);
749
750 /* Reset status for future attempt. */
751 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
752 } else
753 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
754}
755
756static int
757vtpci_alloc_interrupts(struct vtpci_softc *sc, int flags, int nvqs,
758 struct vq_alloc_info *vq_info)
759{
760 int i, nvectors, error;
761
762 /*
763 * Only allocate a vector for virtqueues that are actually
764 * expecting an interrupt.
765 */
766 for (nvectors = 0, i = 0; i < nvqs; i++)
767 if (vq_info[i].vqai_intr != NULL)
768 nvectors++;
769
770 if (vtpci_disable_msix != 0 ||
771 sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSIX ||
772 flags & VIRTIO_ALLOC_VQS_DISABLE_MSIX ||
773 vtpci_alloc_msix(sc, nvectors) != 0) {
774 /*
775 * Use MSI interrupts if available. Otherwise, we fallback
776 * to legacy interrupts.
777 */
778 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSI) == 0 &&
779 vtpci_alloc_msi(sc) == 0)
780 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSI;
781
782 sc->vtpci_nintr_res = 1;
783 }
784
785 error = vtpci_alloc_intr_resources(sc, nvqs, vq_info);
786
787 return (error);
788}
789
790static int
791vtpci_alloc_intr_resources(struct vtpci_softc *sc, int nvqs,
792 struct vq_alloc_info *vq_info)
793{
794 device_t dev;
795 struct resource *irq;
796 struct vtpci_virtqueue *vqx;
797 int i, rid, flags, res_idx;
798
799 dev = sc->vtpci_dev;
800 flags = RF_ACTIVE;
801
802 if ((sc->vtpci_flags &
803 (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) == 0) {
804 rid = 0;
805 flags |= RF_SHAREABLE;
806 } else
807 rid = 1;
808
809 for (i = 0; i < sc->vtpci_nintr_res; i++) {
810 irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, flags);
811 if (irq == NULL)
812 return (ENXIO);
813
814 sc->vtpci_intr_res[i].irq = irq;
815 sc->vtpci_intr_res[i].rid = rid++;
816 }
817
818 /*
819 * Map the virtqueue into the correct index in vq_intr_res[]. Note the
820 * first index is reserved for configuration changes notifications.
821 */
822 for (i = 0, res_idx = 1; i < nvqs; i++) {
823 vqx = &sc->vtpci_vqx[i];
824
825 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) {
826 if (vq_info[i].vqai_intr == NULL)
827 vqx->ires_idx = -1;
828 else if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX)
829 vqx->ires_idx = res_idx;
830 else
831 vqx->ires_idx = res_idx++;
832 } else
833 vqx->ires_idx = -1;
834 }
835
836 return (0);
837}
838
839static int
840vtpci_alloc_msi(struct vtpci_softc *sc)
841{
842 device_t dev;
9aa0ab36 843 int nmsi;
11447b59
VS
844 u_int irq_flags;
845
846 dev = sc->vtpci_dev;
847 nmsi = pci_msi_count(dev);
848
849 if (nmsi < 1)
850 return (1);
851
11447b59
VS
852 sc->vtpci_irq_rid = 0;
853 sc->vtpci_irq_type = pci_alloc_1intr(dev, 1,
854 &sc->vtpci_irq_rid, &irq_flags);
855
856
857 return (1);
858}
859
860static int
861vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
862{
863 /* XXX(vsrinivas): Huh? Is this how MSI-X works?*/
864 /* XXX(vsrinivas): All of this was disabled... */
865#ifdef OLD_MSI
866 device_t dev;
867 int nmsix, cnt, required;
868
869 dev = sc->vtpci_dev;
870
871 nmsix = pci_msix_count(dev);
872 if (nmsix < 1)
873 return (1);
874
875 /* An additional vector is needed for the config changes. */
876 required = nvectors + 1;
877 if (nmsix >= required) {
878 cnt = required;
879 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required)
880 goto out;
881
882 pci_release_msi(dev);
883 }
884
885 /* Attempt shared MSIX configuration. */
886 required = 2;
887 if (nmsix >= required) {
888 cnt = required;
889 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
890 sc->vtpci_flags |= VIRTIO_PCI_FLAG_SHARED_MSIX;
891 goto out;
892 }
893
894 pci_release_msi(dev);
895 }
896
897 return (1);
898
899out:
900 sc->vtpci_nintr_res = required;
901 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSIX;
902
903 if (bootverbose) {
904 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX)
905 device_printf(dev, "using shared virtqueue MSIX\n");
906 else
907 device_printf(dev, "using per virtqueue MSIX\n");
908 }
909#endif
910 return (0);
911}
912
913static int
914vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx)
915{
916 device_t dev;
917 uint16_t vector;
918
919 dev = sc->vtpci_dev;
920
921 if (offset != VIRTIO_MSI_CONFIG_VECTOR &&
922 offset != VIRTIO_MSI_QUEUE_VECTOR)
923 return (EINVAL);
924
925 if (res_idx != -1) {
926 /* Map from rid to host vector. */
927 vector = sc->vtpci_intr_res[res_idx].rid - 1;
928 } else
929 vector = VIRTIO_MSI_NO_VECTOR;
930
931 /* The first resource is special; make sure it is used correctly. */
932 if (res_idx == 0) {
933 KASSERT(vector == 0, ("unexpected config vector"));
934 KASSERT(offset == VIRTIO_MSI_CONFIG_VECTOR,
935 ("unexpected config offset"));
936 }
937
938 vtpci_write_config_2(sc, offset, vector);
939
940 if (vtpci_read_config_2(sc, offset) != vector) {
941 device_printf(dev, "insufficient host resources for "
942 "MSIX interrupts\n");
943 return (ENODEV);
944 }
945
946 return (0);
947}
948
949static void
950vtpci_free_interrupts(struct vtpci_softc *sc)
951{
952 device_t dev;
953 struct vtpci_intr_resource *ires;
954 int i;
955
956 dev = sc->vtpci_dev;
957 sc->vtpci_nintr_res = 0;
958
959 if (sc->vtpci_flags & (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) {
960 pci_release_msi(dev);
961 sc->vtpci_flags &= ~(VIRTIO_PCI_FLAG_MSI |
962 VIRTIO_PCI_FLAG_MSIX | VIRTIO_PCI_FLAG_SHARED_MSIX);
963 }
964
965 for (i = 0; i < 1 + VIRTIO_MAX_VIRTQUEUES; i++) {
966 ires = &sc->vtpci_intr_res[i];
967
968 if (ires->intrhand != NULL) {
969 bus_teardown_intr(dev, ires->irq, ires->intrhand);
970 ires->intrhand = NULL;
971 }
972
973 if (ires->irq != NULL) {
974 bus_release_resource(dev, SYS_RES_IRQ, ires->rid,
975 ires->irq);
976 ires->irq = NULL;
977 }
978
979 ires->rid = -1;
980 }
981}
982
983static void
984vtpci_free_virtqueues(struct vtpci_softc *sc)
985{
986 struct vtpci_virtqueue *vqx;
987 int i;
988
989 sc->vtpci_nvqs = 0;
990
991 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) {
992 vqx = &sc->vtpci_vqx[i];
993
994 if (vqx->vq != NULL) {
995 virtqueue_free(vqx->vq);
996 vqx->vq = NULL;
997 }
998 }
999}
1000
1001static void
1002vtpci_release_child_resources(struct vtpci_softc *sc)
1003{
1004
1005 vtpci_free_interrupts(sc);
1006 vtpci_free_virtqueues(sc);
1007}
1008
1009static void
1010vtpci_reset(struct vtpci_softc *sc)
1011{
1012
1013 /*
1014 * Setting the status to RESET sets the host device to
1015 * the original, uninitialized state.
1016 */
1017 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1018}
1019
1020static int
1021vtpci_legacy_intr(void *xsc)
1022{
1023 struct vtpci_softc *sc;
1024 struct vtpci_virtqueue *vqx;
1025 int i;
1026 uint8_t isr;
1027
1028 sc = xsc;
1029 vqx = &sc->vtpci_vqx[0];
1030
1031 /* Reading the ISR also clears it. */
1032 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1033
1034 if (isr & VIRTIO_PCI_ISR_CONFIG)
1035 vtpci_config_intr(sc);
1036
1037 if (isr & VIRTIO_PCI_ISR_INTR)
1038 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
1039 virtqueue_intr(vqx->vq);
1040
1041 return isr;
1042}
1043
1044static int
1045vtpci_vq_shared_intr(void *xsc)
1046{
1047 struct vtpci_softc *sc;
1048 struct vtpci_virtqueue *vqx;
1049 int i, rc;
1050
1051 rc = 0;
1052 sc = xsc;
1053 vqx = &sc->vtpci_vqx[0];
1054
1055 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
1056 rc |= virtqueue_intr(vqx->vq);
1057
1058 return rc;
1059}
1060
1061static int
1062vtpci_vq_intr(void *xvq)
1063{
1064 struct virtqueue *vq;
1065 int rc;
1066
1067 vq = xvq;
1068 rc = virtqueue_intr(vq);
1069
1070 return rc;
1071}
1072
1073static int
1074vtpci_config_intr(void *xsc)
1075{
1076 struct vtpci_softc *sc;
1077 device_t child;
1078 int rc;
1079
1080 rc = 0;
1081 sc = xsc;
1082 child = sc->vtpci_child_dev;
1083
1084 if (child != NULL)
1085 rc = VIRTIO_CONFIG_CHANGE(child);
1086
1087 return rc;
1088}