kernel: Use NULL for pointers in DRIVER_MODULE().
[dragonfly.git] / sys / dev / virtual / virtio / balloon / virtio_balloon.c
CommitLineData
95fbc42e
DG
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * $FreeBSD: head/sys/dev/virtio/balloon/virtio_balloon.c 326255 2017-11-27 14:52:40Z pfg $
29 */
30
31/*
32 * Copyright (c) 2018 The DragonFly Project. All rights reserved.
33 *
34 * This code is derived from software contributed to The DragonFly Project
35 * by Diederik de Groot <info@talon.nl>
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 *
41 * 1. Redistributions of source code must retain the above copyright
42 * notice, this list of conditions and the following disclaimer.
43 * 2. Redistributions in binary form must reproduce the above copyright
44 * notice, this list of conditions and the following disclaimer in
45 * the documentation and/or other materials provided with the
46 * distribution.
47 * 3. Neither the name of The DragonFly Project nor the names of its
48 * contributors may be used to endorse or promote products derived
49 * from this software without specific, prior written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
52 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
53 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
54 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
55 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
56 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
57 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
58 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
59 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
60 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
61 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62 * SUCH DAMAGE.
63 */
64
65/* Driver for VirtIO memory balloon devices. */
66
67#include <sys/cdefs.h>
68#include <sys/param.h>
69#include <sys/systm.h>
70#include <sys/kernel.h>
71#include <sys/endian.h>
72#include <sys/kthread.h>
73#include <sys/malloc.h>
74#include <sys/module.h>
75#include <sys/sglist.h>
76#include <sys/sysctl.h>
77#include <sys/lock.h>
78#include <sys/mutex.h>
79#include <sys/queue.h>
80
81#include <vm/vm.h>
82#include <vm/vm_page.h>
83#include <sys/bus.h>
84#include <sys/rman.h>
85
86#include <dev/virtual/virtio/virtio/virtio.h>
87#include <dev/virtual/virtio/virtio/virtqueue.h>
88#include <dev/virtual/virtio/balloon/virtio_balloon.h>
89
90struct vtballoon_softc {
91 device_t vtballoon_dev;
92 struct lwkt_serialize vtballoon_slz;
93 uint64_t vtballoon_features;
94 uint32_t vtballoon_flags;
95#define VTBALLOON_FLAG_DETACH 0x01
96
97 struct virtqueue *vtballoon_inflate_vq;
98 struct virtqueue *vtballoon_deflate_vq;
99
100 uint32_t vtballoon_desired_npages;
101 uint32_t vtballoon_current_npages;
102 TAILQ_HEAD(,vm_page) vtballoon_pages;
103
104 struct thread *vtballoon_td;
105 uint32_t *vtballoon_page_frames;
106 int vtballoon_pagereq;
107 int vtballoon_timeout;
108 int vtballoon_nintr;
109 int vtballoon_debug;
110#define VTBALLOON_INFO 0x01
111#define VTBALLOON_ERROR 0x02
112#define VTBALLOON_DEBUG 0x04
113#define VTBALLOON_TRACE 0x08
114
115 struct virtqueue *vtballoon_stats_vq;
116 struct vtballoon_stat vtballoon_stats[VTBALLOON_S_NR];
117 bool vtballoon_update_stats;
118};
119
120static struct virtio_feature_desc vtballoon_feature_desc[] = {
121 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
122 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
123 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOutOfMemory" },
124 { 0, NULL }
125};
126
127#define vtballoon_dprintf(_sc, _level, _msg, _args ...) do { \
128 if ((_sc)->vtballoon_debug & (_level)) \
129 device_printf((_sc)->vtballoon_dev, "%s:%d: "_msg, \
130 __FUNCTION__, __LINE__, ##_args); \
131} while (0)
132
133static int vtballoon_probe(device_t);
134static int vtballoon_attach(device_t);
135static int vtballoon_detach(device_t);
136
137static int vtballoon_alloc_intrs(struct vtballoon_softc *sc);
138
139static void vtballoon_negotiate_features(struct vtballoon_softc *);
140static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
141
142static void vtballoon_config_change_intr(void *);
143
144static void vtballoon_update_stats(struct vtballoon_softc *sc);
145static void vtballoon_stats_vq_intr(void *);
146
147static void vtballoon_inflate_vq_intr(void *);
148static void vtballoon_deflate_vq_intr(void *);
149static void vtballoon_inflate(struct vtballoon_softc *, int);
150static void vtballoon_deflate(struct vtballoon_softc *, int);
151
152static void vtballoon_send_page_frames(struct vtballoon_softc *,
153 struct virtqueue *, int);
154
155static void vtballoon_pop(struct vtballoon_softc *);
156static void vtballoon_stop(struct vtballoon_softc *);
157
158static vm_page_t vtballoon_alloc_page(struct vtballoon_softc *);
159static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
160
161static int vtballoon_sleep(struct vtballoon_softc *);
162static void vtballoon_thread(void *);
163static void vtballoon_get_tunables(struct vtballoon_softc *);
164static void vtballoon_add_sysctl(struct vtballoon_softc *);
165
166/*
167 * Features desired/implemented by this driver.
168 * VIRTIO_BALLOON_F_STATS_VQ | VIRTIO_BALLOON_F_MUST_TELL_HOST
169 */
170#define VTBALLOON_FEATURES VIRTIO_BALLOON_F_STATS_VQ
171
172/* Timeout between retries when the balloon needs inflating. */
173#define VTBALLOON_LOWMEM_TIMEOUT hz * 100
174
175/* vm_page_alloc flags */
176#define VTBALLOON_REGULAR_ALLOC VM_ALLOC_NORMAL
177#define VTBALLOON_LOWMEM_ALLOC VM_ALLOC_SYSTEM
178
179/*
180 * Maximum number of pages we'll request to inflate or deflate
181 * the balloon in one virtqueue request. Both Linux and NetBSD
182 * have settled on 256, doing up to 1MB at a time.
183 */
184#define VTBALLOON_PAGES_PER_REQUEST 256
185
186/*
187 * Default Debug Level
188 * VTBALLOON_INFO | VTBALLOON_ERROR | VTBALLOON_DEBUG | VTBALLOON_TRACE
189 */
190#define VTBALLOON_DEFAULT_DEBUG_LEVEL VTBALLOON_INFO | VTBALLOON_ERROR
191
192/*
193 * Maximum number of interrupts to request
194 */
195#define VTBALLOON_MAX_INTERRUPTS 4
196
197/* Must be able to fix all pages frames in one page (segment). */
198CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
199
200#define VTBALLOON_SLZ(_sc) &(_sc)->vtballoon_slz
201#define VTBALLOON_ENTER_SLZ(_sc) lwkt_serialize_enter(VTBALLOON_SLZ(sc));
202#define VTBALLOON_EXIT_SLZ(_sc) lwkt_serialize_exit(VTBALLOON_SLZ(sc));
203
204static device_method_t vtballoon_methods[] = {
205 /* Device methods. */
206 DEVMETHOD(device_probe, vtballoon_probe),
207 DEVMETHOD(device_attach, vtballoon_attach),
208 DEVMETHOD(device_detach, vtballoon_detach),
209
210 DEVMETHOD_END
211};
212
213static driver_t vtballoon_driver = {
214 "vtballoon",
215 vtballoon_methods,
216 sizeof(struct vtballoon_softc)
217};
218static devclass_t vtballoon_devclass;
219
220DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
17975de1 221 vtballoon_devclass, NULL, NULL);
95fbc42e
DG
222MODULE_VERSION(virtio_balloon, 1);
223MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
224
225static int
226vtballoon_probe(device_t dev)
227{
228 struct vtballoon_softc *sc = device_get_softc(dev);
229 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
230 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
231 return (ENXIO);
232
233 device_set_desc(dev, "VirtIO Balloon Adapter");
234
235 return (BUS_PROBE_DEFAULT);
236}
237
238struct irqmap {
239 int irq;
240 int idx;
241 driver_intr_t *handler;
242 const char * handler_name;
243};
244
245static int
246vtballoon_attach(device_t dev)
247{
248 struct vtballoon_softc *sc;
249 int error, i;
250
251 sc = device_get_softc(dev);
252 sc->vtballoon_dev = dev;
253 sc->vtballoon_debug = VTBALLOON_DEFAULT_DEBUG_LEVEL;
254
255 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
256
257 lwkt_serialize_init(VTBALLOON_SLZ(sc));
258 TAILQ_INIT(&sc->vtballoon_pages);
259
260 vtballoon_get_tunables(sc);
261 vtballoon_add_sysctl(sc);
262
263 virtio_set_feature_desc(dev, vtballoon_feature_desc);
264 vtballoon_negotiate_features(sc);
265
266 sc->vtballoon_page_frames = contigmalloc(VTBALLOON_PAGES_PER_REQUEST *
267 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO, 0, BUS_SPACE_MAXADDR, 16, 0);
268 if (sc->vtballoon_page_frames == NULL) {
269 error = ENOMEM;
270 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate page frame request array (error:%d)\n", error);
271 goto fail;
272 }
273 error = vtballoon_alloc_intrs(sc);
274 if (error) {
275 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate interrupts (error:%d)\n", error);
276 goto fail;
277 }
278
279 error = vtballoon_alloc_virtqueues(sc);
280 if (error) {
281 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate virtqueues (error:%d)\n", error);
282 goto fail;
283 }
284
285 int nrhandlers = virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ) ? 4 : 3;
286 struct irqmap info[4];
287
288 /* Possible "Virtqueue <-> IRQ" configurations */
289 switch (sc->vtballoon_nintr) {
290 case 1:
291 info[2] = (struct irqmap){0, -1, vtballoon_config_change_intr, "config"};
292 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
293 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"};
294 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"};
295 break;
296 case 2:
297 info[2] = (struct irqmap){1, -1, vtballoon_config_change_intr, "config"};
298 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
299 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"};
300 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"};
301 break;
302 case 3:
303 info[2] = (struct irqmap){2, -1, vtballoon_config_change_intr, "config"};
304 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
305 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"};
306 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"};
307 break;
308 case 4:
309 info[2] = (struct irqmap){3, -1, vtballoon_config_change_intr, "config"};
310 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
311 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"};
312 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"};
313 break;
314 default:
315 vtballoon_dprintf(sc, VTBALLOON_ERROR, "Invalid interrupt vector count: %d\n", sc->vtballoon_nintr);
316 goto fail;
317 }
318 for (i = 0; i < nrhandlers; i++) {
319 error = virtio_bind_intr(sc->vtballoon_dev, info[i].irq, info[i].idx,
320 info[i].handler, sc);
321 if (error) {
322 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot bind virtqueue '%s' handler to IRQ:%d/%d\n",
323 info[i].handler_name, info[i].irq, sc->vtballoon_nintr);
324 goto fail;
325 }
326 }
327
328 for (i = 0; i < sc->vtballoon_nintr; i++) {
329 error = virtio_setup_intr(dev, i, VTBALLOON_SLZ(sc));
330 if (error) {
331 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot setup virtqueue interrupt:%d (error:%d)\n", i, error);
332 goto fail;
333 }
334 }
335
336 error = kthread_create(vtballoon_thread, sc, &sc->vtballoon_td, "virtio_balloon");
337 if (error) {
338 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot create balloon kthread (error:%d)\n", error);
339 goto fail;
340 }
341
342 virtqueue_enable_intr(sc->vtballoon_inflate_vq);
343 virtqueue_enable_intr(sc->vtballoon_deflate_vq);
344
345 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
346 virtqueue_enable_intr(sc->vtballoon_stats_vq);
347#if 0 /* enabling this causes a panic, on asserting ASSERT_SERIALIZED(sc) in vtballoon_update_stats */
348 /*
349 * Prime this stats virtqueue with one buffer so the hypervisor can
350 * use it to signal us later.
351 */
352 VTBALLOON_ENTER_SLZ(sc);
353 vtballoon_update_stats(sc);
354 VTBALLOON_EXIT_SLZ(sc);
355#endif
356 }
357
358fail:
359 if (error)
360 vtballoon_detach(dev);
361
362 return (error);
363}
364
365static int
366vtballoon_detach(device_t dev)
367{
368 struct vtballoon_softc *sc;
369 int i;
370
371 sc = device_get_softc(dev);
372 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
373
374 if (sc->vtballoon_td != NULL) {
375 VTBALLOON_ENTER_SLZ(sc);
376 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
377
378 /* drain */
379 wakeup_one(sc);
380 zsleep(sc->vtballoon_td, VTBALLOON_SLZ(sc), 0, "vtbdth", 0);
381 VTBALLOON_EXIT_SLZ(sc);
382 sc->vtballoon_td = NULL;
383 }
384
385 lwkt_serialize_handler_disable(VTBALLOON_SLZ(sc));
386
387 for (i = 0; i < sc->vtballoon_nintr; i++)
388 virtio_teardown_intr(dev, i);
389
390 if (device_is_attached(dev)) {
391 vtballoon_pop(sc);
392 vtballoon_stop(sc);
393 }
394
395 if (sc->vtballoon_page_frames != NULL) {
396 contigfree(sc->vtballoon_page_frames, VTBALLOON_PAGES_PER_REQUEST *
397 sizeof(uint32_t), M_DEVBUF);
398 sc->vtballoon_page_frames = NULL;
399 }
400 return (0);
401}
402
403static void
404vtballoon_negotiate_features(struct vtballoon_softc *sc)
405{
406 device_t dev;
407 uint64_t features;
408
409 dev = sc->vtballoon_dev;
410 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
411 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
412 sc->vtballoon_features = features;
413}
414
415static int vtballoon_alloc_intrs(struct vtballoon_softc *sc)
416{
417 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
418 int cnt, error;
419 int intrcount = virtio_intr_count(sc->vtballoon_dev);
420 int use_config = 1;
421
422 intrcount = imin(intrcount, VTBALLOON_MAX_INTERRUPTS);
423 if (intrcount < 1)
424 return (ENXIO);
425
426 cnt = intrcount;
427 error = virtio_intr_alloc(sc->vtballoon_dev, &cnt, use_config, NULL);
428 if (error != 0) {
429 virtio_intr_release(sc->vtballoon_dev);
430 return (error);
431 }
432 sc->vtballoon_nintr = cnt;
433 vtballoon_dprintf(sc, VTBALLOON_TRACE, "%d Interrupts Allocated\n", sc->vtballoon_nintr);
434 return (0);
435}
436
437static int
438vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
439{
440 device_t dev;
441 struct vq_alloc_info vq_info[3];
442 int nvqs;
443
444 dev = sc->vtballoon_dev;
445 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
446 nvqs = 2;
447
448 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, &sc->vtballoon_inflate_vq,
449 "%s inflate", device_get_nameunit(dev));
450
451 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, &sc->vtballoon_deflate_vq,
452 "%s deflate", device_get_nameunit(dev));
453
454 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
455 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, &sc->vtballoon_stats_vq,
456 "%s stats", device_get_nameunit(dev));
457 nvqs = 3;
458 }
459 return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
460}
461
462static void
463vtballoon_config_change_intr(void *arg)
464{
465 struct vtballoon_softc *sc = arg;
466 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
467 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
468 wakeup_one(sc);
469}
470
471static inline void
472vtballoon_update_stat(struct vtballoon_softc *sc, int idx,
473 uint16_t tag, uint64_t val)
474{
475 KASSERT(idx >= VTBALLOON_S_NR, ("Stats index out of bounds"));
476 /*
477 * XXX: Required for endianess in the future
478 * sc->vtballoon_stats[idx].tag = virtio_is_little_endian(sc->vtballoon_dev) ? le16toh(tag) : tag;
479 * sc->vtballoon_stats[idx].val = virtio_is_little_endian(sc->vtballoon_dev) ? le64toh(val) : val;
480 * at the moment virtio balloon is always little endian.
481 *
482 */
483 sc->vtballoon_stats[idx].tag = le16toh(tag);
484 sc->vtballoon_stats[idx].val = le64toh(val);
485
486}
487
488/*
489 * collect guest side statistics
490 *
491 * XXX: am i using the correct memory and pagefault values
492 */
493static unsigned int collect_balloon_stats(struct vtballoon_softc *sc)
494{
495 #define pages_to_bytes(x) ((uint64_t)(x) << PAGE_SHIFT)
496 unsigned int idx = 0;
497 struct vmtotal total;
498 struct vmmeter vmm;
499 struct vmstats vms;
500 size_t vmt_size = sizeof(total);
501 size_t vmm_size = sizeof(vmm);
502 size_t vms_size = sizeof(vms);
503
504 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Updating Stats Buffer\n");
505 if (!kernel_sysctlbyname("vm.vmtotal", &total, &vmt_size, NULL, 0, NULL)) {
506 /* Total amount of free memory )*/
507 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMFREE,
508 pages_to_bytes(total.t_rm - total.t_arm));
509 /* Total amount of memory */
510 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMTOT,
511 pages_to_bytes(total.t_rm));
512 /* Available memory as in /proc */
513 vtballoon_update_stat(sc, idx++, VTBALLOON_S_AVAIL,
514 pages_to_bytes(total.t_arm));
515 }
516 if (!kernel_sysctlbyname("vm.vmstats", &vms, &vms_size, NULL, 0, NULL)) {
517 /* Disk caches */
518 vtballoon_update_stat(sc, idx++, VTBALLOON_S_CACHES,
519 pages_to_bytes(vms.v_cache_count));
520 }
521 if (!kernel_sysctlbyname("vm.vmmeter", &vmm, &vmm_size, NULL, 0, NULL)) {
522 /* Amount of memory swapped in */
523 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_IN,
524 pages_to_bytes(vmm.v_swappgsin));
525 /* Amount of memory swapped out */
526 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_OUT,
527 pages_to_bytes(vmm.v_swappgsout));
528 /* Number of major faults */
529 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MAJFLT,
530 vmm.v_vm_faults);
531 /* Number of minor faults */
532 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MINFLT,
533 vmm.v_intrans);
534 }
535
536 if (sc->vtballoon_debug & VTBALLOON_TRACE) {
537 static const char *vt_balloon_names[]=VTBALLOON_S_NAMES;
538 int i;
539 for (i=0; i < idx; i++) {
540 kprintf("\t%s = %lu\n", vt_balloon_names[sc->vtballoon_stats[i].tag], sc->vtballoon_stats[i].val);
541 }
542 }
543
544 return idx;
545}
546
547static void
548vtballoon_update_stats(struct vtballoon_softc *sc)
549{
550 struct virtqueue *vq = sc->vtballoon_stats_vq;
551
552 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
553
554 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stats Requested\n");
555
556 struct sglist sg;
557 struct sglist_seg segs[1];
558 unsigned int num_stats;
559 int error;
560
561 num_stats = collect_balloon_stats(sc);
562
563 sglist_init(&sg, 1, segs);
564 error = sglist_append(&sg, sc->vtballoon_stats, sizeof(sc->vtballoon_stats[0]) * num_stats);
565 KASSERT(error == 0, ("error adding page frames to sglist"));
566
567 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
568 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
569 virtqueue_notify(sc->vtballoon_stats_vq, NULL);
570}
571
572/*
573 * While most virtqueues communicate guest-initiated requests to the hypervisor,
574 * the stats queue operates in reverse. The driver(host) initializes the virtqueue
575 * with a single buffer. From that point forward, all conversations consist of
576 * a hypervisor request (a call to this function) which directs us to refill
577 * the virtqueue with a fresh stats buffer. Since stats collection can sleep,
578 * we delegate the job to the vtballoon_thread which will do the actual stats
579 * collecting work.
580 */
581static void
582vtballoon_stats_vq_intr(void *arg)
583{
584 struct vtballoon_softc *sc = arg;
585 struct virtqueue *vq = sc->vtballoon_stats_vq;
586
587 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
588 if (sc->vtballoon_update_stats || !virtqueue_pending(vq))
589 return;
590
591 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Ballooon Stats Requested\n");
592 sc->vtballoon_update_stats = true;
593 wakeup_one(sc);
594 virtqueue_dequeue(vq, NULL);
595}
596
597static void
598vtballoon_inflate_vq_intr(void *arg)
599{
600 struct vtballoon_softc *sc = arg;
601 struct virtqueue *vq = sc->vtballoon_inflate_vq;
602 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
603 if (!virtqueue_pending(vq))
604 return;
605 wakeup_one(sc);
606}
607
608static void
609vtballoon_deflate_vq_intr(void *arg)
610{
611 struct vtballoon_softc *sc = arg;
612 struct virtqueue *vq = sc->vtballoon_deflate_vq;
613 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
614 if (!virtqueue_pending(vq))
615 return;
616 wakeup_one(sc);
617}
618
619static void
620vtballoon_inflate(struct vtballoon_softc *sc, int npages)
621{
622 struct virtqueue *vq;
623
624 vm_page_t m;
625 int i;
626
627 vq = sc->vtballoon_inflate_vq;
628
629 if (npages > VTBALLOON_PAGES_PER_REQUEST)
630 npages = VTBALLOON_PAGES_PER_REQUEST;
631
632 for (i = 0; i < npages; i++) {
633 if ((m = vtballoon_alloc_page(sc)) == NULL) {
634 /* First allocate usign VTBALLOON_REGULAR_ALLOC and fall back to VTBALLOON_LOWMEM_ALLOC
635 * when the guest is under severe memory pressure. Quickly decrease the
636 * allocation rate, allowing the system to swap out pages.
637 */
638 sc->vtballoon_pagereq = VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT;
639 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
640 break;
641 }
642
643 sc->vtballoon_page_frames[i] =
644 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
645
646 KASSERT(m->queue == PQ_NONE,
647 ("%s: allocated page %p on queue", __func__, m));
648 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
649 }
650
651 if (i > 0)
652 vtballoon_send_page_frames(sc, vq, i);
653}
654
655static void
656vtballoon_deflate(struct vtballoon_softc *sc, int npages)
657{
658 TAILQ_HEAD(, vm_page) free_pages;
659 struct virtqueue *vq;
660 vm_page_t m;
661 int i;
662
663 vq = sc->vtballoon_deflate_vq;
664 TAILQ_INIT(&free_pages);
665
666 if (npages > VTBALLOON_PAGES_PER_REQUEST)
667 npages = VTBALLOON_PAGES_PER_REQUEST;
668
669 for (i = 0; i < npages; i++) {
670 m = TAILQ_FIRST(&sc->vtballoon_pages);
671 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
672
673 sc->vtballoon_page_frames[i] =
674 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
675
676 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
677 TAILQ_INSERT_TAIL(&free_pages, m, pageq);
678 }
679
680 if (i > 0) {
681 /*
682 * Note that if virtio VIRTIO_BALLOON_F_MUST_TELL_HOST
683 * feature is true, we *have* to tell host first
684 * before freeing the pages.
685 */
686 vtballoon_send_page_frames(sc, vq, i);
687
688 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
689 TAILQ_REMOVE(&free_pages, m, pageq);
690 vtballoon_free_page(sc, m);
691 }
692 }
693
694 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
695 sc->vtballoon_current_npages == 0) ||
696 (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
697 sc->vtballoon_current_npages != 0),
698 ("%s: bogus page count %d", __func__,
699 sc->vtballoon_current_npages));
700}
701
702static void
703vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
704 int npages)
705{
706 struct sglist sg;
707 struct sglist_seg segs[1];
708 void *c;
709 int error;
710
711 sglist_init(&sg, 1, segs);
712
713 error = sglist_append(&sg, sc->vtballoon_page_frames,
714 npages * sizeof(uint32_t));
715 KASSERT(error == 0, ("error adding page frames to sglist"));
716
717 error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
718 KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
719 virtqueue_notify(vq, NULL);
720
721 /*
722 * Inflate and deflate operations are done synchronously. The
723 * interrupt handler will wake us up.
724 */
725 VTBALLOON_ENTER_SLZ(sc);
726 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) {
727 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbspf", 0);
728 }
729 VTBALLOON_EXIT_SLZ(sc);
730
731 KASSERT(c == vq, ("unexpected balloon operation response"));
732}
733
734static void
735vtballoon_pop(struct vtballoon_softc *sc)
736{
737 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Popping\n");
738
739 while (!TAILQ_EMPTY(&sc->vtballoon_pages))
740 vtballoon_deflate(sc, sc->vtballoon_current_npages);
741}
742
743static void
744vtballoon_stop(struct vtballoon_softc *sc)
745{
746 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stopping\n");
747
748 virtqueue_disable_intr(sc->vtballoon_inflate_vq);
749 virtqueue_disable_intr(sc->vtballoon_deflate_vq);
750/*
751 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
752 virtqueue_disable_intr(sc->vtballoon_stats_vq);
753 }
754*/
755 virtio_stop(sc->vtballoon_dev);
756}
757
758static vm_page_t
759vtballoon_alloc_page(struct vtballoon_softc *sc)
760{
761 vm_page_t m;
762
763 m = vm_page_alloc(NULL, 0, sc->vtballoon_pagereq);
764 if (m != NULL)
765 sc->vtballoon_current_npages++;
766
767 return (m);
768}
769
770static void
771vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
772{
773 vm_page_free_toq(m);
774 sc->vtballoon_current_npages--;
775}
776
777static uint32_t
778vtballoon_desired_size(struct vtballoon_softc *sc)
779{
780 uint32_t desired;
781
782 desired = virtio_read_dev_config_4(sc->vtballoon_dev,
783 offsetof(struct virtio_balloon_config, num_pages));
784
785 return (le32toh(desired));
786}
787
788static void
789vtballoon_update_size(struct vtballoon_softc *sc)
790{
791 virtio_write_dev_config_4(sc->vtballoon_dev,
792 offsetof(struct virtio_balloon_config, actual),
793 htole32(sc->vtballoon_current_npages));
794}
795
796static int
797vtballoon_sleep(struct vtballoon_softc *sc)
798{
799 int rc, timeout;
800 uint32_t current, desired;
801
802 rc = 0;
803 current = sc->vtballoon_current_npages;
804 sc->vtballoon_pagereq = VM_ALLOC_NORMAL | VM_ALLOC_INTERRUPT;
805
806 VTBALLOON_ENTER_SLZ(sc);
807 for (;;) {
808 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
809 rc = 1;
810 break;
811 }
812
813 desired = vtballoon_desired_size(sc);
814 if (desired != sc->vtballoon_desired_npages)
815 vtballoon_dprintf(sc, VTBALLOON_DEBUG, "balloon %s %d -> %d (4K pages)\n",
816 desired < sc->vtballoon_desired_npages ? "deflating" : "inflating",
817 current, desired);
818
819 sc->vtballoon_desired_npages = desired;
820
821 /*
822 * If given, use non-zero timeout on the first time through
823 * the loop. On subsequent times, timeout will be zero so
824 * we will reevaluate the desired size of the balloon and
825 * break out to retry if needed.
826 */
827 timeout = sc->vtballoon_timeout;
828 sc->vtballoon_timeout = 0;
829
830 if (current > desired)
831 break;
832 else if (current < desired && timeout == 0)
833 break;
834 else if (sc->vtballoon_update_stats)
835 break;
836 else if (!timeout)
837 vtballoon_dprintf(sc, VTBALLOON_TRACE, "balloon %d (4K pages) reached\n", current);
838
839 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbslp", timeout);
840 }
841 VTBALLOON_EXIT_SLZ(sc);
842
843 return (rc);
844}
845
846static void
847vtballoon_thread(void *arg)
848{
849 struct vtballoon_softc *sc = arg;
850 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Thread started.\n");
851
852 uint32_t current, desired;
853 for (;;) {
854 if (vtballoon_sleep(sc) != 0)
855 break;
856
857 current = sc->vtballoon_current_npages;
858 desired = sc->vtballoon_desired_npages;
859
860 if (desired != current) {
861 if (desired > current)
862 vtballoon_inflate(sc, desired - current);
863 else
864 vtballoon_deflate(sc, current - desired);
865
866 vtballoon_update_size(sc);
867 }
868 if (sc->vtballoon_update_stats) {
869 vtballoon_update_stats(sc);
870 sc->vtballoon_update_stats = false;
871 }
872 }
873
874 kthread_exit();
875}
876
877static void
878vtballoon_get_tunables(struct vtballoon_softc *sc)
879{
880 char tmpstr[64];
881 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
882
883 TUNABLE_INT_FETCH("hw.vtballoon.debug_level", &sc->vtballoon_debug);
884
885 ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtballoon.%d.debug_level",
886 device_get_unit(sc->vtballoon_dev));
887 TUNABLE_INT_FETCH(tmpstr, &sc->vtballoon_debug);
888}
889
890static void
891vtballoon_add_sysctl(struct vtballoon_softc *sc)
892{
893 device_t dev;
894 struct sysctl_ctx_list *ctx;
895 struct sysctl_oid *tree;
896 struct sysctl_oid_list *child;
897
898 dev = sc->vtballoon_dev;
899 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
900
901 ctx = device_get_sysctl_ctx(dev);
902 tree = device_get_sysctl_tree(dev);
903 child = SYSCTL_CHILDREN(tree);
904
905 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level",
906 CTLFLAG_RW, &sc->vtballoon_debug, 0,
907 "Debug level");
908
909 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
910 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
911 "Desired balloon size in pages");
912
913 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
914 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
915 "Current balloon size in pages");
916}