kernel: Use NULL for pointers in DRIVER_MODULE().
[dragonfly.git] / sys / dev / virtual / virtio / balloon / virtio_balloon.c
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  *
28  * $FreeBSD: head/sys/dev/virtio/balloon/virtio_balloon.c 326255 2017-11-27 14:52:40Z pfg $
29  */
30
31 /*
32  * Copyright (c) 2018 The DragonFly Project.  All rights reserved.
33  *
34  * This code is derived from software contributed to The DragonFly Project
35  * by Diederik de Groot <info@talon.nl>
36  *
37  * Redistribution and use in source and binary forms, with or without
38  * modification, are permitted provided that the following conditions
39  * are met:
40  *
41  * 1. Redistributions of source code must retain the above copyright
42  *    notice, this list of conditions and the following disclaimer.
43  * 2. Redistributions in binary form must reproduce the above copyright
44  *    notice, this list of conditions and the following disclaimer in
45  *    the documentation and/or other materials provided with the
46  *    distribution.
47  * 3. Neither the name of The DragonFly Project nor the names of its
48  *    contributors may be used to endorse or promote products derived
49  *    from this software without specific, prior written permission.
50  *
51  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
52  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
53  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
54  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
55  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
56  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
57  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
58  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
59  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
60  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
61  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
62  * SUCH DAMAGE.
63  */
64
65 /* Driver for VirtIO memory balloon devices. */
66
67 #include <sys/cdefs.h>
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/kernel.h>
71 #include <sys/endian.h>
72 #include <sys/kthread.h>
73 #include <sys/malloc.h>
74 #include <sys/module.h>
75 #include <sys/sglist.h>
76 #include <sys/sysctl.h>
77 #include <sys/lock.h>
78 #include <sys/mutex.h>
79 #include <sys/queue.h>
80
81 #include <vm/vm.h>
82 #include <vm/vm_page.h>
83 #include <sys/bus.h>
84 #include <sys/rman.h>
85
86 #include <dev/virtual/virtio/virtio/virtio.h>
87 #include <dev/virtual/virtio/virtio/virtqueue.h>
88 #include <dev/virtual/virtio/balloon/virtio_balloon.h>
89
90 struct vtballoon_softc {
91         device_t                 vtballoon_dev;
92         struct lwkt_serialize    vtballoon_slz;
93         uint64_t                 vtballoon_features;
94         uint32_t                 vtballoon_flags;
95 #define VTBALLOON_FLAG_DETACH    0x01
96
97         struct virtqueue        *vtballoon_inflate_vq;
98         struct virtqueue        *vtballoon_deflate_vq;
99
100         uint32_t                 vtballoon_desired_npages;
101         uint32_t                 vtballoon_current_npages;
102         TAILQ_HEAD(,vm_page)     vtballoon_pages;
103
104         struct thread           *vtballoon_td;
105         uint32_t                *vtballoon_page_frames;
106         int                      vtballoon_pagereq;
107         int                      vtballoon_timeout;
108         int                      vtballoon_nintr;
109         int                      vtballoon_debug;
110 #define VTBALLOON_INFO           0x01
111 #define VTBALLOON_ERROR          0x02
112 #define VTBALLOON_DEBUG          0x04
113 #define VTBALLOON_TRACE          0x08
114
115         struct virtqueue        *vtballoon_stats_vq;
116         struct vtballoon_stat    vtballoon_stats[VTBALLOON_S_NR];
117         bool                     vtballoon_update_stats;
118 };
119
120 static struct virtio_feature_desc vtballoon_feature_desc[] = {
121         { VIRTIO_BALLOON_F_MUST_TELL_HOST,      "MustTellHost"          },
122         { VIRTIO_BALLOON_F_STATS_VQ,            "StatsVq"               },
123         { VIRTIO_BALLOON_F_DEFLATE_ON_OOM,      "DeflateOnOutOfMemory"  },
124         { 0, NULL }
125 };
126
127 #define vtballoon_dprintf(_sc, _level, _msg, _args ...) do {        \
128         if ((_sc)->vtballoon_debug & (_level))                    \
129                 device_printf((_sc)->vtballoon_dev, "%s:%d: "_msg,      \
130                   __FUNCTION__, __LINE__, ##_args);                  \
131 } while (0)
132
133 static int              vtballoon_probe(device_t);
134 static int              vtballoon_attach(device_t);
135 static int              vtballoon_detach(device_t);
136
137 static int              vtballoon_alloc_intrs(struct vtballoon_softc *sc);
138
139 static void             vtballoon_negotiate_features(struct vtballoon_softc *);
140 static int              vtballoon_alloc_virtqueues(struct vtballoon_softc *);
141
142 static void             vtballoon_config_change_intr(void *);
143
144 static void             vtballoon_update_stats(struct vtballoon_softc *sc);
145 static void             vtballoon_stats_vq_intr(void *);
146
147 static void             vtballoon_inflate_vq_intr(void *);
148 static void             vtballoon_deflate_vq_intr(void *);
149 static void             vtballoon_inflate(struct vtballoon_softc *, int);
150 static void             vtballoon_deflate(struct vtballoon_softc *, int);
151
152 static void             vtballoon_send_page_frames(struct vtballoon_softc *,
153                             struct virtqueue *, int);
154
155 static void             vtballoon_pop(struct vtballoon_softc *);
156 static void             vtballoon_stop(struct vtballoon_softc *);
157
158 static vm_page_t        vtballoon_alloc_page(struct vtballoon_softc *);
159 static void             vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
160
161 static int              vtballoon_sleep(struct vtballoon_softc *);
162 static void             vtballoon_thread(void *);
163 static void             vtballoon_get_tunables(struct vtballoon_softc *);
164 static void             vtballoon_add_sysctl(struct vtballoon_softc *);
165
166 /*
167  * Features desired/implemented by this driver.
168  * VIRTIO_BALLOON_F_STATS_VQ | VIRTIO_BALLOON_F_MUST_TELL_HOST
169  */
170 #define VTBALLOON_FEATURES              VIRTIO_BALLOON_F_STATS_VQ
171
172 /* Timeout between retries when the balloon needs inflating. */
173 #define VTBALLOON_LOWMEM_TIMEOUT        hz * 100
174
175 /* vm_page_alloc flags */
176 #define VTBALLOON_REGULAR_ALLOC         VM_ALLOC_NORMAL
177 #define VTBALLOON_LOWMEM_ALLOC          VM_ALLOC_SYSTEM
178
179 /*
180  * Maximum number of pages we'll request to inflate or deflate
181  * the balloon in one virtqueue request. Both Linux and NetBSD
182  * have settled on 256, doing up to 1MB at a time.
183  */
184 #define VTBALLOON_PAGES_PER_REQUEST     256
185
186 /*
187  * Default Debug Level
188  * VTBALLOON_INFO | VTBALLOON_ERROR | VTBALLOON_DEBUG | VTBALLOON_TRACE
189  */
190 #define VTBALLOON_DEFAULT_DEBUG_LEVEL   VTBALLOON_INFO | VTBALLOON_ERROR
191
192 /*
193  * Maximum number of interrupts to request
194  */
195 #define VTBALLOON_MAX_INTERRUPTS        4
196
197 /* Must be able to fix all pages frames in one page (segment). */
198 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
199
200 #define VTBALLOON_SLZ(_sc)              &(_sc)->vtballoon_slz
201 #define VTBALLOON_ENTER_SLZ(_sc)        lwkt_serialize_enter(VTBALLOON_SLZ(sc));
202 #define VTBALLOON_EXIT_SLZ(_sc)         lwkt_serialize_exit(VTBALLOON_SLZ(sc));
203
204 static device_method_t vtballoon_methods[] = {
205         /* Device methods. */
206         DEVMETHOD(device_probe,         vtballoon_probe),
207         DEVMETHOD(device_attach,        vtballoon_attach),
208         DEVMETHOD(device_detach,        vtballoon_detach),
209
210         DEVMETHOD_END
211 };
212
213 static driver_t vtballoon_driver = {
214         "vtballoon",
215         vtballoon_methods,
216         sizeof(struct vtballoon_softc)
217 };
218 static devclass_t vtballoon_devclass;
219
220 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
221     vtballoon_devclass, NULL, NULL);
222 MODULE_VERSION(virtio_balloon, 1);
223 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
224
225 static int
226 vtballoon_probe(device_t dev)
227 {
228         struct vtballoon_softc *sc = device_get_softc(dev);
229         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
230         if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
231                 return (ENXIO);
232
233         device_set_desc(dev, "VirtIO Balloon Adapter");
234
235         return (BUS_PROBE_DEFAULT);
236 }
237
238 struct irqmap {
239         int irq;
240         int idx;
241         driver_intr_t *handler;
242         const char * handler_name;
243 };
244
245 static int
246 vtballoon_attach(device_t dev)
247 {
248         struct vtballoon_softc *sc;
249         int error, i;
250
251         sc = device_get_softc(dev);
252         sc->vtballoon_dev = dev;
253         sc->vtballoon_debug = VTBALLOON_DEFAULT_DEBUG_LEVEL;
254
255         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
256
257         lwkt_serialize_init(VTBALLOON_SLZ(sc));
258         TAILQ_INIT(&sc->vtballoon_pages);
259
260         vtballoon_get_tunables(sc);
261         vtballoon_add_sysctl(sc);
262
263         virtio_set_feature_desc(dev, vtballoon_feature_desc);
264         vtballoon_negotiate_features(sc);
265
266         sc->vtballoon_page_frames = contigmalloc(VTBALLOON_PAGES_PER_REQUEST *
267             sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO, 0, BUS_SPACE_MAXADDR, 16, 0);
268         if (sc->vtballoon_page_frames == NULL) {
269                 error = ENOMEM;
270                 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate page frame request array (error:%d)\n", error);
271                 goto fail;
272         }
273         error = vtballoon_alloc_intrs(sc);
274         if (error) {
275                 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate interrupts (error:%d)\n", error);
276                 goto fail;
277         }
278
279         error = vtballoon_alloc_virtqueues(sc);
280         if (error) {
281                 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate virtqueues (error:%d)\n", error);
282                 goto fail;
283         }
284
285         int nrhandlers = virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ) ? 4 : 3;
286         struct irqmap info[4];
287
288         /* Possible "Virtqueue <-> IRQ" configurations */
289         switch (sc->vtballoon_nintr) {
290         case 1:
291                 info[2] = (struct irqmap){0, -1, vtballoon_config_change_intr, "config"};
292                 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
293                 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"};
294                 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"};
295                 break;
296         case 2:
297                 info[2] = (struct irqmap){1, -1, vtballoon_config_change_intr, "config"};
298                 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
299                 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"};
300                 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"};
301                 break;
302         case 3:
303                 info[2] = (struct irqmap){2, -1, vtballoon_config_change_intr, "config"};
304                 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
305                 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"};
306                 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"};
307                 break;
308         case 4:
309                 info[2] = (struct irqmap){3, -1, vtballoon_config_change_intr, "config"};
310                 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"};
311                 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"};
312                 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"};
313                 break;
314         default:
315                 vtballoon_dprintf(sc, VTBALLOON_ERROR, "Invalid interrupt vector count: %d\n", sc->vtballoon_nintr);
316                 goto fail;
317         }
318         for (i = 0; i < nrhandlers; i++) {
319                 error = virtio_bind_intr(sc->vtballoon_dev, info[i].irq, info[i].idx,
320                     info[i].handler, sc);
321                 if (error) {
322                         vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot bind virtqueue '%s' handler to IRQ:%d/%d\n", 
323                                 info[i].handler_name, info[i].irq, sc->vtballoon_nintr);
324                         goto fail;
325                 }
326         }
327
328         for (i = 0; i < sc->vtballoon_nintr; i++) {
329                 error = virtio_setup_intr(dev, i, VTBALLOON_SLZ(sc));
330                 if (error) {
331                         vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot setup virtqueue interrupt:%d (error:%d)\n", i, error);
332                         goto fail;
333                 }
334         }
335
336         error = kthread_create(vtballoon_thread, sc, &sc->vtballoon_td, "virtio_balloon");
337         if (error) {
338                 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot create balloon kthread (error:%d)\n", error);
339                 goto fail;
340         }
341
342         virtqueue_enable_intr(sc->vtballoon_inflate_vq);
343         virtqueue_enable_intr(sc->vtballoon_deflate_vq);
344
345         if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
346                 virtqueue_enable_intr(sc->vtballoon_stats_vq);
347 #if 0           /* enabling this causes a panic, on asserting ASSERT_SERIALIZED(sc) in vtballoon_update_stats */
348                 /*
349                  * Prime this stats virtqueue with one buffer so the hypervisor can
350                  * use it to signal us later.
351                  */
352                 VTBALLOON_ENTER_SLZ(sc);
353                 vtballoon_update_stats(sc);
354                 VTBALLOON_EXIT_SLZ(sc);
355 #endif
356         }
357
358 fail:
359         if (error)
360                 vtballoon_detach(dev);
361
362         return (error);
363 }
364
365 static int
366 vtballoon_detach(device_t dev)
367 {
368         struct vtballoon_softc *sc;
369         int i;
370
371         sc = device_get_softc(dev);
372         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
373
374         if (sc->vtballoon_td != NULL) {
375                 VTBALLOON_ENTER_SLZ(sc);
376                 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
377
378                 /* drain */
379                 wakeup_one(sc);
380                 zsleep(sc->vtballoon_td, VTBALLOON_SLZ(sc), 0, "vtbdth", 0);
381                 VTBALLOON_EXIT_SLZ(sc);
382                 sc->vtballoon_td = NULL;
383         }
384
385         lwkt_serialize_handler_disable(VTBALLOON_SLZ(sc));
386
387         for (i = 0; i < sc->vtballoon_nintr; i++)
388                 virtio_teardown_intr(dev, i);
389
390         if (device_is_attached(dev)) {
391                 vtballoon_pop(sc);
392                 vtballoon_stop(sc);
393         }
394
395         if (sc->vtballoon_page_frames != NULL) {
396                 contigfree(sc->vtballoon_page_frames, VTBALLOON_PAGES_PER_REQUEST *
397                         sizeof(uint32_t), M_DEVBUF);
398                 sc->vtballoon_page_frames = NULL;
399         }
400         return (0);
401 }
402
403 static void
404 vtballoon_negotiate_features(struct vtballoon_softc *sc)
405 {
406         device_t dev;
407         uint64_t features;
408
409         dev = sc->vtballoon_dev;
410         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
411         features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
412         sc->vtballoon_features = features;
413 }
414
415 static int vtballoon_alloc_intrs(struct vtballoon_softc *sc)
416 {
417         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
418         int cnt, error;
419         int intrcount = virtio_intr_count(sc->vtballoon_dev);
420         int use_config = 1;
421
422         intrcount = imin(intrcount, VTBALLOON_MAX_INTERRUPTS);
423         if (intrcount < 1)
424                 return (ENXIO);
425
426         cnt = intrcount;
427         error = virtio_intr_alloc(sc->vtballoon_dev, &cnt, use_config, NULL);
428         if (error != 0) {
429                 virtio_intr_release(sc->vtballoon_dev);
430                 return (error);
431         }
432         sc->vtballoon_nintr = cnt;
433         vtballoon_dprintf(sc, VTBALLOON_TRACE, "%d Interrupts Allocated\n", sc->vtballoon_nintr);
434         return (0);
435 }
436
437 static int
438 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
439 {
440         device_t dev;
441         struct vq_alloc_info vq_info[3];
442         int nvqs;
443
444         dev = sc->vtballoon_dev;
445         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
446         nvqs = 2;
447
448         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, &sc->vtballoon_inflate_vq,
449                 "%s inflate", device_get_nameunit(dev));
450
451         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, &sc->vtballoon_deflate_vq,
452                 "%s deflate", device_get_nameunit(dev));
453
454         if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
455                 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, &sc->vtballoon_stats_vq,
456                         "%s stats", device_get_nameunit(dev));
457                 nvqs = 3;
458         }
459         return (virtio_alloc_virtqueues(dev, nvqs, vq_info));
460 }
461
462 static void
463 vtballoon_config_change_intr(void *arg)
464 {
465         struct vtballoon_softc *sc = arg;
466         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
467         ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
468         wakeup_one(sc);
469 }
470
471 static inline void
472 vtballoon_update_stat(struct vtballoon_softc *sc, int idx,
473         uint16_t tag, uint64_t val)
474 {
475         KASSERT(idx >= VTBALLOON_S_NR, ("Stats index out of bounds"));
476         /*
477          * XXX: Required for endianess in the future
478          * sc->vtballoon_stats[idx].tag = virtio_is_little_endian(sc->vtballoon_dev) ? le16toh(tag) : tag;
479          * sc->vtballoon_stats[idx].val = virtio_is_little_endian(sc->vtballoon_dev) ? le64toh(val) : val;
480          * at the moment virtio balloon is always little endian.
481          * 
482          */
483         sc->vtballoon_stats[idx].tag = le16toh(tag);
484         sc->vtballoon_stats[idx].val = le64toh(val);
485
486 }
487
488 /*
489  * collect guest side statistics
490  *
491  * XXX: am i using the correct memory and pagefault values
492  */
493 static unsigned int collect_balloon_stats(struct vtballoon_softc *sc)
494 {
495         #define pages_to_bytes(x) ((uint64_t)(x) << PAGE_SHIFT)
496         unsigned int idx = 0;
497         struct vmtotal total;
498         struct vmmeter vmm;
499         struct vmstats vms;
500         size_t vmt_size = sizeof(total);
501         size_t vmm_size = sizeof(vmm);
502         size_t vms_size = sizeof(vms);
503
504         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Updating Stats Buffer\n");
505         if (!kernel_sysctlbyname("vm.vmtotal", &total, &vmt_size, NULL, 0, NULL)) {
506                 /* Total amount of free memory )*/
507                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMFREE,
508                                         pages_to_bytes(total.t_rm - total.t_arm));
509                 /* Total amount of memory */
510                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMTOT,
511                                         pages_to_bytes(total.t_rm));
512                 /* Available memory as in /proc */
513                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_AVAIL,
514                                         pages_to_bytes(total.t_arm));
515         }
516         if (!kernel_sysctlbyname("vm.vmstats", &vms, &vms_size, NULL, 0, NULL)) {
517                 /* Disk caches */
518                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_CACHES,
519                                         pages_to_bytes(vms.v_cache_count));
520         }
521         if (!kernel_sysctlbyname("vm.vmmeter", &vmm, &vmm_size, NULL, 0, NULL)) {
522                 /* Amount of memory swapped in */
523                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_IN,
524                                         pages_to_bytes(vmm.v_swappgsin));
525                 /* Amount of memory swapped out */
526                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_OUT,
527                                         pages_to_bytes(vmm.v_swappgsout));
528                 /* Number of major faults */
529                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MAJFLT,
530                                         vmm.v_vm_faults);
531                 /* Number of minor faults */
532                 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MINFLT,
533                                         vmm.v_intrans);
534         }
535
536         if (sc->vtballoon_debug & VTBALLOON_TRACE)  {
537                 static const char *vt_balloon_names[]=VTBALLOON_S_NAMES;
538                 int i;
539                 for (i=0; i < idx; i++) {
540                         kprintf("\t%s = %lu\n", vt_balloon_names[sc->vtballoon_stats[i].tag], sc->vtballoon_stats[i].val);
541                 }
542         }
543
544         return idx;
545 }
546
547 static void
548 vtballoon_update_stats(struct vtballoon_softc *sc)
549 {
550         struct virtqueue *vq = sc->vtballoon_stats_vq;
551
552         ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
553
554         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stats Requested\n");
555
556         struct sglist sg;
557         struct sglist_seg segs[1];
558         unsigned int num_stats;
559         int error;
560
561         num_stats = collect_balloon_stats(sc);
562
563         sglist_init(&sg, 1, segs);
564         error = sglist_append(&sg, sc->vtballoon_stats, sizeof(sc->vtballoon_stats[0]) * num_stats);
565         KASSERT(error == 0, ("error adding page frames to sglist"));
566
567         error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
568         KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
569         virtqueue_notify(sc->vtballoon_stats_vq, NULL);
570 }
571
572 /*
573  * While most virtqueues communicate guest-initiated requests to the hypervisor,
574  * the stats queue operates in reverse.  The driver(host) initializes the virtqueue
575  * with a single buffer. From that point forward, all conversations consist of
576  * a hypervisor request (a call to this function) which directs us to refill
577  * the virtqueue with a fresh stats buffer. Since stats collection can sleep,
578  * we delegate the job to the vtballoon_thread which will do the actual stats
579  * collecting work.
580  */
581 static void
582 vtballoon_stats_vq_intr(void *arg)
583 {
584         struct vtballoon_softc *sc = arg;
585         struct virtqueue *vq = sc->vtballoon_stats_vq;
586
587         ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
588         if (sc->vtballoon_update_stats || !virtqueue_pending(vq))
589                 return;
590
591         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Ballooon Stats Requested\n");
592         sc->vtballoon_update_stats = true;
593         wakeup_one(sc);
594         virtqueue_dequeue(vq, NULL);
595 }
596
597 static void
598 vtballoon_inflate_vq_intr(void *arg)
599 {
600         struct vtballoon_softc *sc = arg;
601         struct virtqueue *vq = sc->vtballoon_inflate_vq;
602         ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
603         if (!virtqueue_pending(vq))
604                 return;
605         wakeup_one(sc);
606 }
607
608 static void
609 vtballoon_deflate_vq_intr(void *arg)
610 {
611         struct vtballoon_softc *sc = arg;
612         struct virtqueue *vq = sc->vtballoon_deflate_vq;
613         ASSERT_SERIALIZED(VTBALLOON_SLZ(sc));
614         if (!virtqueue_pending(vq))
615                 return;
616         wakeup_one(sc);
617 }
618
619 static void
620 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
621 {
622         struct virtqueue *vq;
623
624         vm_page_t m;
625         int i;
626
627         vq = sc->vtballoon_inflate_vq;
628
629         if (npages > VTBALLOON_PAGES_PER_REQUEST)
630                 npages = VTBALLOON_PAGES_PER_REQUEST;
631
632         for (i = 0; i < npages; i++) {
633                 if ((m = vtballoon_alloc_page(sc)) == NULL) {
634                         /* First allocate usign VTBALLOON_REGULAR_ALLOC and fall back to VTBALLOON_LOWMEM_ALLOC
635                          * when the guest is under severe memory pressure. Quickly decrease the
636                          * allocation rate, allowing the system to swap out pages.
637                          */
638                         sc->vtballoon_pagereq = VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT;
639                         sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
640                         break;
641                 }
642
643                 sc->vtballoon_page_frames[i] =
644                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
645
646                 KASSERT(m->queue == PQ_NONE,
647                     ("%s: allocated page %p on queue", __func__, m));
648                 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
649         }
650
651         if (i > 0)
652                 vtballoon_send_page_frames(sc, vq, i);
653 }
654
655 static void
656 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
657 {
658         TAILQ_HEAD(, vm_page) free_pages;
659         struct virtqueue *vq;
660         vm_page_t m;
661         int i;
662
663         vq = sc->vtballoon_deflate_vq;
664         TAILQ_INIT(&free_pages);
665
666         if (npages > VTBALLOON_PAGES_PER_REQUEST)
667                 npages = VTBALLOON_PAGES_PER_REQUEST;
668
669         for (i = 0; i < npages; i++) {
670                 m = TAILQ_FIRST(&sc->vtballoon_pages);
671                 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
672
673                 sc->vtballoon_page_frames[i] =
674                     VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
675
676                 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
677                 TAILQ_INSERT_TAIL(&free_pages, m, pageq);
678         }
679
680         if (i > 0) {
681                 /*
682                  * Note that if virtio VIRTIO_BALLOON_F_MUST_TELL_HOST
683                  * feature is true, we *have* to tell host first
684                  * before freeing the pages.
685                  */
686                 vtballoon_send_page_frames(sc, vq, i);
687
688                 while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
689                         TAILQ_REMOVE(&free_pages, m, pageq);
690                         vtballoon_free_page(sc, m);
691                 }
692         }
693
694         KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
695             sc->vtballoon_current_npages == 0) ||
696             (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
697             sc->vtballoon_current_npages != 0),
698             ("%s: bogus page count %d", __func__,
699             sc->vtballoon_current_npages));
700 }
701
702 static void
703 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
704     int npages)
705 {
706         struct sglist sg;
707         struct sglist_seg segs[1];
708         void *c;
709         int error;
710
711         sglist_init(&sg, 1, segs);
712
713         error = sglist_append(&sg, sc->vtballoon_page_frames,
714             npages * sizeof(uint32_t));
715         KASSERT(error == 0, ("error adding page frames to sglist"));
716
717         error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
718         KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
719         virtqueue_notify(vq, NULL);
720
721         /*
722          * Inflate and deflate operations are done synchronously. The
723          * interrupt handler will wake us up.
724          */
725         VTBALLOON_ENTER_SLZ(sc);
726         while ((c = virtqueue_dequeue(vq, NULL)) == NULL) {
727                 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbspf", 0);
728         }
729         VTBALLOON_EXIT_SLZ(sc);
730
731         KASSERT(c == vq, ("unexpected balloon operation response"));
732 }
733
734 static void
735 vtballoon_pop(struct vtballoon_softc *sc)
736 {
737         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Popping\n");
738
739         while (!TAILQ_EMPTY(&sc->vtballoon_pages))
740                 vtballoon_deflate(sc, sc->vtballoon_current_npages);
741 }
742
743 static void
744 vtballoon_stop(struct vtballoon_softc *sc)
745 {
746         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stopping\n");
747
748         virtqueue_disable_intr(sc->vtballoon_inflate_vq);
749         virtqueue_disable_intr(sc->vtballoon_deflate_vq);
750 /*
751         if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) {
752                 virtqueue_disable_intr(sc->vtballoon_stats_vq);
753         }
754 */
755         virtio_stop(sc->vtballoon_dev);
756 }
757
758 static vm_page_t
759 vtballoon_alloc_page(struct vtballoon_softc *sc)
760 {
761         vm_page_t m;
762
763         m = vm_page_alloc(NULL, 0, sc->vtballoon_pagereq);
764         if (m != NULL)
765                 sc->vtballoon_current_npages++;
766
767         return (m);
768 }
769
770 static void
771 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
772 {
773         vm_page_free_toq(m);
774         sc->vtballoon_current_npages--;
775 }
776
777 static uint32_t
778 vtballoon_desired_size(struct vtballoon_softc *sc)
779 {
780         uint32_t desired;
781
782         desired = virtio_read_dev_config_4(sc->vtballoon_dev,
783             offsetof(struct virtio_balloon_config, num_pages));
784
785         return (le32toh(desired));
786 }
787
788 static void
789 vtballoon_update_size(struct vtballoon_softc *sc)
790 {
791         virtio_write_dev_config_4(sc->vtballoon_dev,
792             offsetof(struct virtio_balloon_config, actual),
793             htole32(sc->vtballoon_current_npages));
794 }
795
796 static int
797 vtballoon_sleep(struct vtballoon_softc *sc)
798 {
799         int rc, timeout;
800         uint32_t current, desired;
801
802         rc = 0;
803         current = sc->vtballoon_current_npages;
804         sc->vtballoon_pagereq = VM_ALLOC_NORMAL | VM_ALLOC_INTERRUPT;
805
806         VTBALLOON_ENTER_SLZ(sc);
807         for (;;) {
808                 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
809                         rc = 1;
810                         break;
811                 }
812
813                 desired = vtballoon_desired_size(sc);
814                 if (desired != sc->vtballoon_desired_npages)
815                         vtballoon_dprintf(sc, VTBALLOON_DEBUG, "balloon %s %d -> %d (4K pages)\n",
816                                 desired < sc->vtballoon_desired_npages ? "deflating" : "inflating",
817                                 current, desired);
818
819                 sc->vtballoon_desired_npages = desired;
820
821                 /*
822                  * If given, use non-zero timeout on the first time through
823                  * the loop. On subsequent times, timeout will be zero so
824                  * we will reevaluate the desired size of the balloon and
825                  * break out to retry if needed.
826                  */
827                 timeout = sc->vtballoon_timeout;
828                 sc->vtballoon_timeout = 0;
829
830                 if (current > desired)
831                         break;
832                 else if (current < desired && timeout == 0)
833                         break;
834                 else if (sc->vtballoon_update_stats)
835                         break;
836                 else if (!timeout)
837                         vtballoon_dprintf(sc, VTBALLOON_TRACE, "balloon %d (4K pages) reached\n", current);
838
839                 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbslp", timeout);
840         }
841         VTBALLOON_EXIT_SLZ(sc);
842
843         return (rc);
844 }
845
846 static void
847 vtballoon_thread(void *arg)
848 {
849         struct vtballoon_softc *sc = arg;
850         vtballoon_dprintf(sc, VTBALLOON_TRACE, "Thread started.\n");
851
852         uint32_t current, desired;
853         for (;;) {
854                 if (vtballoon_sleep(sc) != 0)
855                         break;
856
857                 current = sc->vtballoon_current_npages;
858                 desired = sc->vtballoon_desired_npages;
859
860                 if (desired != current) {
861                         if (desired > current)
862                                 vtballoon_inflate(sc, desired - current);
863                         else
864                                 vtballoon_deflate(sc, current - desired);
865
866                         vtballoon_update_size(sc);
867                 }
868                 if (sc->vtballoon_update_stats) {
869                         vtballoon_update_stats(sc);
870                         sc->vtballoon_update_stats = false;
871                 }
872         }
873
874         kthread_exit();
875 }
876
877 static void
878 vtballoon_get_tunables(struct vtballoon_softc *sc)
879 {
880         char tmpstr[64];
881         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
882
883         TUNABLE_INT_FETCH("hw.vtballoon.debug_level", &sc->vtballoon_debug);
884
885         ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtballoon.%d.debug_level",
886             device_get_unit(sc->vtballoon_dev));
887         TUNABLE_INT_FETCH(tmpstr, &sc->vtballoon_debug);
888 }
889
890 static void
891 vtballoon_add_sysctl(struct vtballoon_softc *sc)
892 {
893         device_t dev;
894         struct sysctl_ctx_list *ctx;
895         struct sysctl_oid *tree;
896         struct sysctl_oid_list *child;
897
898         dev = sc->vtballoon_dev;
899         vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n");
900
901         ctx = device_get_sysctl_ctx(dev);
902         tree = device_get_sysctl_tree(dev);
903         child = SYSCTL_CHILDREN(tree);
904
905         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level",
906             CTLFLAG_RW, &sc->vtballoon_debug, 0,
907             "Debug level");
908
909         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
910             CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
911             "Desired balloon size in pages");
912
913         SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
914             CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
915             "Current balloon size in pages");
916 }