sys/dev/virtual/virtio/virtio/virtqueue.c

   1 /*-
   2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * $FreeBSD: src/sys/dev/virtio/virtqueue.c,v 1.2 2012/04/14 05:48:04 grehan Exp $
  27  */
  28
  29 /*
  30  * Implements the virtqueue interface as basically described
  31  * in the original VirtIO paper.
  32  */
  33
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/malloc.h>
  38 #include <sys/sglist.h>
  39 #include <sys/serialize.h>
  40 #include <vm/vm.h>
  41 #include <vm/pmap.h>
  42
  43 #include <machine/cpu.h>
  44 #include <machine/atomic.h>
  45 #include <sys/bus.h>
  46 #include <sys/rman.h>
  47
  48 #include "virtio.h"
  49 #include "virtqueue.h"
  50 #include "virtio_ring.h"
  51
  52 #include "virtio_bus_if.h"
  53
  54 struct virtqueue {
  55         device_t                 vq_dev;
  56         char                     vq_name[VIRTQUEUE_MAX_NAME_SZ];
  57         uint16_t                 vq_queue_index;
  58         uint16_t                 vq_nentries;
  59         uint32_t                 vq_flags;
  60
  61 #define VIRTQUEUE_FLAG_INDIRECT  0x0001
  62 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
  63
  64         int                      vq_alignment;
  65         int                      vq_ring_size;
  66         void                    *vq_ring_mem;
  67         int                      vq_max_indirect_size;
  68         int                      vq_indirect_mem_size;
  69
  70         struct vring             vq_ring;
  71         uint16_t                 vq_free_cnt;
  72         uint16_t                 vq_queued_cnt;
  73         /*
  74          * Head of the free chain in the descriptor table. If
  75          * there are no free descriptors, this will be set to
  76          * VQ_RING_DESC_CHAIN_END.
  77          */
  78         uint16_t                 vq_desc_head_idx;
  79         /*
  80          * Last consumed descriptor in the used table,
  81          * trails vq_ring.used->idx.
  82          */
  83         uint16_t                 vq_used_cons_idx;
  84
  85         struct vq_desc_extra {
  86                 void              *cookie;
  87                 struct vring_desc *indirect;
  88                 vm_paddr_t         indirect_paddr;
  89                 uint16_t           ndescs;
  90         } vq_descx[0];
  91 };
  92
  93 /*
  94  * The maximum virtqueue size is 2^15. Use that value as the end of
  95  * descriptor chain terminator since it will never be a valid index
  96  * in the descriptor table. This is used to verify we are correctly
  97  * handling vq_free_cnt.
  98  */
  99 #define VQ_RING_DESC_CHAIN_END 32768
 100
 101 #define VQASSERT(_vq, _exp, _msg, ...)                          \
 102     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,  \
 103         ##__VA_ARGS__))
 104
 105 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)                     \
 106     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,                \
 107         "invalid ring index: %d, max: %d", (_idx),              \
 108         (_vq)->vq_nentries)
 109
 110 #define VQ_RING_ASSERT_CHAIN_TERM(_vq)                          \
 111     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==                  \
 112         VQ_RING_DESC_CHAIN_END, "full ring terminated "         \
 113         "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
 114
 115 static int      virtqueue_init_indirect(struct virtqueue *vq, int);
 116 static void     virtqueue_free_indirect(struct virtqueue *vq);
 117 static void     virtqueue_init_indirect_list(struct virtqueue *,
 118                     struct vring_desc *);
 119
 120 static void     vq_ring_init(struct virtqueue *);
 121 static void     vq_ring_update_avail(struct virtqueue *, uint16_t);
 122 static uint16_t vq_ring_enqueue_segments(struct virtqueue *,
 123                     struct vring_desc *, uint16_t, struct sglist *, int, int);
 124 static int      vq_ring_use_indirect(struct virtqueue *, int);
 125 static void     vq_ring_enqueue_indirect(struct virtqueue *, void *,
 126                     struct sglist *, int, int);
 127 static int      vq_ring_must_notify_host(struct virtqueue *);
 128 static void     vq_ring_notify_host(struct virtqueue *);
 129 static void     vq_ring_free_chain(struct virtqueue *, uint16_t);
 130
 131 uint64_t
 132 virtqueue_filter_features(uint64_t features)
 133 {
 134         uint64_t mask;
 135
 136         mask = (1 << VIRTIO_TRANSPORT_F_START) - 1;
 137         mask |= VIRTIO_RING_F_INDIRECT_DESC;
 138         mask |= VIRTIO_RING_F_EVENT_IDX;
 139
 140         return (features & mask);
 141 }
 142
 143 int
 144 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
 145     vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
 146 {
 147         struct virtqueue *vq;
 148         int error;
 149
 150         *vqp = NULL;
 151         error = 0;
 152
 153         if (size == 0) {
 154                 device_printf(dev,
 155                     "virtqueue %d (%s) does not exist (size is zero)\n",
 156                     queue, info->vqai_name);
 157                 return (ENODEV);
 158         } else if (!powerof2(size)) {
 159                 device_printf(dev,
 160                     "virtqueue %d (%s) size is not a power of 2: %d\n",
 161                     queue, info->vqai_name, size);
 162                 return (ENXIO);
 163         } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
 164                 device_printf(dev, "virtqueue %d (%s) requested too many "
 165                     "indirect descriptors: %d, max %d\n",
 166                     queue, info->vqai_name, info->vqai_maxindirsz,
 167                     VIRTIO_MAX_INDIRECT);
 168                 return (EINVAL);
 169         }
 170
 171         vq = kmalloc(sizeof(struct virtqueue) +
 172             size * sizeof(struct vq_desc_extra), M_DEVBUF, M_INTWAIT | M_ZERO);
 173         if (vq == NULL) {
 174                 device_printf(dev, "cannot allocate virtqueue\n");
 175                 return (ENOMEM);
 176         }
 177
 178         vq->vq_dev = dev;
 179         strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
 180         vq->vq_queue_index = queue;
 181         vq->vq_alignment = align;
 182         vq->vq_nentries = size;
 183         vq->vq_free_cnt = size;
 184
 185         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
 186                 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
 187
 188         if (info->vqai_maxindirsz > 1) {
 189                 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
 190                 if (error)
 191                         goto fail;
 192         }
 193
 194         vq->vq_ring_size = round_page(vring_size(size, align));
 195         vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
 196             M_WAITOK | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
 197         if (vq->vq_ring_mem == NULL) {
 198                 device_printf(dev,
 199                     "cannot allocate memory for virtqueue ring\n");
 200                 error = ENOMEM;
 201                 goto fail;
 202         }
 203
 204         vq_ring_init(vq);
 205         virtqueue_disable_intr(vq);
 206
 207         *vqp = vq;
 208
 209 fail:
 210         if (error)
 211                 virtqueue_free(vq);
 212
 213         return (error);
 214 }
 215
 216 static int
 217 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
 218 {
 219         device_t dev;
 220         struct vq_desc_extra *dxp;
 221         int i, size;
 222
 223         dev = vq->vq_dev;
 224
 225         if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
 226                 /*
 227                  * Indirect descriptors requested by the driver but not
 228                  * negotiated. Return zero to keep the initialization
 229                  * going: we'll run fine without.
 230                  */
 231                 if (bootverbose)
 232                         device_printf(dev, "virtqueue %d (%s) requested "
 233                             "indirect descriptors but not negotiated\n",
 234                             vq->vq_queue_index, vq->vq_name);
 235                 return (0);
 236         }
 237
 238         size = indirect_size * sizeof(struct vring_desc);
 239         vq->vq_max_indirect_size = indirect_size;
 240         vq->vq_indirect_mem_size = size;
 241         vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
 242
 243         for (i = 0; i < vq->vq_nentries; i++) {
 244                 dxp = &vq->vq_descx[i];
 245
 246                 dxp->indirect = contigmalloc(size, M_DEVBUF, M_WAITOK,
 247                     0, BUS_SPACE_MAXADDR, 16, 0);
 248                 if (dxp->indirect == NULL) {
 249                         device_printf(dev, "cannot allocate indirect list\n");
 250                         return (ENOMEM);
 251                 }
 252
 253                 dxp->indirect_paddr = vtophys(dxp->indirect);
 254                 virtqueue_init_indirect_list(vq, dxp->indirect);
 255         }
 256
 257         return (0);
 258 }
 259
 260 static void
 261 virtqueue_free_indirect(struct virtqueue *vq)
 262 {
 263         struct vq_desc_extra *dxp;
 264         int i;
 265
 266         for (i = 0; i < vq->vq_nentries; i++) {
 267                 dxp = &vq->vq_descx[i];
 268
 269                 if (dxp->indirect == NULL)
 270                         break;
 271
 272                 contigfree(dxp->indirect, vq->vq_indirect_mem_size, M_DEVBUF);
 273                 dxp->indirect = NULL;
 274                 dxp->indirect_paddr = 0;
 275         }
 276
 277         vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
 278         vq->vq_indirect_mem_size = 0;
 279 }
 280
 281 static void
 282 virtqueue_init_indirect_list(struct virtqueue *vq,
 283     struct vring_desc *indirect)
 284 {
 285         int i;
 286
 287         bzero(indirect, vq->vq_indirect_mem_size);
 288
 289         for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
 290                 indirect[i].next = i + 1;
 291         indirect[i].next = VQ_RING_DESC_CHAIN_END;
 292 }
 293
 294 int
 295 virtqueue_reinit(struct virtqueue *vq, uint16_t size)
 296 {
 297         struct vq_desc_extra *dxp;
 298         int i;
 299
 300         if (vq->vq_nentries != size) {
 301                 device_printf(vq->vq_dev,
 302                     "%s: '%s' changed size; old=%hu, new=%hu\n",
 303                     __func__, vq->vq_name, vq->vq_nentries, size);
 304                 return (EINVAL);
 305         }
 306
 307         /* Warn if the virtqueue was not properly cleaned up. */
 308         if (vq->vq_free_cnt != vq->vq_nentries) {
 309                 device_printf(vq->vq_dev,
 310                     "%s: warning, '%s' virtqueue not empty, "
 311                     "leaking %d entries\n", __func__, vq->vq_name,
 312                     vq->vq_nentries - vq->vq_free_cnt);
 313         }
 314
 315         vq->vq_desc_head_idx = 0;
 316         vq->vq_used_cons_idx = 0;
 317         vq->vq_queued_cnt = 0;
 318         vq->vq_free_cnt = vq->vq_nentries;
 319
 320         /* To be safe, reset all our allocated memory. */
 321         bzero(vq->vq_ring_mem, vq->vq_ring_size);
 322         for (i = 0; i < vq->vq_nentries; i++) {
 323                 dxp = &vq->vq_descx[i];
 324                 dxp->cookie = NULL;
 325                 dxp->ndescs = 0;
 326                 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 327                         virtqueue_init_indirect_list(vq, dxp->indirect);
 328         }
 329
 330         vq_ring_init(vq);
 331         virtqueue_disable_intr(vq);
 332
 333         return (0);
 334 }
 335
 336 void
 337 virtqueue_free(struct virtqueue *vq)
 338 {
 339
 340         if (vq->vq_free_cnt != vq->vq_nentries) {
 341                 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
 342                     "leaking %d entries\n", vq->vq_name,
 343                     vq->vq_nentries - vq->vq_free_cnt);
 344         }
 345
 346         if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
 347                 virtqueue_free_indirect(vq);
 348
 349         if (vq->vq_ring_mem != NULL) {
 350                 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF);
 351                 vq->vq_ring_size = 0;
 352                 vq->vq_ring_mem = NULL;
 353         }
 354
 355         kfree(vq, M_DEVBUF);
 356 }
 357
 358 vm_paddr_t
 359 virtqueue_paddr(struct virtqueue *vq)
 360 {
 361         return (vtophys(vq->vq_ring_mem));
 362 }
 363
 364 int
 365 virtqueue_size(struct virtqueue *vq)
 366 {
 367         return (vq->vq_nentries);
 368 }
 369
 370 int
 371 virtqueue_empty(struct virtqueue *vq)
 372 {
 373
 374         return (vq->vq_nentries == vq->vq_free_cnt);
 375 }
 376
 377 int
 378 virtqueue_full(struct virtqueue *vq)
 379 {
 380
 381         return (vq->vq_free_cnt == 0);
 382 }
 383
 384 void
 385 virtqueue_notify(struct virtqueue *vq, lwkt_serialize_t interlock)
 386 {
 387         /* Ensure updated avail->idx is visible to host. */
 388         cpu_mfence();
 389
 390         if (vq_ring_must_notify_host(vq)) {
 391                 if (interlock != NULL)
 392                         lwkt_serialize_exit(interlock);
 393                 vq_ring_notify_host(vq);
 394                 if (interlock != NULL)
 395                         lwkt_serialize_enter(interlock);
 396         }
 397         vq->vq_queued_cnt = 0;
 398 }
 399
 400 int
 401 virtqueue_nused(struct virtqueue *vq)
 402 {
 403         uint16_t used_idx, nused;
 404
 405         used_idx = vq->vq_ring.used->idx;
 406         nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
 407         VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
 408
 409         return (nused);
 410 }
 411
 412 int
 413 virtqueue_pending(struct virtqueue *vq)
 414 {
 415         return (vq->vq_used_cons_idx != vq->vq_ring.used->idx);
 416 }
 417
 418 /*
 419  * Enable interrupts on a given virtqueue. Returns 1 if there are
 420  * additional entries to process on the virtqueue after we return.
 421  */
 422 int
 423 virtqueue_enable_intr(struct virtqueue *vq)
 424 {
 425         /*
 426          * Enable interrupts, making sure we get the latest
 427          * index of what's already been consumed.
 428          */
 429         vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 430         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 431                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx;
 432         } else {
 433                vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 434         }
 435
 436         cpu_mfence();
 437
 438         /*
 439          * Additional items may have been consumed in the time between
 440          * since we last checked and enabled interrupts above. Let our
 441          * caller know so it processes the new entries.
 442          */
 443         if (vq->vq_used_cons_idx != vq->vq_ring.used->idx)
 444                 return (1);
 445
 446         return (0);
 447 }
 448
 449 int
 450 virtqueue_postpone_intr(struct virtqueue *vq)
 451 {
 452         uint16_t ndesc;
 453
 454         /*
 455          * Postpone until at least half of the available descriptors
 456          * have been consumed.
 457          *
 458          * XXX Adaptive factor? (Linux uses 3/4)
 459          */
 460         ndesc = (uint16_t)(vq->vq_ring.avail->idx - vq->vq_used_cons_idx) / 2;
 461
 462         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
 463                 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
 464         else
 465                 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
 466
 467         cpu_mfence();
 468
 469         /*
 470          * Enough items may have already been consumed to meet our
 471          * threshold since we last checked. Let our caller know so
 472          * it processes the new entries.
 473          */
 474         if (virtqueue_nused(vq) > ndesc)
 475                 return (1);
 476
 477         return (0);
 478 }
 479
 480 void
 481 virtqueue_disable_intr(struct virtqueue *vq)
 482 {
 483         /*
 484          * Note this is only considered a hint to the host.
 485          */
 486         if ((vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) == 0)
 487                 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
 488 }
 489
 490 int
 491 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
 492     int readable, int writable)
 493 {
 494         struct vq_desc_extra *dxp;
 495         int needed;
 496         uint16_t head_idx, idx;
 497
 498         needed = readable + writable;
 499
 500         VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
 501         VQASSERT(vq, needed == sg->sg_nseg,
 502             "segment count mismatch, %d, %d", needed, sg->sg_nseg);
 503         VQASSERT(vq,
 504             needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
 505             "too many segments to enqueue: %d, %d/%d", needed,
 506             vq->vq_nentries, vq->vq_max_indirect_size);
 507
 508         if (needed < 1)
 509                 return (EINVAL);
 510         if (vq->vq_free_cnt == 0)
 511                 return (ENOSPC);
 512
 513         if (vq_ring_use_indirect(vq, needed)) {
 514                 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
 515                 return (0);
 516         } else if (vq->vq_free_cnt < needed)
 517                 return (EMSGSIZE);
 518
 519         head_idx = vq->vq_desc_head_idx;
 520         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 521         dxp = &vq->vq_descx[head_idx];
 522
 523         VQASSERT(vq, dxp->cookie == NULL,
 524             "cookie already exists for index %d", head_idx);
 525         dxp->cookie = cookie;
 526         dxp->ndescs = needed;
 527
 528         idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
 529             sg, readable, writable);
 530
 531         vq->vq_desc_head_idx = idx;
 532         vq->vq_free_cnt -= needed;
 533         if (vq->vq_free_cnt == 0)
 534                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 535         else
 536                 VQ_RING_ASSERT_VALID_IDX(vq, idx);
 537
 538         vq_ring_update_avail(vq, head_idx);
 539
 540         return (0);
 541 }
 542
 543 void *
 544 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
 545 {
 546         struct vring_used_elem *uep;
 547         void *cookie;
 548         uint16_t used_idx, desc_idx;
 549
 550         if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 551                 return (NULL);
 552
 553         used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
 554         uep = &vq->vq_ring.used->ring[used_idx];
 555
 556         cpu_lfence();
 557         desc_idx = (uint16_t) uep->id;
 558         if (len != NULL)
 559                 *len = uep->len;
 560
 561         vq_ring_free_chain(vq, desc_idx);
 562
 563         cookie = vq->vq_descx[desc_idx].cookie;
 564         VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
 565         vq->vq_descx[desc_idx].cookie = NULL;
 566
 567         return (cookie);
 568 }
 569
 570 void *
 571 virtqueue_poll(struct virtqueue *vq, uint32_t *len)
 572 {
 573         void *cookie;
 574
 575         /* We only poll the virtqueue when dumping to virtio-blk */
 576         while ((cookie = virtqueue_dequeue(vq, len)) == NULL)
 577                 ;
 578
 579         return (cookie);
 580 }
 581
 582 void *
 583 virtqueue_drain(struct virtqueue *vq, int *last)
 584 {
 585         void *cookie;
 586         int idx;
 587
 588         cookie = NULL;
 589         idx = *last;
 590
 591         while (idx < vq->vq_nentries && cookie == NULL) {
 592                 if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
 593                         vq->vq_descx[idx].cookie = NULL;
 594                         /* Free chain to keep free count consistent. */
 595                         vq_ring_free_chain(vq, idx);
 596                 }
 597                 idx++;
 598         }
 599
 600         *last = idx;
 601
 602         return (cookie);
 603 }
 604
 605 void
 606 virtqueue_dump(struct virtqueue *vq)
 607 {
 608
 609         if (vq == NULL)
 610                 return;
 611
 612         kprintf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
 613             "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
 614             "used.idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
 615             vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
 616             virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
 617             vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
 618             vq->vq_ring.used->idx, vq->vq_ring.avail->flags,
 619             vq->vq_ring.used->flags);
 620 }
 621
 622 static void
 623 vq_ring_init(struct virtqueue *vq)
 624 {
 625         struct vring *vr;
 626         char *ring_mem;
 627         int i, size;
 628
 629         ring_mem = vq->vq_ring_mem;
 630         size = vq->vq_nentries;
 631         vr = &vq->vq_ring;
 632
 633         vring_init(vr, size, ring_mem, vq->vq_alignment);
 634
 635         for (i = 0; i < size - 1; i++)
 636                 vr->desc[i].next = i + 1;
 637         vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
 638 }
 639
 640 static void
 641 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
 642 {
 643         uint16_t avail_idx;
 644
 645         /*
 646          * Place the head of the descriptor chain into the next slot and make
 647          * it usable to the host. The chain is made available now rather than
 648          * deferring to virtqueue_notify() in the hopes that if the host is
 649          * currently running on another CPU, we can keep it processing the new
 650          * descriptor.
 651          */
 652         avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
 653         vq->vq_ring.avail->ring[avail_idx] = desc_idx;
 654
 655         cpu_sfence();
 656         vq->vq_ring.avail->idx++;
 657
 658         /* Keep pending count until virtqueue_notify() for debugging. */
 659         vq->vq_queued_cnt++;
 660 }
 661
 662 static uint16_t
 663 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
 664     uint16_t head_idx, struct sglist *sg, int readable, int writable)
 665 {
 666         struct sglist_seg *seg;
 667         struct vring_desc *dp;
 668         int i, needed;
 669         uint16_t idx;
 670
 671         needed = readable + writable;
 672
 673         for (i = 0, idx = head_idx, seg = sg->sg_segs;
 674              i < needed;
 675              i++, idx = dp->next, seg++) {
 676                 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
 677                     "premature end of free desc chain");
 678
 679                 dp = &desc[idx];
 680                 dp->addr = seg->ss_paddr;
 681                 dp->len = seg->ss_len;
 682                 dp->flags = 0;
 683
 684                 if (i < needed - 1)
 685                         dp->flags |= VRING_DESC_F_NEXT;
 686                 if (i >= readable)
 687                         dp->flags |= VRING_DESC_F_WRITE;
 688         }
 689
 690         return (idx);
 691 }
 692
 693 static int
 694 vq_ring_use_indirect(struct virtqueue *vq, int needed)
 695 {
 696
 697         if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
 698                 return (0);
 699
 700         if (vq->vq_max_indirect_size < needed)
 701                 return (0);
 702
 703         if (needed < 2)
 704                 return (0);
 705
 706         return (1);
 707 }
 708
 709 static void
 710 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
 711     struct sglist *sg, int readable, int writable)
 712 {
 713         struct vring_desc *dp;
 714         struct vq_desc_extra *dxp;
 715         int needed;
 716         uint16_t head_idx;
 717
 718         needed = readable + writable;
 719         VQASSERT(vq, needed <= vq->vq_max_indirect_size,
 720             "enqueuing too many indirect descriptors");
 721
 722         head_idx = vq->vq_desc_head_idx;
 723         VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
 724         dp = &vq->vq_ring.desc[head_idx];
 725         dxp = &vq->vq_descx[head_idx];
 726
 727         VQASSERT(vq, dxp->cookie == NULL,
 728             "cookie already exists for index %d", head_idx);
 729         dxp->cookie = cookie;
 730         dxp->ndescs = 1;
 731
 732         dp->addr = dxp->indirect_paddr;
 733         dp->len = needed * sizeof(struct vring_desc);
 734         dp->flags = VRING_DESC_F_INDIRECT;
 735
 736         vq_ring_enqueue_segments(vq, dxp->indirect, 0,
 737             sg, readable, writable);
 738
 739         vq->vq_desc_head_idx = dp->next;
 740         vq->vq_free_cnt--;
 741         if (vq->vq_free_cnt == 0)
 742                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 743         else
 744                 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
 745
 746         vq_ring_update_avail(vq, head_idx);
 747 }
 748
 749 static int
 750 vq_ring_must_notify_host(struct virtqueue *vq)
 751 {
 752         uint16_t new_idx, prev_idx, event_idx;
 753
 754         if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
 755                 new_idx = vq->vq_ring.avail->idx;
 756                 prev_idx = new_idx - vq->vq_queued_cnt;
 757                 event_idx = vring_avail_event(&vq->vq_ring);
 758
 759                 return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
 760         }
 761
 762         return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
 763 }
 764
 765 static void
 766 vq_ring_notify_host(struct virtqueue *vq)
 767 {
 768         VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
 769 }
 770
 771 static void
 772 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
 773 {
 774         struct vring_desc *dp;
 775         struct vq_desc_extra *dxp;
 776
 777         VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
 778         dp = &vq->vq_ring.desc[desc_idx];
 779         dxp = &vq->vq_descx[desc_idx];
 780
 781         if (vq->vq_free_cnt == 0)
 782                 VQ_RING_ASSERT_CHAIN_TERM(vq);
 783
 784         vq->vq_free_cnt += dxp->ndescs;
 785         dxp->ndescs--;
 786
 787         if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
 788                 while (dp->flags & VRING_DESC_F_NEXT) {
 789                         VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
 790                         dp = &vq->vq_ring.desc[dp->next];
 791                         dxp->ndescs--;
 792                 }
 793         }
 794         VQASSERT(vq, dxp->ndescs == 0, "failed to free entire desc chain");
 795
 796         /*
 797          * We must append the existing free chain, if any, to the end of
 798          * newly freed chain. If the virtqueue was completely used, then
 799          * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
 800          */
 801         dp->next = vq->vq_desc_head_idx;
 802         vq->vq_desc_head_idx = desc_idx;
 803 }