kernel - Improve contiguous DMA memory allocation
authorMatthew Dillon <dillon@apollo.backplane.com>
Thu, 7 Dec 2017 03:09:20 +0000 (19:09 -0800)
committerMatthew Dillon <dillon@apollo.backplane.com>
Thu, 7 Dec 2017 03:09:20 +0000 (19:09 -0800)
* vm_page_alloc_contig() will use vm_page_alloc() for degenerate
  requests instead of the free page reserve.  This case often
  occurs when called from kmem_alloc_attr().

* vm_contig_pg_alloc() will scan the VM page queues first for
  unrestricted memory requests, falling back to vm_page_alloc_contig()
  if this fails.

  vm_contig_pg_alloc() will call vm_page_alloc_contig() first if the
  memory request has an address range restriction, falling back to
  a VM page queue scan if it fails.

* This should hopefully reduce instances where device drivers probed
  prior to e.g. NATA eat up all available low-memory DMA space,
  leaving none for NATA (or other drivers) that might need low-memory
  DMA space.

sys/vm/vm_contig.c
sys/vm/vm_kern.c
sys/vm/vm_page.c

index 6640448..4b75258 100644 (file)
 #include <sys/spinlock2.h>
 #include <vm/vm_page2.h>
 
+#include <machine/bus_dma.h>
+
 static void vm_contig_pg_free(vm_pindex_t start, u_long size);
 
 /*
@@ -269,11 +271,17 @@ vm_contig_pg_alloc(unsigned long size, vm_paddr_t low, vm_paddr_t high,
         * See if we can get the pages from the contiguous page reserve
         * alist.  The returned pages will be allocated and wired but not
         * busied.
+        *
+        * If high is not set to BUS_SPACE_MAXADDR we try using our
+        * free memory reserve first, otherwise we try it last.
         */
-       m = vm_page_alloc_contig(
-               low, high, alignment, boundary, size, VM_MEMATTR_DEFAULT);
-       if (m)
-               return (m - &pga[0]);
+       if (high != BUS_SPACE_MAXADDR) {
+               m = vm_page_alloc_contig(
+                       low, high, alignment, boundary,
+                       size, VM_MEMATTR_DEFAULT);
+               if (m)
+                       return (m - &pga[0]);
+       }
 
        /*
         * Three passes (0, 1, 2).  Each pass scans the VM page list for
@@ -295,7 +303,8 @@ again:
                        if (((pqtype == PQ_FREE) || (pqtype == PQ_CACHE)) &&
                            (phys >= low) && (phys < high) &&
                            ((phys & (alignment - 1)) == 0) &&
-                           (((phys ^ (phys + size - 1)) & ~(boundary - 1)) == 0) &&
+                           (((phys ^ (phys + size - 1)) & /* bitwise and */
+                            ~(boundary - 1)) == 0) &&
                            m->wire_count == 0 && m->hold_count == 0 &&
                            (m->busy_count &
                             (PBUSY_LOCKED | PBUSY_MASK)) == 0 &&
@@ -431,6 +440,18 @@ again:
                return (start); /* aka &pga[start] */
        }
 
+       /*
+        * Failed, if we haven't already tried, allocate from our reserved
+        * dma memory.
+        */
+       if (high == BUS_SPACE_MAXADDR) {
+               m = vm_page_alloc_contig(
+                       low, high, alignment, boundary,
+                       size, VM_MEMATTR_DEFAULT);
+               if (m)
+                       return (m - &pga[0]);
+       }
+
        /*
         * Failed.
         */
index f94a0a6..c44bd41 100644 (file)
@@ -436,7 +436,8 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, vm_subsys_t id,
        vm_map_entry_release(count);
        vm_object_drop(&kernel_object);
        for (i = 0; i < size; i += PAGE_SIZE) {
-               m = vm_page_alloc_contig(low, high, PAGE_SIZE, 0, PAGE_SIZE, memattr);
+               m = vm_page_alloc_contig(low, high, PAGE_SIZE, 0,
+                                        PAGE_SIZE, memattr);
                if (!m) {
                        return (0);
                }
index dd9a9a9..2fee368 100644 (file)
@@ -91,6 +91,7 @@
 #include <machine/inttypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
+#include <machine/bus_dma.h>
 
 #include <vm/vm_page2.h>
 #include <sys/spinlock2.h>
@@ -1966,6 +1967,7 @@ vm_page_alloc_contig(vm_paddr_t low, vm_paddr_t high,
        alist_blk_t blk;
        vm_page_t m;
        vm_pindex_t i;
+       static vm_pindex_t contig_rover;
 
        alignment >>= PAGE_SHIFT;
        if (alignment == 0)
@@ -1975,35 +1977,55 @@ vm_page_alloc_contig(vm_paddr_t low, vm_paddr_t high,
                boundary = 1;
        size = (size + PAGE_MASK) >> PAGE_SHIFT;
 
-       spin_lock(&vm_contig_spin);
-       blk = alist_alloc(&vm_contig_alist, 0, size);
-       if (blk == ALIST_BLOCK_NONE) {
-               spin_unlock(&vm_contig_spin);
-               if (bootverbose) {
-                       kprintf("vm_page_alloc_contig: %ldk nospace\n",
-                               (size + PAGE_MASK) * (PAGE_SIZE / 1024));
+       if (high == BUS_SPACE_MAXADDR && alignment <= PAGE_SIZE &&
+           boundary <= PAGE_SIZE && size == 1 &&
+           memattr == VM_MEMATTR_DEFAULT) {
+               /*
+                * Any page will work, use vm_page_alloc()
+                * (e.g. when used from kmem_alloc_attr())
+                */
+               m = vm_page_alloc(NULL, (contig_rover++) & 0x7FFFFFFF,
+                                 VM_ALLOC_NORMAL | VM_ALLOC_SYSTEM |
+                                 VM_ALLOC_INTERRUPT);
+               m->valid = VM_PAGE_BITS_ALL;
+               vm_page_wire(m);
+               vm_page_wakeup(m);
+       } else {
+               /*
+                * Use the low-memory dma reserve
+                */
+               spin_lock(&vm_contig_spin);
+               blk = alist_alloc(&vm_contig_alist, 0, size);
+               if (blk == ALIST_BLOCK_NONE) {
+                       spin_unlock(&vm_contig_spin);
+                       if (bootverbose) {
+                               kprintf("vm_page_alloc_contig: %ldk nospace\n",
+                                       (size << PAGE_SHIFT) / 1024);
+                               print_backtrace(5);
+                       }
+                       return(NULL);
                }
-               return(NULL);
-       }
-       if (high && ((vm_paddr_t)(blk + size) << PAGE_SHIFT) > high) {
-               alist_free(&vm_contig_alist, blk, size);
-               spin_unlock(&vm_contig_spin);
-               if (bootverbose) {
-                       kprintf("vm_page_alloc_contig: %ldk high "
-                               "%016jx failed\n",
-                               (size + PAGE_MASK) * (PAGE_SIZE / 1024),
-                               (intmax_t)high);
+               if (high && ((vm_paddr_t)(blk + size) << PAGE_SHIFT) > high) {
+                       alist_free(&vm_contig_alist, blk, size);
+                       spin_unlock(&vm_contig_spin);
+                       if (bootverbose) {
+                               kprintf("vm_page_alloc_contig: %ldk high "
+                                       "%016jx failed\n",
+                                       (size << PAGE_SHIFT) / 1024,
+                                       (intmax_t)high);
+                       }
+                       return(NULL);
                }
-               return(NULL);
+               spin_unlock(&vm_contig_spin);
+               m = PHYS_TO_VM_PAGE((vm_paddr_t)blk << PAGE_SHIFT);
        }
-       spin_unlock(&vm_contig_spin);
        if (vm_contig_verbose) {
-               kprintf("vm_page_alloc_contig: %016jx/%ldk\n",
-                       (intmax_t)(vm_paddr_t)blk << PAGE_SHIFT,
-                       (size + PAGE_MASK) * (PAGE_SIZE / 1024));
+               kprintf("vm_page_alloc_contig: %016jx/%ldk "
+                       "(%016jx-%016jx al=%lu bo=%lu pgs=%lu attr=%d\n",
+                       (intmax_t)m->phys_addr,
+                       (size << PAGE_SHIFT) / 1024,
+                       low, high, alignment, boundary, size, memattr);
        }
-
-       m = PHYS_TO_VM_PAGE((vm_paddr_t)blk << PAGE_SHIFT);
        if (memattr != VM_MEMATTR_DEFAULT) {
                for (i = 0;i < size; i++)
                        pmap_page_set_memattr(&m[i], memattr);