kernel - Refactor swapcache heuristic
authorMatthew Dillon <dillon@apollo.backplane.com>
Sun, 24 Mar 2019 05:57:02 +0000 (22:57 -0700)
committerMatthew Dillon <dillon@apollo.backplane.com>
Wed, 27 Mar 2019 03:32:47 +0000 (20:32 -0700)
* Refactor the swapcache inactive queue heuristic to remove a
  write to a global variable that is in the critical path, and
  to improve operation.  This should reduce cpu cache ping-ponging.

* Change vpgqueues.lcnt from an int to a long, change misc use
  cases in the pageout code to a long.

* Use __aligned(64) to 64-byte-align vm_page_queues[].  It was
  previously only 32-byte aligned.

sys/vm/swap_pager.h
sys/vm/vm_page.c
sys/vm/vm_page.h
sys/vm/vm_pageout.c
sys/vm/vm_swapcache.c

index 3bb1414..63904ca 100644 (file)
@@ -96,7 +96,6 @@ extern swblk_t vm_swap_max;
 extern swblk_t vm_swap_cache_use;
 extern swblk_t vm_swap_anon_use;
 extern int vm_swapcache_read_enable;
-extern int vm_swapcache_inactive_heuristic;
 extern int vm_swapcache_use_chflags;
 
 extern struct blist *swapblist;
index 21a3a4a..630c39f 100644 (file)
@@ -124,7 +124,7 @@ static void vm_numa_add_topology_mem(cpu_node_t *cpup, int physid, long bytes);
 /*
  * Array of tailq lists
  */
-__cachealign struct vpgqueues vm_page_queues[PQ_COUNT];
+struct vpgqueues vm_page_queues[PQ_COUNT];
 
 static volatile int vm_pages_waiting;
 static struct alist vm_contig_alist;
@@ -2796,7 +2796,6 @@ vm_page_unwire(vm_page_t m, int activate)
                                                        PQ_INACTIVE + m->pc, 0);
                                _vm_page_and_queue_spin_unlock(m);
 #endif
-                               ++vm_swapcache_inactive_heuristic;
                        }
                }
        }
@@ -2831,8 +2830,10 @@ _vm_page_deactivate_locked(vm_page_t m, int athead)
                        mycpu->gd_cnt.v_reactivated++;
                vm_page_flag_clear(m, PG_WINATCFLS);
                _vm_page_add_queue_spinlocked(m, PQ_INACTIVE + m->pc, athead);
-               if (athead == 0)
-                       ++vm_swapcache_inactive_heuristic;
+               if (athead == 0) {
+                       atomic_add_long(
+                               &vm_page_queues[PQ_INACTIVE + m->pc].adds, 1);
+               }
        }
        /* NOTE: PQ_NONE if condition not taken */
        _vm_page_queue_spin_unlock(m);
@@ -3581,25 +3582,25 @@ DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
        int i;
        db_printf("PQ_FREE:");
        for (i = 0; i < PQ_L2_SIZE; i++) {
-               db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
+               db_printf(" %ld", vm_page_queues[PQ_FREE + i].lcnt);
        }
        db_printf("\n");
                
        db_printf("PQ_CACHE:");
        for(i = 0; i < PQ_L2_SIZE; i++) {
-               db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
+               db_printf(" %ld", vm_page_queues[PQ_CACHE + i].lcnt);
        }
        db_printf("\n");
 
        db_printf("PQ_ACTIVE:");
        for(i = 0; i < PQ_L2_SIZE; i++) {
-               db_printf(" %d", vm_page_queues[PQ_ACTIVE + i].lcnt);
+               db_printf(" %ld", vm_page_queues[PQ_ACTIVE + i].lcnt);
        }
        db_printf("\n");
 
        db_printf("PQ_INACTIVE:");
        for(i = 0; i < PQ_L2_SIZE; i++) {
-               db_printf(" %d", vm_page_queues[PQ_INACTIVE + i].lcnt);
+               db_printf(" %ld", vm_page_queues[PQ_INACTIVE + i].lcnt);
        }
        db_printf("\n");
 }
index 550f2a3..791c8b6 100644 (file)
@@ -228,14 +228,10 @@ int rb_vm_page_scancmp(struct vm_page *, void *);
 struct vpgqueues {
        struct spinlock spin;
        struct pglist pl;
+       long    lcnt;
+       long    adds;           /* heuristic, add operations */
        int     cnt_offset;     /* offset into vmstats structure (int) */
-       int     lcnt;
-       int     flipflop;       /* probably not the best place */
-       int     unused00;
-       int     unused01;
-       char    unused[64 - sizeof(struct pglist) -
-                       sizeof(int *) - sizeof(int) * 4];
-};
+} __aligned(64);
 
 extern struct vpgqueues vm_page_queues[PQ_COUNT];
 
index 88d270d..3bd6a60 100644 (file)
@@ -740,7 +740,7 @@ vm_pageout_scan_inactive(int pass, int q, long avail_shortage,
        vm_page_t m;
        struct vm_page marker;
        struct vnode *vpfailed;         /* warning, allowed to be stale */
-       int maxscan;
+       long maxscan;
        long delta = 0;
        long max_launder;
        int isep;
@@ -958,7 +958,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp,
                                &vm_page_queues[m->queue].pl, m, pageq);
                        TAILQ_INSERT_TAIL(
                                &vm_page_queues[m->queue].pl, m, pageq);
-                       ++vm_swapcache_inactive_heuristic;
                }
                vm_page_and_queue_spin_unlock(m);
                vm_page_wakeup(m);
@@ -1061,7 +1060,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp,
                                &vm_page_queues[m->queue].pl, m, pageq);
                        TAILQ_INSERT_TAIL(
                                &vm_page_queues[m->queue].pl, m, pageq);
-                       ++vm_swapcache_inactive_heuristic;
                }
                vm_page_and_queue_spin_unlock(m);
                vm_page_wakeup(m);
@@ -1105,7 +1103,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp,
                                TAILQ_INSERT_TAIL(
                                    &vm_page_queues[m->queue].pl,
                                    m, pageq);
-                               ++vm_swapcache_inactive_heuristic;
                        }
                        vm_page_and_queue_spin_unlock(m);
                        vm_page_wakeup(m);
@@ -1216,7 +1213,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp,
                                if (m->queue - m->pc == PQ_INACTIVE) {
                                        TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq);
                                        TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, pageq);
-                                       ++vm_swapcache_inactive_heuristic;
                                }
                                vm_page_and_queue_spin_unlock(m);
                                if (object->flags & OBJ_MIGHTBEDIRTY)
index 0250320..f58eef1 100644 (file)
@@ -92,7 +92,7 @@ struct thread *swapcached_thread;
 SYSCTL_NODE(_vm, OID_AUTO, swapcache, CTLFLAG_RW, NULL, NULL);
 
 int vm_swapcache_read_enable;
-int vm_swapcache_inactive_heuristic;
+static long vm_swapcache_wtrigger;
 static int vm_swapcache_sleep;
 static int vm_swapcache_maxscan = PQ_L2_SIZE * 8;
 static int vm_swapcache_maxlaunder = PQ_L2_SIZE * 4;
@@ -204,7 +204,7 @@ vm_swapcached_thread(void)
 
        vm_swapcache_min_hysteresis = 1024;
        vm_swapcache_hysteresis = vm_swapcache_min_hysteresis;
-       vm_swapcache_inactive_heuristic = -vm_swapcache_hysteresis;
+       vm_swapcache_wtrigger = -vm_swapcache_hysteresis;
 
        /*
         * Initialize our marker for the vm_object scan (SWAPC_CLEANING)
@@ -308,8 +308,7 @@ vm_swapcached_thread(void)
                        burst = SWAPB_BURSTING;
                }
                if (reached_end == PQ_L2_SIZE) {
-                       vm_swapcache_inactive_heuristic =
-                               -vm_swapcache_hysteresis;
+                       vm_swapcache_wtrigger = -vm_swapcache_hysteresis;
                }
        }
 
@@ -351,6 +350,8 @@ static int
 vm_swapcache_writing_heuristic(void)
 {
        int hyst;
+       int q;
+       long adds;
 
        hyst = vmstats.v_inactive_count / 4;
        if (hyst < vm_swapcache_min_hysteresis)
@@ -358,10 +359,14 @@ vm_swapcache_writing_heuristic(void)
        cpu_ccfence();
        vm_swapcache_hysteresis = hyst;
 
-       if (vm_swapcache_inactive_heuristic < -hyst)
-               vm_swapcache_inactive_heuristic = -hyst;
-
-       return (vm_swapcache_inactive_heuristic >= 0);
+       adds = 0;
+       for (q = PQ_INACTIVE; q < PQ_INACTIVE + PQ_L2_SIZE; ++q) {
+               adds += atomic_swap_long(&vm_page_queues[q].adds, 0);
+       }
+       vm_swapcache_wtrigger += adds;
+       if (vm_swapcache_wtrigger < -hyst)
+               vm_swapcache_wtrigger = -hyst;
+       return (vm_swapcache_wtrigger >= 0);
 }
 
 /*