From b396bb03905035e2d0e69f9993b2ccc47e77ae73 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sat, 23 Mar 2019 22:57:02 -0700 Subject: [PATCH] kernel - Refactor swapcache heuristic * Refactor the swapcache inactive queue heuristic to remove a write to a global variable that is in the critical path, and to improve operation. This should reduce cpu cache ping-ponging. * Change vpgqueues.lcnt from an int to a long, change misc use cases in the pageout code to a long. * Use __aligned(64) to 64-byte-align vm_page_queues[]. It was previously only 32-byte aligned. --- sys/vm/swap_pager.h | 1 - sys/vm/vm_page.c | 17 +++++++++-------- sys/vm/vm_page.h | 10 +++------- sys/vm/vm_pageout.c | 6 +----- sys/vm/vm_swapcache.c | 21 +++++++++++++-------- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/sys/vm/swap_pager.h b/sys/vm/swap_pager.h index 3bb1414a44..63904ca37f 100644 --- a/sys/vm/swap_pager.h +++ b/sys/vm/swap_pager.h @@ -96,7 +96,6 @@ extern swblk_t vm_swap_max; extern swblk_t vm_swap_cache_use; extern swblk_t vm_swap_anon_use; extern int vm_swapcache_read_enable; -extern int vm_swapcache_inactive_heuristic; extern int vm_swapcache_use_chflags; extern struct blist *swapblist; diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 21a3a4a9d8..630c39f6de 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -124,7 +124,7 @@ static void vm_numa_add_topology_mem(cpu_node_t *cpup, int physid, long bytes); /* * Array of tailq lists */ -__cachealign struct vpgqueues vm_page_queues[PQ_COUNT]; +struct vpgqueues vm_page_queues[PQ_COUNT]; static volatile int vm_pages_waiting; static struct alist vm_contig_alist; @@ -2796,7 +2796,6 @@ vm_page_unwire(vm_page_t m, int activate) PQ_INACTIVE + m->pc, 0); _vm_page_and_queue_spin_unlock(m); #endif - ++vm_swapcache_inactive_heuristic; } } } @@ -2831,8 +2830,10 @@ _vm_page_deactivate_locked(vm_page_t m, int athead) mycpu->gd_cnt.v_reactivated++; vm_page_flag_clear(m, PG_WINATCFLS); _vm_page_add_queue_spinlocked(m, PQ_INACTIVE + m->pc, athead); - if (athead == 0) - ++vm_swapcache_inactive_heuristic; + if (athead == 0) { + atomic_add_long( + &vm_page_queues[PQ_INACTIVE + m->pc].adds, 1); + } } /* NOTE: PQ_NONE if condition not taken */ _vm_page_queue_spin_unlock(m); @@ -3581,25 +3582,25 @@ DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) int i; db_printf("PQ_FREE:"); for (i = 0; i < PQ_L2_SIZE; i++) { - db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt); + db_printf(" %ld", vm_page_queues[PQ_FREE + i].lcnt); } db_printf("\n"); db_printf("PQ_CACHE:"); for(i = 0; i < PQ_L2_SIZE; i++) { - db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt); + db_printf(" %ld", vm_page_queues[PQ_CACHE + i].lcnt); } db_printf("\n"); db_printf("PQ_ACTIVE:"); for(i = 0; i < PQ_L2_SIZE; i++) { - db_printf(" %d", vm_page_queues[PQ_ACTIVE + i].lcnt); + db_printf(" %ld", vm_page_queues[PQ_ACTIVE + i].lcnt); } db_printf("\n"); db_printf("PQ_INACTIVE:"); for(i = 0; i < PQ_L2_SIZE; i++) { - db_printf(" %d", vm_page_queues[PQ_INACTIVE + i].lcnt); + db_printf(" %ld", vm_page_queues[PQ_INACTIVE + i].lcnt); } db_printf("\n"); } diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 550f2a3dbd..791c8b61a9 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -228,14 +228,10 @@ int rb_vm_page_scancmp(struct vm_page *, void *); struct vpgqueues { struct spinlock spin; struct pglist pl; + long lcnt; + long adds; /* heuristic, add operations */ int cnt_offset; /* offset into vmstats structure (int) */ - int lcnt; - int flipflop; /* probably not the best place */ - int unused00; - int unused01; - char unused[64 - sizeof(struct pglist) - - sizeof(int *) - sizeof(int) * 4]; -}; +} __aligned(64); extern struct vpgqueues vm_page_queues[PQ_COUNT]; diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 88d270dddd..3bd6a607f2 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -740,7 +740,7 @@ vm_pageout_scan_inactive(int pass, int q, long avail_shortage, vm_page_t m; struct vm_page marker; struct vnode *vpfailed; /* warning, allowed to be stale */ - int maxscan; + long maxscan; long delta = 0; long max_launder; int isep; @@ -958,7 +958,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp, &vm_page_queues[m->queue].pl, m, pageq); TAILQ_INSERT_TAIL( &vm_page_queues[m->queue].pl, m, pageq); - ++vm_swapcache_inactive_heuristic; } vm_page_and_queue_spin_unlock(m); vm_page_wakeup(m); @@ -1061,7 +1060,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp, &vm_page_queues[m->queue].pl, m, pageq); TAILQ_INSERT_TAIL( &vm_page_queues[m->queue].pl, m, pageq); - ++vm_swapcache_inactive_heuristic; } vm_page_and_queue_spin_unlock(m); vm_page_wakeup(m); @@ -1105,7 +1103,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp, TAILQ_INSERT_TAIL( &vm_page_queues[m->queue].pl, m, pageq); - ++vm_swapcache_inactive_heuristic; } vm_page_and_queue_spin_unlock(m); vm_page_wakeup(m); @@ -1216,7 +1213,6 @@ vm_pageout_page(vm_page_t m, long *max_launderp, long *vnodes_skippedp, if (m->queue - m->pc == PQ_INACTIVE) { TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq); TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, pageq); - ++vm_swapcache_inactive_heuristic; } vm_page_and_queue_spin_unlock(m); if (object->flags & OBJ_MIGHTBEDIRTY) diff --git a/sys/vm/vm_swapcache.c b/sys/vm/vm_swapcache.c index 0250320516..f58eef11af 100644 --- a/sys/vm/vm_swapcache.c +++ b/sys/vm/vm_swapcache.c @@ -92,7 +92,7 @@ struct thread *swapcached_thread; SYSCTL_NODE(_vm, OID_AUTO, swapcache, CTLFLAG_RW, NULL, NULL); int vm_swapcache_read_enable; -int vm_swapcache_inactive_heuristic; +static long vm_swapcache_wtrigger; static int vm_swapcache_sleep; static int vm_swapcache_maxscan = PQ_L2_SIZE * 8; static int vm_swapcache_maxlaunder = PQ_L2_SIZE * 4; @@ -204,7 +204,7 @@ vm_swapcached_thread(void) vm_swapcache_min_hysteresis = 1024; vm_swapcache_hysteresis = vm_swapcache_min_hysteresis; - vm_swapcache_inactive_heuristic = -vm_swapcache_hysteresis; + vm_swapcache_wtrigger = -vm_swapcache_hysteresis; /* * Initialize our marker for the vm_object scan (SWAPC_CLEANING) @@ -308,8 +308,7 @@ vm_swapcached_thread(void) burst = SWAPB_BURSTING; } if (reached_end == PQ_L2_SIZE) { - vm_swapcache_inactive_heuristic = - -vm_swapcache_hysteresis; + vm_swapcache_wtrigger = -vm_swapcache_hysteresis; } } @@ -351,6 +350,8 @@ static int vm_swapcache_writing_heuristic(void) { int hyst; + int q; + long adds; hyst = vmstats.v_inactive_count / 4; if (hyst < vm_swapcache_min_hysteresis) @@ -358,10 +359,14 @@ vm_swapcache_writing_heuristic(void) cpu_ccfence(); vm_swapcache_hysteresis = hyst; - if (vm_swapcache_inactive_heuristic < -hyst) - vm_swapcache_inactive_heuristic = -hyst; - - return (vm_swapcache_inactive_heuristic >= 0); + adds = 0; + for (q = PQ_INACTIVE; q < PQ_INACTIVE + PQ_L2_SIZE; ++q) { + adds += atomic_swap_long(&vm_page_queues[q].adds, 0); + } + vm_swapcache_wtrigger += adds; + if (vm_swapcache_wtrigger < -hyst) + vm_swapcache_wtrigger = -hyst; + return (vm_swapcache_wtrigger >= 0); } /* -- 2.41.0