From 635c9c159eb758c1328d8a014b7cb1d0255d00ab Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Sun, 5 Feb 2017 18:51:24 -0800 Subject: [PATCH] kernel - Refactor vm_page_alloc() to improve hot pages * Since we do not pre-zero pages any more, nothing uses PG_ZERO and we no longer have to use the head and tail of the free queue as an indication of a possibly freed page being zerod or not. For PQ_FREE, refactor the use case to always be LIFO, improving chances that vm_page_alloc() will retrieve a page that is already hot in the cache. * Should improve the host, and in particular will improve the vkernel, preventing it from unnecessarily cycling new pages (faulting them in on the host) when normal operation is in steady-state, and also reducing unnecessary initial/fresh faults. --- sys/vm/vm_page.c | 40 +++++++++++++++++----------------------- sys/vm/vm_page.h | 2 +- sys/vm/vm_pageout.c | 3 +-- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 863543907d..15c1a4faaf 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -837,7 +837,10 @@ _vm_page_rem_queue_spinlocked(vm_page_t m) } /* - * Helper function places the vm_page on the specified queue. + * Helper function places the vm_page on the specified queue. Generally + * speaking only PQ_FREE pages are placed at the head, to allow them to + * be allocated sooner rather than later on the assumption that they + * are cache-hot. * * The vm_page must be spinlocked. * This function will return with both the page and the queue locked. @@ -1129,7 +1132,7 @@ vm_page_unhold(vm_page_t m) if (m->hold_count == 0 && m->queue - m->pc == PQ_HOLD) { _vm_page_queue_spin_lock(m); _vm_page_rem_queue_spinlocked(m); - _vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 0); + _vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 1); _vm_page_queue_spin_unlock(m); } vm_page_spin_unlock(m); @@ -1572,17 +1575,12 @@ vm_page_unqueue(vm_page_t m) */ static __inline vm_page_t -_vm_page_list_find(int basequeue, int index, boolean_t prefer_zero) +_vm_page_list_find(int basequeue, int index) { vm_page_t m; for (;;) { - if (prefer_zero) { - m = TAILQ_LAST(&vm_page_queues[basequeue+index].pl, - pglist); - } else { - m = TAILQ_FIRST(&vm_page_queues[basequeue+index].pl); - } + m = TAILQ_FIRST(&vm_page_queues[basequeue+index].pl); if (m == NULL) { m = _vm_page_list_find2(basequeue, index); return(m); @@ -1647,9 +1645,9 @@ _vm_page_list_find2(int basequeue, int index) * Returns a spinlocked vm_page that has been removed from its queue. */ vm_page_t -vm_page_list_find(int basequeue, int index, boolean_t prefer_zero) +vm_page_list_find(int basequeue, int index) { - return(_vm_page_list_find(basequeue, index, prefer_zero)); + return(_vm_page_list_find(basequeue, index)); } /* @@ -1668,7 +1666,7 @@ vm_page_select_cache(u_short pg_color) vm_page_t m; for (;;) { - m = _vm_page_list_find(PQ_CACHE, pg_color & PQ_L2_MASK, FALSE); + m = _vm_page_list_find(PQ_CACHE, pg_color & PQ_L2_MASK); if (m == NULL) break; /* @@ -1706,20 +1704,19 @@ vm_page_select_cache(u_short pg_color) } /* - * Find a free or zero page, with specified preference. We attempt to - * inline the nominal case and fall back to _vm_page_select_free() - * otherwise. A busied page is removed from the queue and returned. + * Find a free page. We attempt to inline the nominal case and fall back + * to _vm_page_select_free() otherwise. A busied page is removed from + * the queue and returned. * * This routine may not block. */ static __inline vm_page_t -vm_page_select_free(u_short pg_color, boolean_t prefer_zero) +vm_page_select_free(u_short pg_color) { vm_page_t m; for (;;) { - m = _vm_page_list_find(PQ_FREE, pg_color & PQ_L2_MASK, - prefer_zero); + m = _vm_page_list_find(PQ_FREE, pg_color & PQ_L2_MASK); if (m == NULL) break; if (vm_page_busy_try(m, TRUE)) { @@ -1870,10 +1867,7 @@ loop: /* * The free queue has sufficient free pages to take one out. */ - if (page_req & (VM_ALLOC_ZERO | VM_ALLOC_FORCE_ZERO)) - m = vm_page_select_free(pg_color, TRUE); - else - m = vm_page_select_free(pg_color, FALSE); + m = vm_page_select_free(pg_color); } else if (page_req & VM_ALLOC_NORMAL) { /* * Allocatable from the cache (non-interrupt only). On @@ -2389,7 +2383,7 @@ vm_page_free_toq(vm_page_t m) if (m->hold_count != 0) { _vm_page_add_queue_spinlocked(m, PQ_HOLD + m->pc, 0); } else { - _vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 0); + _vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 1); } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index ce229904ef..120532804d 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -446,7 +446,7 @@ void vm_page_set_invalid (vm_page_t, int, int); int vm_page_is_valid (vm_page_t, int, int); void vm_page_test_dirty (vm_page_t); int vm_page_bits (int, int); -vm_page_t vm_page_list_find(int basequeue, int index, boolean_t prefer_zero); +vm_page_t vm_page_list_find(int basequeue, int index); void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid); void vm_page_free_toq(vm_page_t m); void vm_page_free_contig(vm_page_t m, unsigned long size); diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 41eaa9d4a2..60e6ace5af 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -1443,8 +1443,7 @@ vm_pageout_scan_cache(int avail_shortage, int pass, */ static int cache_rover = 0; - m = vm_page_list_find(PQ_CACHE, - cache_rover & PQ_L2_MASK, FALSE); + m = vm_page_list_find(PQ_CACHE, cache_rover & PQ_L2_MASK); if (m == NULL) break; /* page is returned removed from its queue and spinlocked */ -- 2.41.0