vm_page_queues[PQ_FREE+i].cnt = &vmstats.v_free_count;
for (i = 0; i < PQ_L2_SIZE; i++)
vm_page_queues[PQ_CACHE+i].cnt = &vmstats.v_cache_count;
-
- vm_page_queues[PQ_INACTIVE].cnt = &vmstats.v_inactive_count;
- vm_page_queues[PQ_ACTIVE].cnt = &vmstats.v_active_count;
- vm_page_queues[PQ_HOLD].cnt = &vmstats.v_active_count;
+ for (i = 0; i < PQ_L2_SIZE; i++)
+ vm_page_queues[PQ_INACTIVE+i].cnt = &vmstats.v_inactive_count;
+ for (i = 0; i < PQ_L2_SIZE; i++)
+ vm_page_queues[PQ_ACTIVE+i].cnt = &vmstats.v_active_count;
+ for (i = 0; i < PQ_L2_SIZE; i++)
+ vm_page_queues[PQ_HOLD+i].cnt = &vmstats.v_active_count;
/* PQ_NONE has no queue */
for (i = 0; i < PQ_COUNT; i++) {
if (m->queue - m->pc == PQ_FREE) {
_vm_page_queue_spin_lock(m);
_vm_page_rem_queue_spinlocked(m);
- _vm_page_add_queue_spinlocked(m, PQ_HOLD, 0);
+ _vm_page_add_queue_spinlocked(m, PQ_HOLD + m->pc, 0);
_vm_page_queue_spin_unlock(m);
}
vm_page_spin_unlock(m);
{
vm_page_spin_lock(m);
atomic_add_int(&m->hold_count, -1);
- if (m->hold_count == 0 && m->queue == PQ_HOLD) {
+ if (m->hold_count == 0 && m->queue - m->pc == PQ_HOLD) {
_vm_page_queue_spin_lock(m);
_vm_page_rem_queue_spinlocked(m);
_vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 0);
u_short oqueue;
vm_page_spin_lock(m);
- if (m->queue != PQ_ACTIVE) {
+ if (m->queue - m->pc != PQ_ACTIVE) {
_vm_page_queue_spin_lock(m);
oqueue = _vm_page_rem_queue_spinlocked(m);
/* page is left spinlocked, queue is unlocked */
if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;
- _vm_page_add_queue_spinlocked(m, PQ_ACTIVE, 0);
+ _vm_page_add_queue_spinlocked(m, PQ_ACTIVE + m->pc, 0);
}
_vm_page_and_queue_spin_unlock(m);
if (oqueue == PQ_CACHE || oqueue == PQ_FREE)
if (m->hold_count != 0) {
vm_page_flag_clear(m, PG_ZERO);
- _vm_page_add_queue_spinlocked(m, PQ_HOLD, 0);
+ _vm_page_add_queue_spinlocked(m, PQ_HOLD + m->pc, 0);
} else {
_vm_page_add_queue_spinlocked(m, PQ_FREE + m->pc, 0);
}
;
} else if (activate) {
vm_page_spin_lock(m);
- _vm_page_add_queue_spinlocked(m, PQ_ACTIVE, 0);
+ _vm_page_add_queue_spinlocked(m,
+ PQ_ACTIVE + m->pc, 0);
_vm_page_and_queue_spin_unlock(m);
} else {
vm_page_spin_lock(m);
vm_page_flag_clear(m, PG_WINATCFLS);
- _vm_page_add_queue_spinlocked(m, PQ_INACTIVE,
- 0);
+ _vm_page_add_queue_spinlocked(m,
+ PQ_INACTIVE + m->pc, 0);
++vm_swapcache_inactive_heuristic;
_vm_page_and_queue_spin_unlock(m);
}
/*
* Ignore if already inactive.
*/
- if (m->queue == PQ_INACTIVE)
+ if (m->queue - m->pc == PQ_INACTIVE)
return;
_vm_page_queue_spin_lock(m);
oqueue = _vm_page_rem_queue_spinlocked(m);
if (oqueue == PQ_CACHE)
mycpu->gd_cnt.v_reactivated++;
vm_page_flag_clear(m, PG_WINATCFLS);
- _vm_page_add_queue_spinlocked(m, PQ_INACTIVE, athead);
+ _vm_page_add_queue_spinlocked(m, PQ_INACTIVE + m->pc, athead);
if (athead == 0)
++vm_swapcache_inactive_heuristic;
}
* occassionally leave the page alone
*/
if ((dnw & 0x01F0) == 0 ||
- m->queue == PQ_INACTIVE ||
+ m->queue - m->pc == PQ_INACTIVE ||
m->queue - m->pc == PQ_CACHE
) {
if (m->act_count >= ACT_INIT)
}
db_printf("\n");
- db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
- vm_page_queues[PQ_ACTIVE].lcnt,
- vm_page_queues[PQ_INACTIVE].lcnt);
+ db_printf("PQ_ACTIVE:");
+ for(i=0;i<PQ_L2_SIZE;i++) {
+ db_printf(" %d", vm_page_queues[PQ_ACTIVE + i].lcnt);
+ }
+ db_printf("\n");
+
+ db_printf("PQ_INACTIVE:");
+ for(i=0;i<PQ_L2_SIZE;i++) {
+ db_printf(" %d", vm_page_queues[PQ_INACTIVE + i].lcnt);
+ }
+ db_printf("\n");
}
#endif /* DDB */
/* the kernel process "vm_pageout"*/
static int vm_pageout_clean (vm_page_t);
-static int vm_pageout_scan (int pass);
static int vm_pageout_free_page_calc (vm_size_t count);
struct thread *pagethread;
static freeer_fcn_t vm_pageout_object_deactivate_pages;
static void vm_req_vmdaemon (void);
#endif
-static void vm_pageout_page_stats(void);
+static void vm_pageout_page_stats(int q);
/*
* Update vm_load to slow down faulting processes.
}
vm_page_test_dirty(p);
if ((p->dirty & p->valid) == 0 ||
- p->queue != PQ_INACTIVE ||
+ p->queue - p->pc != PQ_INACTIVE ||
p->wire_count != 0 || /* may be held by buf cache */
p->hold_count != 0) { /* may be undergoing I/O */
vm_page_wakeup(p);
}
vm_page_test_dirty(p);
if ((p->dirty & p->valid) == 0 ||
- p->queue != PQ_INACTIVE ||
+ p->queue - p->pc != PQ_INACTIVE ||
p->wire_count != 0 || /* may be held by buf cache */
p->hold_count != 0) { /* may be undergoing I/O */
vm_page_wakeup(p);
}
vm_page_and_queue_spin_lock(p);
- if (p->queue != PQ_ACTIVE && (p->flags & PG_REFERENCED)) {
+ if (p->queue - p->pc != PQ_ACTIVE && (p->flags & PG_REFERENCED)) {
vm_page_and_queue_spin_unlock(p);
vm_page_activate(p);
p->act_count += actcount;
vm_page_flag_clear(p, PG_REFERENCED);
- } else if (p->queue == PQ_ACTIVE) {
+ } else if (p->queue - p->pc == PQ_ACTIVE) {
if ((p->flags & PG_REFERENCED) == 0) {
p->act_count -= min(p->act_count, ACT_DECLINE);
if (!info->limit &&
vm_page_protect(p, VM_PROT_NONE);
vm_page_deactivate(p);
} else {
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
+ TAILQ_REMOVE(&vm_page_queues[p->queue].pl,
+ p, pageq);
+ TAILQ_INSERT_TAIL(&vm_page_queues[p->queue].pl,
+ p, pageq);
vm_page_and_queue_spin_unlock(p);
}
} else {
vm_page_flag_clear(p, PG_REFERENCED);
vm_page_and_queue_spin_lock(p);
- if (p->queue == PQ_ACTIVE) {
+ if (p->queue - p->pc == PQ_ACTIVE) {
if (p->act_count < (ACT_MAX - ACT_ADVANCE))
p->act_count += ACT_ADVANCE;
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
+ TAILQ_REMOVE(&vm_page_queues[p->queue].pl,
+ p, pageq);
+ TAILQ_INSERT_TAIL(&vm_page_queues[p->queue].pl,
+ p, pageq);
}
vm_page_and_queue_spin_unlock(p);
}
- } else if (p->queue == PQ_INACTIVE) {
+ } else if (p->queue - p->pc == PQ_INACTIVE) {
vm_page_and_queue_spin_unlock(p);
vm_page_protect(p, VM_PROT_NONE);
} else {
static int vm_pageout_scan_callback(struct proc *p, void *data);
static int
-vm_pageout_scan(int pass)
+vm_pageout_scan_inactive(int pass, int q, int inactive_shortage,
+ int *vnodes_skippedp)
{
- struct vm_pageout_scan_info info;
vm_page_t m;
struct vm_page marker;
struct vnode *vpfailed; /* warning, allowed to be stale */
- int maxscan, pcount;
- int recycle_count;
- int inactive_shortage, active_shortage;
- int inactive_original_shortage;
+ int maxscan;
+ int delta = 0;
vm_object_t object;
int actcount;
- int vnodes_skipped = 0;
int maxlaunder;
/*
- * Do whatever cleanup that the pmap code can.
- */
- pmap_collect();
-
- /*
- * Calculate our target for the number of free+cache pages we
- * want to get to. This is higher then the number that causes
- * allocations to stall (severe) in order to provide hysteresis,
- * and if we don't make it all the way but get to the minimum
- * we're happy.
- */
- inactive_shortage = vm_paging_target() + vm_pageout_deficit;
- inactive_original_shortage = inactive_shortage;
- vm_pageout_deficit = 0;
-
- /*
* Start scanning the inactive queue for pages we can move to the
* cache or free. The scan will stop when the target is reached or
* we have scanned the entire inactive queue. Note that m->act_count
*/
bzero(&marker, sizeof(marker));
marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
- marker.queue = PQ_INACTIVE;
+ marker.queue = PQ_INACTIVE + q;
+ marker.pc = q;
marker.wire_count = 1;
/*
*/
vpfailed = NULL;
- vm_page_queues_spin_lock(PQ_INACTIVE);
- TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, &marker, pageq);
+ vm_page_queues_spin_lock(PQ_INACTIVE + q);
+ TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE + q].pl, &marker, pageq);
maxscan = vmstats.v_inactive_count;
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ vm_page_queues_spin_unlock(PQ_INACTIVE + q);
while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
- maxscan-- > 0 && inactive_shortage > 0)
+ maxscan-- > 0 && inactive_shortage - delta > 0)
{
vm_page_and_queue_spin_lock(m);
if (m != TAILQ_NEXT(&marker, pageq)) {
++maxscan;
continue;
}
- KKASSERT(m->queue == PQ_INACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl,
+ KKASSERT(m->queue - m->pc == PQ_INACTIVE);
+ TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE + q].pl,
&marker, pageq);
- TAILQ_INSERT_AFTER(&vm_page_queues[PQ_INACTIVE].pl, m,
+ TAILQ_INSERT_AFTER(&vm_page_queues[PQ_INACTIVE + q].pl, m,
&marker, pageq);
mycpu->gd_cnt.v_pdpages++;
continue;
}
vm_page_and_queue_spin_unlock(m);
- KKASSERT(m->queue == PQ_INACTIVE);
+ KKASSERT(m->queue - m->pc == PQ_INACTIVE);
lwkt_yield();
*/
if (m->hold_count) {
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_INACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl,
- m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl,
- m, pageq);
+ if (m->queue - m->pc == PQ_INACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
++vm_swapcache_inactive_heuristic;
*/
vm_pageout_page_free(m);
mycpu->gd_cnt.v_dfree++;
- --inactive_shortage;
+ ++delta;
} else if (m->dirty == 0) {
/*
* Clean pages can be placed onto the cache queue.
* This effectively frees them.
*/
vm_page_cache(m);
- --inactive_shortage;
+ ++delta;
} else if ((m->flags & PG_WINATCFLS) == 0 && pass == 0) {
/*
* Dirty pages need to be paged out, but flushing
*/
vm_page_flag_set(m, PG_WINATCFLS);
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_INACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+ if (m->queue - m->pc == PQ_INACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
++vm_swapcache_inactive_heuristic;
*/
if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) {
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_INACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+ if (m->queue - m->pc == PQ_INACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
++vm_swapcache_inactive_heuristic;
vpfailed = vp;
++pageout_lock_miss;
if (object->flags & OBJ_MIGHTBEDIRTY)
- vnodes_skipped++;
+ ++*vnodes_skippedp;
vm_page_unhold(m);
continue;
}
* reused for another vnode. The object might
* have been reused for another vnode.
*/
- if (m->queue != PQ_INACTIVE ||
+ if (m->queue - m->pc != PQ_INACTIVE ||
m->object != object ||
object->handle != vp) {
if (object->flags & OBJ_MIGHTBEDIRTY)
- vnodes_skipped++;
+ ++*vnodes_skippedp;
vput(vp);
vm_page_unhold(m);
continue;
*/
if (m->hold_count) {
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_INACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+ if (m->queue - m->pc == PQ_INACTIVE) {
+ TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE + q].pl, m, pageq);
+ TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE + q].pl, m, pageq);
}
vm_page_and_queue_spin_unlock(m);
++vm_swapcache_inactive_heuristic;
if (object->flags & OBJ_MIGHTBEDIRTY)
- vnodes_skipped++;
+ ++*vnodes_skippedp;
vm_page_wakeup(m);
vput(vp);
continue;
* pages.
*/
if (vm_pageout_clean(m) != 0) {
- --inactive_shortage;
+ ++delta;
--maxlaunder;
}
/* clean ate busy, page no longer accessible */
vm_page_wakeup(m);
}
}
- vm_page_queues_spin_lock(PQ_INACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, &marker, pageq);
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(PQ_INACTIVE + q);
+ TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE + q].pl, &marker, pageq);
+ vm_page_queues_spin_unlock(PQ_INACTIVE + q);
+
+ return (delta);
+}
+
+static int
+vm_pageout_scan_active(int pass, int q,
+ int inactive_shortage, int active_shortage,
+ int *recycle_countp)
+{
+ struct vm_page marker;
+ vm_page_t m;
+ int actcount;
+ int delta = 0;
+ int pcount;
/*
* We want to move pages from the active queue to the inactive
* NOTE: Both variables can end up negative.
* NOTE: We are still in a critical section.
*/
- active_shortage = vmstats.v_inactive_target - vmstats.v_inactive_count;
- if (inactive_original_shortage < vmstats.v_inactive_target / 10)
- inactive_original_shortage = vmstats.v_inactive_target / 10;
- if (inactive_shortage <= 0 &&
- active_shortage > inactive_original_shortage * 2) {
- active_shortage = inactive_original_shortage * 2;
- }
- recycle_count = 0;
- marker.queue = PQ_ACTIVE;
+ bzero(&marker, sizeof(marker));
+ marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
+ marker.queue = PQ_ACTIVE + q;
+ marker.pc = q;
+ marker.wire_count = 1;
- vm_page_queues_spin_lock(PQ_ACTIVE);
- TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE].pl, &marker, pageq);
- vm_page_queues_spin_unlock(PQ_ACTIVE);
+ vm_page_queues_spin_lock(PQ_ACTIVE + q);
+ TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
+ vm_page_queues_spin_unlock(PQ_ACTIVE + q);
pcount = vmstats.v_active_count;
while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
- pcount-- > 0 && (inactive_shortage > 0 || active_shortage > 0))
+ pcount-- > 0 && (inactive_shortage - delta > 0 ||
+ active_shortage > 0))
{
vm_page_and_queue_spin_lock(m);
if (m != TAILQ_NEXT(&marker, pageq)) {
++pcount;
continue;
}
- KKASSERT(m->queue == PQ_ACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl,
+ KKASSERT(m->queue - m->pc == PQ_ACTIVE);
+ TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl,
&marker, pageq);
- TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE].pl, m,
+ TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE + q].pl, m,
&marker, pageq);
/*
* busy them. (XXX why not?)
*/
if (m->hold_count != 0) {
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl,
+ TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl,
m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl,
+ TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE + q].pl,
m, pageq);
vm_page_and_queue_spin_unlock(m);
vm_page_wakeup(m);
*/
if (actcount && m->object->ref_count != 0) {
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_ACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
+ if (m->queue - m->pc == PQ_ACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
vm_page_wakeup(m);
* gigabytes being moved.
*/
--active_shortage;
- if (inactive_shortage > 0 ||
+ if (inactive_shortage - delta > 0 ||
m->object->ref_count == 0) {
- if (inactive_shortage > 0)
- ++recycle_count;
+ if (inactive_shortage - delta > 0)
+ ++*recycle_countp;
vm_page_protect(m, VM_PROT_NONE);
if (m->dirty == 0 &&
- inactive_shortage > 0) {
- --inactive_shortage;
+ inactive_shortage - delta > 0) {
+ ++delta;
vm_page_cache(m);
} else {
vm_page_deactivate(m);
}
} else {
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_ACTIVE) {
+ if (m->queue - m->pc == PQ_ACTIVE) {
TAILQ_REMOVE(
- &vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
TAILQ_INSERT_TAIL(
- &vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
vm_page_wakeup(m);
/*
* Clean out our local marker.
*/
- vm_page_queues_spin_lock(PQ_ACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, &marker, pageq);
- vm_page_queues_spin_unlock(PQ_ACTIVE);
+ vm_page_queues_spin_lock(PQ_ACTIVE + q);
+ TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
+ vm_page_queues_spin_unlock(PQ_ACTIVE + q);
+
+ return (delta);
+}
+
+/*
+ * The number of actually free pages can drop down to v_free_reserved,
+ * we try to build the free count back above v_free_min. Note that
+ * vm_paging_needed() also returns TRUE if v_free_count is not at
+ * least v_free_min so that is the minimum we must build the free
+ * count to.
+ *
+ * We use a slightly higher target to improve hysteresis,
+ * ((v_free_target + v_free_min) / 2). Since v_free_target
+ * is usually the same as v_cache_min this maintains about
+ * half the pages in the free queue as are in the cache queue,
+ * providing pretty good pipelining for pageout operation.
+ *
+ * The system operator can manipulate vm.v_cache_min and
+ * vm.v_free_target to tune the pageout demon. Be sure
+ * to keep vm.v_free_min < vm.v_free_target.
+ *
+ * Note that the original paging target is to get at least
+ * (free_min + cache_min) into (free + cache). The slightly
+ * higher target will shift additional pages from cache to free
+ * without effecting the original paging target in order to
+ * maintain better hysteresis and not have the free count always
+ * be dead-on v_free_min.
+ *
+ * NOTE: we are still in a critical section.
+ *
+ * Pages moved from PQ_CACHE to totally free are not counted in the
+ * pages_freed counter.
+ */
+static void
+vm_pageout_scan_cache(int inactive_shortage,
+ int vnodes_skipped, int recycle_count)
+{
+ struct vm_pageout_scan_info info;
+ vm_page_t m;
- /*
- * The number of actually free pages can drop down to v_free_reserved,
- * we try to build the free count back above v_free_min. Note that
- * vm_paging_needed() also returns TRUE if v_free_count is not at
- * least v_free_min so that is the minimum we must build the free
- * count to.
- *
- * We use a slightly higher target to improve hysteresis,
- * ((v_free_target + v_free_min) / 2). Since v_free_target
- * is usually the same as v_cache_min this maintains about
- * half the pages in the free queue as are in the cache queue,
- * providing pretty good pipelining for pageout operation.
- *
- * The system operator can manipulate vm.v_cache_min and
- * vm.v_free_target to tune the pageout demon. Be sure
- * to keep vm.v_free_min < vm.v_free_target.
- *
- * Note that the original paging target is to get at least
- * (free_min + cache_min) into (free + cache). The slightly
- * higher target will shift additional pages from cache to free
- * without effecting the original paging target in order to
- * maintain better hysteresis and not have the free count always
- * be dead-on v_free_min.
- *
- * NOTE: we are still in a critical section.
- *
- * Pages moved from PQ_CACHE to totally free are not counted in the
- * pages_freed counter.
- */
while (vmstats.v_free_count <
(vmstats.v_free_min + vmstats.v_free_target) / 2) {
/*
PRELE(info.bigproc);
}
}
- return(inactive_shortage);
}
/*
* helps the situation where paging just starts to occur.
*/
static void
-vm_pageout_page_stats(void)
+vm_pageout_page_stats(int q)
{
static int fullintervalcount = 0;
struct vm_page marker;
bzero(&marker, sizeof(marker));
marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
- marker.queue = PQ_ACTIVE;
+ marker.queue = PQ_ACTIVE + q;
+ marker.pc = q;
marker.wire_count = 1;
- vm_page_queues_spin_lock(PQ_ACTIVE);
- TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE].pl, &marker, pageq);
- vm_page_queues_spin_unlock(PQ_ACTIVE);
+ vm_page_queues_spin_lock(PQ_ACTIVE + q);
+ TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
+ vm_page_queues_spin_unlock(PQ_ACTIVE + q);
while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
pcount-- > 0)
++pcount;
continue;
}
- KKASSERT(m->queue == PQ_ACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, &marker, pageq);
- TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE].pl, m,
+ KKASSERT(m->queue - m->pc == PQ_ACTIVE);
+ TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
+ TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE + q].pl, m,
&marker, pageq);
/*
continue;
}
vm_page_and_queue_spin_unlock(m);
- KKASSERT(m->queue == PQ_ACTIVE);
+ KKASSERT(m->queue - m->pc == PQ_ACTIVE);
/*
* We now have a safely busied page, the page and queue
if (m->act_count > ACT_MAX)
m->act_count = ACT_MAX;
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_ACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
+ if (m->queue - m->pc == PQ_ACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
vm_page_wakeup(m);
} else {
m->act_count -= min(m->act_count, ACT_DECLINE);
vm_page_and_queue_spin_lock(m);
- if (m->queue == PQ_ACTIVE) {
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl,
- m, pageq);
+ if (m->queue - m->pc == PQ_ACTIVE) {
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
+ TAILQ_INSERT_TAIL(
+ &vm_page_queues[PQ_ACTIVE + q].pl,
+ m, pageq);
}
vm_page_and_queue_spin_unlock(m);
}
/*
* Remove our local marker
*/
- vm_page_queues_spin_lock(PQ_ACTIVE);
- TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, &marker, pageq);
- vm_page_queues_spin_unlock(PQ_ACTIVE);
-
+ vm_page_queues_spin_lock(PQ_ACTIVE + q);
+ TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
+ vm_page_queues_spin_unlock(PQ_ACTIVE + q);
}
static int
vm_pageout_thread(void)
{
int pass;
- int inactive_shortage;
+ int q;
/*
* Initialize some paging parameters.
*/
while (TRUE) {
int error;
+ int delta1;
+ int delta2;
+ int inactive_shortage;
+ int active_shortage;
+ int vnodes_skipped = 0;
+ int recycle_count = 0;
+ int tmp;
/*
* Wait for an action request. If we timeout check to
if (error &&
vm_paging_needed() == 0 &&
vm_pages_needed == 0) {
- vm_pageout_page_stats();
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q)
+ vm_pageout_page_stats(q);
continue;
}
vm_pages_needed = 1;
mycpu->gd_cnt.v_pdwakeups++;
/*
+ * Do whatever cleanup that the pmap code can.
+ */
+ pmap_collect();
+
+ /*
* Scan for pageout. Try to avoid thrashing the system
* with activity.
+ *
+ * Calculate our target for the number of free+cache pages we
+ * want to get to. This is higher then the number that causes
+ * allocations to stall (severe) in order to provide hysteresis,
+ * and if we don't make it all the way but get to the minimum
+ * we're happy.
+ */
+ inactive_shortage = vm_paging_target() + vm_pageout_deficit;
+ vm_pageout_deficit = 0;
+ delta1 = 0;
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ delta1 += vm_pageout_scan_inactive(
+ pass, q,
+ inactive_shortage / PQ_MAXL2_SIZE + 1,
+ &vnodes_skipped);
+ }
+
+ /*
+ * Figure out how many active pages we must deactivate. If
+ * we were able to reach our target with just the inactive
+ * scan above we limit the number of active pages we
+ * deactivate to reduce unnecessary work.
+ */
+ active_shortage = vmstats.v_inactive_target -
+ vmstats.v_inactive_count;
+
+ tmp = inactive_shortage;
+ if (tmp < vmstats.v_inactive_target / 10)
+ tmp = vmstats.v_inactive_target / 10;
+ inactive_shortage -= delta1;
+ if (inactive_shortage <= 0 && active_shortage > tmp * 2)
+ active_shortage = tmp * 2;
+
+ delta2 = 0;
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ delta2 += vm_pageout_scan_active(
+ pass, q,
+ inactive_shortage / PQ_MAXL2_SIZE + 1,
+ active_shortage / PQ_MAXL2_SIZE + 1,
+ &recycle_count);
+ }
+
+ /*
+ * Finally free enough cache pages to meet our free page
+ * requirement and take more drastic measures if we are
+ * still in trouble.
+ */
+ inactive_shortage -= delta2;
+ vm_pageout_scan_cache(inactive_shortage, vnodes_skipped,
+ recycle_count);
+
+ /*
+ * Wait for more work.
*/
- inactive_shortage = vm_pageout_scan(pass);
if (inactive_shortage > 0) {
++pass;
if (swap_pager_full) {
#include <sys/spinlock2.h>
#include <vm/vm_page2.h>
-#define INACTIVE_LIST (&vm_page_queues[PQ_INACTIVE].pl)
-
/* the kernel process "vm_pageout"*/
static int vm_swapcached_flush (vm_page_t m, int isblkdev);
static int vm_swapcache_test(vm_page_t m);
{
enum { SWAPC_WRITING, SWAPC_CLEANING } state = SWAPC_WRITING;
enum { SWAPB_BURSTING, SWAPB_RECOVERING } burst = SWAPB_BURSTING;
- struct vm_page page_marker;
- struct vm_object object_marker;
+ static struct vm_page page_marker[PQ_MAXL2_SIZE];
+ static struct vm_object object_marker;
+ int q;
/*
* Thread setup
* Initialize our marker for the inactive scan (SWAPC_WRITING)
*/
bzero(&page_marker, sizeof(page_marker));
- page_marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
- page_marker.queue = PQ_INACTIVE;
- page_marker.wire_count = 1;
-
- vm_page_queues_spin_lock(PQ_INACTIVE);
- TAILQ_INSERT_HEAD(INACTIVE_LIST, &page_marker, pageq);
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ page_marker[q].flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
+ page_marker[q].queue = PQ_INACTIVE + q;
+ page_marker[q].pc = q;
+ page_marker[q].wire_count = 1;
+ vm_page_queues_spin_lock(PQ_INACTIVE + q);
+ TAILQ_INSERT_HEAD(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ &page_marker[q], pageq);
+ vm_page_queues_spin_unlock(PQ_INACTIVE + q);
+ }
vm_swapcache_hysteresis = vmstats.v_inactive_target / 2;
vm_swapcache_inactive_heuristic = -vm_swapcache_hysteresis;
if (state == SWAPC_WRITING) {
if (vm_swapcache_curburst >= vm_swapcache_accrate) {
if (burst == SWAPB_BURSTING) {
- vm_swapcache_writing(&page_marker);
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ vm_swapcache_writing(
+ &page_marker[q]);
+ }
if (vm_swapcache_curburst <= 0)
burst = SWAPB_RECOVERING;
} else if (vm_swapcache_curburst >
vm_swapcache_minburst) {
- vm_swapcache_writing(&page_marker);
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ vm_swapcache_writing(
+ &page_marker[q]);
+ }
burst = SWAPB_BURSTING;
}
}
/*
* Cleanup (NOT REACHED)
*/
- vm_page_queues_spin_lock(PQ_INACTIVE);
- TAILQ_REMOVE(INACTIVE_LIST, &page_marker, pageq);
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ for (q = 0; q < PQ_MAXL2_SIZE; ++q) {
+ vm_page_queues_spin_lock(PQ_INACTIVE + q);
+ TAILQ_REMOVE(
+ &vm_page_queues[PQ_INACTIVE + q].pl,
+ &page_marker[q], pageq);
+ vm_page_queues_spin_unlock(PQ_INACTIVE + q);
+ }
lwkt_gettoken(&vmobj_token);
TAILQ_REMOVE(&vm_object_list, &object_marker, object_list);
*/
count = vm_swapcache_maxlaunder;
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
while ((m = TAILQ_NEXT(marker, pageq)) != NULL && count-- > 0) {
- KKASSERT(m->queue == PQ_INACTIVE);
+ KKASSERT(m->queue == marker->queue);
if (vm_swapcache_curburst < 0)
break;
- TAILQ_REMOVE(INACTIVE_LIST, marker, pageq);
- TAILQ_INSERT_AFTER(INACTIVE_LIST, m, marker, pageq);
+ TAILQ_REMOVE(
+ &vm_page_queues[marker->queue].pl, marker, pageq);
+ TAILQ_INSERT_AFTER(
+ &vm_page_queues[marker->queue].pl, m, marker, pageq);
if (m->flags & (PG_MARKER | PG_SWAPPED)) {
++count;
continue;
}
if (vm_page_busy_try(m, TRUE))
continue;
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ vm_page_queues_spin_unlock(marker->queue);
if ((object = m->object) == NULL) {
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
vm_object_hold(object);
if (m->object != object) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
if (vm_swapcache_test(m)) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
if (vp == NULL) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
if (m->flags & PG_NOTMETA) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
vm_swapcache_use_chflags)) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
if (vm_swapcache_maxfilesize &&
(vm_swapcache_maxfilesize >> PAGE_SHIFT)) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
isblkdev = 0;
if (m->flags & PG_NOTMETA) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
if (vm_swapcache_meta_enable == 0) {
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
isblkdev = 1;
default:
vm_object_drop(object);
vm_page_wakeup(m);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
continue;
}
* Setup for next loop using marker.
*/
vm_object_drop(object);
- vm_page_queues_spin_lock(PQ_INACTIVE);
+ vm_page_queues_spin_lock(marker->queue);
}
/*
*/
if (m == NULL)
vm_swapcache_inactive_heuristic = -vm_swapcache_hysteresis;
- vm_page_queues_spin_unlock(PQ_INACTIVE);
+ vm_page_queues_spin_unlock(marker->queue);
}
/*
lentferj's projects / dragonfly.git / commitdiff