kernel - Cleanup token code, add simple exclusive priority
[dragonfly.git] / sys / vm / vm_pageout.c
CommitLineData
984263bc
MD
1/*
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
dc71b7ab 20 * 3. Neither the name of the University nor the names of its contributors
984263bc
MD
21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * from: @(#)vm_pageout.c 7.4 (Berkeley) 5/7/91
37 *
38 *
39 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40 * All rights reserved.
41 *
42 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43 *
44 * Permission to use, copy, modify and distribute this software and
45 * its documentation is hereby granted, provided that both the copyright
46 * notice and this permission notice appear in all copies of the
47 * software, derivative works or modified versions, and any portions
48 * thereof, and that both notices appear in supporting documentation.
49 *
50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53 *
54 * Carnegie Mellon requests users of this software to return to
55 *
56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
57 * School of Computer Science
58 * Carnegie Mellon University
59 * Pittsburgh PA 15213-3890
60 *
61 * any improvements or extensions that they make and grant Carnegie the
62 * rights to redistribute these changes.
63 *
64 * $FreeBSD: src/sys/vm/vm_pageout.c,v 1.151.2.15 2002/12/29 18:21:04 dillon Exp $
65 */
66
67/*
68 * The proverbial page-out daemon.
69 */
70
71#include "opt_vm.h"
72#include <sys/param.h>
73#include <sys/systm.h>
74#include <sys/kernel.h>
75#include <sys/proc.h>
76#include <sys/kthread.h>
77#include <sys/resourcevar.h>
78#include <sys/signalvar.h>
79#include <sys/vnode.h>
80#include <sys/vmmeter.h>
32c821cf 81#include <sys/conf.h>
984263bc
MD
82#include <sys/sysctl.h>
83
84#include <vm/vm.h>
85#include <vm/vm_param.h>
86#include <sys/lock.h>
87#include <vm/vm_object.h>
88#include <vm/vm_page.h>
89#include <vm/vm_map.h>
90#include <vm/vm_pageout.h>
91#include <vm/vm_pager.h>
92#include <vm/swap_pager.h>
93#include <vm/vm_extern.h>
5fd012e0
MD
94
95#include <sys/thread2.h>
b12defdc 96#include <sys/spinlock2.h>
12e4aaff 97#include <vm/vm_page2.h>
984263bc
MD
98
99/*
100 * System initialization
101 */
102
103/* the kernel process "vm_pageout"*/
534ee349
MD
104static int vm_pageout_page(vm_page_t m, int *max_launderp,
105 int *vnodes_skippedp, struct vnode **vpfailedp,
106 int pass, int vmflush_flags);
107static int vm_pageout_clean_helper (vm_page_t, int);
1388df65 108static int vm_pageout_free_page_calc (vm_size_t count);
534ee349 109static void vm_pageout_page_free(vm_page_t m) ;
32c821cf 110struct thread *emergpager;
bc6dffab 111struct thread *pagethread;
c3feb36a 112static int sequence_emerg_pager;
984263bc 113
984263bc
MD
114#if !defined(NO_SWAPPING)
115/* the kernel process "vm_daemon"*/
1388df65 116static void vm_daemon (void);
bc6dffab 117static struct thread *vmthread;
984263bc
MD
118
119static struct kproc_desc vm_kp = {
120 "vmdaemon",
121 vm_daemon,
bc6dffab 122 &vmthread
984263bc 123};
f3f3eadb 124SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
984263bc
MD
125#endif
126
534ee349
MD
127int vm_pages_needed = 0; /* Event on which pageout daemon sleeps */
128int vm_pageout_deficit = 0; /* Estimated number of pages deficit */
129int vm_pageout_pages_needed = 0;/* pageout daemon needs pages */
77d1fb91 130int vm_page_free_hysteresis = 16;
c3feb36a 131static int vm_pagedaemon_time;
984263bc
MD
132
133#if !defined(NO_SWAPPING)
534ee349 134static int vm_pageout_req_swapout;
984263bc
MD
135static int vm_daemon_needed;
136#endif
79ce07d0 137static int vm_max_launder = 4096;
ead23175 138static int vm_emerg_launder = 100;
984263bc
MD
139static int vm_pageout_stats_max=0, vm_pageout_stats_interval = 0;
140static int vm_pageout_full_stats_interval = 0;
141static int vm_pageout_stats_free_max=0, vm_pageout_algorithm=0;
142static int defer_swap_pageouts=0;
143static int disable_swap_pageouts=0;
38587548
MD
144static u_int vm_anonmem_decline = ACT_DECLINE;
145static u_int vm_filemem_decline = ACT_DECLINE * 2;
984263bc
MD
146
147#if defined(NO_SWAPPING)
148static int vm_swap_enabled=0;
149static int vm_swap_idle_enabled=0;
150#else
151static int vm_swap_enabled=1;
152static int vm_swap_idle_enabled=0;
153#endif
486b807a 154int vm_pageout_memuse_mode=1; /* 0-disable, 1-passive, 2-active swp*/
984263bc 155
38587548
MD
156SYSCTL_UINT(_vm, VM_PAGEOUT_ALGORITHM, anonmem_decline,
157 CTLFLAG_RW, &vm_anonmem_decline, 0, "active->inactive anon memory");
158
159SYSCTL_INT(_vm, VM_PAGEOUT_ALGORITHM, filemem_decline,
160 CTLFLAG_RW, &vm_filemem_decline, 0, "active->inactive file cache");
984263bc 161
77d1fb91
MD
162SYSCTL_INT(_vm, OID_AUTO, page_free_hysteresis,
163 CTLFLAG_RW, &vm_page_free_hysteresis, 0,
164 "Free more pages than the minimum required");
165
984263bc
MD
166SYSCTL_INT(_vm, OID_AUTO, max_launder,
167 CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
ead23175
MD
168SYSCTL_INT(_vm, OID_AUTO, emerg_launder,
169 CTLFLAG_RW, &vm_emerg_launder, 0, "Emergency pager minimum");
984263bc
MD
170
171SYSCTL_INT(_vm, OID_AUTO, pageout_stats_max,
172 CTLFLAG_RW, &vm_pageout_stats_max, 0, "Max pageout stats scan length");
173
174SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats_interval,
175 CTLFLAG_RW, &vm_pageout_full_stats_interval, 0, "Interval for full stats scan");
176
177SYSCTL_INT(_vm, OID_AUTO, pageout_stats_interval,
178 CTLFLAG_RW, &vm_pageout_stats_interval, 0, "Interval for partial stats scan");
179
180SYSCTL_INT(_vm, OID_AUTO, pageout_stats_free_max,
181 CTLFLAG_RW, &vm_pageout_stats_free_max, 0, "Not implemented");
486b807a
MD
182SYSCTL_INT(_vm, OID_AUTO, pageout_memuse_mode,
183 CTLFLAG_RW, &vm_pageout_memuse_mode, 0, "memoryuse resource mode");
984263bc
MD
184
185#if defined(NO_SWAPPING)
186SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
187 CTLFLAG_RD, &vm_swap_enabled, 0, "");
188SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
189 CTLFLAG_RD, &vm_swap_idle_enabled, 0, "");
190#else
191SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
192 CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
193SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
194 CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
195#endif
196
197SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
198 CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
199
200SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
201 CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
202
203static int pageout_lock_miss;
204SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
205 CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
206
984263bc
MD
207int vm_page_max_wired; /* XXX max # of wired pages system-wide */
208
209#if !defined(NO_SWAPPING)
1388df65 210static void vm_req_vmdaemon (void);
984263bc 211#endif
027193eb 212static void vm_pageout_page_stats(int q);
984263bc 213
9cd626ca
MD
214/*
215 * Calculate approximately how many pages on each queue to try to
216 * clean. An exact calculation creates an edge condition when the
217 * queues are unbalanced so add significant slop. The queue scans
218 * will stop early when targets are reached and will start where they
219 * left off on the next pass.
1f79526e
MD
220 *
221 * We need to be generous here because there are all sorts of loading
222 * conditions that can cause edge cases if try to average over all queues.
223 * In particular, storage subsystems have become so fast that paging
224 * activity can become quite frantic. Eventually we will probably need
225 * two paging threads, one for dirty pages and one for clean, to deal
226 * with the bandwidth requirements.
227
228 * So what we do is calculate a value that can be satisfied nominally by
229 * only having to scan half the queues.
9cd626ca 230 */
51c99c61
MD
231static __inline int
232PQAVERAGE(int n)
233{
9cd626ca
MD
234 int avg;
235
236 if (n >= 0) {
1f79526e 237 avg = ((n + (PQ_L2_SIZE - 1)) / (PQ_L2_SIZE / 2) + 1);
9cd626ca 238 } else {
1f79526e 239 avg = ((n - (PQ_L2_SIZE - 1)) / (PQ_L2_SIZE / 2) - 1);
9cd626ca
MD
240 }
241 return avg;
51c99c61
MD
242}
243
984263bc 244/*
534ee349 245 * vm_pageout_clean_helper:
984263bc 246 *
95270b7e
MD
247 * Clean the page and remove it from the laundry. The page must be busied
248 * by the caller and will be disposed of (put away, flushed) by this routine.
984263bc 249 */
984263bc 250static int
534ee349 251vm_pageout_clean_helper(vm_page_t m, int vmflush_flags)
984263bc 252{
5f910b2f 253 vm_object_t object;
79ce07d0 254 vm_page_t mc[BLIST_MAX_ALLOC];
b12defdc 255 int error;
984263bc
MD
256 int ib, is, page_base;
257 vm_pindex_t pindex = m->pindex;
258
259 object = m->object;
260
261 /*
95270b7e 262 * Don't mess with the page if it's held or special.
b12defdc
MD
263 *
264 * XXX do we really need to check hold_count here? hold_count
265 * isn't supposed to mess with vm_page ops except prevent the
266 * page from being reused.
984263bc 267 */
b12defdc
MD
268 if (m->hold_count != 0 || (m->flags & PG_UNMANAGED)) {
269 vm_page_wakeup(m);
984263bc
MD
270 return 0;
271 }
272
79ce07d0
MD
273 /*
274 * Place page in cluster. Align cluster for optimal swap space
275 * allocation (whether it is swap or not). This is typically ~16-32
276 * pages, which also tends to align the cluster to multiples of the
277 * filesystem block size if backed by a filesystem.
278 */
279 page_base = pindex % BLIST_MAX_ALLOC;
280 mc[page_base] = m;
281 ib = page_base - 1;
282 is = page_base + 1;
984263bc
MD
283
284 /*
285 * Scan object for clusterable pages.
286 *
287 * We can cluster ONLY if: ->> the page is NOT
288 * clean, wired, busy, held, or mapped into a
289 * buffer, and one of the following:
290 * 1) The page is inactive, or a seldom used
291 * active page.
292 * -or-
293 * 2) we force the issue.
294 *
295 * During heavy mmap/modification loads the pageout
296 * daemon can really fragment the underlying file
297 * due to flushing pages out of order and not trying
298 * align the clusters (which leave sporatic out-of-order
299 * holes). To solve this problem we do the reverse scan
300 * first and attempt to align our cluster, then do a
301 * forward scan if room remains.
302 */
398c240d 303 vm_object_hold(object);
534ee349 304
79ce07d0 305 while (ib >= 0) {
984263bc
MD
306 vm_page_t p;
307
79ce07d0
MD
308 p = vm_page_lookup_busy_try(object, pindex - page_base + ib,
309 TRUE, &error);
310 if (error || p == NULL)
984263bc 311 break;
b12defdc
MD
312 if ((p->queue - p->pc) == PQ_CACHE ||
313 (p->flags & PG_UNMANAGED)) {
314 vm_page_wakeup(p);
984263bc
MD
315 break;
316 }
317 vm_page_test_dirty(p);
9bf025db
MD
318 if (((p->dirty & p->valid) == 0 &&
319 (p->flags & PG_NEED_COMMIT) == 0) ||
984263bc
MD
320 p->wire_count != 0 || /* may be held by buf cache */
321 p->hold_count != 0) { /* may be undergoing I/O */
b12defdc 322 vm_page_wakeup(p);
984263bc
MD
323 break;
324 }
534ee349
MD
325 if (p->queue - p->pc != PQ_INACTIVE) {
326 if (p->queue - p->pc != PQ_ACTIVE ||
327 (vmflush_flags & VM_PAGER_ALLOW_ACTIVE) == 0) {
328 vm_page_wakeup(p);
329 break;
330 }
331 }
332
333 /*
334 * Try to maintain page groupings in the cluster.
335 */
336 if (m->flags & PG_WINATCFLS)
337 vm_page_flag_set(p, PG_WINATCFLS);
338 else
339 vm_page_flag_clear(p, PG_WINATCFLS);
340 p->act_count = m->act_count;
341
79ce07d0
MD
342 mc[ib] = p;
343 --ib;
984263bc 344 }
79ce07d0 345 ++ib; /* fixup */
984263bc 346
79ce07d0
MD
347 while (is < BLIST_MAX_ALLOC &&
348 pindex - page_base + is < object->size) {
984263bc
MD
349 vm_page_t p;
350
79ce07d0
MD
351 p = vm_page_lookup_busy_try(object, pindex - page_base + is,
352 TRUE, &error);
b12defdc 353 if (error || p == NULL)
984263bc
MD
354 break;
355 if (((p->queue - p->pc) == PQ_CACHE) ||
534ee349 356 (p->flags & PG_UNMANAGED)) {
b12defdc 357 vm_page_wakeup(p);
984263bc
MD
358 break;
359 }
360 vm_page_test_dirty(p);
9bf025db
MD
361 if (((p->dirty & p->valid) == 0 &&
362 (p->flags & PG_NEED_COMMIT) == 0) ||
984263bc
MD
363 p->wire_count != 0 || /* may be held by buf cache */
364 p->hold_count != 0) { /* may be undergoing I/O */
b12defdc 365 vm_page_wakeup(p);
984263bc
MD
366 break;
367 }
534ee349
MD
368 if (p->queue - p->pc != PQ_INACTIVE) {
369 if (p->queue - p->pc != PQ_ACTIVE ||
370 (vmflush_flags & VM_PAGER_ALLOW_ACTIVE) == 0) {
371 vm_page_wakeup(p);
372 break;
373 }
374 }
375
376 /*
377 * Try to maintain page groupings in the cluster.
378 */
379 if (m->flags & PG_WINATCFLS)
380 vm_page_flag_set(p, PG_WINATCFLS);
381 else
382 vm_page_flag_clear(p, PG_WINATCFLS);
383 p->act_count = m->act_count;
384
79ce07d0 385 mc[is] = p;
984263bc
MD
386 ++is;
387 }
388
398c240d
VS
389 vm_object_drop(object);
390
984263bc
MD
391 /*
392 * we allow reads during pageouts...
393 */
534ee349 394 return vm_pageout_flush(&mc[ib], is - ib, vmflush_flags);
984263bc
MD
395}
396
397/*
398 * vm_pageout_flush() - launder the given pages
399 *
400 * The given pages are laundered. Note that we setup for the start of
401 * I/O ( i.e. busy the page ), mark it read-only, and bump the object
402 * reference count all in here rather then in the parent. If we want
403 * the parent to do more sophisticated things we may have to change
404 * the ordering.
99ad9bc4 405 *
b12defdc
MD
406 * The pages in the array must be busied by the caller and will be
407 * unbusied by this function.
984263bc 408 */
984263bc 409int
534ee349 410vm_pageout_flush(vm_page_t *mc, int count, int vmflush_flags)
984263bc 411{
5f910b2f 412 vm_object_t object;
984263bc
MD
413 int pageout_status[count];
414 int numpagedout = 0;
415 int i;
416
417 /*
17cde63e
MD
418 * Initiate I/O. Bump the vm_page_t->busy counter.
419 */
420 for (i = 0; i < count; i++) {
b12defdc
MD
421 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL,
422 ("vm_pageout_flush page %p index %d/%d: partially "
423 "invalid page", mc[i], i, count));
17cde63e
MD
424 vm_page_io_start(mc[i]);
425 }
426
427 /*
4530a3aa
MD
428 * We must make the pages read-only. This will also force the
429 * modified bit in the related pmaps to be cleared. The pager
430 * cannot clear the bit for us since the I/O completion code
431 * typically runs from an interrupt. The act of making the page
432 * read-only handles the case for us.
b12defdc
MD
433 *
434 * Then we can unbusy the pages, we still hold a reference by virtue
435 * of our soft-busy.
984263bc 436 */
984263bc 437 for (i = 0; i < count; i++) {
534ee349
MD
438 if (vmflush_flags & VM_PAGER_TRY_TO_CACHE)
439 vm_page_protect(mc[i], VM_PROT_NONE);
440 else
441 vm_page_protect(mc[i], VM_PROT_READ);
b12defdc 442 vm_page_wakeup(mc[i]);
984263bc
MD
443 }
444
445 object = mc[0]->object;
446 vm_object_pip_add(object, count);
447
448 vm_pager_put_pages(object, mc, count,
95270b7e
MD
449 (vmflush_flags |
450 ((object == &kernel_object) ?
451 VM_PAGER_PUT_SYNC : 0)),
452 pageout_status);
984263bc
MD
453
454 for (i = 0; i < count; i++) {
455 vm_page_t mt = mc[i];
456
457 switch (pageout_status[i]) {
458 case VM_PAGER_OK:
459 numpagedout++;
460 break;
461 case VM_PAGER_PEND:
462 numpagedout++;
463 break;
464 case VM_PAGER_BAD:
465 /*
466 * Page outside of range of object. Right now we
467 * essentially lose the changes by pretending it
468 * worked.
469 */
b12defdc 470 vm_page_busy_wait(mt, FALSE, "pgbad");
984263bc
MD
471 pmap_clear_modify(mt);
472 vm_page_undirty(mt);
b12defdc 473 vm_page_wakeup(mt);
984263bc
MD
474 break;
475 case VM_PAGER_ERROR:
476 case VM_PAGER_FAIL:
477 /*
c84c24da
MD
478 * A page typically cannot be paged out when we
479 * have run out of swap. We leave the page
480 * marked inactive and will try to page it out
481 * again later.
482 *
483 * Starvation of the active page list is used to
484 * determine when the system is massively memory
485 * starved.
984263bc 486 */
984263bc
MD
487 break;
488 case VM_PAGER_AGAIN:
489 break;
490 }
491
492 /*
534ee349
MD
493 * If not PENDing this was a synchronous operation and we
494 * clean up after the I/O. If it is PENDing the mess is
495 * cleaned up asynchronously.
496 *
497 * Also nominally act on the caller's wishes if the caller
498 * wants to try to really clean (cache or free) the page.
93afe6be 499 *
534ee349
MD
500 * Also nominally deactivate the page if the system is
501 * memory-stressed.
984263bc
MD
502 */
503 if (pageout_status[i] != VM_PAGER_PEND) {
b12defdc 504 vm_page_busy_wait(mt, FALSE, "pgouw");
a491077e 505 vm_page_io_finish(mt);
534ee349
MD
506 if (vmflush_flags & VM_PAGER_TRY_TO_CACHE) {
507 vm_page_try_to_cache(mt);
508 } else if (vm_page_count_severe()) {
509 vm_page_deactivate(mt);
510 vm_page_wakeup(mt);
511 } else {
512 vm_page_wakeup(mt);
513 }
a491077e 514 vm_object_pip_wakeup(object);
984263bc
MD
515 }
516 }
517 return numpagedout;
518}
519
520#if !defined(NO_SWAPPING)
534ee349 521
984263bc 522/*
a7a03a5f
MD
523 * Callback function, page busied for us. We must dispose of the busy
524 * condition. Any related pmap pages may be held but will not be locked.
534ee349
MD
525 */
526static
527int
a7a03a5f
MD
528vm_pageout_mdp_callback(struct pmap_pgscan_info *info, vm_offset_t va,
529 vm_page_t p)
1f804340 530{
1f804340 531 int actcount;
534ee349 532 int cleanit = 0;
984263bc 533
534ee349
MD
534 /*
535 * Basic tests - There should never be a marker, and we can stop
536 * once the RSS is below the required level.
537 */
538 KKASSERT((p->flags & PG_MARKER) == 0);
a7a03a5f
MD
539 if (pmap_resident_tlnw_count(info->pmap) <= info->limit) {
540 vm_page_wakeup(p);
1f804340
MD
541 return(-1);
542 }
534ee349 543
1f804340 544 mycpu->gd_cnt.v_pdpages++;
b12defdc 545
534ee349
MD
546 if (p->wire_count || p->hold_count || (p->flags & PG_UNMANAGED)) {
547 vm_page_wakeup(p);
548 goto done;
549 }
984263bc 550
a7a03a5f
MD
551 ++info->actioncount;
552
553 /*
554 * Check if the page has been referened recently. If it has,
555 * activate it and skip.
556 */
1f804340
MD
557 actcount = pmap_ts_referenced(p);
558 if (actcount) {
559 vm_page_flag_set(p, PG_REFERENCED);
560 } else if (p->flags & PG_REFERENCED) {
561 actcount = 1;
562 }
563
a7a03a5f
MD
564 if (actcount) {
565 if (p->queue - p->pc != PQ_ACTIVE) {
566 vm_page_and_queue_spin_lock(p);
567 if (p->queue - p->pc != PQ_ACTIVE) {
b12defdc 568 vm_page_and_queue_spin_unlock(p);
a7a03a5f 569 vm_page_activate(p);
1f804340 570 } else {
b12defdc 571 vm_page_and_queue_spin_unlock(p);
984263bc 572 }
1f804340 573 } else {
a7a03a5f
MD
574 p->act_count += actcount;
575 if (p->act_count > ACT_MAX)
576 p->act_count = ACT_MAX;
577 }
578 vm_page_flag_clear(p, PG_REFERENCED);
579 vm_page_wakeup(p);
580 goto done;
581 }
b12defdc 582
a7a03a5f
MD
583 /*
584 * Remove the page from this particular pmap. Once we do this, our
585 * pmap scans will not see it again (unless it gets faulted in), so
586 * we must actively dispose of or deal with the page.
587 */
588 pmap_remove_specific(info->pmap, p);
589
590 /*
591 * If the page is not mapped to another process (i.e. as would be
592 * typical if this were a shared page from a library) then deactivate
593 * the page and clean it in two passes only.
594 *
595 * If the page hasn't been referenced since the last check, remove it
596 * from the pmap. If it is no longer mapped, deactivate it
597 * immediately, accelerating the normal decline.
598 *
599 * Once the page has been removed from the pmap the RSS code no
600 * longer tracks it so we have to make sure that it is staged for
601 * potential flush action.
602 */
603 if ((p->flags & PG_MAPPED) == 0) {
604 if (p->queue - p->pc == PQ_ACTIVE) {
605 vm_page_deactivate(p);
984263bc 606 }
a7a03a5f 607 if (p->queue - p->pc == PQ_INACTIVE) {
534ee349
MD
608 cleanit = 1;
609 }
984263bc 610 }
534ee349
MD
611
612 /*
613 * Ok, try to fully clean the page and any nearby pages such that at
614 * least the requested page is freed or moved to the cache queue.
615 *
616 * We usually do this synchronously to allow us to get the page into
617 * the CACHE queue quickly, which will prevent memory exhaustion if
618 * a process with a memoryuse limit is running away. However, the
619 * sysadmin may desire to set vm.swap_user_async which relaxes this
620 * and improves write performance.
621 */
622 if (cleanit) {
623 int max_launder = 0x7FFF;
624 int vnodes_skipped = 0;
625 int vmflush_flags;
626 struct vnode *vpfailed = NULL;
627
a7a03a5f 628 info->offset = va;
534ee349 629
486b807a 630 if (vm_pageout_memuse_mode >= 2) {
a7a03a5f
MD
631 vmflush_flags = VM_PAGER_TRY_TO_CACHE |
632 VM_PAGER_ALLOW_ACTIVE;
633 if (swap_user_async == 0)
634 vmflush_flags |= VM_PAGER_PUT_SYNC;
486b807a 635 vm_page_flag_set(p, PG_WINATCFLS);
a7a03a5f
MD
636 info->cleancount +=
637 vm_pageout_page(p, &max_launder,
638 &vnodes_skipped,
639 &vpfailed, 1, vmflush_flags);
486b807a 640 } else {
486b807a 641 vm_page_wakeup(p);
a7a03a5f 642 ++info->cleancount;
486b807a 643 }
534ee349
MD
644 } else {
645 vm_page_wakeup(p);
646 }
07540d37
MD
647
648 /*
649 * Must be at end to avoid SMP races.
650 */
534ee349
MD
651done:
652 lwkt_user_yield();
a7a03a5f 653 return 0;
984263bc
MD
654}
655
656/*
534ee349 657 * Deactivate some number of pages in a map due to set RLIMIT_RSS limits.
a7a03a5f
MD
658 * that is relatively difficult to do. We try to keep track of where we
659 * left off last time to reduce scan overhead.
486b807a
MD
660 *
661 * Called when vm_pageout_memuse_mode is >= 1.
984263bc 662 */
534ee349
MD
663void
664vm_pageout_map_deactivate_pages(vm_map_t map, vm_pindex_t limit)
984263bc 665{
a7a03a5f
MD
666 vm_offset_t pgout_offset;
667 struct pmap_pgscan_info info;
534ee349 668 int retries = 3;
984263bc 669
534ee349
MD
670 pgout_offset = map->pgout_offset;
671again:
a7a03a5f
MD
672#if 0
673 kprintf("%016jx ", pgout_offset);
674#endif
675 if (pgout_offset < VM_MIN_USER_ADDRESS)
676 pgout_offset = VM_MIN_USER_ADDRESS;
677 if (pgout_offset >= VM_MAX_USER_ADDRESS)
534ee349 678 pgout_offset = 0;
a7a03a5f
MD
679 info.pmap = vm_map_pmap(map);
680 info.limit = limit;
681 info.beg_addr = pgout_offset;
682 info.end_addr = VM_MAX_USER_ADDRESS;
683 info.callback = vm_pageout_mdp_callback;
684 info.cleancount = 0;
685 info.actioncount = 0;
686 info.busycount = 0;
687
688 pmap_pgscan(&info);
689 pgout_offset = info.offset;
690#if 0
691 kprintf("%016jx %08lx %08lx\n", pgout_offset,
692 info.cleancount, info.actioncount);
693#endif
984263bc 694
a7a03a5f
MD
695 if (pgout_offset != VM_MAX_USER_ADDRESS &&
696 pmap_resident_tlnw_count(vm_map_pmap(map)) > limit) {
697 goto again;
698 } else if (retries &&
699 pmap_resident_tlnw_count(vm_map_pmap(map)) > limit) {
700 --retries;
701 goto again;
702 }
534ee349 703 map->pgout_offset = pgout_offset;
984263bc
MD
704}
705#endif
706
707/*
a5fc46c9
MD
708 * Called when the pageout scan wants to free a page. We no longer
709 * try to cycle the vm_object here with a reference & dealloc, which can
710 * cause a non-trivial object collapse in a critical path.
99ad9bc4 711 *
a5fc46c9
MD
712 * It is unclear why we cycled the ref_count in the past, perhaps to try
713 * to optimize shadow chain collapses but I don't quite see why it would
714 * be necessary. An OBJ_DEAD object should terminate any and all vm_pages
715 * synchronously and not have to be kicked-start.
984263bc 716 */
99ad9bc4 717static void
95813af0
MD
718vm_pageout_page_free(vm_page_t m)
719{
984263bc
MD
720 vm_page_protect(m, VM_PROT_NONE);
721 vm_page_free(m);
984263bc
MD
722}
723
724/*
20479584 725 * vm_pageout_scan does the dirty work for the pageout daemon.
984263bc 726 */
8fa76237
MD
727struct vm_pageout_scan_info {
728 struct proc *bigproc;
729 vm_offset_t bigsize;
730};
731
732static int vm_pageout_scan_callback(struct proc *p, void *data);
733
c3feb36a
MD
734/*
735 * Scan inactive queue
736 *
737 * WARNING! Can be called from two pagedaemon threads simultaneously.
738 */
20479584 739static int
51c99c61 740vm_pageout_scan_inactive(int pass, int q, int avail_shortage,
534ee349 741 int *vnodes_skipped)
984263bc 742{
b12defdc 743 vm_page_t m;
984263bc 744 struct vm_page marker;
5d6a945b 745 struct vnode *vpfailed; /* warning, allowed to be stale */
027193eb
MD
746 int maxscan;
747 int delta = 0;
534ee349 748 int max_launder;
c3feb36a
MD
749 int isep;
750
751 isep = (curthread == emergpager);
984263bc 752
984263bc
MD
753 /*
754 * Start scanning the inactive queue for pages we can move to the
755 * cache or free. The scan will stop when the target is reached or
756 * we have scanned the entire inactive queue. Note that m->act_count
757 * is not used to form decisions for the inactive queue, only for the
758 * active queue.
759 *
534ee349 760 * max_launder limits the number of dirty pages we flush per scan.
984263bc
MD
761 * For most systems a smaller value (16 or 32) is more robust under
762 * extreme memory and disk pressure because any unnecessary writes
763 * to disk can result in extreme performance degredation. However,
764 * systems with excessive dirty pages (especially when MAP_NOSYNC is
765 * used) will die horribly with limited laundering. If the pageout
766 * daemon cannot clean enough pages in the first pass, we let it go
767 * all out in succeeding passes.
c3feb36a
MD
768 *
769 * NOTE! THE EMERGENCY PAGER (isep) DOES NOT LAUNDER VNODE-BACKED
770 * PAGES.
984263bc 771 */
534ee349
MD
772 if ((max_launder = vm_max_launder) <= 1)
773 max_launder = 1;
984263bc 774 if (pass)
534ee349 775 max_launder = 10000;
984263bc 776
06ecca5a 777 /*
b12defdc
MD
778 * Initialize our marker
779 */
780 bzero(&marker, sizeof(marker));
bc0aa189
MD
781 marker.flags = PG_FICTITIOUS | PG_MARKER;
782 marker.busy_count = PBUSY_LOCKED;
027193eb
MD
783 marker.queue = PQ_INACTIVE + q;
784 marker.pc = q;
b12defdc
MD
785 marker.wire_count = 1;
786
787 /*
788 * Inactive queue scan.
789 *
790 * NOTE: The vm_page must be spinlocked before the queue to avoid
791 * deadlocks, so it is easiest to simply iterate the loop
792 * with the queue unlocked at the top.
06ecca5a 793 */
5d6a945b 794 vpfailed = NULL;
b12defdc 795
027193eb
MD
796 vm_page_queues_spin_lock(PQ_INACTIVE + q);
797 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE + q].pl, &marker, pageq);
51c99c61 798 maxscan = vm_page_queues[PQ_INACTIVE + q].lcnt;
b12defdc 799
bb0d6093
MD
800 /*
801 * Queue locked at top of loop to avoid stack marker issues.
802 */
b12defdc 803 while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
51c99c61 804 maxscan-- > 0 && avail_shortage - delta > 0)
b12defdc 805 {
534ee349
MD
806 int count;
807
808 KKASSERT(m->queue == PQ_INACTIVE + q);
027193eb 809 TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE + q].pl,
b12defdc 810 &marker, pageq);
027193eb 811 TAILQ_INSERT_AFTER(&vm_page_queues[PQ_INACTIVE + q].pl, m,
b12defdc 812 &marker, pageq);
12e4aaff 813 mycpu->gd_cnt.v_pdpages++;
984263bc 814
06ecca5a 815 /*
bb0d6093
MD
816 * Skip marker pages (atomic against other markers to avoid
817 * infinite hop-over scans).
06ecca5a 818 */
bb0d6093 819 if (m->flags & PG_MARKER)
b12defdc 820 continue;
984263bc
MD
821
822 /*
b12defdc
MD
823 * Try to busy the page. Don't mess with pages which are
824 * already busy or reorder them in the queue.
984263bc 825 */
bb0d6093 826 if (vm_page_busy_try(m, TRUE))
984263bc 827 continue;
d2d8515b 828
984263bc 829 /*
bb0d6093
MD
830 * Remaining operations run with the page busy and neither
831 * the page or the queue will be spin-locked.
984263bc 832 */
bb0d6093 833 vm_page_queues_spin_unlock(PQ_INACTIVE + q);
534ee349
MD
834 KKASSERT(m->queue == PQ_INACTIVE + q);
835
c3feb36a
MD
836 /*
837 * The emergency pager runs when the primary pager gets
838 * stuck, which typically means the primary pager deadlocked
839 * on a vnode-backed page. Therefore, the emergency pager
32c821cf
MD
840 * must skip any complex objects.
841 *
842 * We disallow VNODEs unless they are VCHR whos device ops
843 * does not flag D_NOEMERGPGR.
c3feb36a 844 */
32c821cf
MD
845 if (isep && m->object) {
846 struct vnode *vp;
847
848 switch(m->object->type) {
849 case OBJT_DEFAULT:
850 case OBJT_SWAP:
851 /*
852 * Allow anonymous memory and assume that
853 * swap devices are not complex, since its
854 * kinda worthless if we can't swap out dirty
855 * anonymous pages.
856 */
857 break;
858 case OBJT_VNODE:
859 /*
860 * Allow VCHR device if the D_NOEMERGPGR
861 * flag is not set, deny other vnode types
862 * as being too complex.
863 */
864 vp = m->object->handle;
865 if (vp && vp->v_type == VCHR &&
866 vp->v_rdev && vp->v_rdev->si_ops &&
867 (vp->v_rdev->si_ops->head.flags &
868 D_NOEMERGPGR) == 0) {
869 break;
870 }
871 /* Deny - fall through */
872 default:
873 /*
874 * Deny
875 */
c3feb36a
MD
876 vm_page_wakeup(m);
877 vm_page_queues_spin_lock(PQ_INACTIVE + q);
878 lwkt_yield();
879 continue;
880 }
881 }
882
883 /*
884 * Try to pageout the page and perhaps other nearby pages.
885 */
534ee349
MD
886 count = vm_pageout_page(m, &max_launder, vnodes_skipped,
887 &vpfailed, pass, 0);
888 delta += count;
06ecca5a 889
90244566 890 /*
534ee349
MD
891 * Systems with a ton of memory can wind up with huge
892 * deactivation counts. Because the inactive scan is
893 * doing a lot of flushing, the combination can result
894 * in excessive paging even in situations where other
895 * unrelated threads free up sufficient VM.
896 *
897 * To deal with this we abort the nominal active->inactive
898 * scan before we hit the inactive target when free+cache
899 * levels have reached a reasonable target.
900 *
901 * When deciding to stop early we need to add some slop to
902 * the test and we need to return full completion to the caller
903 * to prevent the caller from thinking there is something
904 * wrong and issuing a low-memory+swap warning or pkill.
905 *
906 * A deficit forces paging regardless of the state of the
907 * VM page queues (used for RSS enforcement).
90244566 908 */
534ee349
MD
909 lwkt_yield();
910 vm_page_queues_spin_lock(PQ_INACTIVE + q);
911 if (vm_paging_target() < -vm_max_launder) {
912 /*
913 * Stopping early, return full completion to caller.
914 */
915 if (delta < avail_shortage)
916 delta = avail_shortage;
917 break;
918 }
919 }
920
921 /* page queue still spin-locked */
922 TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE + q].pl, &marker, pageq);
923 vm_page_queues_spin_unlock(PQ_INACTIVE + q);
924
925 return (delta);
926}
927
928/*
929 * Pageout the specified page, return the total number of pages paged out
930 * (this routine may cluster).
931 *
932 * The page must be busied and soft-busied by the caller and will be disposed
933 * of by this function.
934 */
935static int
936vm_pageout_page(vm_page_t m, int *max_launderp, int *vnodes_skippedp,
937 struct vnode **vpfailedp, int pass, int vmflush_flags)
938{
939 vm_object_t object;
940 int actcount;
941 int count = 0;
942
943 /*
944 * It is possible for a page to be busied ad-hoc (e.g. the
945 * pmap_collect() code) and wired and race against the
946 * allocation of a new page. vm_page_alloc() may be forced
947 * to deactivate the wired page in which case it winds up
948 * on the inactive queue and must be handled here. We
949 * correct the problem simply by unqueuing the page.
950 */
951 if (m->wire_count) {
952 vm_page_unqueue_nowakeup(m);
953 vm_page_wakeup(m);
954 kprintf("WARNING: pagedaemon: wired page on "
955 "inactive queue %p\n", m);
956 return 0;
957 }
958
959 /*
960 * A held page may be undergoing I/O, so skip it.
961 */
962 if (m->hold_count) {
963 vm_page_and_queue_spin_lock(m);
964 if (m->queue - m->pc == PQ_INACTIVE) {
965 TAILQ_REMOVE(
966 &vm_page_queues[m->queue].pl, m, pageq);
967 TAILQ_INSERT_TAIL(
968 &vm_page_queues[m->queue].pl, m, pageq);
969 ++vm_swapcache_inactive_heuristic;
90244566 970 }
534ee349
MD
971 vm_page_and_queue_spin_unlock(m);
972 vm_page_wakeup(m);
973 return 0;
974 }
90244566 975
534ee349 976 if (m->object == NULL || m->object->ref_count == 0) {
984263bc 977 /*
534ee349
MD
978 * If the object is not being used, we ignore previous
979 * references.
984263bc 980 */
534ee349
MD
981 vm_page_flag_clear(m, PG_REFERENCED);
982 pmap_clear_reference(m);
983 /* fall through to end */
984 } else if (((m->flags & PG_REFERENCED) == 0) &&
985 (actcount = pmap_ts_referenced(m))) {
986 /*
987 * Otherwise, if the page has been referenced while
988 * in the inactive queue, we bump the "activation
989 * count" upwards, making it less likely that the
990 * page will be added back to the inactive queue
991 * prematurely again. Here we check the page tables
992 * (or emulated bits, if any), given the upper level
993 * VM system not knowing anything about existing
994 * references.
995 */
996 vm_page_activate(m);
997 m->act_count += (actcount + ACT_ADVANCE);
998 vm_page_wakeup(m);
999 return 0;
1000 }
984263bc 1001
534ee349
MD
1002 /*
1003 * (m) is still busied.
1004 *
1005 * If the upper level VM system knows about any page
1006 * references, we activate the page. We also set the
1007 * "activation count" higher than normal so that we will less
1008 * likely place pages back onto the inactive queue again.
1009 */
1010 if ((m->flags & PG_REFERENCED) != 0) {
1011 vm_page_flag_clear(m, PG_REFERENCED);
1012 actcount = pmap_ts_referenced(m);
1013 vm_page_activate(m);
1014 m->act_count += (actcount + ACT_ADVANCE + 1);
1015 vm_page_wakeup(m);
1016 return 0;
1017 }
1018
1019 /*
1020 * If the upper level VM system doesn't know anything about
1021 * the page being dirty, we have to check for it again. As
1022 * far as the VM code knows, any partially dirty pages are
1023 * fully dirty.
1024 *
1025 * Pages marked PG_WRITEABLE may be mapped into the user
1026 * address space of a process running on another cpu. A
1027 * user process (without holding the MP lock) running on
1028 * another cpu may be able to touch the page while we are
1029 * trying to remove it. vm_page_cache() will handle this
1030 * case for us.
1031 */
1032 if (m->dirty == 0) {
1033 vm_page_test_dirty(m);
1034 } else {
1035 vm_page_dirty(m);
1036 }
984263bc 1037
534ee349 1038 if (m->valid == 0 && (m->flags & PG_NEED_COMMIT) == 0) {
984263bc 1039 /*
534ee349 1040 * Invalid pages can be easily freed
984263bc 1041 */
534ee349
MD
1042 vm_pageout_page_free(m);
1043 mycpu->gd_cnt.v_dfree++;
1044 ++count;
1045 } else if (m->dirty == 0 && (m->flags & PG_NEED_COMMIT) == 0) {
1046 /*
1047 * Clean pages can be placed onto the cache queue.
1048 * This effectively frees them.
1049 */
1050 vm_page_cache(m);
1051 ++count;
1052 } else if ((m->flags & PG_WINATCFLS) == 0 && pass == 0) {
1053 /*
1054 * Dirty pages need to be paged out, but flushing
1055 * a page is extremely expensive verses freeing
1056 * a clean page. Rather then artificially limiting
1057 * the number of pages we can flush, we instead give
1058 * dirty pages extra priority on the inactive queue
1059 * by forcing them to be cycled through the queue
1060 * twice before being flushed, after which the
1061 * (now clean) page will cycle through once more
1062 * before being freed. This significantly extends
1063 * the thrash point for a heavily loaded machine.
1064 */
1065 vm_page_flag_set(m, PG_WINATCFLS);
1066 vm_page_and_queue_spin_lock(m);
1067 if (m->queue - m->pc == PQ_INACTIVE) {
1068 TAILQ_REMOVE(
1069 &vm_page_queues[m->queue].pl, m, pageq);
1070 TAILQ_INSERT_TAIL(
1071 &vm_page_queues[m->queue].pl, m, pageq);
1072 ++vm_swapcache_inactive_heuristic;
984263bc 1073 }
534ee349
MD
1074 vm_page_and_queue_spin_unlock(m);
1075 vm_page_wakeup(m);
1076 } else if (*max_launderp > 0) {
984263bc 1077 /*
534ee349
MD
1078 * We always want to try to flush some dirty pages if
1079 * we encounter them, to keep the system stable.
1080 * Normally this number is small, but under extreme
1081 * pressure where there are insufficient clean pages
1082 * on the inactive queue, we may have to go all out.
984263bc 1083 */
534ee349
MD
1084 int swap_pageouts_ok;
1085 struct vnode *vp = NULL;
1086
1087 swap_pageouts_ok = 0;
1088 object = m->object;
1089 if (object &&
1090 (object->type != OBJT_SWAP) &&
1091 (object->type != OBJT_DEFAULT)) {
1092 swap_pageouts_ok = 1;
984263bc 1093 } else {
c3feb36a
MD
1094 swap_pageouts_ok = !(defer_swap_pageouts ||
1095 disable_swap_pageouts);
1096 swap_pageouts_ok |= (!disable_swap_pageouts &&
1097 defer_swap_pageouts &&
1098 vm_page_count_min(0));
984263bc
MD
1099 }
1100
534ee349
MD
1101 /*
1102 * We don't bother paging objects that are "dead".
1103 * Those objects are in a "rundown" state.
1104 */
1105 if (!swap_pageouts_ok ||
1106 (object == NULL) ||
1107 (object->flags & OBJ_DEAD)) {
b12defdc 1108 vm_page_and_queue_spin_lock(m);
027193eb
MD
1109 if (m->queue - m->pc == PQ_INACTIVE) {
1110 TAILQ_REMOVE(
534ee349
MD
1111 &vm_page_queues[m->queue].pl,
1112 m, pageq);
027193eb 1113 TAILQ_INSERT_TAIL(
534ee349
MD
1114 &vm_page_queues[m->queue].pl,
1115 m, pageq);
64949baa 1116 ++vm_swapcache_inactive_heuristic;
b12defdc
MD
1117 }
1118 vm_page_and_queue_spin_unlock(m);
b12defdc 1119 vm_page_wakeup(m);
534ee349
MD
1120 return 0;
1121 }
1122
1123 /*
1124 * (m) is still busied.
1125 *
1126 * The object is already known NOT to be dead. It
1127 * is possible for the vget() to block the whole
1128 * pageout daemon, but the new low-memory handling
1129 * code should prevent it.
1130 *
1131 * The previous code skipped locked vnodes and, worse,
1132 * reordered pages in the queue. This results in
1133 * completely non-deterministic operation because,
1134 * quite often, a vm_fault has initiated an I/O and
1135 * is holding a locked vnode at just the point where
1136 * the pageout daemon is woken up.
1137 *
1138 * We can't wait forever for the vnode lock, we might
1139 * deadlock due to a vn_read() getting stuck in
1140 * vm_wait while holding this vnode. We skip the
1141 * vnode if we can't get it in a reasonable amount
1142 * of time.
1143 *
1144 * vpfailed is used to (try to) avoid the case where
1145 * a large number of pages are associated with a
1146 * locked vnode, which could cause the pageout daemon
1147 * to stall for an excessive amount of time.
1148 */
1149 if (object->type == OBJT_VNODE) {
1150 int flags;
1151
1152 vp = object->handle;
1153 flags = LK_EXCLUSIVE;
1154 if (vp == *vpfailedp)
1155 flags |= LK_NOWAIT;
1156 else
1157 flags |= LK_TIMELOCK;
1158 vm_page_hold(m);
1159 vm_page_wakeup(m);
1160
984263bc 1161 /*
534ee349
MD
1162 * We have unbusied (m) temporarily so we can
1163 * acquire the vp lock without deadlocking.
1164 * (m) is held to prevent destruction.
984263bc 1165 */
534ee349
MD
1166 if (vget(vp, flags) != 0) {
1167 *vpfailedp = vp;
1168 ++pageout_lock_miss;
1169 if (object->flags & OBJ_MIGHTBEDIRTY)
1170 ++*vnodes_skippedp;
1171 vm_page_unhold(m);
1172 return 0;
984263bc
MD
1173 }
1174
1175 /*
534ee349
MD
1176 * The page might have been moved to another
1177 * queue during potential blocking in vget()
1178 * above. The page might have been freed and
1179 * reused for another vnode. The object might
1180 * have been reused for another vnode.
984263bc 1181 */
534ee349
MD
1182 if (m->queue - m->pc != PQ_INACTIVE ||
1183 m->object != object ||
1184 object->handle != vp) {
1185 if (object->flags & OBJ_MIGHTBEDIRTY)
1186 ++*vnodes_skippedp;
1187 vput(vp);
1188 vm_page_unhold(m);
1189 return 0;
984263bc
MD
1190 }
1191
1192 /*
534ee349
MD
1193 * The page may have been busied during the
1194 * blocking in vput(); We don't move the
1195 * page back onto the end of the queue so that
1196 * statistics are more correct if we don't.
984263bc 1197 */
534ee349
MD
1198 if (vm_page_busy_try(m, TRUE)) {
1199 vput(vp);
b12defdc 1200 vm_page_unhold(m);
534ee349 1201 return 0;
984263bc 1202 }
534ee349 1203 vm_page_unhold(m);
984263bc
MD
1204
1205 /*
534ee349 1206 * (m) is busied again
b12defdc 1207 *
534ee349
MD
1208 * We own the busy bit and remove our hold
1209 * bit. If the page is still held it
1210 * might be undergoing I/O, so skip it.
79ce07d0 1211 */
534ee349
MD
1212 if (m->hold_count) {
1213 vm_page_and_queue_spin_lock(m);
1214 if (m->queue - m->pc == PQ_INACTIVE) {
1215 TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq);
1216 TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, pageq);
1217 ++vm_swapcache_inactive_heuristic;
1218 }
1219 vm_page_and_queue_spin_unlock(m);
1220 if (object->flags & OBJ_MIGHTBEDIRTY)
1221 ++*vnodes_skippedp;
1222 vm_page_wakeup(m);
984263bc 1223 vput(vp);
534ee349
MD
1224 return 0;
1225 }
1226 /* (m) is left busied as we fall through */
984263bc 1227 }
1e1efe39
MD
1228
1229 /*
534ee349 1230 * page is busy and not held here.
1e1efe39 1231 *
534ee349
MD
1232 * If a page is dirty, then it is either being washed
1233 * (but not yet cleaned) or it is still in the
1234 * laundry. If it is still in the laundry, then we
1235 * start the cleaning operation.
1e1efe39 1236 *
534ee349
MD
1237 * decrement inactive_shortage on success to account
1238 * for the (future) cleaned page. Otherwise we
1239 * could wind up laundering or cleaning too many
1240 * pages.
1241 *
1242 * NOTE: Cleaning the page here does not cause
1243 * force_deficit to be adjusted, because the
1244 * page is not being freed or moved to the
1245 * cache.
1e1efe39 1246 */
534ee349
MD
1247 count = vm_pageout_clean_helper(m, vmflush_flags);
1248 *max_launderp -= count;
79ce07d0 1249
534ee349
MD
1250 /*
1251 * Clean ate busy, page no longer accessible
1252 */
1253 if (vp != NULL)
1254 vput(vp);
1255 } else {
1256 vm_page_wakeup(m);
1257 }
1258 return count;
027193eb
MD
1259}
1260
c3feb36a
MD
1261/*
1262 * Scan active queue
1263 *
1264 * WARNING! Can be called from two pagedaemon threads simultaneously.
1265 */
027193eb
MD
1266static int
1267vm_pageout_scan_active(int pass, int q,
51c99c61 1268 int avail_shortage, int inactive_shortage,
027193eb
MD
1269 int *recycle_countp)
1270{
1271 struct vm_page marker;
1272 vm_page_t m;
1273 int actcount;
1274 int delta = 0;
51c99c61 1275 int maxscan;
c3feb36a
MD
1276 int isep;
1277
1278 isep = (curthread == emergpager);
984263bc
MD
1279
1280 /*
20479584
MD
1281 * We want to move pages from the active queue to the inactive
1282 * queue to get the inactive queue to the inactive target. If
1283 * we still have a page shortage from above we try to directly free
1284 * clean pages instead of moving them.
06ecca5a 1285 *
20479584
MD
1286 * If we do still have a shortage we keep track of the number of
1287 * pages we free or cache (recycle_count) as a measure of thrashing
1288 * between the active and inactive queues.
1289 *
51db7ca2
MD
1290 * If we were able to completely satisfy the free+cache targets
1291 * from the inactive pool we limit the number of pages we move
1292 * from the active pool to the inactive pool to 2x the pages we
e6e9a0c3
MD
1293 * had removed from the inactive pool (with a minimum of 1/5 the
1294 * inactive target). If we were not able to completely satisfy
1295 * the free+cache targets we go for the whole target aggressively.
20479584
MD
1296 *
1297 * NOTE: Both variables can end up negative.
1298 * NOTE: We are still in a critical section.
c3feb36a
MD
1299 *
1300 * NOTE! THE EMERGENCY PAGER (isep) DOES NOT LAUNDER VNODE-BACKED
1301 * PAGES.
984263bc 1302 */
20479584 1303
027193eb 1304 bzero(&marker, sizeof(marker));
bc0aa189
MD
1305 marker.flags = PG_FICTITIOUS | PG_MARKER;
1306 marker.busy_count = PBUSY_LOCKED;
027193eb
MD
1307 marker.queue = PQ_ACTIVE + q;
1308 marker.pc = q;
1309 marker.wire_count = 1;
b12defdc 1310
027193eb
MD
1311 vm_page_queues_spin_lock(PQ_ACTIVE + q);
1312 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
51c99c61 1313 maxscan = vm_page_queues[PQ_ACTIVE + q].lcnt;
b12defdc 1314
bb0d6093
MD
1315 /*
1316 * Queue locked at top of loop to avoid stack marker issues.
1317 */
b12defdc 1318 while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
51c99c61
MD
1319 maxscan-- > 0 && (avail_shortage - delta > 0 ||
1320 inactive_shortage > 0))
b12defdc 1321 {
534ee349 1322 KKASSERT(m->queue == PQ_ACTIVE + q);
027193eb 1323 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl,
b12defdc 1324 &marker, pageq);
027193eb 1325 TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE + q].pl, m,
b12defdc 1326 &marker, pageq);
984263bc 1327
984263bc 1328 /*
bb0d6093
MD
1329 * Skip marker pages (atomic against other markers to avoid
1330 * infinite hop-over scans).
984263bc 1331 */
bb0d6093 1332 if (m->flags & PG_MARKER)
b12defdc 1333 continue;
06ecca5a 1334
984263bc 1335 /*
b12defdc
MD
1336 * Try to busy the page. Don't mess with pages which are
1337 * already busy or reorder them in the queue.
984263bc 1338 */
bb0d6093 1339 if (vm_page_busy_try(m, TRUE))
984263bc 1340 continue;
bb0d6093
MD
1341
1342 /*
1343 * Remaining operations run with the page busy and neither
1344 * the page or the queue will be spin-locked.
1345 */
1346 vm_page_queues_spin_unlock(PQ_ACTIVE + q);
534ee349 1347 KKASSERT(m->queue == PQ_ACTIVE + q);
984263bc 1348
b12defdc
MD
1349 /*
1350 * Don't deactivate pages that are held, even if we can
1351 * busy them. (XXX why not?)
1352 */
1353 if (m->hold_count != 0) {
bb0d6093
MD
1354 vm_page_and_queue_spin_lock(m);
1355 if (m->queue - m->pc == PQ_ACTIVE) {
1356 TAILQ_REMOVE(
1357 &vm_page_queues[PQ_ACTIVE + q].pl,
1358 m, pageq);
1359 TAILQ_INSERT_TAIL(
1360 &vm_page_queues[PQ_ACTIVE + q].pl,
1361 m, pageq);
1362 }
b12defdc
MD
1363 vm_page_and_queue_spin_unlock(m);
1364 vm_page_wakeup(m);
bb0d6093 1365 goto next;
b12defdc 1366 }
b12defdc 1367
c3feb36a
MD
1368 /*
1369 * The emergency pager ignores vnode-backed pages as these
1370 * are the pages that probably bricked the main pager.
1371 */
1372 if (isep && m->object && m->object->type == OBJT_VNODE) {
1373 vm_page_and_queue_spin_lock(m);
1374 if (m->queue - m->pc == PQ_ACTIVE) {
1375 TAILQ_REMOVE(
1376 &vm_page_queues[PQ_ACTIVE + q].pl,
1377 m, pageq);
1378 TAILQ_INSERT_TAIL(
1379 &vm_page_queues[PQ_ACTIVE + q].pl,
1380 m, pageq);
1381 }
1382 vm_page_and_queue_spin_unlock(m);
1383 vm_page_wakeup(m);
1384 goto next;
1385 }
1386
984263bc
MD
1387 /*
1388 * The count for pagedaemon pages is done after checking the
1389 * page for eligibility...
1390 */
12e4aaff 1391 mycpu->gd_cnt.v_pdpages++;
984263bc
MD
1392
1393 /*
20479584
MD
1394 * Check to see "how much" the page has been used and clear
1395 * the tracking access bits. If the object has no references
1396 * don't bother paying the expense.
984263bc
MD
1397 */
1398 actcount = 0;
19cd98ea 1399 if (m->object && m->object->ref_count != 0) {
20479584
MD
1400 if (m->flags & PG_REFERENCED)
1401 ++actcount;
984263bc
MD
1402 actcount += pmap_ts_referenced(m);
1403 if (actcount) {
1404 m->act_count += ACT_ADVANCE + actcount;
1405 if (m->act_count > ACT_MAX)
1406 m->act_count = ACT_MAX;
1407 }
1408 }
984263bc
MD
1409 vm_page_flag_clear(m, PG_REFERENCED);
1410
1411 /*
20479584 1412 * actcount is only valid if the object ref_count is non-zero.
19cd98ea 1413 * If the page does not have an object, actcount will be zero.
984263bc 1414 */
20479584 1415 if (actcount && m->object->ref_count != 0) {
b12defdc 1416 vm_page_and_queue_spin_lock(m);
027193eb
MD
1417 if (m->queue - m->pc == PQ_ACTIVE) {
1418 TAILQ_REMOVE(
1419 &vm_page_queues[PQ_ACTIVE + q].pl,
1420 m, pageq);
1421 TAILQ_INSERT_TAIL(
1422 &vm_page_queues[PQ_ACTIVE + q].pl,
1423 m, pageq);
b12defdc
MD
1424 }
1425 vm_page_and_queue_spin_unlock(m);
1426 vm_page_wakeup(m);
984263bc 1427 } else {
38587548
MD
1428 switch(m->object->type) {
1429 case OBJT_DEFAULT:
1430 case OBJT_SWAP:
1431 m->act_count -= min(m->act_count,
1432 vm_anonmem_decline);
1433 break;
1434 default:
1435 m->act_count -= min(m->act_count,
1436 vm_filemem_decline);
1437 break;
1438 }
984263bc 1439 if (vm_pageout_algorithm ||
19cd98ea
VS
1440 (m->object == NULL) ||
1441 (m->object && (m->object->ref_count == 0)) ||
20479584
MD
1442 m->act_count < pass + 1
1443 ) {
1444 /*
1445 * Deactivate the page. If we had a
1446 * shortage from our inactive scan try to
1447 * free (cache) the page instead.
e6e9a0c3
MD
1448 *
1449 * Don't just blindly cache the page if
1450 * we do not have a shortage from the
1451 * inactive scan, that could lead to
1452 * gigabytes being moved.
20479584 1453 */
51c99c61
MD
1454 --inactive_shortage;
1455 if (avail_shortage - delta > 0 ||
19cd98ea
VS
1456 (m->object && (m->object->ref_count == 0)))
1457 {
51c99c61 1458 if (avail_shortage - delta > 0)
027193eb 1459 ++*recycle_countp;
984263bc 1460 vm_page_protect(m, VM_PROT_NONE);
e6e9a0c3 1461 if (m->dirty == 0 &&
9bf025db 1462 (m->flags & PG_NEED_COMMIT) == 0 &&
51c99c61 1463 avail_shortage - delta > 0) {
984263bc 1464 vm_page_cache(m);
c84c24da 1465 } else {
984263bc 1466 vm_page_deactivate(m);
a491077e 1467 vm_page_wakeup(m);
c84c24da 1468 }
984263bc
MD
1469 } else {
1470 vm_page_deactivate(m);
b12defdc 1471 vm_page_wakeup(m);
984263bc 1472 }
51c99c61 1473 ++delta;
984263bc 1474 } else {
b12defdc 1475 vm_page_and_queue_spin_lock(m);
027193eb 1476 if (m->queue - m->pc == PQ_ACTIVE) {
b12defdc 1477 TAILQ_REMOVE(
027193eb
MD
1478 &vm_page_queues[PQ_ACTIVE + q].pl,
1479 m, pageq);
b12defdc 1480 TAILQ_INSERT_TAIL(
027193eb
MD
1481 &vm_page_queues[PQ_ACTIVE + q].pl,
1482 m, pageq);
b12defdc
MD
1483 }
1484 vm_page_and_queue_spin_unlock(m);
1485 vm_page_wakeup(m);
984263bc
MD
1486 }
1487 }
bb0d6093 1488next:
534ee349 1489 lwkt_yield();
bb0d6093 1490 vm_page_queues_spin_lock(PQ_ACTIVE + q);
984263bc
MD
1491 }
1492
b12defdc
MD
1493 /*
1494 * Clean out our local marker.
bb0d6093
MD
1495 *
1496 * Page queue still spin-locked.
b12defdc 1497 */
027193eb
MD
1498 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
1499 vm_page_queues_spin_unlock(PQ_ACTIVE + q);
1500
1501 return (delta);
1502}
1503
1504/*
1505 * The number of actually free pages can drop down to v_free_reserved,
1506 * we try to build the free count back above v_free_min. Note that
1507 * vm_paging_needed() also returns TRUE if v_free_count is not at
1508 * least v_free_min so that is the minimum we must build the free
1509 * count to.
1510 *
1511 * We use a slightly higher target to improve hysteresis,
1512 * ((v_free_target + v_free_min) / 2). Since v_free_target
1513 * is usually the same as v_cache_min this maintains about
1514 * half the pages in the free queue as are in the cache queue,
1515 * providing pretty good pipelining for pageout operation.
1516 *
1517 * The system operator can manipulate vm.v_cache_min and
1518 * vm.v_free_target to tune the pageout demon. Be sure
1519 * to keep vm.v_free_min < vm.v_free_target.
1520 *
1521 * Note that the original paging target is to get at least
1522 * (free_min + cache_min) into (free + cache). The slightly
1523 * higher target will shift additional pages from cache to free
1524 * without effecting the original paging target in order to
1525 * maintain better hysteresis and not have the free count always
1526 * be dead-on v_free_min.
1527 *
1528 * NOTE: we are still in a critical section.
1529 *
1530 * Pages moved from PQ_CACHE to totally free are not counted in the
1531 * pages_freed counter.
c3feb36a
MD
1532 *
1533 * WARNING! Can be called from two pagedaemon threads simultaneously.
027193eb
MD
1534 */
1535static void
09eff544
MD
1536vm_pageout_scan_cache(int avail_shortage, int pass,
1537 int vnodes_skipped, int recycle_count)
027193eb 1538{
9cd626ca 1539 static int lastkillticks;
027193eb
MD
1540 struct vm_pageout_scan_info info;
1541 vm_page_t m;
c3feb36a
MD
1542 int isep;
1543
1544 isep = (curthread == emergpager);
b12defdc 1545
cd3c66bd
MD
1546 while (vmstats.v_free_count <
1547 (vmstats.v_free_min + vmstats.v_free_target) / 2) {
1548 /*
b12defdc 1549 * This steals some code from vm/vm_page.c
cd3c66bd 1550 */
984263bc 1551 static int cache_rover = 0;
b12defdc 1552
635c9c15 1553 m = vm_page_list_find(PQ_CACHE, cache_rover & PQ_L2_MASK);
20479584 1554 if (m == NULL)
984263bc 1555 break;
b12defdc
MD
1556 /* page is returned removed from its queue and spinlocked */
1557 if (vm_page_busy_try(m, TRUE)) {
1558 vm_page_deactivate_locked(m);
1559 vm_page_spin_unlock(m);
b12defdc
MD
1560 continue;
1561 }
1562 vm_page_spin_unlock(m);
1563 pagedaemon_wakeup();
d2d8515b 1564 lwkt_yield();
b12defdc
MD
1565
1566 /*
bb0d6093
MD
1567 * Remaining operations run with the page busy and neither
1568 * the page or the queue will be spin-locked.
b12defdc 1569 */
9bf025db 1570 if ((m->flags & (PG_UNMANAGED | PG_NEED_COMMIT)) ||
b12defdc
MD
1571 m->hold_count ||
1572 m->wire_count) {
984263bc 1573 vm_page_deactivate(m);
b12defdc 1574 vm_page_wakeup(m);
984263bc
MD
1575 continue;
1576 }
17cde63e
MD
1577 KKASSERT((m->flags & PG_MAPPED) == 0);
1578 KKASSERT(m->dirty == 0);
b12defdc 1579 cache_rover += PQ_PRIME2;
984263bc 1580 vm_pageout_page_free(m);
12e4aaff 1581 mycpu->gd_cnt.v_dfree++;
984263bc 1582 }
06ecca5a 1583
984263bc
MD
1584#if !defined(NO_SWAPPING)
1585 /*
1586 * Idle process swapout -- run once per second.
1587 */
1588 if (vm_swap_idle_enabled) {
cec73927
MD
1589 static time_t lsec;
1590 if (time_uptime != lsec) {
534ee349 1591 atomic_set_int(&vm_pageout_req_swapout, VM_SWAP_IDLE);
984263bc 1592 vm_req_vmdaemon();
cec73927 1593 lsec = time_uptime;
984263bc
MD
1594 }
1595 }
1596#endif
1597
1598 /*
1599 * If we didn't get enough free pages, and we have skipped a vnode
1600 * in a writeable object, wakeup the sync daemon. And kick swapout
1601 * if we did not get enough free pages.
1602 */
1603 if (vm_paging_target() > 0) {
20479584 1604 if (vnodes_skipped && vm_page_count_min(0))
cf6a53ca 1605 speedup_syncer(NULL);
984263bc
MD
1606#if !defined(NO_SWAPPING)
1607 if (vm_swap_enabled && vm_page_count_target()) {
534ee349 1608 atomic_set_int(&vm_pageout_req_swapout, VM_SWAP_NORMAL);
984263bc 1609 vm_req_vmdaemon();
984263bc
MD
1610 }
1611#endif
1612 }
1613
1614 /*
20479584
MD
1615 * Handle catastrophic conditions. Under good conditions we should
1616 * be at the target, well beyond our minimum. If we could not even
09eff544
MD
1617 * reach our minimum the system is under heavy stress. But just being
1618 * under heavy stress does not trigger process killing.
20479584 1619 *
09eff544
MD
1620 * We consider ourselves to have run out of memory if the swap pager
1621 * is full and avail_shortage is still positive. The secondary check
1622 * ensures that we do not kill processes if the instantanious
1623 * availability is good, even if the pageout demon pass says it
1624 * couldn't get to the target.
c3feb36a
MD
1625 *
1626 * NOTE! THE EMERGENCY PAGER (isep) DOES NOT HANDLE SWAP FULL
1627 * SITUATIONS.
984263bc 1628 */
09eff544 1629 if (swap_pager_almost_full &&
9cd626ca 1630 pass > 0 &&
c3feb36a 1631 isep == 0 &&
09eff544
MD
1632 (vm_page_count_min(recycle_count) || avail_shortage > 0)) {
1633 kprintf("Warning: system low on memory+swap "
1634 "shortage %d for %d ticks!\n",
1635 avail_shortage, ticks - swap_fail_ticks);
d081ceac
MD
1636 if (bootverbose)
1637 kprintf("Metrics: spaf=%d spf=%d pass=%d avail=%d target=%d last=%u\n",
1638 swap_pager_almost_full,
1639 swap_pager_full,
1640 pass,
1641 avail_shortage,
1642 vm_paging_target(),
1643 (unsigned int)(ticks - lastkillticks));
09eff544
MD
1644 }
1645 if (swap_pager_full &&
9cd626ca 1646 pass > 1 &&
c3feb36a 1647 isep == 0 &&
09eff544 1648 avail_shortage > 0 &&
9cd626ca
MD
1649 vm_paging_target() > 0 &&
1650 (unsigned int)(ticks - lastkillticks) >= hz) {
20479584 1651 /*
9cd626ca
MD
1652 * Kill something, maximum rate once per second to give
1653 * the process time to free up sufficient memory.
20479584 1654 */
9cd626ca 1655 lastkillticks = ticks;
8fa76237
MD
1656 info.bigproc = NULL;
1657 info.bigsize = 0;
586c4308 1658 allproc_scan(vm_pageout_scan_callback, &info, 0);
8fa76237 1659 if (info.bigproc != NULL) {
d081ceac
MD
1660 kprintf("Try to kill process %d %s\n",
1661 info.bigproc->p_pid, info.bigproc->p_comm);
8fa76237 1662 info.bigproc->p_nice = PRIO_MIN;
08f2f1bb
SS
1663 info.bigproc->p_usched->resetpriority(
1664 FIRST_LWP_IN_PROC(info.bigproc));
2c9e2984 1665 atomic_set_int(&info.bigproc->p_flags, P_LOWMEMKILL);
9cd626ca 1666 killproc(info.bigproc, "out of swap space");
12e4aaff 1667 wakeup(&vmstats.v_free_count);
8fa76237 1668 PRELE(info.bigproc);
984263bc
MD
1669 }
1670 }
1671}
1672
8fa76237
MD
1673static int
1674vm_pageout_scan_callback(struct proc *p, void *data)
1675{
1676 struct vm_pageout_scan_info *info = data;
1677 vm_offset_t size;
1678
1679 /*
20479584
MD
1680 * Never kill system processes or init. If we have configured swap
1681 * then try to avoid killing low-numbered pids.
8fa76237 1682 */
4643740a 1683 if ((p->p_flags & P_SYSTEM) || (p->p_pid == 1) ||
8fa76237
MD
1684 ((p->p_pid < 48) && (vm_swap_size != 0))) {
1685 return (0);
1686 }
1687
a8d3ab53
MD
1688 lwkt_gettoken(&p->p_token);
1689
8fa76237
MD
1690 /*
1691 * if the process is in a non-running type state,
1692 * don't touch it.
1693 */
f5b92db7 1694 if (p->p_stat != SACTIVE && p->p_stat != SSTOP && p->p_stat != SCORE) {
a8d3ab53 1695 lwkt_reltoken(&p->p_token);
8fa76237 1696 return (0);
a8d3ab53 1697 }
8fa76237
MD
1698
1699 /*
20479584
MD
1700 * Get the approximate process size. Note that anonymous pages
1701 * with backing swap will be counted twice, but there should not
1702 * be too many such pages due to the stress the VM system is
1703 * under at this point.
8fa76237 1704 */
20479584 1705 size = vmspace_anonymous_count(p->p_vmspace) +
8fa76237
MD
1706 vmspace_swap_count(p->p_vmspace);
1707
1708 /*
1709 * If the this process is bigger than the biggest one
1710 * remember it.
1711 */
20479584 1712 if (info->bigsize < size) {
8fa76237
MD
1713 if (info->bigproc)
1714 PRELE(info->bigproc);
1715 PHOLD(p);
1716 info->bigproc = p;
1717 info->bigsize = size;
1718 }
a8d3ab53 1719 lwkt_reltoken(&p->p_token);
d2d8515b 1720 lwkt_yield();
a8d3ab53 1721
8fa76237
MD
1722 return(0);
1723}
1724
984263bc
MD
1725/*
1726 * This routine tries to maintain the pseudo LRU active queue,
1727 * so that during long periods of time where there is no paging,
1728 * that some statistic accumulation still occurs. This code
1729 * helps the situation where paging just starts to occur.
1730 */
1731static void
027193eb 1732vm_pageout_page_stats(int q)
984263bc 1733{
984263bc 1734 static int fullintervalcount = 0;
b12defdc
MD
1735 struct vm_page marker;
1736 vm_page_t m;
1737 int pcount, tpcount; /* Number of pages to check */
984263bc 1738 int page_shortage;
984263bc 1739
b12defdc
MD
1740 page_shortage = (vmstats.v_inactive_target + vmstats.v_cache_max +
1741 vmstats.v_free_min) -
1742 (vmstats.v_free_count + vmstats.v_inactive_count +
1743 vmstats.v_cache_count);
984263bc
MD
1744
1745 if (page_shortage <= 0)
1746 return;
1747
51c99c61 1748 pcount = vm_page_queues[PQ_ACTIVE + q].lcnt;
984263bc
MD
1749 fullintervalcount += vm_pageout_stats_interval;
1750 if (fullintervalcount < vm_pageout_full_stats_interval) {
51c99c61
MD
1751 tpcount = (vm_pageout_stats_max * pcount) /
1752 vmstats.v_page_count + 1;
984263bc
MD
1753 if (pcount > tpcount)
1754 pcount = tpcount;
1755 } else {
1756 fullintervalcount = 0;
1757 }
1758
b12defdc 1759 bzero(&marker, sizeof(marker));
bc0aa189
MD
1760 marker.flags = PG_FICTITIOUS | PG_MARKER;
1761 marker.busy_count = PBUSY_LOCKED;
027193eb
MD
1762 marker.queue = PQ_ACTIVE + q;
1763 marker.pc = q;
b12defdc
MD
1764 marker.wire_count = 1;
1765
027193eb
MD
1766 vm_page_queues_spin_lock(PQ_ACTIVE + q);
1767 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
b12defdc 1768
bb0d6093
MD
1769 /*
1770 * Queue locked at top of loop to avoid stack marker issues.
1771 */
b12defdc
MD
1772 while ((m = TAILQ_NEXT(&marker, pageq)) != NULL &&
1773 pcount-- > 0)
1774 {
984263bc
MD
1775 int actcount;
1776
534ee349 1777 KKASSERT(m->queue == PQ_ACTIVE + q);
027193eb
MD
1778 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
1779 TAILQ_INSERT_AFTER(&vm_page_queues[PQ_ACTIVE + q].pl, m,
b12defdc 1780 &marker, pageq);
984263bc 1781
984263bc 1782 /*
bb0d6093
MD
1783 * Skip marker pages (atomic against other markers to avoid
1784 * infinite hop-over scans).
984263bc 1785 */
bb0d6093 1786 if (m->flags & PG_MARKER)
984263bc 1787 continue;
984263bc 1788
b12defdc
MD
1789 /*
1790 * Ignore pages we can't busy
1791 */
bb0d6093 1792 if (vm_page_busy_try(m, TRUE))
b12defdc 1793 continue;
bb0d6093
MD
1794
1795 /*
1796 * Remaining operations run with the page busy and neither
1797 * the page or the queue will be spin-locked.
1798 */
1799 vm_page_queues_spin_unlock(PQ_ACTIVE + q);
534ee349 1800 KKASSERT(m->queue == PQ_ACTIVE + q);
b12defdc
MD
1801
1802 /*
1803 * We now have a safely busied page, the page and queue
1804 * spinlocks have been released.
1805 *
1806 * Ignore held pages
1807 */
1808 if (m->hold_count) {
1809 vm_page_wakeup(m);
bb0d6093 1810 goto next;
b12defdc
MD
1811 }
1812
1813 /*
1814 * Calculate activity
1815 */
984263bc
MD
1816 actcount = 0;
1817 if (m->flags & PG_REFERENCED) {
1818 vm_page_flag_clear(m, PG_REFERENCED);
1819 actcount += 1;
1820 }
984263bc 1821 actcount += pmap_ts_referenced(m);
b12defdc
MD
1822
1823 /*
1824 * Update act_count and move page to end of queue.
1825 */
984263bc
MD
1826 if (actcount) {
1827 m->act_count += ACT_ADVANCE + actcount;
1828 if (m->act_count > ACT_MAX)
1829 m->act_count = ACT_MAX;
b12defdc 1830 vm_page_and_queue_spin_lock(m);
027193eb
MD
1831 if (m->queue - m->pc == PQ_ACTIVE) {
1832 TAILQ_REMOVE(
1833 &vm_page_queues[PQ_ACTIVE + q].pl,
1834 m, pageq);
1835 TAILQ_INSERT_TAIL(
1836 &vm_page_queues[PQ_ACTIVE + q].pl,
1837 m, pageq);
984263bc 1838 }
b12defdc
MD
1839 vm_page_and_queue_spin_unlock(m);
1840 vm_page_wakeup(m);
bb0d6093 1841 goto next;
984263bc
MD
1842 }
1843
b12defdc
MD
1844 if (m->act_count == 0) {
1845 /*
1846 * We turn off page access, so that we have
1847 * more accurate RSS stats. We don't do this
1848 * in the normal page deactivation when the
1849 * system is loaded VM wise, because the
1850 * cost of the large number of page protect
1851 * operations would be higher than the value
1852 * of doing the operation.
1853 *
1854 * We use the marker to save our place so
1855 * we can release the spin lock. both (m)
1856 * and (next) will be invalid.
1857 */
1858 vm_page_protect(m, VM_PROT_NONE);
1859 vm_page_deactivate(m);
1860 } else {
1861 m->act_count -= min(m->act_count, ACT_DECLINE);
1862 vm_page_and_queue_spin_lock(m);
027193eb
MD
1863 if (m->queue - m->pc == PQ_ACTIVE) {
1864 TAILQ_REMOVE(
1865 &vm_page_queues[PQ_ACTIVE + q].pl,
1866 m, pageq);
1867 TAILQ_INSERT_TAIL(
1868 &vm_page_queues[PQ_ACTIVE + q].pl,
1869 m, pageq);
b12defdc
MD
1870 }
1871 vm_page_and_queue_spin_unlock(m);
1872 }
1873 vm_page_wakeup(m);
bb0d6093
MD
1874next:
1875 vm_page_queues_spin_lock(PQ_ACTIVE + q);
984263bc 1876 }
b12defdc
MD
1877
1878 /*
1879 * Remove our local marker
bb0d6093
MD
1880 *
1881 * Page queue still spin-locked.
b12defdc 1882 */
027193eb
MD
1883 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE + q].pl, &marker, pageq);
1884 vm_page_queues_spin_unlock(PQ_ACTIVE + q);
984263bc
MD
1885}
1886
1887static int
57e43348 1888vm_pageout_free_page_calc(vm_size_t count)
984263bc 1889{
12e4aaff 1890 if (count < vmstats.v_page_count)
984263bc
MD
1891 return 0;
1892 /*
1893 * free_reserved needs to include enough for the largest swap pager
1894 * structures plus enough for any pv_entry structs when paging.
0a4d4828
MD
1895 *
1896 * v_free_min normal allocations
1897 * v_free_reserved system allocations
1898 * v_pageout_free_min allocations by pageout daemon
1899 * v_interrupt_free_min low level allocations (e.g swap structures)
984263bc 1900 */
12e4aaff 1901 if (vmstats.v_page_count > 1024)
0a4d4828 1902 vmstats.v_free_min = 64 + (vmstats.v_page_count - 1024) / 200;
984263bc 1903 else
0a4d4828 1904 vmstats.v_free_min = 64;
55a7854b
MD
1905
1906 /*
1907 * Make sure the vmmeter slop can't blow out our global minimums.
8178c126
MD
1908 *
1909 * However, to accomodate weird configurations (vkernels with many
1910 * cpus and little memory, or artifically reduced hw.physmem), do
1911 * not allow v_free_min to exceed 1/20 of ram or the pageout demon
1912 * will go out of control.
55a7854b
MD
1913 */
1914 if (vmstats.v_free_min < VMMETER_SLOP_COUNT * ncpus * 10)
1915 vmstats.v_free_min = VMMETER_SLOP_COUNT * ncpus * 10;
8178c126
MD
1916 if (vmstats.v_free_min > vmstats.v_page_count / 20)
1917 vmstats.v_free_min = vmstats.v_page_count / 20;
55a7854b 1918
0a4d4828
MD
1919 vmstats.v_free_reserved = vmstats.v_free_min * 4 / 8 + 7;
1920 vmstats.v_free_severe = vmstats.v_free_min * 4 / 8 + 0;
1921 vmstats.v_pageout_free_min = vmstats.v_free_min * 2 / 8 + 7;
1922 vmstats.v_interrupt_free_min = vmstats.v_free_min * 1 / 8 + 7;
1923
984263bc
MD
1924 return 1;
1925}
1926
1927
1928/*
c3feb36a
MD
1929 * vm_pageout is the high level pageout daemon. TWO kernel threads run
1930 * this daemon, the primary pageout daemon and the emergency pageout daemon.
99ad9bc4 1931 *
c3feb36a
MD
1932 * The emergency pageout daemon takes over when the primary pageout daemon
1933 * deadlocks. The emergency pageout daemon ONLY pages out to swap, thus
1934 * avoiding the many low-memory deadlocks which can occur when paging out
1935 * to VFS's.
984263bc
MD
1936 */
1937static void
cd8ab232 1938vm_pageout_thread(void)
984263bc
MD
1939{
1940 int pass;
027193eb 1941 int q;
79ce07d0
MD
1942 int q1iterator = 0;
1943 int q2iterator = 0;
c3feb36a 1944 int isep;
c3feb36a
MD
1945
1946 curthread->td_flags |= TDF_SYSTHREAD;
984263bc
MD
1947
1948 /*
c3feb36a 1949 * We only need to setup once.
984263bc 1950 */
c3feb36a 1951 isep = 0;
c3feb36a
MD
1952 if (curthread == emergpager) {
1953 isep = 1;
1954 goto skip_setup;
1955 }
984263bc 1956
c3feb36a
MD
1957 /*
1958 * Initialize some paging parameters.
1959 */
12e4aaff 1960 vm_pageout_free_page_calc(vmstats.v_page_count);
20479584 1961
984263bc
MD
1962 /*
1963 * v_free_target and v_cache_min control pageout hysteresis. Note
1964 * that these are more a measure of the VM cache queue hysteresis
1965 * then the VM free queue. Specifically, v_free_target is the
1966 * high water mark (free+cache pages).
1967 *
1968 * v_free_reserved + v_cache_min (mostly means v_cache_min) is the
1969 * low water mark, while v_free_min is the stop. v_cache_min must
1970 * be big enough to handle memory needs while the pageout daemon
1971 * is signalled and run to free more pages.
1972 */
12e4aaff 1973 if (vmstats.v_free_count > 6144)
8178c126
MD
1974 vmstats.v_free_target = 4 * vmstats.v_free_min +
1975 vmstats.v_free_reserved;
984263bc 1976 else
8178c126
MD
1977 vmstats.v_free_target = 2 * vmstats.v_free_min +
1978 vmstats.v_free_reserved;
984263bc 1979
0e8bd897
MD
1980 /*
1981 * NOTE: With the new buffer cache b_act_count we want the default
1982 * inactive target to be a percentage of available memory.
1983 *
1984 * The inactive target essentially determines the minimum
1985 * number of 'temporary' pages capable of caching one-time-use
1986 * files when the VM system is otherwise full of pages
1987 * belonging to multi-time-use files or active program data.
51db7ca2
MD
1988 *
1989 * NOTE: The inactive target is aggressively persued only if the
1990 * inactive queue becomes too small. If the inactive queue
1991 * is large enough to satisfy page movement to free+cache
1992 * then it is repopulated more slowly from the active queue.
e15708fc 1993 * This allows a general inactive_target default to be set.
51db7ca2
MD
1994 *
1995 * There is an issue here for processes which sit mostly idle
1996 * 'overnight', such as sshd, tcsh, and X. Any movement from
1997 * the active queue will eventually cause such pages to
1998 * recycle eventually causing a lot of paging in the morning.
1999 * To reduce the incidence of this pages cycled out of the
2000 * buffer cache are moved directly to the inactive queue if
e15708fc
MD
2001 * they were only used once or twice.
2002 *
2003 * The vfs.vm_cycle_point sysctl can be used to adjust this.
2004 * Increasing the value (up to 64) increases the number of
2005 * buffer recyclements which go directly to the inactive queue.
0e8bd897 2006 */
12e4aaff
MD
2007 if (vmstats.v_free_count > 2048) {
2008 vmstats.v_cache_min = vmstats.v_free_target;
2009 vmstats.v_cache_max = 2 * vmstats.v_cache_min;
984263bc 2010 } else {
12e4aaff
MD
2011 vmstats.v_cache_min = 0;
2012 vmstats.v_cache_max = 0;
984263bc 2013 }
e15708fc 2014 vmstats.v_inactive_target = vmstats.v_free_count / 4;
984263bc
MD
2015
2016 /* XXX does not really belong here */
2017 if (vm_page_max_wired == 0)
12e4aaff 2018 vm_page_max_wired = vmstats.v_free_count / 3;
984263bc
MD
2019
2020 if (vm_pageout_stats_max == 0)
12e4aaff 2021 vm_pageout_stats_max = vmstats.v_free_target;
984263bc
MD
2022
2023 /*
2024 * Set interval in seconds for stats scan.
2025 */
2026 if (vm_pageout_stats_interval == 0)
2027 vm_pageout_stats_interval = 5;
2028 if (vm_pageout_full_stats_interval == 0)
2029 vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;
2030
2031
2032 /*
2033 * Set maximum free per pass
2034 */
2035 if (vm_pageout_stats_free_max == 0)
2036 vm_pageout_stats_free_max = 5;
2037
2038 swap_pager_swap_init();
2039 pass = 0;
20479584 2040
c3feb36a
MD
2041 atomic_swap_int(&sequence_emerg_pager, 1);
2042 wakeup(&sequence_emerg_pager);
2043
2044skip_setup:
2045 /*
2046 * Sequence emergency pager startup
2047 */
2048 if (isep) {
2049 while (sequence_emerg_pager == 0)
2050 tsleep(&sequence_emerg_pager, 0, "pstartup", hz);
2051 }
2052
984263bc
MD
2053 /*
2054 * The pageout daemon is never done, so loop forever.
c3feb36a
MD
2055 *
2056 * WARNING! This code is being executed by two kernel threads
2057 * potentially simultaneously.
984263bc
MD
2058 */
2059 while (TRUE) {
2060 int error;
51c99c61 2061 int avail_shortage;
027193eb 2062 int inactive_shortage;
027193eb
MD
2063 int vnodes_skipped = 0;
2064 int recycle_count = 0;
2065 int tmp;
984263bc 2066
12d8aca7 2067 /*
1bfac262
MD
2068 * Wait for an action request. If we timeout check to
2069 * see if paging is needed (in case the normal wakeup
2070 * code raced us).
12d8aca7 2071 */
c3feb36a
MD
2072 if (isep) {
2073 /*
2074 * Emergency pagedaemon monitors the primary
2075 * pagedaemon while vm_pages_needed != 0.
2076 *
2077 * The emergency pagedaemon only runs if VM paging
2078 * is needed and the primary pagedaemon has not
2079 * updated vm_pagedaemon_time for more than 2 seconds.
2080 */
2081 if (vm_pages_needed)
2082 tsleep(&vm_pagedaemon_time, 0, "psleep", hz);
2083 else
2084 tsleep(&vm_pagedaemon_time, 0, "psleep", hz*10);
2085 if (vm_pages_needed == 0) {
2086 pass = 0;
2087 continue;
2088 }
2089 if ((int)(ticks - vm_pagedaemon_time) < hz * 2) {
c3feb36a 2090 pass = 0;
984263bc
MD
2091 continue;
2092 }
c3feb36a
MD
2093 } else {
2094 /*
2095 * Primary pagedaemon
2096 */
2097 if (vm_pages_needed == 0) {
2098 error = tsleep(&vm_pages_needed,
2099 0, "psleep",
2100 vm_pageout_stats_interval * hz);
2101 if (error &&
2102 vm_paging_needed() == 0 &&
2103 vm_pages_needed == 0) {
2104 for (q = 0; q < PQ_L2_SIZE; ++q)
2105 vm_pageout_page_stats(q);
2106 continue;
2107 }
2108 vm_pagedaemon_time = ticks;
2109 vm_pages_needed = 1;
2110
2111 /*
2112 * Wake the emergency pagedaemon up so it
2113 * can monitor us. It will automatically
2114 * go back into a long sleep when
2115 * vm_pages_needed returns to 0.
2116 */
2117 wakeup(&vm_pagedaemon_time);
2118 }
984263bc
MD
2119 }
2120
20479584 2121 mycpu->gd_cnt.v_pdwakeups++;
20479584 2122
027193eb 2123 /*
534ee349
MD
2124 * Scan for INACTIVE->CLEAN/PAGEOUT
2125 *
2126 * This routine tries to avoid thrashing the system with
2127 * unnecessary activity.
027193eb
MD
2128 *
2129 * Calculate our target for the number of free+cache pages we
2130 * want to get to. This is higher then the number that causes
2131 * allocations to stall (severe) in order to provide hysteresis,
2132 * and if we don't make it all the way but get to the minimum
47891af8
MD
2133 * we're happy. Goose it a bit if there are multiple requests
2134 * for memory.
2135 *
2136 * Don't reduce avail_shortage inside the loop or the
2137 * PQAVERAGE() calculation will break.
534ee349
MD
2138 *
2139 * NOTE! deficit is differentiated from avail_shortage as
2140 * REQUIRING at least (deficit) pages to be cleaned,
2141 * even if the page queues are in good shape. This
2142 * is used primarily for handling per-process
2143 * RLIMIT_RSS and may also see small values when
2144 * processes block due to low memory.
027193eb 2145 */
5ba14d44 2146 vmstats_rollup();
c3feb36a
MD
2147 if (isep == 0)
2148 vm_pagedaemon_time = ticks;
51c99c61 2149 avail_shortage = vm_paging_target() + vm_pageout_deficit;
027193eb 2150 vm_pageout_deficit = 0;
79ce07d0 2151
51c99c61 2152 if (avail_shortage > 0) {
47891af8
MD
2153 int delta = 0;
2154
51c99c61 2155 for (q = 0; q < PQ_L2_SIZE; ++q) {
47891af8 2156 delta += vm_pageout_scan_inactive(
79ce07d0
MD
2157 pass,
2158 (q + q1iterator) & PQ_L2_MASK,
51c99c61
MD
2159 PQAVERAGE(avail_shortage),
2160 &vnodes_skipped);
47891af8 2161 if (avail_shortage - delta <= 0)
79ce07d0 2162 break;
51c99c61 2163 }
47891af8 2164 avail_shortage -= delta;
79ce07d0 2165 q1iterator = q + 1;
027193eb
MD
2166 }
2167
2168 /*
2169 * Figure out how many active pages we must deactivate. If
2170 * we were able to reach our target with just the inactive
2171 * scan above we limit the number of active pages we
2172 * deactivate to reduce unnecessary work.
2173 */
5ba14d44 2174 vmstats_rollup();
c3feb36a
MD
2175 if (isep == 0)
2176 vm_pagedaemon_time = ticks;
51c99c61
MD
2177 inactive_shortage = vmstats.v_inactive_target -
2178 vmstats.v_inactive_count;
027193eb 2179
3038a8ca
MD
2180 /*
2181 * If we were unable to free sufficient inactive pages to
2182 * satisfy the free/cache queue requirements then simply
2183 * reaching the inactive target may not be good enough.
2184 * Try to deactivate pages in excess of the target based
2185 * on the shortfall.
2186 *
2187 * However to prevent thrashing the VM system do not
2188 * deactivate more than an additional 1/10 the inactive
2189 * target's worth of active pages.
2190 */
51c99c61
MD
2191 if (avail_shortage > 0) {
2192 tmp = avail_shortage * 2;
3038a8ca
MD
2193 if (tmp > vmstats.v_inactive_target / 10)
2194 tmp = vmstats.v_inactive_target / 10;
51c99c61 2195 inactive_shortage += tmp;
3038a8ca 2196 }
027193eb 2197
a5c1251f 2198 /*
534ee349
MD
2199 * Only trigger a pmap cleanup on inactive shortage.
2200 */
c3feb36a 2201 if (isep == 0 && inactive_shortage > 0) {
534ee349
MD
2202 pmap_collect();
2203 }
2204
2205 /*
2206 * Scan for ACTIVE->INACTIVE
2207 *
a5c1251f
MD
2208 * Only trigger on inactive shortage. Triggering on
2209 * avail_shortage can starve the active queue with
2210 * unnecessary active->inactive transitions and destroy
2211 * performance.
ead23175
MD
2212 *
2213 * If this is the emergency pager, always try to move
2214 * a few pages from active to inactive because the inactive
2215 * queue might have enough pages, but not enough anonymous
2216 * pages.
a5c1251f 2217 */
ead23175
MD
2218 if (isep && inactive_shortage < vm_emerg_launder)
2219 inactive_shortage = vm_emerg_launder;
2220
a5c1251f 2221 if (/*avail_shortage > 0 ||*/ inactive_shortage > 0) {
47891af8 2222 int delta = 0;
79ce07d0 2223
51c99c61 2224 for (q = 0; q < PQ_L2_SIZE; ++q) {
47891af8 2225 delta += vm_pageout_scan_active(
79ce07d0
MD
2226 pass,
2227 (q + q2iterator) & PQ_L2_MASK,
51c99c61
MD
2228 PQAVERAGE(avail_shortage),
2229 PQAVERAGE(inactive_shortage),
2230 &recycle_count);
47891af8
MD
2231 if (inactive_shortage - delta <= 0 &&
2232 avail_shortage - delta <= 0) {
79ce07d0
MD
2233 break;
2234 }
51c99c61 2235 }
47891af8
MD
2236 inactive_shortage -= delta;
2237 avail_shortage -= delta;
79ce07d0 2238 q2iterator = q + 1;
027193eb
MD
2239 }
2240
2241 /*
534ee349
MD
2242 * Scan for CACHE->FREE
2243 *
027193eb
MD
2244 * Finally free enough cache pages to meet our free page
2245 * requirement and take more drastic measures if we are
2246 * still in trouble.
2247 */
5ba14d44 2248 vmstats_rollup();
c3feb36a
MD
2249 if (isep == 0)
2250 vm_pagedaemon_time = ticks;
09eff544
MD
2251 vm_pageout_scan_cache(avail_shortage, pass,
2252 vnodes_skipped, recycle_count);
027193eb
MD
2253
2254 /*
2255 * Wait for more work.
20479584 2256 */
51c99c61 2257 if (avail_shortage > 0) {
20479584 2258 ++pass;
9cd626ca 2259 if (pass < 10 && vm_pages_needed > 1) {
20479584
MD
2260 /*
2261 * Normal operation, additional processes
9cd626ca
MD
2262 * have already kicked us. Retry immediately
2263 * unless swap space is completely full in
2264 * which case delay a bit.
20479584 2265 */
9cd626ca
MD
2266 if (swap_pager_full) {
2267 tsleep(&vm_pages_needed, 0, "pdelay",
2268 hz / 5);
2269 } /* else immediate retry */
20479584
MD
2270 } else if (pass < 10) {
2271 /*
2272 * Normal operation, fewer processes. Delay
ead23175
MD
2273 * a bit but allow wakeups. vm_pages_needed
2274 * is only adjusted against the primary
2275 * pagedaemon here.
20479584 2276 */
ead23175
MD
2277 if (isep == 0)
2278 vm_pages_needed = 0;
20479584 2279 tsleep(&vm_pages_needed, 0, "pdelay", hz / 10);
ead23175
MD
2280 if (isep == 0)
2281 vm_pages_needed = 1;
9cd626ca 2282 } else if (swap_pager_full == 0) {
20479584
MD
2283 /*
2284 * We've taken too many passes, forced delay.
2285 */
2286 tsleep(&vm_pages_needed, 0, "pdelay", hz / 10);
9cd626ca
MD
2287 } else {
2288 /*
2289 * Running out of memory, catastrophic
2290 * back-off to one-second intervals.
2291 */
2292 tsleep(&vm_pages_needed, 0, "pdelay", hz);
20479584 2293 }
77d1fb91 2294 } else if (vm_pages_needed) {
12d8aca7 2295 /*
77d1fb91
MD
2296 * Interlocked wakeup of waiters (non-optional).
2297 *
2298 * Similar to vm_page_free_wakeup() in vm_page.c,
2299 * wake
12d8aca7 2300 */
20479584 2301 pass = 0;
77d1fb91
MD
2302 if (!vm_page_count_min(vm_page_free_hysteresis) ||
2303 !vm_page_count_target()) {
12d8aca7 2304 vm_pages_needed = 0;
77d1fb91 2305 wakeup(&vmstats.v_free_count);
12d8aca7 2306 }
77d1fb91
MD
2307 } else {
2308 pass = 0;
20479584 2309 }
984263bc
MD
2310 }
2311}
2312
c3feb36a 2313static struct kproc_desc pg1_kp = {
cd8ab232
MD
2314 "pagedaemon",
2315 vm_pageout_thread,
2316 &pagethread
2317};
c3feb36a
MD
2318SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &pg1_kp);
2319
2320static struct kproc_desc pg2_kp = {
2321 "emergpager",
2322 vm_pageout_thread,
2323 &emergpager
2324};
2325SYSINIT(emergpager, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY, kproc_start, &pg2_kp);
cd8ab232
MD
2326
2327
20479584
MD
2328/*
2329 * Called after allocating a page out of the cache or free queue
2330 * to possibly wake the pagedaemon up to replentish our supply.
2331 *
2332 * We try to generate some hysteresis by waking the pagedaemon up
1bfac262
MD
2333 * when our free+cache pages go below the free_min+cache_min level.
2334 * The pagedaemon tries to get the count back up to at least the
2335 * minimum, and through to the target level if possible.
20479584
MD
2336 *
2337 * If the pagedaemon is already active bump vm_pages_needed as a hint
2338 * that there are even more requests pending.
99ad9bc4
MD
2339 *
2340 * SMP races ok?
2341 * No requirements.
20479584 2342 */
984263bc 2343void
57e43348 2344pagedaemon_wakeup(void)
984263bc 2345{
1bfac262 2346 if (vm_paging_needed() && curthread != pagethread) {
20479584 2347 if (vm_pages_needed == 0) {
1bfac262 2348 vm_pages_needed = 1; /* SMP race ok */
20479584
MD
2349 wakeup(&vm_pages_needed);
2350 } else if (vm_page_count_min(0)) {
1bfac262 2351 ++vm_pages_needed; /* SMP race ok */
20479584 2352 }
984263bc
MD
2353 }
2354}
2355
2356#if !defined(NO_SWAPPING)
99ad9bc4
MD
2357
2358/*
2359 * SMP races ok?
2360 * No requirements.
2361 */
984263bc 2362static void
57e43348 2363vm_req_vmdaemon(void)
984263bc
MD
2364{
2365 static int lastrun = 0;
2366
2367 if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
2368 wakeup(&vm_daemon_needed);
2369 lastrun = ticks;
2370 }
2371}
2372
8fa76237
MD
2373static int vm_daemon_callback(struct proc *p, void *data __unused);
2374
99ad9bc4
MD
2375/*
2376 * No requirements.
2377 */
984263bc 2378static void
57e43348 2379vm_daemon(void)
984263bc 2380{
534ee349
MD
2381 int req_swapout;
2382
984263bc 2383 while (TRUE) {
377d4740 2384 tsleep(&vm_daemon_needed, 0, "psleep", 0);
534ee349
MD
2385 req_swapout = atomic_swap_int(&vm_pageout_req_swapout, 0);
2386
2387 /*
2388 * forced swapouts
2389 */
2390 if (req_swapout)
984263bc 2391 swapout_procs(vm_pageout_req_swapout);
534ee349 2392
984263bc
MD
2393 /*
2394 * scan the processes for exceeding their rlimits or if
2395 * process is swapped out -- deactivate pages
2396 */
586c4308 2397 allproc_scan(vm_daemon_callback, NULL, 0);
8fa76237
MD
2398 }
2399}
984263bc 2400
8fa76237
MD
2401static int
2402vm_daemon_callback(struct proc *p, void *data __unused)
2403{
38189316 2404 struct vmspace *vm;
8fa76237 2405 vm_pindex_t limit, size;
984263bc 2406
8fa76237
MD
2407 /*
2408 * if this is a system process or if we have already
2409 * looked at this process, skip it.
2410 */
a8d3ab53
MD
2411 lwkt_gettoken(&p->p_token);
2412
2413 if (p->p_flags & (P_SYSTEM | P_WEXIT)) {
2414 lwkt_reltoken(&p->p_token);
8fa76237 2415 return (0);
a8d3ab53 2416 }
984263bc 2417
8fa76237
MD
2418 /*
2419 * if the process is in a non-running type state,
2420 * don't touch it.
2421 */
f5b92db7 2422 if (p->p_stat != SACTIVE && p->p_stat != SSTOP && p->p_stat != SCORE) {
a8d3ab53 2423 lwkt_reltoken(&p->p_token);
8fa76237 2424 return (0);
a8d3ab53 2425 }
984263bc 2426
8fa76237
MD
2427 /*
2428 * get a limit
2429 */
2430 limit = OFF_TO_IDX(qmin(p->p_rlimit[RLIMIT_RSS].rlim_cur,
2431 p->p_rlimit[RLIMIT_RSS].rlim_max));
2432
2433 /*
2434 * let processes that are swapped out really be
2435 * swapped out. Set the limit to nothing to get as
2436 * many pages out to swap as possible.
2437 */
4643740a 2438 if (p->p_flags & P_SWAPPEDOUT)
8fa76237
MD
2439 limit = 0;
2440
38189316
MD
2441 vm = p->p_vmspace;
2442 vmspace_hold(vm);
534ee349 2443 size = pmap_resident_tlnw_count(&vm->vm_pmap);
a7a03a5f
MD
2444 if (limit >= 0 && size > 4096 &&
2445 size - 4096 >= limit && vm_pageout_memuse_mode >= 1) {
38189316 2446 vm_pageout_map_deactivate_pages(&vm->vm_map, limit);
984263bc 2447 }
38189316 2448 vmspace_drop(vm);
a8d3ab53
MD
2449
2450 lwkt_reltoken(&p->p_token);
2451
8fa76237 2452 return (0);
984263bc 2453}
8fa76237 2454
984263bc 2455#endif