691824bc013d4d59fd3550c88346d501ca4c00ce
[dragonfly.git] / sys / vm / vm_pageout.c
1 /*
2  * Copyright (c) 1991 Regents of the University of California.
3  * All rights reserved.
4  * Copyright (c) 1994 John S. Dyson
5  * All rights reserved.
6  * Copyright (c) 1994 David Greenman
7  * All rights reserved.
8  *
9  * This code is derived from software contributed to Berkeley by
10  * The Mach Operating System project at Carnegie-Mellon University.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  * 3. All advertising materials mentioning features or use of this software
21  *    must display the following acknowledgement:
22  *      This product includes software developed by the University of
23  *      California, Berkeley and its contributors.
24  * 4. Neither the name of the University nor the names of its contributors
25  *    may be used to endorse or promote products derived from this software
26  *    without specific prior written permission.
27  *
28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38  * SUCH DAMAGE.
39  *
40  *      from: @(#)vm_pageout.c  7.4 (Berkeley) 5/7/91
41  *
42  *
43  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
44  * All rights reserved.
45  *
46  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
47  *
48  * Permission to use, copy, modify and distribute this software and
49  * its documentation is hereby granted, provided that both the copyright
50  * notice and this permission notice appear in all copies of the
51  * software, derivative works or modified versions, and any portions
52  * thereof, and that both notices appear in supporting documentation.
53  *
54  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57  *
58  * Carnegie Mellon requests users of this software to return to
59  *
60  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
61  *  School of Computer Science
62  *  Carnegie Mellon University
63  *  Pittsburgh PA 15213-3890
64  *
65  * any improvements or extensions that they make and grant Carnegie the
66  * rights to redistribute these changes.
67  *
68  * $FreeBSD: src/sys/vm/vm_pageout.c,v 1.151.2.15 2002/12/29 18:21:04 dillon Exp $
69  * $DragonFly: src/sys/vm/vm_pageout.c,v 1.36 2008/07/01 02:02:56 dillon Exp $
70  */
71
72 /*
73  *      The proverbial page-out daemon.
74  */
75
76 #include "opt_vm.h"
77 #include <sys/param.h>
78 #include <sys/systm.h>
79 #include <sys/kernel.h>
80 #include <sys/proc.h>
81 #include <sys/kthread.h>
82 #include <sys/resourcevar.h>
83 #include <sys/signalvar.h>
84 #include <sys/vnode.h>
85 #include <sys/vmmeter.h>
86 #include <sys/sysctl.h>
87
88 #include <vm/vm.h>
89 #include <vm/vm_param.h>
90 #include <sys/lock.h>
91 #include <vm/vm_object.h>
92 #include <vm/vm_page.h>
93 #include <vm/vm_map.h>
94 #include <vm/vm_pageout.h>
95 #include <vm/vm_pager.h>
96 #include <vm/swap_pager.h>
97 #include <vm/vm_extern.h>
98
99 #include <sys/thread2.h>
100 #include <vm/vm_page2.h>
101
102 /*
103  * System initialization
104  */
105
106 /* the kernel process "vm_pageout"*/
107 static void vm_pageout (void);
108 static int vm_pageout_clean (vm_page_t);
109 static int vm_pageout_scan (int pass);
110 static int vm_pageout_free_page_calc (vm_size_t count);
111 struct thread *pagethread;
112
113 static struct kproc_desc page_kp = {
114         "pagedaemon",
115         vm_pageout,
116         &pagethread
117 };
118 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &page_kp)
119
120 #if !defined(NO_SWAPPING)
121 /* the kernel process "vm_daemon"*/
122 static void vm_daemon (void);
123 static struct   thread *vmthread;
124
125 static struct kproc_desc vm_kp = {
126         "vmdaemon",
127         vm_daemon,
128         &vmthread
129 };
130 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp)
131 #endif
132
133
134 int vm_pages_needed=0;          /* Event on which pageout daemon sleeps */
135 int vm_pageout_deficit=0;       /* Estimated number of pages deficit */
136 int vm_pageout_pages_needed=0;  /* flag saying that the pageout daemon needs pages */
137
138 #if !defined(NO_SWAPPING)
139 static int vm_pageout_req_swapout;      /* XXX */
140 static int vm_daemon_needed;
141 #endif
142 static int vm_max_launder = 32;
143 static int vm_pageout_stats_max=0, vm_pageout_stats_interval = 0;
144 static int vm_pageout_full_stats_interval = 0;
145 static int vm_pageout_stats_free_max=0, vm_pageout_algorithm=0;
146 static int defer_swap_pageouts=0;
147 static int disable_swap_pageouts=0;
148
149 #if defined(NO_SWAPPING)
150 static int vm_swap_enabled=0;
151 static int vm_swap_idle_enabled=0;
152 #else
153 static int vm_swap_enabled=1;
154 static int vm_swap_idle_enabled=0;
155 #endif
156
157 SYSCTL_INT(_vm, VM_PAGEOUT_ALGORITHM, pageout_algorithm,
158         CTLFLAG_RW, &vm_pageout_algorithm, 0, "LRU page mgmt");
159
160 SYSCTL_INT(_vm, OID_AUTO, max_launder,
161         CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
162
163 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_max,
164         CTLFLAG_RW, &vm_pageout_stats_max, 0, "Max pageout stats scan length");
165
166 SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats_interval,
167         CTLFLAG_RW, &vm_pageout_full_stats_interval, 0, "Interval for full stats scan");
168
169 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_interval,
170         CTLFLAG_RW, &vm_pageout_stats_interval, 0, "Interval for partial stats scan");
171
172 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_free_max,
173         CTLFLAG_RW, &vm_pageout_stats_free_max, 0, "Not implemented");
174
175 #if defined(NO_SWAPPING)
176 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
177         CTLFLAG_RD, &vm_swap_enabled, 0, "");
178 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
179         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "");
180 #else
181 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
182         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
183 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
184         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
185 #endif
186
187 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
188         CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
189
190 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
191         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
192
193 static int pageout_lock_miss;
194 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
195         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
196
197 int vm_load;
198 SYSCTL_INT(_vm, OID_AUTO, vm_load,
199         CTLFLAG_RD, &vm_load, 0, "load on the VM system");
200 int vm_load_enable = 1;
201 SYSCTL_INT(_vm, OID_AUTO, vm_load_enable,
202         CTLFLAG_RW, &vm_load_enable, 0, "enable vm_load rate limiting");
203 #ifdef INVARIANTS
204 int vm_load_debug;
205 SYSCTL_INT(_vm, OID_AUTO, vm_load_debug,
206         CTLFLAG_RW, &vm_load_debug, 0, "debug vm_load");
207 #endif
208
209 #define VM_PAGEOUT_PAGE_COUNT 16
210 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
211
212 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
213
214 #if !defined(NO_SWAPPING)
215 typedef void freeer_fcn_t (vm_map_t, vm_object_t, vm_pindex_t, int);
216 static void vm_pageout_map_deactivate_pages (vm_map_t, vm_pindex_t);
217 static freeer_fcn_t vm_pageout_object_deactivate_pages;
218 static void vm_req_vmdaemon (void);
219 #endif
220 static void vm_pageout_page_stats(void);
221
222 /*
223  * Update vm_load to slow down faulting processes.
224  */
225 void
226 vm_fault_ratecheck(void)
227 {
228         if (vm_pages_needed) {
229                 if (vm_load < 1000)
230                         ++vm_load;
231         } else {
232                 if (vm_load > 0)
233                         --vm_load;
234         }
235 }
236
237 /*
238  * vm_pageout_clean:
239  *
240  * Clean the page and remove it from the laundry.  The page must not be
241  * busy on-call.
242  * 
243  * We set the busy bit to cause potential page faults on this page to
244  * block.  Note the careful timing, however, the busy bit isn't set till
245  * late and we cannot do anything that will mess with the page.
246  */
247
248 static int
249 vm_pageout_clean(vm_page_t m)
250 {
251         vm_object_t object;
252         vm_page_t mc[2*vm_pageout_page_count];
253         int pageout_count;
254         int ib, is, page_base;
255         vm_pindex_t pindex = m->pindex;
256
257         object = m->object;
258
259         /*
260          * It doesn't cost us anything to pageout OBJT_DEFAULT or OBJT_SWAP
261          * with the new swapper, but we could have serious problems paging
262          * out other object types if there is insufficient memory.  
263          *
264          * Unfortunately, checking free memory here is far too late, so the
265          * check has been moved up a procedural level.
266          */
267
268         /*
269          * Don't mess with the page if it's busy, held, or special
270          */
271         if ((m->hold_count != 0) ||
272             ((m->busy != 0) || (m->flags & (PG_BUSY|PG_UNMANAGED)))) {
273                 return 0;
274         }
275
276         mc[vm_pageout_page_count] = m;
277         pageout_count = 1;
278         page_base = vm_pageout_page_count;
279         ib = 1;
280         is = 1;
281
282         /*
283          * Scan object for clusterable pages.
284          *
285          * We can cluster ONLY if: ->> the page is NOT
286          * clean, wired, busy, held, or mapped into a
287          * buffer, and one of the following:
288          * 1) The page is inactive, or a seldom used
289          *    active page.
290          * -or-
291          * 2) we force the issue.
292          *
293          * During heavy mmap/modification loads the pageout
294          * daemon can really fragment the underlying file
295          * due to flushing pages out of order and not trying
296          * align the clusters (which leave sporatic out-of-order
297          * holes).  To solve this problem we do the reverse scan
298          * first and attempt to align our cluster, then do a 
299          * forward scan if room remains.
300          */
301
302 more:
303         while (ib && pageout_count < vm_pageout_page_count) {
304                 vm_page_t p;
305
306                 if (ib > pindex) {
307                         ib = 0;
308                         break;
309                 }
310
311                 if ((p = vm_page_lookup(object, pindex - ib)) == NULL) {
312                         ib = 0;
313                         break;
314                 }
315                 if (((p->queue - p->pc) == PQ_CACHE) ||
316                     (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) {
317                         ib = 0;
318                         break;
319                 }
320                 vm_page_test_dirty(p);
321                 if ((p->dirty & p->valid) == 0 ||
322                     p->queue != PQ_INACTIVE ||
323                     p->wire_count != 0 ||       /* may be held by buf cache */
324                     p->hold_count != 0) {       /* may be undergoing I/O */
325                         ib = 0;
326                         break;
327                 }
328                 mc[--page_base] = p;
329                 ++pageout_count;
330                 ++ib;
331                 /*
332                  * alignment boundry, stop here and switch directions.  Do
333                  * not clear ib.
334                  */
335                 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
336                         break;
337         }
338
339         while (pageout_count < vm_pageout_page_count && 
340             pindex + is < object->size) {
341                 vm_page_t p;
342
343                 if ((p = vm_page_lookup(object, pindex + is)) == NULL)
344                         break;
345                 if (((p->queue - p->pc) == PQ_CACHE) ||
346                     (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) {
347                         break;
348                 }
349                 vm_page_test_dirty(p);
350                 if ((p->dirty & p->valid) == 0 ||
351                     p->queue != PQ_INACTIVE ||
352                     p->wire_count != 0 ||       /* may be held by buf cache */
353                     p->hold_count != 0) {       /* may be undergoing I/O */
354                         break;
355                 }
356                 mc[page_base + pageout_count] = p;
357                 ++pageout_count;
358                 ++is;
359         }
360
361         /*
362          * If we exhausted our forward scan, continue with the reverse scan
363          * when possible, even past a page boundry.  This catches boundry
364          * conditions.
365          */
366         if (ib && pageout_count < vm_pageout_page_count)
367                 goto more;
368
369         /*
370          * we allow reads during pageouts...
371          */
372         return vm_pageout_flush(&mc[page_base], pageout_count, 0);
373 }
374
375 /*
376  * vm_pageout_flush() - launder the given pages
377  *
378  *      The given pages are laundered.  Note that we setup for the start of
379  *      I/O ( i.e. busy the page ), mark it read-only, and bump the object
380  *      reference count all in here rather then in the parent.  If we want
381  *      the parent to do more sophisticated things we may have to change
382  *      the ordering.
383  */
384 int
385 vm_pageout_flush(vm_page_t *mc, int count, int flags)
386 {
387         vm_object_t object;
388         int pageout_status[count];
389         int numpagedout = 0;
390         int i;
391
392         /*
393          * Initiate I/O.  Bump the vm_page_t->busy counter.
394          */
395         for (i = 0; i < count; i++) {
396                 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL, ("vm_pageout_flush page %p index %d/%d: partially invalid page", mc[i], i, count));
397                 vm_page_io_start(mc[i]);
398         }
399
400         /*
401          * We must make the pages read-only.  This will also force the
402          * modified bit in the related pmaps to be cleared.  The pager
403          * cannot clear the bit for us since the I/O completion code
404          * typically runs from an interrupt.  The act of making the page
405          * read-only handles the case for us.
406          */
407         for (i = 0; i < count; i++) {
408                 vm_page_protect(mc[i], VM_PROT_READ);
409         }
410
411         object = mc[0]->object;
412         vm_object_pip_add(object, count);
413
414         vm_pager_put_pages(object, mc, count,
415             (flags | ((object == &kernel_object) ? VM_PAGER_PUT_SYNC : 0)),
416             pageout_status);
417
418         for (i = 0; i < count; i++) {
419                 vm_page_t mt = mc[i];
420
421                 switch (pageout_status[i]) {
422                 case VM_PAGER_OK:
423                         numpagedout++;
424                         break;
425                 case VM_PAGER_PEND:
426                         numpagedout++;
427                         break;
428                 case VM_PAGER_BAD:
429                         /*
430                          * Page outside of range of object. Right now we
431                          * essentially lose the changes by pretending it
432                          * worked.
433                          */
434                         pmap_clear_modify(mt);
435                         vm_page_undirty(mt);
436                         break;
437                 case VM_PAGER_ERROR:
438                 case VM_PAGER_FAIL:
439                         /*
440                          * A page typically cannot be paged out when we
441                          * have run out of swap.  We leave the page
442                          * marked inactive and will try to page it out
443                          * again later.
444                          *
445                          * Starvation of the active page list is used to
446                          * determine when the system is massively memory
447                          * starved.
448                          */
449                         break;
450                 case VM_PAGER_AGAIN:
451                         break;
452                 }
453
454                 /*
455                  * If the operation is still going, leave the page busy to
456                  * block all other accesses. Also, leave the paging in
457                  * progress indicator set so that we don't attempt an object
458                  * collapse.
459                  *
460                  * For any pages which have completed synchronously, 
461                  * deactivate the page if we are under a severe deficit.
462                  * Do not try to enter them into the cache, though, they
463                  * might still be read-heavy.
464                  */
465                 if (pageout_status[i] != VM_PAGER_PEND) {
466                         vm_object_pip_wakeup(object);
467                         vm_page_io_finish(mt);
468                         if (vm_page_count_severe())
469                                 vm_page_deactivate(mt);
470 #if 0
471                         if (!vm_page_count_severe() || !vm_page_try_to_cache(mt))
472                                 vm_page_protect(mt, VM_PROT_READ);
473 #endif
474                 }
475         }
476         return numpagedout;
477 }
478
479 #if !defined(NO_SWAPPING)
480 /*
481  *      vm_pageout_object_deactivate_pages
482  *
483  *      deactivate enough pages to satisfy the inactive target
484  *      requirements or if vm_page_proc_limit is set, then
485  *      deactivate all of the pages in the object and its
486  *      backing_objects.
487  *
488  *      The object and map must be locked.
489  */
490 static int vm_pageout_object_deactivate_pages_callback(vm_page_t, void *);
491
492 static void
493 vm_pageout_object_deactivate_pages(vm_map_t map, vm_object_t object,
494         vm_pindex_t desired, int map_remove_only)
495 {
496         struct rb_vm_page_scan_info info;
497         int remove_mode;
498
499         if (object->type == OBJT_DEVICE || object->type == OBJT_PHYS)
500                 return;
501
502         while (object) {
503                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
504                         return;
505                 if (object->paging_in_progress)
506                         return;
507
508                 remove_mode = map_remove_only;
509                 if (object->shadow_count > 1)
510                         remove_mode = 1;
511
512                 /*
513                  * scan the objects entire memory queue.  spl protection is
514                  * required to avoid an interrupt unbusy/free race against
515                  * our busy check.
516                  */
517                 crit_enter();
518                 info.limit = remove_mode;
519                 info.map = map;
520                 info.desired = desired;
521                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
522                                 vm_pageout_object_deactivate_pages_callback,
523                                 &info
524                 );
525                 crit_exit();
526                 object = object->backing_object;
527         }
528 }
529                                         
530 static int
531 vm_pageout_object_deactivate_pages_callback(vm_page_t p, void *data)
532 {
533         struct rb_vm_page_scan_info *info = data;
534         int actcount;
535
536         if (pmap_resident_count(vm_map_pmap(info->map)) <= info->desired) {
537                 return(-1);
538         }
539         mycpu->gd_cnt.v_pdpages++;
540         if (p->wire_count != 0 || p->hold_count != 0 || p->busy != 0 ||
541             (p->flags & (PG_BUSY|PG_UNMANAGED)) ||
542             !pmap_page_exists_quick(vm_map_pmap(info->map), p)) {
543                 return(0);
544         }
545
546         actcount = pmap_ts_referenced(p);
547         if (actcount) {
548                 vm_page_flag_set(p, PG_REFERENCED);
549         } else if (p->flags & PG_REFERENCED) {
550                 actcount = 1;
551         }
552
553         if ((p->queue != PQ_ACTIVE) &&
554                 (p->flags & PG_REFERENCED)) {
555                 vm_page_activate(p);
556                 p->act_count += actcount;
557                 vm_page_flag_clear(p, PG_REFERENCED);
558         } else if (p->queue == PQ_ACTIVE) {
559                 if ((p->flags & PG_REFERENCED) == 0) {
560                         p->act_count -= min(p->act_count, ACT_DECLINE);
561                         if (!info->limit && (vm_pageout_algorithm || (p->act_count == 0))) {
562                                 vm_page_busy(p);
563                                 vm_page_protect(p, VM_PROT_NONE);
564                                 vm_page_wakeup(p);
565                                 vm_page_deactivate(p);
566                         } else {
567                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
568                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
569                         }
570                 } else {
571                         vm_page_activate(p);
572                         vm_page_flag_clear(p, PG_REFERENCED);
573                         if (p->act_count < (ACT_MAX - ACT_ADVANCE))
574                                 p->act_count += ACT_ADVANCE;
575                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
576                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
577                 }
578         } else if (p->queue == PQ_INACTIVE) {
579                 vm_page_busy(p);
580                 vm_page_protect(p, VM_PROT_NONE);
581                 vm_page_wakeup(p);
582         }
583         return(0);
584 }
585
586 /*
587  * deactivate some number of pages in a map, try to do it fairly, but
588  * that is really hard to do.
589  */
590 static void
591 vm_pageout_map_deactivate_pages(vm_map_t map, vm_pindex_t desired)
592 {
593         vm_map_entry_t tmpe;
594         vm_object_t obj, bigobj;
595         int nothingwired;
596
597         if (lockmgr(&map->lock, LK_EXCLUSIVE | LK_NOWAIT)) {
598                 return;
599         }
600
601         bigobj = NULL;
602         nothingwired = TRUE;
603
604         /*
605          * first, search out the biggest object, and try to free pages from
606          * that.
607          */
608         tmpe = map->header.next;
609         while (tmpe != &map->header) {
610                 switch(tmpe->maptype) {
611                 case VM_MAPTYPE_NORMAL:
612                 case VM_MAPTYPE_VPAGETABLE:
613                         obj = tmpe->object.vm_object;
614                         if ((obj != NULL) && (obj->shadow_count <= 1) &&
615                                 ((bigobj == NULL) ||
616                                  (bigobj->resident_page_count < obj->resident_page_count))) {
617                                 bigobj = obj;
618                         }
619                         break;
620                 default:
621                         break;
622                 }
623                 if (tmpe->wired_count > 0)
624                         nothingwired = FALSE;
625                 tmpe = tmpe->next;
626         }
627
628         if (bigobj)
629                 vm_pageout_object_deactivate_pages(map, bigobj, desired, 0);
630
631         /*
632          * Next, hunt around for other pages to deactivate.  We actually
633          * do this search sort of wrong -- .text first is not the best idea.
634          */
635         tmpe = map->header.next;
636         while (tmpe != &map->header) {
637                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
638                         break;
639                 switch(tmpe->maptype) {
640                 case VM_MAPTYPE_NORMAL:
641                 case VM_MAPTYPE_VPAGETABLE:
642                         obj = tmpe->object.vm_object;
643                         if (obj)
644                                 vm_pageout_object_deactivate_pages(map, obj, desired, 0);
645                         break;
646                 default:
647                         break;
648                 }
649                 tmpe = tmpe->next;
650         };
651
652         /*
653          * Remove all mappings if a process is swapped out, this will free page
654          * table pages.
655          */
656         if (desired == 0 && nothingwired)
657                 pmap_remove(vm_map_pmap(map),
658                             VM_MIN_USER_ADDRESS, VM_MAX_USER_ADDRESS);
659         vm_map_unlock(map);
660 }
661 #endif
662
663 /*
664  * Don't try to be fancy - being fancy can lead to vnode deadlocks.   We
665  * only do it for OBJT_DEFAULT and OBJT_SWAP objects which we know can
666  * be trivially freed.
667  */
668 void
669 vm_pageout_page_free(vm_page_t m) 
670 {
671         vm_object_t object = m->object;
672         int type = object->type;
673
674         if (type == OBJT_SWAP || type == OBJT_DEFAULT)
675                 vm_object_reference(object);
676         vm_page_busy(m);
677         vm_page_protect(m, VM_PROT_NONE);
678         vm_page_free(m);
679         if (type == OBJT_SWAP || type == OBJT_DEFAULT)
680                 vm_object_deallocate(object);
681 }
682
683 /*
684  * vm_pageout_scan does the dirty work for the pageout daemon.
685  */
686 struct vm_pageout_scan_info {
687         struct proc *bigproc;
688         vm_offset_t bigsize;
689 };
690
691 static int vm_pageout_scan_callback(struct proc *p, void *data);
692
693 static int
694 vm_pageout_scan(int pass)
695 {
696         struct vm_pageout_scan_info info;
697         vm_page_t m, next;
698         struct vm_page marker;
699         int maxscan, pcount;
700         int recycle_count;
701         int inactive_shortage, active_shortage;
702         int inactive_original_shortage;
703         vm_object_t object;
704         int actcount;
705         int vnodes_skipped = 0;
706         int maxlaunder;
707
708         /*
709          * Do whatever cleanup that the pmap code can.
710          */
711         pmap_collect();
712
713         /*
714          * Calculate our target for the number of free+cache pages we
715          * want to get to.  This is higher then the number that causes
716          * allocations to stall (severe) in order to provide hysteresis,
717          * and if we don't make it all the way but get to the minimum
718          * we're happy.
719          */
720         inactive_shortage = vm_paging_target() + vm_pageout_deficit;
721         inactive_original_shortage = inactive_shortage;
722         vm_pageout_deficit = 0;
723
724         /*
725          * Initialize our marker
726          */
727         bzero(&marker, sizeof(marker));
728         marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
729         marker.queue = PQ_INACTIVE;
730         marker.wire_count = 1;
731
732         /*
733          * Start scanning the inactive queue for pages we can move to the
734          * cache or free.  The scan will stop when the target is reached or
735          * we have scanned the entire inactive queue.  Note that m->act_count
736          * is not used to form decisions for the inactive queue, only for the
737          * active queue.
738          *
739          * maxlaunder limits the number of dirty pages we flush per scan.
740          * For most systems a smaller value (16 or 32) is more robust under
741          * extreme memory and disk pressure because any unnecessary writes
742          * to disk can result in extreme performance degredation.  However,
743          * systems with excessive dirty pages (especially when MAP_NOSYNC is
744          * used) will die horribly with limited laundering.  If the pageout
745          * daemon cannot clean enough pages in the first pass, we let it go
746          * all out in succeeding passes.
747          */
748         if ((maxlaunder = vm_max_launder) <= 1)
749                 maxlaunder = 1;
750         if (pass)
751                 maxlaunder = 10000;
752
753         /*
754          * We will generally be in a critical section throughout the 
755          * scan, but we can release it temporarily when we are sitting on a
756          * non-busy page without fear.  this is required to prevent an
757          * interrupt from unbusying or freeing a page prior to our busy
758          * check, leaving us on the wrong queue or checking the wrong
759          * page.
760          */
761         crit_enter();
762 rescan0:
763         maxscan = vmstats.v_inactive_count;
764         for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
765              m != NULL && maxscan-- > 0 && inactive_shortage > 0;
766              m = next
767          ) {
768                 mycpu->gd_cnt.v_pdpages++;
769
770                 /*
771                  * Give interrupts a chance
772                  */
773                 crit_exit();
774                 crit_enter();
775
776                 /*
777                  * It's easier for some of the conditions below to just loop
778                  * and catch queue changes here rather then check everywhere
779                  * else.
780                  */
781                 if (m->queue != PQ_INACTIVE)
782                         goto rescan0;
783                 next = TAILQ_NEXT(m, pageq);
784
785                 /*
786                  * skip marker pages
787                  */
788                 if (m->flags & PG_MARKER)
789                         continue;
790
791                 /*
792                  * A held page may be undergoing I/O, so skip it.
793                  */
794                 if (m->hold_count) {
795                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
796                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
797                         ++vm_swapcache_inactive_heuristic;
798                         continue;
799                 }
800
801                 /*
802                  * Dont mess with busy pages, keep in the front of the
803                  * queue, most likely are being paged out.
804                  */
805                 if (m->busy || (m->flags & PG_BUSY)) {
806                         continue;
807                 }
808
809                 if (m->object->ref_count == 0) {
810                         /*
811                          * If the object is not being used, we ignore previous 
812                          * references.
813                          */
814                         vm_page_flag_clear(m, PG_REFERENCED);
815                         pmap_clear_reference(m);
816
817                 } else if (((m->flags & PG_REFERENCED) == 0) &&
818                             (actcount = pmap_ts_referenced(m))) {
819                         /*
820                          * Otherwise, if the page has been referenced while 
821                          * in the inactive queue, we bump the "activation
822                          * count" upwards, making it less likely that the
823                          * page will be added back to the inactive queue
824                          * prematurely again.  Here we check the page tables
825                          * (or emulated bits, if any), given the upper level
826                          * VM system not knowing anything about existing 
827                          * references.
828                          */
829                         vm_page_activate(m);
830                         m->act_count += (actcount + ACT_ADVANCE);
831                         continue;
832                 }
833
834                 /*
835                  * If the upper level VM system knows about any page 
836                  * references, we activate the page.  We also set the 
837                  * "activation count" higher than normal so that we will less 
838                  * likely place pages back onto the inactive queue again.
839                  */
840                 if ((m->flags & PG_REFERENCED) != 0) {
841                         vm_page_flag_clear(m, PG_REFERENCED);
842                         actcount = pmap_ts_referenced(m);
843                         vm_page_activate(m);
844                         m->act_count += (actcount + ACT_ADVANCE + 1);
845                         continue;
846                 }
847
848                 /*
849                  * If the upper level VM system doesn't know anything about 
850                  * the page being dirty, we have to check for it again.  As 
851                  * far as the VM code knows, any partially dirty pages are 
852                  * fully dirty.
853                  *
854                  * Pages marked PG_WRITEABLE may be mapped into the user
855                  * address space of a process running on another cpu.  A
856                  * user process (without holding the MP lock) running on
857                  * another cpu may be able to touch the page while we are
858                  * trying to remove it.  vm_page_cache() will handle this
859                  * case for us.
860                  */
861                 if (m->dirty == 0) {
862                         vm_page_test_dirty(m);
863                 } else {
864                         vm_page_dirty(m);
865                 }
866
867                 if (m->valid == 0) {
868                         /*
869                          * Invalid pages can be easily freed
870                          */
871                         vm_pageout_page_free(m);
872                         mycpu->gd_cnt.v_dfree++;
873                         --inactive_shortage;
874                 } else if (m->dirty == 0) {
875                         /*
876                          * Clean pages can be placed onto the cache queue.
877                          * This effectively frees them.
878                          */
879                         vm_page_cache(m);
880                         --inactive_shortage;
881                 } else if ((m->flags & PG_WINATCFLS) == 0 && pass == 0) {
882                         /*
883                          * Dirty pages need to be paged out, but flushing
884                          * a page is extremely expensive verses freeing
885                          * a clean page.  Rather then artificially limiting
886                          * the number of pages we can flush, we instead give
887                          * dirty pages extra priority on the inactive queue
888                          * by forcing them to be cycled through the queue
889                          * twice before being flushed, after which the 
890                          * (now clean) page will cycle through once more
891                          * before being freed.  This significantly extends
892                          * the thrash point for a heavily loaded machine.
893                          */
894                         vm_page_flag_set(m, PG_WINATCFLS);
895                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
896                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
897                         ++vm_swapcache_inactive_heuristic;
898                 } else if (maxlaunder > 0) {
899                         /*
900                          * We always want to try to flush some dirty pages if
901                          * we encounter them, to keep the system stable.
902                          * Normally this number is small, but under extreme
903                          * pressure where there are insufficient clean pages
904                          * on the inactive queue, we may have to go all out.
905                          */
906                         int swap_pageouts_ok;
907                         struct vnode *vp = NULL;
908
909                         object = m->object;
910
911                         if ((object->type != OBJT_SWAP) && (object->type != OBJT_DEFAULT)) {
912                                 swap_pageouts_ok = 1;
913                         } else {
914                                 swap_pageouts_ok = !(defer_swap_pageouts || disable_swap_pageouts);
915                                 swap_pageouts_ok |= (!disable_swap_pageouts && defer_swap_pageouts &&
916                                 vm_page_count_min(0));
917                                                                                 
918                         }
919
920                         /*
921                          * We don't bother paging objects that are "dead".  
922                          * Those objects are in a "rundown" state.
923                          */
924                         if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) {
925                                 TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
926                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
927                                 ++vm_swapcache_inactive_heuristic;
928                                 continue;
929                         }
930
931                         /*
932                          * The object is already known NOT to be dead.   It
933                          * is possible for the vget() to block the whole
934                          * pageout daemon, but the new low-memory handling
935                          * code should prevent it.
936                          *
937                          * The previous code skipped locked vnodes and, worse,
938                          * reordered pages in the queue.  This results in
939                          * completely non-deterministic operation because,
940                          * quite often, a vm_fault has initiated an I/O and
941                          * is holding a locked vnode at just the point where
942                          * the pageout daemon is woken up.
943                          *
944                          * We can't wait forever for the vnode lock, we might
945                          * deadlock due to a vn_read() getting stuck in
946                          * vm_wait while holding this vnode.  We skip the 
947                          * vnode if we can't get it in a reasonable amount
948                          * of time.
949                          */
950
951                         if (object->type == OBJT_VNODE) {
952                                 vp = object->handle;
953
954                                 if (vget(vp, LK_EXCLUSIVE|LK_NOOBJ|LK_TIMELOCK)) {
955                                         ++pageout_lock_miss;
956                                         if (object->flags & OBJ_MIGHTBEDIRTY)
957                                                     vnodes_skipped++;
958                                         continue;
959                                 }
960
961                                 /*
962                                  * The page might have been moved to another
963                                  * queue during potential blocking in vget()
964                                  * above.  The page might have been freed and
965                                  * reused for another vnode.  The object might
966                                  * have been reused for another vnode.
967                                  */
968                                 if (m->queue != PQ_INACTIVE ||
969                                     m->object != object ||
970                                     object->handle != vp) {
971                                         if (object->flags & OBJ_MIGHTBEDIRTY)
972                                                 vnodes_skipped++;
973                                         vput(vp);
974                                         continue;
975                                 }
976         
977                                 /*
978                                  * The page may have been busied during the
979                                  * blocking in vput();  We don't move the
980                                  * page back onto the end of the queue so that
981                                  * statistics are more correct if we don't.
982                                  */
983                                 if (m->busy || (m->flags & PG_BUSY)) {
984                                         vput(vp);
985                                         continue;
986                                 }
987
988                                 /*
989                                  * If the page has become held it might
990                                  * be undergoing I/O, so skip it
991                                  */
992                                 if (m->hold_count) {
993                                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
994                                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
995                                         ++vm_swapcache_inactive_heuristic;
996                                         if (object->flags & OBJ_MIGHTBEDIRTY)
997                                                 vnodes_skipped++;
998                                         vput(vp);
999                                         continue;
1000                                 }
1001                         }
1002
1003                         /*
1004                          * If a page is dirty, then it is either being washed
1005                          * (but not yet cleaned) or it is still in the
1006                          * laundry.  If it is still in the laundry, then we
1007                          * start the cleaning operation. 
1008                          *
1009                          * This operation may cluster, invalidating the 'next'
1010                          * pointer.  To prevent an inordinate number of
1011                          * restarts we use our marker to remember our place.
1012                          *
1013                          * decrement inactive_shortage on success to account
1014                          * for the (future) cleaned page.  Otherwise we
1015                          * could wind up laundering or cleaning too many
1016                          * pages.
1017                          */
1018                         TAILQ_INSERT_AFTER(&vm_page_queues[PQ_INACTIVE].pl, m, &marker, pageq);
1019                         if (vm_pageout_clean(m) != 0) {
1020                                 --inactive_shortage;
1021                                 --maxlaunder;
1022                         }
1023                         next = TAILQ_NEXT(&marker, pageq);
1024                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, &marker, pageq);
1025                         if (vp != NULL)
1026                                 vput(vp);
1027                 }
1028         }
1029
1030         /*
1031          * We want to move pages from the active queue to the inactive
1032          * queue to get the inactive queue to the inactive target.  If
1033          * we still have a page shortage from above we try to directly free
1034          * clean pages instead of moving them.
1035          *
1036          * If we do still have a shortage we keep track of the number of
1037          * pages we free or cache (recycle_count) as a measure of thrashing
1038          * between the active and inactive queues.
1039          *
1040          * If we were able to completely satisfy the free+cache targets
1041          * from the inactive pool we limit the number of pages we move
1042          * from the active pool to the inactive pool to 2x the pages we
1043          * had removed from the inactive pool (with a minimum of 1/5 the
1044          * inactive target).  If we were not able to completely satisfy
1045          * the free+cache targets we go for the whole target aggressively.
1046          *
1047          * NOTE: Both variables can end up negative.
1048          * NOTE: We are still in a critical section.
1049          */
1050         active_shortage = vmstats.v_inactive_target - vmstats.v_inactive_count;
1051         if (inactive_original_shortage < vmstats.v_inactive_target / 10)
1052                 inactive_original_shortage = vmstats.v_inactive_target / 10;
1053         if (inactive_shortage <= 0 &&
1054             active_shortage > inactive_original_shortage * 2) {
1055                 active_shortage = inactive_original_shortage * 2;
1056         }
1057
1058         pcount = vmstats.v_active_count;
1059         recycle_count = 0;
1060         m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1061
1062         while ((m != NULL) && (pcount-- > 0) &&
1063                (inactive_shortage > 0 || active_shortage > 0)
1064         ) {
1065                 /*
1066                  * Give interrupts a chance.
1067                  */
1068                 crit_exit();
1069                 crit_enter();
1070
1071                 /*
1072                  * If the page was ripped out from under us, just stop.
1073                  */
1074                 if (m->queue != PQ_ACTIVE)
1075                         break;
1076                 next = TAILQ_NEXT(m, pageq);
1077
1078                 /*
1079                  * Don't deactivate pages that are busy.
1080                  */
1081                 if ((m->busy != 0) ||
1082                     (m->flags & PG_BUSY) ||
1083                     (m->hold_count != 0)) {
1084                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1085                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1086                         m = next;
1087                         continue;
1088                 }
1089
1090                 /*
1091                  * The count for pagedaemon pages is done after checking the
1092                  * page for eligibility...
1093                  */
1094                 mycpu->gd_cnt.v_pdpages++;
1095
1096                 /*
1097                  * Check to see "how much" the page has been used and clear
1098                  * the tracking access bits.  If the object has no references
1099                  * don't bother paying the expense.
1100                  */
1101                 actcount = 0;
1102                 if (m->object->ref_count != 0) {
1103                         if (m->flags & PG_REFERENCED)
1104                                 ++actcount;
1105                         actcount += pmap_ts_referenced(m);
1106                         if (actcount) {
1107                                 m->act_count += ACT_ADVANCE + actcount;
1108                                 if (m->act_count > ACT_MAX)
1109                                         m->act_count = ACT_MAX;
1110                         }
1111                 }
1112                 vm_page_flag_clear(m, PG_REFERENCED);
1113
1114                 /*
1115                  * actcount is only valid if the object ref_count is non-zero.
1116                  */
1117                 if (actcount && m->object->ref_count != 0) {
1118                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1119                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1120                 } else {
1121                         m->act_count -= min(m->act_count, ACT_DECLINE);
1122                         if (vm_pageout_algorithm ||
1123                             m->object->ref_count == 0 ||
1124                             m->act_count < pass + 1
1125                         ) {
1126                                 /*
1127                                  * Deactivate the page.  If we had a
1128                                  * shortage from our inactive scan try to
1129                                  * free (cache) the page instead.
1130                                  *
1131                                  * Don't just blindly cache the page if
1132                                  * we do not have a shortage from the
1133                                  * inactive scan, that could lead to
1134                                  * gigabytes being moved.
1135                                  */
1136                                 --active_shortage;
1137                                 if (inactive_shortage > 0 ||
1138                                     m->object->ref_count == 0) {
1139                                         if (inactive_shortage > 0)
1140                                                 ++recycle_count;
1141                                         vm_page_busy(m);
1142                                         vm_page_protect(m, VM_PROT_NONE);
1143                                         vm_page_wakeup(m);
1144                                         if (m->dirty == 0 &&
1145                                             inactive_shortage > 0) {
1146                                                 --inactive_shortage;
1147                                                 vm_page_cache(m);
1148                                         } else {
1149                                                 vm_page_deactivate(m);
1150                                         }
1151                                 } else {
1152                                         vm_page_deactivate(m);
1153                                 }
1154                         } else {
1155                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1156                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1157                         }
1158                 }
1159                 m = next;
1160         }
1161
1162         /*
1163          * We try to maintain some *really* free pages, this allows interrupt
1164          * code to be guaranteed space.  Since both cache and free queues 
1165          * are considered basically 'free', moving pages from cache to free
1166          * does not effect other calculations.
1167          *
1168          * NOTE: we are still in a critical section.
1169          *
1170          * Pages moved from PQ_CACHE to totally free are not counted in the
1171          * pages_freed counter.
1172          */
1173         while (vmstats.v_free_count < vmstats.v_free_reserved) {
1174                 static int cache_rover = 0;
1175                 m = vm_page_list_find(PQ_CACHE, cache_rover, FALSE);
1176                 if (m == NULL)
1177                         break;
1178                 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || 
1179                     m->busy || 
1180                     m->hold_count || 
1181                     m->wire_count) {
1182 #ifdef INVARIANTS
1183                         kprintf("Warning: busy page %p found in cache\n", m);
1184 #endif
1185                         vm_page_deactivate(m);
1186                         continue;
1187                 }
1188                 KKASSERT((m->flags & PG_MAPPED) == 0);
1189                 KKASSERT(m->dirty == 0);
1190                 cache_rover = (cache_rover + PQ_PRIME2) & PQ_L2_MASK;
1191                 vm_pageout_page_free(m);
1192                 mycpu->gd_cnt.v_dfree++;
1193         }
1194
1195         crit_exit();
1196
1197 #if !defined(NO_SWAPPING)
1198         /*
1199          * Idle process swapout -- run once per second.
1200          */
1201         if (vm_swap_idle_enabled) {
1202                 static long lsec;
1203                 if (time_second != lsec) {
1204                         vm_pageout_req_swapout |= VM_SWAP_IDLE;
1205                         vm_req_vmdaemon();
1206                         lsec = time_second;
1207                 }
1208         }
1209 #endif
1210                 
1211         /*
1212          * If we didn't get enough free pages, and we have skipped a vnode
1213          * in a writeable object, wakeup the sync daemon.  And kick swapout
1214          * if we did not get enough free pages.
1215          */
1216         if (vm_paging_target() > 0) {
1217                 if (vnodes_skipped && vm_page_count_min(0))
1218                         speedup_syncer();
1219 #if !defined(NO_SWAPPING)
1220                 if (vm_swap_enabled && vm_page_count_target()) {
1221                         vm_req_vmdaemon();
1222                         vm_pageout_req_swapout |= VM_SWAP_NORMAL;
1223                 }
1224 #endif
1225         }
1226
1227         /*
1228          * Handle catastrophic conditions.  Under good conditions we should
1229          * be at the target, well beyond our minimum.  If we could not even
1230          * reach our minimum the system is under heavy stress.
1231          *
1232          * Determine whether we have run out of memory.  This occurs when
1233          * swap_pager_full is TRUE and the only pages left in the page
1234          * queues are dirty.  We will still likely have page shortages.
1235          *
1236          * - swap_pager_full is set if insufficient swap was
1237          *   available to satisfy a requested pageout.
1238          *
1239          * - the inactive queue is bloated (4 x size of active queue),
1240          *   meaning it is unable to get rid of dirty pages and.
1241          *
1242          * - vm_page_count_min() without counting pages recycled from the
1243          *   active queue (recycle_count) means we could not recover
1244          *   enough pages to meet bare minimum needs.  This test only
1245          *   works if the inactive queue is bloated.
1246          *
1247          * - due to a positive inactive_shortage we shifted the remaining
1248          *   dirty pages from the active queue to the inactive queue
1249          *   trying to find clean ones to free.
1250          */
1251         if (swap_pager_full && vm_page_count_min(recycle_count))
1252                 kprintf("Warning: system low on memory+swap!\n");
1253         if (swap_pager_full && vm_page_count_min(recycle_count) &&
1254             vmstats.v_inactive_count > vmstats.v_active_count * 4 &&
1255             inactive_shortage > 0) {
1256                 /*
1257                  * Kill something.
1258                  */
1259                 info.bigproc = NULL;
1260                 info.bigsize = 0;
1261                 allproc_scan(vm_pageout_scan_callback, &info);
1262                 if (info.bigproc != NULL) {
1263                         killproc(info.bigproc, "out of swap space");
1264                         info.bigproc->p_nice = PRIO_MIN;
1265                         info.bigproc->p_usched->resetpriority(
1266                                 FIRST_LWP_IN_PROC(info.bigproc));
1267                         wakeup(&vmstats.v_free_count);
1268                         PRELE(info.bigproc);
1269                 }
1270         }
1271         return(inactive_shortage);
1272 }
1273
1274 static int
1275 vm_pageout_scan_callback(struct proc *p, void *data)
1276 {
1277         struct vm_pageout_scan_info *info = data;
1278         vm_offset_t size;
1279
1280         /*
1281          * Never kill system processes or init.  If we have configured swap
1282          * then try to avoid killing low-numbered pids.
1283          */
1284         if ((p->p_flag & P_SYSTEM) || (p->p_pid == 1) ||
1285             ((p->p_pid < 48) && (vm_swap_size != 0))) {
1286                 return (0);
1287         }
1288
1289         /*
1290          * if the process is in a non-running type state,
1291          * don't touch it.
1292          */
1293         if (p->p_stat != SACTIVE && p->p_stat != SSTOP)
1294                 return (0);
1295
1296         /*
1297          * Get the approximate process size.  Note that anonymous pages
1298          * with backing swap will be counted twice, but there should not
1299          * be too many such pages due to the stress the VM system is
1300          * under at this point.
1301          */
1302         size = vmspace_anonymous_count(p->p_vmspace) +
1303                 vmspace_swap_count(p->p_vmspace);
1304
1305         /*
1306          * If the this process is bigger than the biggest one
1307          * remember it.
1308          */
1309         if (info->bigsize < size) {
1310                 if (info->bigproc)
1311                         PRELE(info->bigproc);
1312                 PHOLD(p);
1313                 info->bigproc = p;
1314                 info->bigsize = size;
1315         }
1316         return(0);
1317 }
1318
1319 /*
1320  * This routine tries to maintain the pseudo LRU active queue,
1321  * so that during long periods of time where there is no paging,
1322  * that some statistic accumulation still occurs.  This code
1323  * helps the situation where paging just starts to occur.
1324  */
1325 static void
1326 vm_pageout_page_stats(void)
1327 {
1328         vm_page_t m,next;
1329         int pcount,tpcount;             /* Number of pages to check */
1330         static int fullintervalcount = 0;
1331         int page_shortage;
1332
1333         page_shortage = 
1334             (vmstats.v_inactive_target + vmstats.v_cache_max + vmstats.v_free_min) -
1335             (vmstats.v_free_count + vmstats.v_inactive_count + vmstats.v_cache_count);
1336
1337         if (page_shortage <= 0)
1338                 return;
1339
1340         crit_enter();
1341
1342         pcount = vmstats.v_active_count;
1343         fullintervalcount += vm_pageout_stats_interval;
1344         if (fullintervalcount < vm_pageout_full_stats_interval) {
1345                 tpcount = (vm_pageout_stats_max * vmstats.v_active_count) / vmstats.v_page_count;
1346                 if (pcount > tpcount)
1347                         pcount = tpcount;
1348         } else {
1349                 fullintervalcount = 0;
1350         }
1351
1352         m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1353         while ((m != NULL) && (pcount-- > 0)) {
1354                 int actcount;
1355
1356                 if (m->queue != PQ_ACTIVE) {
1357                         break;
1358                 }
1359
1360                 next = TAILQ_NEXT(m, pageq);
1361                 /*
1362                  * Don't deactivate pages that are busy.
1363                  */
1364                 if ((m->busy != 0) ||
1365                     (m->flags & PG_BUSY) ||
1366                     (m->hold_count != 0)) {
1367                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1368                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1369                         m = next;
1370                         continue;
1371                 }
1372
1373                 actcount = 0;
1374                 if (m->flags & PG_REFERENCED) {
1375                         vm_page_flag_clear(m, PG_REFERENCED);
1376                         actcount += 1;
1377                 }
1378
1379                 actcount += pmap_ts_referenced(m);
1380                 if (actcount) {
1381                         m->act_count += ACT_ADVANCE + actcount;
1382                         if (m->act_count > ACT_MAX)
1383                                 m->act_count = ACT_MAX;
1384                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1385                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1386                 } else {
1387                         if (m->act_count == 0) {
1388                                 /*
1389                                  * We turn off page access, so that we have
1390                                  * more accurate RSS stats.  We don't do this
1391                                  * in the normal page deactivation when the
1392                                  * system is loaded VM wise, because the
1393                                  * cost of the large number of page protect
1394                                  * operations would be higher than the value
1395                                  * of doing the operation.
1396                                  */
1397                                 vm_page_busy(m);
1398                                 vm_page_protect(m, VM_PROT_NONE);
1399                                 vm_page_wakeup(m);
1400                                 vm_page_deactivate(m);
1401                         } else {
1402                                 m->act_count -= min(m->act_count, ACT_DECLINE);
1403                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1404                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1405                         }
1406                 }
1407
1408                 m = next;
1409         }
1410         crit_exit();
1411 }
1412
1413 static int
1414 vm_pageout_free_page_calc(vm_size_t count)
1415 {
1416         if (count < vmstats.v_page_count)
1417                  return 0;
1418         /*
1419          * free_reserved needs to include enough for the largest swap pager
1420          * structures plus enough for any pv_entry structs when paging.
1421          */
1422         if (vmstats.v_page_count > 1024)
1423                 vmstats.v_free_min = 4 + (vmstats.v_page_count - 1024) / 200;
1424         else
1425                 vmstats.v_free_min = 4;
1426         vmstats.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1427                 vmstats.v_interrupt_free_min;
1428         vmstats.v_free_reserved = vm_pageout_page_count +
1429                 vmstats.v_pageout_free_min + (count / 768) + PQ_L2_SIZE;
1430         vmstats.v_free_severe = vmstats.v_free_min / 2;
1431         vmstats.v_free_min += vmstats.v_free_reserved;
1432         vmstats.v_free_severe += vmstats.v_free_reserved;
1433         return 1;
1434 }
1435
1436
1437 /*
1438  * vm_pageout is the high level pageout daemon.
1439  */
1440 static void
1441 vm_pageout(void)
1442 {
1443         int pass;
1444         int inactive_shortage;
1445
1446         /*
1447          * Initialize some paging parameters.
1448          */
1449         curthread->td_flags |= TDF_SYSTHREAD;
1450
1451         vmstats.v_interrupt_free_min = 2;
1452         if (vmstats.v_page_count < 2000)
1453                 vm_pageout_page_count = 8;
1454
1455         vm_pageout_free_page_calc(vmstats.v_page_count);
1456
1457         /*
1458          * v_free_target and v_cache_min control pageout hysteresis.  Note
1459          * that these are more a measure of the VM cache queue hysteresis
1460          * then the VM free queue.  Specifically, v_free_target is the
1461          * high water mark (free+cache pages).
1462          *
1463          * v_free_reserved + v_cache_min (mostly means v_cache_min) is the
1464          * low water mark, while v_free_min is the stop.  v_cache_min must
1465          * be big enough to handle memory needs while the pageout daemon
1466          * is signalled and run to free more pages.
1467          */
1468         if (vmstats.v_free_count > 6144)
1469                 vmstats.v_free_target = 4 * vmstats.v_free_min + vmstats.v_free_reserved;
1470         else
1471                 vmstats.v_free_target = 2 * vmstats.v_free_min + vmstats.v_free_reserved;
1472
1473         /*
1474          * NOTE: With the new buffer cache b_act_count we want the default
1475          *       inactive target to be a percentage of available memory.
1476          *
1477          *       The inactive target essentially determines the minimum
1478          *       number of 'temporary' pages capable of caching one-time-use
1479          *       files when the VM system is otherwise full of pages
1480          *       belonging to multi-time-use files or active program data.
1481          *
1482          * NOTE: The inactive target is aggressively persued only if the
1483          *       inactive queue becomes too small.  If the inactive queue
1484          *       is large enough to satisfy page movement to free+cache
1485          *       then it is repopulated more slowly from the active queue.
1486          *       This allows a general inactive_target default to be set.
1487          *
1488          *       There is an issue here for processes which sit mostly idle
1489          *       'overnight', such as sshd, tcsh, and X.  Any movement from
1490          *       the active queue will eventually cause such pages to
1491          *       recycle eventually causing a lot of paging in the morning.
1492          *       To reduce the incidence of this pages cycled out of the
1493          *       buffer cache are moved directly to the inactive queue if
1494          *       they were only used once or twice.
1495          *
1496          *       The vfs.vm_cycle_point sysctl can be used to adjust this.
1497          *       Increasing the value (up to 64) increases the number of
1498          *       buffer recyclements which go directly to the inactive queue.
1499          */
1500         if (vmstats.v_free_count > 2048) {
1501                 vmstats.v_cache_min = vmstats.v_free_target;
1502                 vmstats.v_cache_max = 2 * vmstats.v_cache_min;
1503         } else {
1504                 vmstats.v_cache_min = 0;
1505                 vmstats.v_cache_max = 0;
1506         }
1507         vmstats.v_inactive_target = vmstats.v_free_count / 4;
1508
1509         /* XXX does not really belong here */
1510         if (vm_page_max_wired == 0)
1511                 vm_page_max_wired = vmstats.v_free_count / 3;
1512
1513         if (vm_pageout_stats_max == 0)
1514                 vm_pageout_stats_max = vmstats.v_free_target;
1515
1516         /*
1517          * Set interval in seconds for stats scan.
1518          */
1519         if (vm_pageout_stats_interval == 0)
1520                 vm_pageout_stats_interval = 5;
1521         if (vm_pageout_full_stats_interval == 0)
1522                 vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;
1523         
1524
1525         /*
1526          * Set maximum free per pass
1527          */
1528         if (vm_pageout_stats_free_max == 0)
1529                 vm_pageout_stats_free_max = 5;
1530
1531         swap_pager_swap_init();
1532         pass = 0;
1533
1534         /*
1535          * The pageout daemon is never done, so loop forever.
1536          */
1537         while (TRUE) {
1538                 int error;
1539
1540                 /*
1541                  * Wait for an action request
1542                  */
1543                 crit_enter();
1544                 if (vm_pages_needed == 0) {
1545                         error = tsleep(&vm_pages_needed,
1546                                        0, "psleep",
1547                                        vm_pageout_stats_interval * hz);
1548                         if (error && vm_pages_needed == 0) {
1549                                 vm_pageout_page_stats();
1550                                 continue;
1551                         }
1552                         vm_pages_needed = 1;
1553                 }
1554                 crit_exit();
1555
1556                 /*
1557                  * If we have enough free memory, wakeup waiters.
1558                  * (This is optional here)
1559                  */
1560                 crit_enter();
1561                 if (!vm_page_count_min(0))
1562                         wakeup(&vmstats.v_free_count);
1563                 mycpu->gd_cnt.v_pdwakeups++;
1564                 crit_exit();
1565
1566                 /*
1567                  * Scan for pageout.  Try to avoid thrashing the system
1568                  * with activity.
1569                  */
1570                 inactive_shortage = vm_pageout_scan(pass);
1571                 if (inactive_shortage > 0) {
1572                         ++pass;
1573                         if (swap_pager_full) {
1574                                 /*
1575                                  * Running out of memory, catastrophic back-off
1576                                  * to one-second intervals.
1577                                  */
1578                                 tsleep(&vm_pages_needed, 0, "pdelay", hz);
1579                         } else if (pass < 10 && vm_pages_needed > 1) {
1580                                 /*
1581                                  * Normal operation, additional processes
1582                                  * have already kicked us.  Retry immediately.
1583                                  */
1584                         } else if (pass < 10) {
1585                                 /*
1586                                  * Normal operation, fewer processes.  Delay
1587                                  * a bit but allow wakeups.
1588                                  */
1589                                 vm_pages_needed = 0;
1590                                 tsleep(&vm_pages_needed, 0, "pdelay", hz / 10);
1591                                 vm_pages_needed = 1;
1592                         } else {
1593                                 /*
1594                                  * We've taken too many passes, forced delay.
1595                                  */
1596                                 tsleep(&vm_pages_needed, 0, "pdelay", hz / 10);
1597                         }
1598                 } else {
1599                         /*
1600                          * Interlocked wakeup of waiters (non-optional)
1601                          */
1602                         pass = 0;
1603                         if (vm_pages_needed && !vm_page_count_min(0)) {
1604                                 wakeup(&vmstats.v_free_count);
1605                                 vm_pages_needed = 0;
1606                         }
1607                 }
1608         }
1609 }
1610
1611 /*
1612  * Called after allocating a page out of the cache or free queue
1613  * to possibly wake the pagedaemon up to replentish our supply.
1614  *
1615  * We try to generate some hysteresis by waking the pagedaemon up
1616  * when our free+cache pages go below the severe level.  The pagedaemon
1617  * tries to get the count back up to at least the minimum, and through
1618  * to the target level if possible.
1619  *
1620  * If the pagedaemon is already active bump vm_pages_needed as a hint
1621  * that there are even more requests pending.
1622  */
1623 void
1624 pagedaemon_wakeup(void)
1625 {
1626         if (vm_page_count_severe() && curthread != pagethread) {
1627                 if (vm_pages_needed == 0) {
1628                         vm_pages_needed = 1;
1629                         wakeup(&vm_pages_needed);
1630                 } else if (vm_page_count_min(0)) {
1631                         ++vm_pages_needed;
1632                 }
1633         }
1634 }
1635
1636 #if !defined(NO_SWAPPING)
1637 static void
1638 vm_req_vmdaemon(void)
1639 {
1640         static int lastrun = 0;
1641
1642         if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1643                 wakeup(&vm_daemon_needed);
1644                 lastrun = ticks;
1645         }
1646 }
1647
1648 static int vm_daemon_callback(struct proc *p, void *data __unused);
1649
1650 static void
1651 vm_daemon(void)
1652 {
1653         while (TRUE) {
1654                 tsleep(&vm_daemon_needed, 0, "psleep", 0);
1655                 if (vm_pageout_req_swapout) {
1656                         swapout_procs(vm_pageout_req_swapout);
1657                         vm_pageout_req_swapout = 0;
1658                 }
1659                 /*
1660                  * scan the processes for exceeding their rlimits or if
1661                  * process is swapped out -- deactivate pages
1662                  */
1663                 allproc_scan(vm_daemon_callback, NULL);
1664         }
1665 }
1666
1667 static int
1668 vm_daemon_callback(struct proc *p, void *data __unused)
1669 {
1670         vm_pindex_t limit, size;
1671
1672         /*
1673          * if this is a system process or if we have already
1674          * looked at this process, skip it.
1675          */
1676         if (p->p_flag & (P_SYSTEM | P_WEXIT))
1677                 return (0);
1678
1679         /*
1680          * if the process is in a non-running type state,
1681          * don't touch it.
1682          */
1683         if (p->p_stat != SACTIVE && p->p_stat != SSTOP)
1684                 return (0);
1685
1686         /*
1687          * get a limit
1688          */
1689         limit = OFF_TO_IDX(qmin(p->p_rlimit[RLIMIT_RSS].rlim_cur,
1690                                 p->p_rlimit[RLIMIT_RSS].rlim_max));
1691
1692         /*
1693          * let processes that are swapped out really be
1694          * swapped out.  Set the limit to nothing to get as
1695          * many pages out to swap as possible.
1696          */
1697         if (p->p_flag & P_SWAPPEDOUT)
1698                 limit = 0;
1699
1700         size = vmspace_resident_count(p->p_vmspace);
1701         if (limit >= 0 && size >= limit) {
1702                 vm_pageout_map_deactivate_pages(
1703                     &p->p_vmspace->vm_map, limit);
1704         }
1705         return (0);
1706 }
1707
1708 #endif