sys/vm/vm_pageout.c

   1 /*
   2  * Copyright (c) 1991 Regents of the University of California.
   3  * All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  *
   9  * This code is derived from software contributed to Berkeley by
  10  * The Mach Operating System project at Carnegie-Mellon University.
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions
  14  * are met:
  15  * 1. Redistributions of source code must retain the above copyright
  16  *    notice, this list of conditions and the following disclaimer.
  17  * 2. Redistributions in binary form must reproduce the above copyright
  18  *    notice, this list of conditions and the following disclaimer in the
  19  *    documentation and/or other materials provided with the distribution.
  20  * 3. All advertising materials mentioning features or use of this software
  21  *    must display the following acknowledgement:
  22  *      This product includes software developed by the University of
  23  *      California, Berkeley and its contributors.
  24  * 4. Neither the name of the University nor the names of its contributors
  25  *    may be used to endorse or promote products derived from this software
  26  *    without specific prior written permission.
  27  *
  28  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  29  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  30  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  31  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  32  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  36  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  37  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  38  * SUCH DAMAGE.
  39  *
  40  *      from: @(#)vm_pageout.c  7.4 (Berkeley) 5/7/91
  41  *
  42  *
  43  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  44  * All rights reserved.
  45  *
  46  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  47  *
  48  * Permission to use, copy, modify and distribute this software and
  49  * its documentation is hereby granted, provided that both the copyright
  50  * notice and this permission notice appear in all copies of the
  51  * software, derivative works or modified versions, and any portions
  52  * thereof, and that both notices appear in supporting documentation.
  53  *
  54  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  55  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  56  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  57  *
  58  * Carnegie Mellon requests users of this software to return to
  59  *
  60  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  61  *  School of Computer Science
  62  *  Carnegie Mellon University
  63  *  Pittsburgh PA 15213-3890
  64  *
  65  * any improvements or extensions that they make and grant Carnegie the
  66  * rights to redistribute these changes.
  67  *
  68  * $FreeBSD: src/sys/vm/vm_pageout.c,v 1.151.2.15 2002/12/29 18:21:04 dillon Exp $
  69  * $DragonFly: src/sys/vm/vm_pageout.c,v 1.9 2004/03/01 06:33:24 dillon Exp $
  70  */
  71
  72 /*
  73  *      The proverbial page-out daemon.
  74  */
  75
  76 #include "opt_vm.h"
  77 #include <sys/param.h>
  78 #include <sys/systm.h>
  79 #include <sys/kernel.h>
  80 #include <sys/proc.h>
  81 #include <sys/kthread.h>
  82 #include <sys/resourcevar.h>
  83 #include <sys/signalvar.h>
  84 #include <sys/vnode.h>
  85 #include <sys/vmmeter.h>
  86 #include <sys/sysctl.h>
  87
  88 #include <vm/vm.h>
  89 #include <vm/vm_param.h>
  90 #include <sys/lock.h>
  91 #include <vm/vm_object.h>
  92 #include <vm/vm_page.h>
  93 #include <vm/vm_map.h>
  94 #include <vm/vm_pageout.h>
  95 #include <vm/vm_pager.h>
  96 #include <vm/swap_pager.h>
  97 #include <vm/vm_extern.h>
  98 #include <vm/vm_page2.h>
  99
 100 /*
 101  * System initialization
 102  */
 103
 104 /* the kernel process "vm_pageout"*/
 105 static void vm_pageout (void);
 106 static int vm_pageout_clean (vm_page_t);
 107 static void vm_pageout_scan (int pass);
 108 static int vm_pageout_free_page_calc (vm_size_t count);
 109 struct thread *pagethread;
 110
 111 static struct kproc_desc page_kp = {
 112         "pagedaemon",
 113         vm_pageout,
 114         &pagethread
 115 };
 116 SYSINIT(pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, kproc_start, &page_kp)
 117
 118 #if !defined(NO_SWAPPING)
 119 /* the kernel process "vm_daemon"*/
 120 static void vm_daemon (void);
 121 static struct   thread *vmthread;
 122
 123 static struct kproc_desc vm_kp = {
 124         "vmdaemon",
 125         vm_daemon,
 126         &vmthread
 127 };
 128 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp)
 129 #endif
 130
 131
 132 int vm_pages_needed=0;          /* Event on which pageout daemon sleeps */
 133 int vm_pageout_deficit=0;       /* Estimated number of pages deficit */
 134 int vm_pageout_pages_needed=0;  /* flag saying that the pageout daemon needs pages */
 135
 136 #if !defined(NO_SWAPPING)
 137 static int vm_pageout_req_swapout;      /* XXX */
 138 static int vm_daemon_needed;
 139 #endif
 140 extern int vm_swap_size;
 141 static int vm_max_launder = 32;
 142 static int vm_pageout_stats_max=0, vm_pageout_stats_interval = 0;
 143 static int vm_pageout_full_stats_interval = 0;
 144 static int vm_pageout_stats_free_max=0, vm_pageout_algorithm=0;
 145 static int defer_swap_pageouts=0;
 146 static int disable_swap_pageouts=0;
 147
 148 #if defined(NO_SWAPPING)
 149 static int vm_swap_enabled=0;
 150 static int vm_swap_idle_enabled=0;
 151 #else
 152 static int vm_swap_enabled=1;
 153 static int vm_swap_idle_enabled=0;
 154 #endif
 155
 156 SYSCTL_INT(_vm, VM_PAGEOUT_ALGORITHM, pageout_algorithm,
 157         CTLFLAG_RW, &vm_pageout_algorithm, 0, "LRU page mgmt");
 158
 159 SYSCTL_INT(_vm, OID_AUTO, max_launder,
 160         CTLFLAG_RW, &vm_max_launder, 0, "Limit dirty flushes in pageout");
 161
 162 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_max,
 163         CTLFLAG_RW, &vm_pageout_stats_max, 0, "Max pageout stats scan length");
 164
 165 SYSCTL_INT(_vm, OID_AUTO, pageout_full_stats_interval,
 166         CTLFLAG_RW, &vm_pageout_full_stats_interval, 0, "Interval for full stats scan");
 167
 168 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_interval,
 169         CTLFLAG_RW, &vm_pageout_stats_interval, 0, "Interval for partial stats scan");
 170
 171 SYSCTL_INT(_vm, OID_AUTO, pageout_stats_free_max,
 172         CTLFLAG_RW, &vm_pageout_stats_free_max, 0, "Not implemented");
 173
 174 #if defined(NO_SWAPPING)
 175 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 176         CTLFLAG_RD, &vm_swap_enabled, 0, "");
 177 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 178         CTLFLAG_RD, &vm_swap_idle_enabled, 0, "");
 179 #else
 180 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled,
 181         CTLFLAG_RW, &vm_swap_enabled, 0, "Enable entire process swapout");
 182 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled,
 183         CTLFLAG_RW, &vm_swap_idle_enabled, 0, "Allow swapout on idle criteria");
 184 #endif
 185
 186 SYSCTL_INT(_vm, OID_AUTO, defer_swapspace_pageouts,
 187         CTLFLAG_RW, &defer_swap_pageouts, 0, "Give preference to dirty pages in mem");
 188
 189 SYSCTL_INT(_vm, OID_AUTO, disable_swapspace_pageouts,
 190         CTLFLAG_RW, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages");
 191
 192 static int pageout_lock_miss;
 193 SYSCTL_INT(_vm, OID_AUTO, pageout_lock_miss,
 194         CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout");
 195
 196 #define VM_PAGEOUT_PAGE_COUNT 16
 197 int vm_pageout_page_count = VM_PAGEOUT_PAGE_COUNT;
 198
 199 int vm_page_max_wired;          /* XXX max # of wired pages system-wide */
 200
 201 #if !defined(NO_SWAPPING)
 202 typedef void freeer_fcn_t (vm_map_t, vm_object_t, vm_pindex_t, int);
 203 static void vm_pageout_map_deactivate_pages (vm_map_t, vm_pindex_t);
 204 static freeer_fcn_t vm_pageout_object_deactivate_pages;
 205 static void vm_req_vmdaemon (void);
 206 #endif
 207 static void vm_pageout_page_stats(void);
 208
 209 /*
 210  * vm_pageout_clean:
 211  *
 212  * Clean the page and remove it from the laundry.
 213  *
 214  * We set the busy bit to cause potential page faults on this page to
 215  * block.  Note the careful timing, however, the busy bit isn't set till
 216  * late and we cannot do anything that will mess with the page.
 217  */
 218
 219 static int
 220 vm_pageout_clean(m)
 221         vm_page_t m;
 222 {
 223         vm_object_t object;
 224         vm_page_t mc[2*vm_pageout_page_count];
 225         int pageout_count;
 226         int ib, is, page_base;
 227         vm_pindex_t pindex = m->pindex;
 228
 229         object = m->object;
 230
 231         /*
 232          * It doesn't cost us anything to pageout OBJT_DEFAULT or OBJT_SWAP
 233          * with the new swapper, but we could have serious problems paging
 234          * out other object types if there is insufficient memory.
 235          *
 236          * Unfortunately, checking free memory here is far too late, so the
 237          * check has been moved up a procedural level.
 238          */
 239
 240         /*
 241          * Don't mess with the page if it's busy, held, or special
 242          */
 243         if ((m->hold_count != 0) ||
 244             ((m->busy != 0) || (m->flags & (PG_BUSY|PG_UNMANAGED)))) {
 245                 return 0;
 246         }
 247
 248         mc[vm_pageout_page_count] = m;
 249         pageout_count = 1;
 250         page_base = vm_pageout_page_count;
 251         ib = 1;
 252         is = 1;
 253
 254         /*
 255          * Scan object for clusterable pages.
 256          *
 257          * We can cluster ONLY if: ->> the page is NOT
 258          * clean, wired, busy, held, or mapped into a
 259          * buffer, and one of the following:
 260          * 1) The page is inactive, or a seldom used
 261          *    active page.
 262          * -or-
 263          * 2) we force the issue.
 264          *
 265          * During heavy mmap/modification loads the pageout
 266          * daemon can really fragment the underlying file
 267          * due to flushing pages out of order and not trying
 268          * align the clusters (which leave sporatic out-of-order
 269          * holes).  To solve this problem we do the reverse scan
 270          * first and attempt to align our cluster, then do a
 271          * forward scan if room remains.
 272          */
 273
 274 more:
 275         while (ib && pageout_count < vm_pageout_page_count) {
 276                 vm_page_t p;
 277
 278                 if (ib > pindex) {
 279                         ib = 0;
 280                         break;
 281                 }
 282
 283                 if ((p = vm_page_lookup(object, pindex - ib)) == NULL) {
 284                         ib = 0;
 285                         break;
 286                 }
 287                 if (((p->queue - p->pc) == PQ_CACHE) ||
 288                     (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) {
 289                         ib = 0;
 290                         break;
 291                 }
 292                 vm_page_test_dirty(p);
 293                 if ((p->dirty & p->valid) == 0 ||
 294                     p->queue != PQ_INACTIVE ||
 295                     p->wire_count != 0 ||       /* may be held by buf cache */
 296                     p->hold_count != 0) {       /* may be undergoing I/O */
 297                         ib = 0;
 298                         break;
 299                 }
 300                 mc[--page_base] = p;
 301                 ++pageout_count;
 302                 ++ib;
 303                 /*
 304                  * alignment boundry, stop here and switch directions.  Do
 305                  * not clear ib.
 306                  */
 307                 if ((pindex - (ib - 1)) % vm_pageout_page_count == 0)
 308                         break;
 309         }
 310
 311         while (pageout_count < vm_pageout_page_count &&
 312             pindex + is < object->size) {
 313                 vm_page_t p;
 314
 315                 if ((p = vm_page_lookup(object, pindex + is)) == NULL)
 316                         break;
 317                 if (((p->queue - p->pc) == PQ_CACHE) ||
 318                     (p->flags & (PG_BUSY|PG_UNMANAGED)) || p->busy) {
 319                         break;
 320                 }
 321                 vm_page_test_dirty(p);
 322                 if ((p->dirty & p->valid) == 0 ||
 323                     p->queue != PQ_INACTIVE ||
 324                     p->wire_count != 0 ||       /* may be held by buf cache */
 325                     p->hold_count != 0) {       /* may be undergoing I/O */
 326                         break;
 327                 }
 328                 mc[page_base + pageout_count] = p;
 329                 ++pageout_count;
 330                 ++is;
 331         }
 332
 333         /*
 334          * If we exhausted our forward scan, continue with the reverse scan
 335          * when possible, even past a page boundry.  This catches boundry
 336          * conditions.
 337          */
 338         if (ib && pageout_count < vm_pageout_page_count)
 339                 goto more;
 340
 341         /*
 342          * we allow reads during pageouts...
 343          */
 344         return vm_pageout_flush(&mc[page_base], pageout_count, 0);
 345 }
 346
 347 /*
 348  * vm_pageout_flush() - launder the given pages
 349  *
 350  *      The given pages are laundered.  Note that we setup for the start of
 351  *      I/O ( i.e. busy the page ), mark it read-only, and bump the object
 352  *      reference count all in here rather then in the parent.  If we want
 353  *      the parent to do more sophisticated things we may have to change
 354  *      the ordering.
 355  */
 356
 357 int
 358 vm_pageout_flush(mc, count, flags)
 359         vm_page_t *mc;
 360         int count;
 361         int flags;
 362 {
 363         vm_object_t object;
 364         int pageout_status[count];
 365         int numpagedout = 0;
 366         int i;
 367
 368         /*
 369          * Initiate I/O.  Bump the vm_page_t->busy counter and
 370          * mark the pages read-only.
 371          *
 372          * We do not have to fixup the clean/dirty bits here... we can
 373          * allow the pager to do it after the I/O completes.
 374          */
 375
 376         for (i = 0; i < count; i++) {
 377                 KASSERT(mc[i]->valid == VM_PAGE_BITS_ALL, ("vm_pageout_flush page %p index %d/%d: partially invalid page", mc[i], i, count));
 378                 vm_page_io_start(mc[i]);
 379                 vm_page_protect(mc[i], VM_PROT_READ);
 380         }
 381
 382         object = mc[0]->object;
 383         vm_object_pip_add(object, count);
 384
 385         vm_pager_put_pages(object, mc, count,
 386             (flags | ((object == kernel_object) ? VM_PAGER_PUT_SYNC : 0)),
 387             pageout_status);
 388
 389         for (i = 0; i < count; i++) {
 390                 vm_page_t mt = mc[i];
 391
 392                 switch (pageout_status[i]) {
 393                 case VM_PAGER_OK:
 394                         numpagedout++;
 395                         break;
 396                 case VM_PAGER_PEND:
 397                         numpagedout++;
 398                         break;
 399                 case VM_PAGER_BAD:
 400                         /*
 401                          * Page outside of range of object. Right now we
 402                          * essentially lose the changes by pretending it
 403                          * worked.
 404                          */
 405                         pmap_clear_modify(mt);
 406                         vm_page_undirty(mt);
 407                         break;
 408                 case VM_PAGER_ERROR:
 409                 case VM_PAGER_FAIL:
 410                         /*
 411                          * If page couldn't be paged out, then reactivate the
 412                          * page so it doesn't clog the inactive list.  (We
 413                          * will try paging out it again later).
 414                          */
 415                         vm_page_activate(mt);
 416                         break;
 417                 case VM_PAGER_AGAIN:
 418                         break;
 419                 }
 420
 421                 /*
 422                  * If the operation is still going, leave the page busy to
 423                  * block all other accesses. Also, leave the paging in
 424                  * progress indicator set so that we don't attempt an object
 425                  * collapse.
 426                  */
 427                 if (pageout_status[i] != VM_PAGER_PEND) {
 428                         vm_object_pip_wakeup(object);
 429                         vm_page_io_finish(mt);
 430                         if (!vm_page_count_severe() || !vm_page_try_to_cache(mt))
 431                                 vm_page_protect(mt, VM_PROT_READ);
 432                 }
 433         }
 434         return numpagedout;
 435 }
 436
 437 #if !defined(NO_SWAPPING)
 438 /*
 439  *      vm_pageout_object_deactivate_pages
 440  *
 441  *      deactivate enough pages to satisfy the inactive target
 442  *      requirements or if vm_page_proc_limit is set, then
 443  *      deactivate all of the pages in the object and its
 444  *      backing_objects.
 445  *
 446  *      The object and map must be locked.
 447  */
 448 static void
 449 vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only)
 450         vm_map_t map;
 451         vm_object_t object;
 452         vm_pindex_t desired;
 453         int map_remove_only;
 454 {
 455         vm_page_t p, next;
 456         int rcount;
 457         int remove_mode;
 458         int s;
 459
 460         if (object->type == OBJT_DEVICE || object->type == OBJT_PHYS)
 461                 return;
 462
 463         while (object) {
 464                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 465                         return;
 466                 if (object->paging_in_progress)
 467                         return;
 468
 469                 remove_mode = map_remove_only;
 470                 if (object->shadow_count > 1)
 471                         remove_mode = 1;
 472         /*
 473          * scan the objects entire memory queue
 474          */
 475                 rcount = object->resident_page_count;
 476                 p = TAILQ_FIRST(&object->memq);
 477                 while (p && (rcount-- > 0)) {
 478                         int actcount;
 479                         if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 480                                 return;
 481                         next = TAILQ_NEXT(p, listq);
 482                         mycpu->gd_cnt.v_pdpages++;
 483                         if (p->wire_count != 0 ||
 484                             p->hold_count != 0 ||
 485                             p->busy != 0 ||
 486                             (p->flags & (PG_BUSY|PG_UNMANAGED)) ||
 487                             !pmap_page_exists_quick(vm_map_pmap(map), p)) {
 488                                 p = next;
 489                                 continue;
 490                         }
 491
 492                         actcount = pmap_ts_referenced(p);
 493                         if (actcount) {
 494                                 vm_page_flag_set(p, PG_REFERENCED);
 495                         } else if (p->flags & PG_REFERENCED) {
 496                                 actcount = 1;
 497                         }
 498
 499                         if ((p->queue != PQ_ACTIVE) &&
 500                                 (p->flags & PG_REFERENCED)) {
 501                                 vm_page_activate(p);
 502                                 p->act_count += actcount;
 503                                 vm_page_flag_clear(p, PG_REFERENCED);
 504                         } else if (p->queue == PQ_ACTIVE) {
 505                                 if ((p->flags & PG_REFERENCED) == 0) {
 506                                         p->act_count -= min(p->act_count, ACT_DECLINE);
 507                                         if (!remove_mode && (vm_pageout_algorithm || (p->act_count == 0))) {
 508                                                 vm_page_protect(p, VM_PROT_NONE);
 509                                                 vm_page_deactivate(p);
 510                                         } else {
 511                                                 s = splvm();
 512                                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 513                                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 514                                                 splx(s);
 515                                         }
 516                                 } else {
 517                                         vm_page_activate(p);
 518                                         vm_page_flag_clear(p, PG_REFERENCED);
 519                                         if (p->act_count < (ACT_MAX - ACT_ADVANCE))
 520                                                 p->act_count += ACT_ADVANCE;
 521                                         s = splvm();
 522                                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 523                                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 524                                         splx(s);
 525                                 }
 526                         } else if (p->queue == PQ_INACTIVE) {
 527                                 vm_page_protect(p, VM_PROT_NONE);
 528                         }
 529                         p = next;
 530                 }
 531                 object = object->backing_object;
 532         }
 533         return;
 534 }
 535
 536 /*
 537  * deactivate some number of pages in a map, try to do it fairly, but
 538  * that is really hard to do.
 539  */
 540 static void
 541 vm_pageout_map_deactivate_pages(map, desired)
 542         vm_map_t map;
 543         vm_pindex_t desired;
 544 {
 545         vm_map_entry_t tmpe;
 546         vm_object_t obj, bigobj;
 547         int nothingwired;
 548
 549         if (lockmgr(&map->lock, LK_EXCLUSIVE | LK_NOWAIT, NULL, curthread)) {
 550                 return;
 551         }
 552
 553         bigobj = NULL;
 554         nothingwired = TRUE;
 555
 556         /*
 557          * first, search out the biggest object, and try to free pages from
 558          * that.
 559          */
 560         tmpe = map->header.next;
 561         while (tmpe != &map->header) {
 562                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 563                         obj = tmpe->object.vm_object;
 564                         if ((obj != NULL) && (obj->shadow_count <= 1) &&
 565                                 ((bigobj == NULL) ||
 566                                  (bigobj->resident_page_count < obj->resident_page_count))) {
 567                                 bigobj = obj;
 568                         }
 569                 }
 570                 if (tmpe->wired_count > 0)
 571                         nothingwired = FALSE;
 572                 tmpe = tmpe->next;
 573         }
 574
 575         if (bigobj)
 576                 vm_pageout_object_deactivate_pages(map, bigobj, desired, 0);
 577
 578         /*
 579          * Next, hunt around for other pages to deactivate.  We actually
 580          * do this search sort of wrong -- .text first is not the best idea.
 581          */
 582         tmpe = map->header.next;
 583         while (tmpe != &map->header) {
 584                 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
 585                         break;
 586                 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
 587                         obj = tmpe->object.vm_object;
 588                         if (obj)
 589                                 vm_pageout_object_deactivate_pages(map, obj, desired, 0);
 590                 }
 591                 tmpe = tmpe->next;
 592         };
 593
 594         /*
 595          * Remove all mappings if a process is swapped out, this will free page
 596          * table pages.
 597          */
 598         if (desired == 0 && nothingwired)
 599                 pmap_remove(vm_map_pmap(map),
 600                         VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
 601         vm_map_unlock(map);
 602         return;
 603 }
 604 #endif
 605
 606 /*
 607  * Don't try to be fancy - being fancy can lead to VOP_LOCK's and therefore
 608  * to vnode deadlocks.  We only do it for OBJT_DEFAULT and OBJT_SWAP objects
 609  * which we know can be trivially freed.
 610  */
 611
 612 void
 613 vm_pageout_page_free(vm_page_t m) {
 614         vm_object_t object = m->object;
 615         int type = object->type;
 616
 617         if (type == OBJT_SWAP || type == OBJT_DEFAULT)
 618                 vm_object_reference(object);
 619         vm_page_busy(m);
 620         vm_page_protect(m, VM_PROT_NONE);
 621         vm_page_free(m);
 622         if (type == OBJT_SWAP || type == OBJT_DEFAULT)
 623                 vm_object_deallocate(object);
 624 }
 625
 626 /*
 627  *      vm_pageout_scan does the dirty work for the pageout daemon.
 628  */
 629 static void
 630 vm_pageout_scan(int pass)
 631 {
 632         vm_page_t m, next;
 633         struct vm_page marker;
 634         int page_shortage, maxscan, pcount;
 635         int addl_page_shortage, addl_page_shortage_init;
 636         struct proc *p, *bigproc;
 637         vm_offset_t size, bigsize;
 638         vm_object_t object;
 639         int actcount;
 640         int vnodes_skipped = 0;
 641         int maxlaunder;
 642         int s;
 643
 644         /*
 645          * Do whatever cleanup that the pmap code can.
 646          */
 647         pmap_collect();
 648
 649         addl_page_shortage_init = vm_pageout_deficit;
 650         vm_pageout_deficit = 0;
 651
 652         /*
 653          * Calculate the number of pages we want to either free or move
 654          * to the cache.
 655          */
 656         page_shortage = vm_paging_target() + addl_page_shortage_init;
 657
 658         /*
 659          * Initialize our marker
 660          */
 661         bzero(&marker, sizeof(marker));
 662         marker.flags = PG_BUSY | PG_FICTITIOUS | PG_MARKER;
 663         marker.queue = PQ_INACTIVE;
 664         marker.wire_count = 1;
 665
 666         /*
 667          * Start scanning the inactive queue for pages we can move to the
 668          * cache or free.  The scan will stop when the target is reached or
 669          * we have scanned the entire inactive queue.  Note that m->act_count
 670          * is not used to form decisions for the inactive queue, only for the
 671          * active queue.
 672          *
 673          * maxlaunder limits the number of dirty pages we flush per scan.
 674          * For most systems a smaller value (16 or 32) is more robust under
 675          * extreme memory and disk pressure because any unnecessary writes
 676          * to disk can result in extreme performance degredation.  However,
 677          * systems with excessive dirty pages (especially when MAP_NOSYNC is
 678          * used) will die horribly with limited laundering.  If the pageout
 679          * daemon cannot clean enough pages in the first pass, we let it go
 680          * all out in succeeding passes.
 681          */
 682         if ((maxlaunder = vm_max_launder) <= 1)
 683                 maxlaunder = 1;
 684         if (pass)
 685                 maxlaunder = 10000;
 686
 687 rescan0:
 688         addl_page_shortage = addl_page_shortage_init;
 689         maxscan = vmstats.v_inactive_count;
 690         for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
 691              m != NULL && maxscan-- > 0 && page_shortage > 0;
 692              m = next) {
 693
 694                 mycpu->gd_cnt.v_pdpages++;
 695
 696                 if (m->queue != PQ_INACTIVE) {
 697                         goto rescan0;
 698                 }
 699
 700                 next = TAILQ_NEXT(m, pageq);
 701
 702                 /*
 703                  * skip marker pages
 704                  */
 705                 if (m->flags & PG_MARKER)
 706                         continue;
 707
 708                 /*
 709                  * A held page may be undergoing I/O, so skip it.
 710                  */
 711                 if (m->hold_count) {
 712                         s = splvm();
 713                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 714                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 715                         splx(s);
 716                         addl_page_shortage++;
 717                         continue;
 718                 }
 719                 /*
 720                  * Dont mess with busy pages, keep in the front of the
 721                  * queue, most likely are being paged out.
 722                  */
 723                 if (m->busy || (m->flags & PG_BUSY)) {
 724                         addl_page_shortage++;
 725                         continue;
 726                 }
 727
 728                 /*
 729                  * If the object is not being used, we ignore previous
 730                  * references.
 731                  */
 732                 if (m->object->ref_count == 0) {
 733                         vm_page_flag_clear(m, PG_REFERENCED);
 734                         pmap_clear_reference(m);
 735
 736                 /*
 737                  * Otherwise, if the page has been referenced while in the
 738                  * inactive queue, we bump the "activation count" upwards,
 739                  * making it less likely that the page will be added back to
 740                  * the inactive queue prematurely again.  Here we check the
 741                  * page tables (or emulated bits, if any), given the upper
 742                  * level VM system not knowing anything about existing
 743                  * references.
 744                  */
 745                 } else if (((m->flags & PG_REFERENCED) == 0) &&
 746                         (actcount = pmap_ts_referenced(m))) {
 747                         vm_page_activate(m);
 748                         m->act_count += (actcount + ACT_ADVANCE);
 749                         continue;
 750                 }
 751
 752                 /*
 753                  * If the upper level VM system knows about any page
 754                  * references, we activate the page.  We also set the
 755                  * "activation count" higher than normal so that we will less
 756                  * likely place pages back onto the inactive queue again.
 757                  */
 758                 if ((m->flags & PG_REFERENCED) != 0) {
 759                         vm_page_flag_clear(m, PG_REFERENCED);
 760                         actcount = pmap_ts_referenced(m);
 761                         vm_page_activate(m);
 762                         m->act_count += (actcount + ACT_ADVANCE + 1);
 763                         continue;
 764                 }
 765
 766                 /*
 767                  * If the upper level VM system doesn't know anything about
 768                  * the page being dirty, we have to check for it again.  As
 769                  * far as the VM code knows, any partially dirty pages are
 770                  * fully dirty.
 771                  *
 772                  * Pages marked PG_WRITEABLE may be mapped into the user
 773                  * address space of a process running on another cpu.  A
 774                  * user process (without holding the MP lock) running on
 775                  * another cpu may be able to touch the page while we are
 776                  * trying to remove it.  To prevent this from occuring we
 777                  * must call pmap_remove_all() or otherwise make the page
 778                  * read-only.  If the race occured pmap_remove_all() is
 779                  * responsible for setting m->dirty.
 780                  */
 781                 if (m->dirty == 0) {
 782                         vm_page_test_dirty(m);
 783 #if 0
 784                         if (m->dirty == 0 && (m->flags & PG_WRITEABLE) != 0)
 785                                 pmap_remove_all(m);
 786 #endif
 787                 } else {
 788                         vm_page_dirty(m);
 789                 }
 790
 791                 if (m->valid == 0) {
 792                         /*
 793                          * Invalid pages can be easily freed
 794                          */
 795                         vm_pageout_page_free(m);
 796                         mycpu->gd_cnt.v_dfree++;
 797                         --page_shortage;
 798                 } else if (m->dirty == 0) {
 799                         /*
 800                          * Clean pages can be placed onto the cache queue.
 801                          * This effectively frees them.
 802                          */
 803                         vm_page_cache(m);
 804                         --page_shortage;
 805                 } else if ((m->flags & PG_WINATCFLS) == 0 && pass == 0) {
 806                         /*
 807                          * Dirty pages need to be paged out, but flushing
 808                          * a page is extremely expensive verses freeing
 809                          * a clean page.  Rather then artificially limiting
 810                          * the number of pages we can flush, we instead give
 811                          * dirty pages extra priority on the inactive queue
 812                          * by forcing them to be cycled through the queue
 813                          * twice before being flushed, after which the
 814                          * (now clean) page will cycle through once more
 815                          * before being freed.  This significantly extends
 816                          * the thrash point for a heavily loaded machine.
 817                          */
 818                         s = splvm();
 819                         vm_page_flag_set(m, PG_WINATCFLS);
 820                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 821                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 822                         splx(s);
 823                 } else if (maxlaunder > 0) {
 824                         /*
 825                          * We always want to try to flush some dirty pages if
 826                          * we encounter them, to keep the system stable.
 827                          * Normally this number is small, but under extreme
 828                          * pressure where there are insufficient clean pages
 829                          * on the inactive queue, we may have to go all out.
 830                          */
 831                         int swap_pageouts_ok;
 832                         struct vnode *vp = NULL;
 833
 834                         object = m->object;
 835
 836                         if ((object->type != OBJT_SWAP) && (object->type != OBJT_DEFAULT)) {
 837                                 swap_pageouts_ok = 1;
 838                         } else {
 839                                 swap_pageouts_ok = !(defer_swap_pageouts || disable_swap_pageouts);
 840                                 swap_pageouts_ok |= (!disable_swap_pageouts && defer_swap_pageouts &&
 841                                 vm_page_count_min());
 842
 843                         }
 844
 845                         /*
 846                          * We don't bother paging objects that are "dead".
 847                          * Those objects are in a "rundown" state.
 848                          */
 849                         if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) {
 850                                 s = splvm();
 851                                 TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 852                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 853                                 splx(s);
 854                                 continue;
 855                         }
 856
 857                         /*
 858                          * The object is already known NOT to be dead.   It
 859                          * is possible for the vget() to block the whole
 860                          * pageout daemon, but the new low-memory handling
 861                          * code should prevent it.
 862                          *
 863                          * The previous code skipped locked vnodes and, worse,
 864                          * reordered pages in the queue.  This results in
 865                          * completely non-deterministic operation because,
 866                          * quite often, a vm_fault has initiated an I/O and
 867                          * is holding a locked vnode at just the point where
 868                          * the pageout daemon is woken up.
 869                          *
 870                          * We can't wait forever for the vnode lock, we might
 871                          * deadlock due to a vn_read() getting stuck in
 872                          * vm_wait while holding this vnode.  We skip the
 873                          * vnode if we can't get it in a reasonable amount
 874                          * of time.
 875                          */
 876
 877                         if (object->type == OBJT_VNODE) {
 878                                 vp = object->handle;
 879
 880                                 if (vget(vp, NULL, LK_EXCLUSIVE|LK_NOOBJ|LK_TIMELOCK, curthread)) {
 881                                         ++pageout_lock_miss;
 882                                         if (object->flags & OBJ_MIGHTBEDIRTY)
 883                                                     vnodes_skipped++;
 884                                         continue;
 885                                 }
 886
 887                                 /*
 888                                  * The page might have been moved to another
 889                                  * queue during potential blocking in vget()
 890                                  * above.  The page might have been freed and
 891                                  * reused for another vnode.  The object might
 892                                  * have been reused for another vnode.
 893                                  */
 894                                 if (m->queue != PQ_INACTIVE ||
 895                                     m->object != object ||
 896                                     object->handle != vp) {
 897                                         if (object->flags & OBJ_MIGHTBEDIRTY)
 898                                                 vnodes_skipped++;
 899                                         vput(vp);
 900                                         continue;
 901                                 }
 902
 903                                 /*
 904                                  * The page may have been busied during the
 905                                  * blocking in vput();  We don't move the
 906                                  * page back onto the end of the queue so that
 907                                  * statistics are more correct if we don't.
 908                                  */
 909                                 if (m->busy || (m->flags & PG_BUSY)) {
 910                                         vput(vp);
 911                                         continue;
 912                                 }
 913
 914                                 /*
 915                                  * If the page has become held it might
 916                                  * be undergoing I/O, so skip it
 917                                  */
 918                                 if (m->hold_count) {
 919                                         s = splvm();
 920                                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 921                                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 922                                         splx(s);
 923                                         if (object->flags & OBJ_MIGHTBEDIRTY)
 924                                                 vnodes_skipped++;
 925                                         vput(vp);
 926                                         continue;
 927                                 }
 928                         }
 929
 930                         /*
 931                          * If a page is dirty, then it is either being washed
 932                          * (but not yet cleaned) or it is still in the
 933                          * laundry.  If it is still in the laundry, then we
 934                          * start the cleaning operation.
 935                          *
 936                          * This operation may cluster, invalidating the 'next'
 937                          * pointer.  To prevent an inordinate number of
 938                          * restarts we use our marker to remember our place.
 939                          *
 940                          * decrement page_shortage on success to account for
 941                          * the (future) cleaned page.  Otherwise we could wind
 942                          * up laundering or cleaning too many pages.
 943                          */
 944                         s = splvm();
 945                         TAILQ_INSERT_AFTER(&vm_page_queues[PQ_INACTIVE].pl, m, &marker, pageq);
 946                         splx(s);
 947                         if (vm_pageout_clean(m) != 0) {
 948                                 --page_shortage;
 949                                 --maxlaunder;
 950                         }
 951                         s = splvm();
 952                         next = TAILQ_NEXT(&marker, pageq);
 953                         TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, &marker, pageq);
 954                         splx(s);
 955                         if (vp != NULL)
 956                                 vput(vp);
 957                 }
 958         }
 959
 960         /*
 961          * Compute the number of pages we want to try to move from the
 962          * active queue to the inactive queue.
 963          */
 964         page_shortage = vm_paging_target() +
 965             vmstats.v_inactive_target - vmstats.v_inactive_count;
 966         page_shortage += addl_page_shortage;
 967
 968         /*
 969          * Scan the active queue for things we can deactivate. We nominally
 970          * track the per-page activity counter and use it to locate
 971          * deactivation candidates.
 972          */
 973
 974         pcount = vmstats.v_active_count;
 975         m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
 976
 977         while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
 978
 979                 /*
 980                  * This is a consistency check, and should likely be a panic
 981                  * or warning.
 982                  */
 983                 if (m->queue != PQ_ACTIVE) {
 984                         break;
 985                 }
 986
 987                 next = TAILQ_NEXT(m, pageq);
 988                 /*
 989                  * Don't deactivate pages that are busy.
 990                  */
 991                 if ((m->busy != 0) ||
 992                     (m->flags & PG_BUSY) ||
 993                     (m->hold_count != 0)) {
 994                         s = splvm();
 995                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 996                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 997                         splx(s);
 998                         m = next;
 999                         continue;
1000                 }
1001
1002                 /*
1003                  * The count for pagedaemon pages is done after checking the
1004                  * page for eligibility...
1005                  */
1006                 mycpu->gd_cnt.v_pdpages++;
1007
1008                 /*
1009                  * Check to see "how much" the page has been used.
1010                  */
1011                 actcount = 0;
1012                 if (m->object->ref_count != 0) {
1013                         if (m->flags & PG_REFERENCED) {
1014                                 actcount += 1;
1015                         }
1016                         actcount += pmap_ts_referenced(m);
1017                         if (actcount) {
1018                                 m->act_count += ACT_ADVANCE + actcount;
1019                                 if (m->act_count > ACT_MAX)
1020                                         m->act_count = ACT_MAX;
1021                         }
1022                 }
1023
1024                 /*
1025                  * Since we have "tested" this bit, we need to clear it now.
1026                  */
1027                 vm_page_flag_clear(m, PG_REFERENCED);
1028
1029                 /*
1030                  * Only if an object is currently being used, do we use the
1031                  * page activation count stats.
1032                  */
1033                 if (actcount && (m->object->ref_count != 0)) {
1034                         s = splvm();
1035                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1036                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1037                         splx(s);
1038                 } else {
1039                         m->act_count -= min(m->act_count, ACT_DECLINE);
1040                         if (vm_pageout_algorithm ||
1041                             m->object->ref_count == 0 ||
1042                             m->act_count == 0) {
1043                                 page_shortage--;
1044                                 if (m->object->ref_count == 0) {
1045                                         vm_page_protect(m, VM_PROT_NONE);
1046                                         if (m->dirty == 0)
1047                                                 vm_page_cache(m);
1048                                         else
1049                                                 vm_page_deactivate(m);
1050                                 } else {
1051                                         vm_page_deactivate(m);
1052                                 }
1053                         } else {
1054                                 s = splvm();
1055                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1056                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1057                                 splx(s);
1058                         }
1059                 }
1060                 m = next;
1061         }
1062
1063         s = splvm();
1064
1065         /*
1066          * We try to maintain some *really* free pages, this allows interrupt
1067          * code to be guaranteed space.  Since both cache and free queues
1068          * are considered basically 'free', moving pages from cache to free
1069          * does not effect other calculations.
1070          */
1071
1072         while (vmstats.v_free_count < vmstats.v_free_reserved) {
1073                 static int cache_rover = 0;
1074                 m = vm_page_list_find(PQ_CACHE, cache_rover, FALSE);
1075                 if (!m)
1076                         break;
1077                 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) ||
1078                     m->busy ||
1079                     m->hold_count ||
1080                     m->wire_count) {
1081 #ifdef INVARIANTS
1082                         printf("Warning: busy page %p found in cache\n", m);
1083 #endif
1084                         vm_page_deactivate(m);
1085                         continue;
1086                 }
1087                 cache_rover = (cache_rover + PQ_PRIME2) & PQ_L2_MASK;
1088                 vm_pageout_page_free(m);
1089                 mycpu->gd_cnt.v_dfree++;
1090         }
1091         splx(s);
1092
1093 #if !defined(NO_SWAPPING)
1094         /*
1095          * Idle process swapout -- run once per second.
1096          */
1097         if (vm_swap_idle_enabled) {
1098                 static long lsec;
1099                 if (time_second != lsec) {
1100                         vm_pageout_req_swapout |= VM_SWAP_IDLE;
1101                         vm_req_vmdaemon();
1102                         lsec = time_second;
1103                 }
1104         }
1105 #endif
1106
1107         /*
1108          * If we didn't get enough free pages, and we have skipped a vnode
1109          * in a writeable object, wakeup the sync daemon.  And kick swapout
1110          * if we did not get enough free pages.
1111          */
1112         if (vm_paging_target() > 0) {
1113                 if (vnodes_skipped && vm_page_count_min())
1114                         (void) speedup_syncer();
1115 #if !defined(NO_SWAPPING)
1116                 if (vm_swap_enabled && vm_page_count_target()) {
1117                         vm_req_vmdaemon();
1118                         vm_pageout_req_swapout |= VM_SWAP_NORMAL;
1119                 }
1120 #endif
1121         }
1122
1123         /*
1124          * If we are out of swap and were not able to reach our paging
1125          * target, kill the largest process.
1126          */
1127         if ((vm_swap_size < 64 && vm_page_count_min()) ||
1128             (swap_pager_full && vm_paging_target() > 0)) {
1129 #if 0
1130         if ((vm_swap_size < 64 || swap_pager_full) && vm_page_count_min()) {
1131 #endif
1132                 bigproc = NULL;
1133                 bigsize = 0;
1134                 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1135                         /*
1136                          * if this is a system process, skip it
1137                          */
1138                         if ((p->p_flag & P_SYSTEM) || (p->p_pid == 1) ||
1139                             ((p->p_pid < 48) && (vm_swap_size != 0))) {
1140                                 continue;
1141                         }
1142                         /*
1143                          * if the process is in a non-running type state,
1144                          * don't touch it.
1145                          */
1146                         if (p->p_stat != SRUN && p->p_stat != SSLEEP) {
1147                                 continue;
1148                         }
1149                         /*
1150                          * get the process size
1151                          */
1152                         size = vmspace_resident_count(p->p_vmspace) +
1153                                 vmspace_swap_count(p->p_vmspace);
1154                         /*
1155                          * if the this process is bigger than the biggest one
1156                          * remember it.
1157                          */
1158                         if (size > bigsize) {
1159                                 bigproc = p;
1160                                 bigsize = size;
1161                         }
1162                 }
1163                 if (bigproc != NULL) {
1164                         killproc(bigproc, "out of swap space");
1165                         bigproc->p_estcpu = 0;
1166                         bigproc->p_nice = PRIO_MIN;
1167                         resetpriority(bigproc);
1168                         wakeup(&vmstats.v_free_count);
1169                 }
1170         }
1171 }
1172
1173 /*
1174  * This routine tries to maintain the pseudo LRU active queue,
1175  * so that during long periods of time where there is no paging,
1176  * that some statistic accumulation still occurs.  This code
1177  * helps the situation where paging just starts to occur.
1178  */
1179 static void
1180 vm_pageout_page_stats()
1181 {
1182         int s;
1183         vm_page_t m,next;
1184         int pcount,tpcount;             /* Number of pages to check */
1185         static int fullintervalcount = 0;
1186         int page_shortage;
1187         int s0;
1188
1189         page_shortage =
1190             (vmstats.v_inactive_target + vmstats.v_cache_max + vmstats.v_free_min) -
1191             (vmstats.v_free_count + vmstats.v_inactive_count + vmstats.v_cache_count);
1192
1193         if (page_shortage <= 0)
1194                 return;
1195
1196         s0 = splvm();
1197
1198         pcount = vmstats.v_active_count;
1199         fullintervalcount += vm_pageout_stats_interval;
1200         if (fullintervalcount < vm_pageout_full_stats_interval) {
1201                 tpcount = (vm_pageout_stats_max * vmstats.v_active_count) / vmstats.v_page_count;
1202                 if (pcount > tpcount)
1203                         pcount = tpcount;
1204         } else {
1205                 fullintervalcount = 0;
1206         }
1207
1208         m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1209         while ((m != NULL) && (pcount-- > 0)) {
1210                 int actcount;
1211
1212                 if (m->queue != PQ_ACTIVE) {
1213                         break;
1214                 }
1215
1216                 next = TAILQ_NEXT(m, pageq);
1217                 /*
1218                  * Don't deactivate pages that are busy.
1219                  */
1220                 if ((m->busy != 0) ||
1221                     (m->flags & PG_BUSY) ||
1222                     (m->hold_count != 0)) {
1223                         s = splvm();
1224                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1225                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1226                         splx(s);
1227                         m = next;
1228                         continue;
1229                 }
1230
1231                 actcount = 0;
1232                 if (m->flags & PG_REFERENCED) {
1233                         vm_page_flag_clear(m, PG_REFERENCED);
1234                         actcount += 1;
1235                 }
1236
1237                 actcount += pmap_ts_referenced(m);
1238                 if (actcount) {
1239                         m->act_count += ACT_ADVANCE + actcount;
1240                         if (m->act_count > ACT_MAX)
1241                                 m->act_count = ACT_MAX;
1242                         s = splvm();
1243                         TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1244                         TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1245                         splx(s);
1246                 } else {
1247                         if (m->act_count == 0) {
1248                                 /*
1249                                  * We turn off page access, so that we have
1250                                  * more accurate RSS stats.  We don't do this
1251                                  * in the normal page deactivation when the
1252                                  * system is loaded VM wise, because the
1253                                  * cost of the large number of page protect
1254                                  * operations would be higher than the value
1255                                  * of doing the operation.
1256                                  */
1257                                 vm_page_protect(m, VM_PROT_NONE);
1258                                 vm_page_deactivate(m);
1259                         } else {
1260                                 m->act_count -= min(m->act_count, ACT_DECLINE);
1261                                 s = splvm();
1262                                 TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1263                                 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
1264                                 splx(s);
1265                         }
1266                 }
1267
1268                 m = next;
1269         }
1270         splx(s0);
1271 }
1272
1273 static int
1274 vm_pageout_free_page_calc(count)
1275 vm_size_t count;
1276 {
1277         if (count < vmstats.v_page_count)
1278                  return 0;
1279         /*
1280          * free_reserved needs to include enough for the largest swap pager
1281          * structures plus enough for any pv_entry structs when paging.
1282          */
1283         if (vmstats.v_page_count > 1024)
1284                 vmstats.v_free_min = 4 + (vmstats.v_page_count - 1024) / 200;
1285         else
1286                 vmstats.v_free_min = 4;
1287         vmstats.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1288                 vmstats.v_interrupt_free_min;
1289         vmstats.v_free_reserved = vm_pageout_page_count +
1290                 vmstats.v_pageout_free_min + (count / 768) + PQ_L2_SIZE;
1291         vmstats.v_free_severe = vmstats.v_free_min / 2;
1292         vmstats.v_free_min += vmstats.v_free_reserved;
1293         vmstats.v_free_severe += vmstats.v_free_reserved;
1294         return 1;
1295 }
1296
1297
1298 /*
1299  *      vm_pageout is the high level pageout daemon.
1300  */
1301 static void
1302 vm_pageout()
1303 {
1304         int pass;
1305
1306         /*
1307          * Initialize some paging parameters.
1308          */
1309
1310         vmstats.v_interrupt_free_min = 2;
1311         if (vmstats.v_page_count < 2000)
1312                 vm_pageout_page_count = 8;
1313
1314         vm_pageout_free_page_calc(vmstats.v_page_count);
1315         /*
1316          * v_free_target and v_cache_min control pageout hysteresis.  Note
1317          * that these are more a measure of the VM cache queue hysteresis
1318          * then the VM free queue.  Specifically, v_free_target is the
1319          * high water mark (free+cache pages).
1320          *
1321          * v_free_reserved + v_cache_min (mostly means v_cache_min) is the
1322          * low water mark, while v_free_min is the stop.  v_cache_min must
1323          * be big enough to handle memory needs while the pageout daemon
1324          * is signalled and run to free more pages.
1325          */
1326         if (vmstats.v_free_count > 6144)
1327                 vmstats.v_free_target = 4 * vmstats.v_free_min + vmstats.v_free_reserved;
1328         else
1329                 vmstats.v_free_target = 2 * vmstats.v_free_min + vmstats.v_free_reserved;
1330
1331         if (vmstats.v_free_count > 2048) {
1332                 vmstats.v_cache_min = vmstats.v_free_target;
1333                 vmstats.v_cache_max = 2 * vmstats.v_cache_min;
1334                 vmstats.v_inactive_target = (3 * vmstats.v_free_target) / 2;
1335         } else {
1336                 vmstats.v_cache_min = 0;
1337                 vmstats.v_cache_max = 0;
1338                 vmstats.v_inactive_target = vmstats.v_free_count / 4;
1339         }
1340         if (vmstats.v_inactive_target > vmstats.v_free_count / 3)
1341                 vmstats.v_inactive_target = vmstats.v_free_count / 3;
1342
1343         /* XXX does not really belong here */
1344         if (vm_page_max_wired == 0)
1345                 vm_page_max_wired = vmstats.v_free_count / 3;
1346
1347         if (vm_pageout_stats_max == 0)
1348                 vm_pageout_stats_max = vmstats.v_free_target;
1349
1350         /*
1351          * Set interval in seconds for stats scan.
1352          */
1353         if (vm_pageout_stats_interval == 0)
1354                 vm_pageout_stats_interval = 5;
1355         if (vm_pageout_full_stats_interval == 0)
1356                 vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;
1357
1358
1359         /*
1360          * Set maximum free per pass
1361          */
1362         if (vm_pageout_stats_free_max == 0)
1363                 vm_pageout_stats_free_max = 5;
1364
1365         swap_pager_swap_init();
1366         pass = 0;
1367         /*
1368          * The pageout daemon is never done, so loop forever.
1369          */
1370         while (TRUE) {
1371                 int error;
1372                 int s = splvm();
1373
1374                 /*
1375                  * If we have enough free memory, wakeup waiters.  Do
1376                  * not clear vm_pages_needed until we reach our target,
1377                  * otherwise we may be woken up over and over again and
1378                  * waste a lot of cpu.
1379                  */
1380                 if (vm_pages_needed && !vm_page_count_min()) {
1381                         if (vm_paging_needed() <= 0)
1382                                 vm_pages_needed = 0;
1383                         wakeup(&vmstats.v_free_count);
1384                 }
1385                 if (vm_pages_needed) {
1386                         /*
1387                          * Still not done, take a second pass without waiting
1388                          * (unlimited dirty cleaning), otherwise sleep a bit
1389                          * and try again.
1390                          */
1391                         ++pass;
1392                         if (pass > 1)
1393                                 tsleep(&vm_pages_needed, 0, "psleep", hz/2);
1394                 } else {
1395                         /*
1396                          * Good enough, sleep & handle stats.  Prime the pass
1397                          * for the next run.
1398                          */
1399                         if (pass > 1)
1400                                 pass = 1;
1401                         else
1402                                 pass = 0;
1403                         error = tsleep(&vm_pages_needed,
1404                                 0, "psleep", vm_pageout_stats_interval * hz);
1405                         if (error && !vm_pages_needed) {
1406                                 splx(s);
1407                                 pass = 0;
1408                                 vm_pageout_page_stats();
1409                                 continue;
1410                         }
1411                 }
1412
1413                 if (vm_pages_needed)
1414                         mycpu->gd_cnt.v_pdwakeups++;
1415                 splx(s);
1416                 vm_pageout_scan(pass);
1417                 vm_pageout_deficit = 0;
1418         }
1419 }
1420
1421 void
1422 pagedaemon_wakeup()
1423 {
1424         if (!vm_pages_needed && curthread != pagethread) {
1425                 vm_pages_needed++;
1426                 wakeup(&vm_pages_needed);
1427         }
1428 }
1429
1430 #if !defined(NO_SWAPPING)
1431 static void
1432 vm_req_vmdaemon()
1433 {
1434         static int lastrun = 0;
1435
1436         if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
1437                 wakeup(&vm_daemon_needed);
1438                 lastrun = ticks;
1439         }
1440 }
1441
1442 static void
1443 vm_daemon()
1444 {
1445         struct proc *p;
1446
1447         while (TRUE) {
1448                 tsleep(&vm_daemon_needed, 0, "psleep", 0);
1449                 if (vm_pageout_req_swapout) {
1450                         swapout_procs(vm_pageout_req_swapout);
1451                         vm_pageout_req_swapout = 0;
1452                 }
1453                 /*
1454                  * scan the processes for exceeding their rlimits or if
1455                  * process is swapped out -- deactivate pages
1456                  */
1457
1458                 for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1459                         vm_pindex_t limit, size;
1460
1461                         /*
1462                          * if this is a system process or if we have already
1463                          * looked at this process, skip it.
1464                          */
1465                         if (p->p_flag & (P_SYSTEM | P_WEXIT)) {
1466                                 continue;
1467                         }
1468                         /*
1469                          * if the process is in a non-running type state,
1470                          * don't touch it.
1471                          */
1472                         if (p->p_stat != SRUN && p->p_stat != SSLEEP) {
1473                                 continue;
1474                         }
1475                         /*
1476                          * get a limit
1477                          */
1478                         limit = OFF_TO_IDX(
1479                             qmin(p->p_rlimit[RLIMIT_RSS].rlim_cur,
1480                                 p->p_rlimit[RLIMIT_RSS].rlim_max));
1481
1482                         /*
1483                          * let processes that are swapped out really be
1484                          * swapped out set the limit to nothing (will force a
1485                          * swap-out.)
1486                          */
1487                         if ((p->p_flag & P_INMEM) == 0)
1488                                 limit = 0;      /* XXX */
1489
1490                         size = vmspace_resident_count(p->p_vmspace);
1491                         if (limit >= 0 && size >= limit) {
1492                                 vm_pageout_map_deactivate_pages(
1493                                     &p->p_vmspace->vm_map, limit);
1494                         }
1495                 }
1496         }
1497 }
1498 #endif