sys/vm/vm_fault.c

   1 /*
   2  * Copyright (c) 1991, 1993
   3  *      The Regents of the University of California.  All rights reserved.
   4  * Copyright (c) 1994 John S. Dyson
   5  * All rights reserved.
   6  * Copyright (c) 1994 David Greenman
   7  * All rights reserved.
   8  *
   9  *
  10  * This code is derived from software contributed to Berkeley by
  11  * The Mach Operating System project at Carnegie-Mellon University.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. All advertising materials mentioning features or use of this software
  22  *    must display the following acknowledgement:
  23  *      This product includes software developed by the University of
  24  *      California, Berkeley and its contributors.
  25  * 4. Neither the name of the University nor the names of its contributors
  26  *    may be used to endorse or promote products derived from this software
  27  *    without specific prior written permission.
  28  *
  29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  39  * SUCH DAMAGE.
  40  *
  41  *      from: @(#)vm_fault.c    8.4 (Berkeley) 1/12/94
  42  *
  43  *
  44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  45  * All rights reserved.
  46  *
  47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  48  *
  49  * Permission to use, copy, modify and distribute this software and
  50  * its documentation is hereby granted, provided that both the copyright
  51  * notice and this permission notice appear in all copies of the
  52  * software, derivative works or modified versions, and any portions
  53  * thereof, and that both notices appear in supporting documentation.
  54  *
  55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  58  *
  59  * Carnegie Mellon requests users of this software to return to
  60  *
  61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  62  *  School of Computer Science
  63  *  Carnegie Mellon University
  64  *  Pittsburgh PA 15213-3890
  65  *
  66  * any improvements or extensions that they make and grant Carnegie the
  67  * rights to redistribute these changes.
  68  *
  69  * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
  70  * $DragonFly: src/sys/vm/vm_fault.c,v 1.47 2008/07/01 02:02:56 dillon Exp $
  71  */
  72
  73 /*
  74  *      Page fault handling module.
  75  */
  76
  77 #include <sys/param.h>
  78 #include <sys/systm.h>
  79 #include <sys/kernel.h>
  80 #include <sys/proc.h>
  81 #include <sys/vnode.h>
  82 #include <sys/resourcevar.h>
  83 #include <sys/vmmeter.h>
  84 #include <sys/vkernel.h>
  85 #include <sys/lock.h>
  86 #include <sys/sysctl.h>
  87
  88 #include <cpu/lwbuf.h>
  89
  90 #include <vm/vm.h>
  91 #include <vm/vm_param.h>
  92 #include <vm/pmap.h>
  93 #include <vm/vm_map.h>
  94 #include <vm/vm_object.h>
  95 #include <vm/vm_page.h>
  96 #include <vm/vm_pageout.h>
  97 #include <vm/vm_kern.h>
  98 #include <vm/vm_pager.h>
  99 #include <vm/vnode_pager.h>
 100 #include <vm/vm_extern.h>
 101
 102 #include <sys/thread2.h>
 103 #include <vm/vm_page2.h>
 104
 105 struct faultstate {
 106         vm_page_t m;
 107         vm_object_t object;
 108         vm_pindex_t pindex;
 109         vm_prot_t prot;
 110         vm_page_t first_m;
 111         vm_object_t first_object;
 112         vm_prot_t first_prot;
 113         vm_map_t map;
 114         vm_map_entry_t entry;
 115         int lookup_still_valid;
 116         int didlimit;
 117         int hardfault;
 118         int fault_flags;
 119         int map_generation;
 120         boolean_t wired;
 121         struct vnode *vp;
 122 };
 123
 124 static int vm_fast_fault = 1;
 125 SYSCTL_INT(_vm, OID_AUTO, fast_fault, CTLFLAG_RW, &vm_fast_fault, 0, "");
 126 static int debug_cluster = 0;
 127 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
 128
 129 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t);
 130 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *, vpte_t, int);
 131 #if 0
 132 static int vm_fault_additional_pages (vm_page_t, int, int, vm_page_t *, int *);
 133 #endif
 134 static int vm_fault_ratelimit(struct vmspace *);
 135 static void vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry,
 136                         int prot);
 137
 138 static __inline void
 139 release_page(struct faultstate *fs)
 140 {
 141         vm_page_deactivate(fs->m);
 142         vm_page_wakeup(fs->m);
 143         fs->m = NULL;
 144 }
 145
 146 static __inline void
 147 unlock_map(struct faultstate *fs)
 148 {
 149         if (fs->lookup_still_valid && fs->map) {
 150                 vm_map_lookup_done(fs->map, fs->entry, 0);
 151                 fs->lookup_still_valid = FALSE;
 152         }
 153 }
 154
 155 /*
 156  * Clean up after a successful call to vm_fault_object() so another call
 157  * to vm_fault_object() can be made.
 158  */
 159 static void
 160 _cleanup_successful_fault(struct faultstate *fs, int relock)
 161 {
 162         if (fs->object != fs->first_object) {
 163                 vm_page_free(fs->first_m);
 164                 vm_object_pip_wakeup(fs->object);
 165                 fs->first_m = NULL;
 166         }
 167         fs->object = fs->first_object;
 168         if (relock && fs->lookup_still_valid == FALSE) {
 169                 if (fs->map)
 170                         vm_map_lock_read(fs->map);
 171                 fs->lookup_still_valid = TRUE;
 172         }
 173 }
 174
 175 static void
 176 _unlock_things(struct faultstate *fs, int dealloc)
 177 {
 178         vm_object_pip_wakeup(fs->first_object);
 179         _cleanup_successful_fault(fs, 0);
 180         if (dealloc) {
 181                 vm_object_deallocate(fs->first_object);
 182                 fs->first_object = NULL;
 183         }
 184         unlock_map(fs);
 185         if (fs->vp != NULL) {
 186                 vput(fs->vp);
 187                 fs->vp = NULL;
 188         }
 189 }
 190
 191 #define unlock_things(fs) _unlock_things(fs, 0)
 192 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
 193 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
 194
 195 /*
 196  * TRYPAGER
 197  *
 198  * Determine if the pager for the current object *might* contain the page.
 199  *
 200  * We only need to try the pager if this is not a default object (default
 201  * objects are zero-fill and have no real pager), and if we are not taking
 202  * a wiring fault or if the FS entry is wired.
 203  */
 204 #define TRYPAGER(fs)    \
 205                 (fs->object->type != OBJT_DEFAULT && \
 206                 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) || fs->wired))
 207
 208 /*
 209  * vm_fault:
 210  *
 211  * Handle a page fault occuring at the given address, requiring the given
 212  * permissions, in the map specified.  If successful, the page is inserted
 213  * into the associated physical map.
 214  *
 215  * NOTE: The given address should be truncated to the proper page address.
 216  *
 217  * KERN_SUCCESS is returned if the page fault is handled; otherwise,
 218  * a standard error specifying why the fault is fatal is returned.
 219  *
 220  * The map in question must be referenced, and remains so.
 221  * The caller may hold no locks.
 222  */
 223 int
 224 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
 225 {
 226         int result;
 227         vm_pindex_t first_pindex;
 228         struct faultstate fs;
 229         int growstack;
 230
 231         mycpu->gd_cnt.v_vm_faults++;
 232
 233         fs.didlimit = 0;
 234         fs.hardfault = 0;
 235         fs.fault_flags = fault_flags;
 236         growstack = 1;
 237
 238 RetryFault:
 239         /*
 240          * Find the vm_map_entry representing the backing store and resolve
 241          * the top level object and page index.  This may have the side
 242          * effect of executing a copy-on-write on the map entry and/or
 243          * creating a shadow object, but will not COW any actual VM pages.
 244          *
 245          * On success fs.map is left read-locked and various other fields
 246          * are initialized but not otherwise referenced or locked.
 247          *
 248          * NOTE!  vm_map_lookup will try to upgrade the fault_type to
 249          * VM_FAULT_WRITE if the map entry is a virtual page table and also
 250          * writable, so we can set the 'A'accessed bit in the virtual page
 251          * table entry.
 252          */
 253         fs.map = map;
 254         result = vm_map_lookup(&fs.map, vaddr, fault_type,
 255                                &fs.entry, &fs.first_object,
 256                                &first_pindex, &fs.first_prot, &fs.wired);
 257
 258         /*
 259          * If the lookup failed or the map protections are incompatible,
 260          * the fault generally fails.  However, if the caller is trying
 261          * to do a user wiring we have more work to do.
 262          */
 263         if (result != KERN_SUCCESS) {
 264                 if (result != KERN_PROTECTION_FAILURE ||
 265                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
 266                 {
 267                         if (result == KERN_INVALID_ADDRESS && growstack &&
 268                             map != &kernel_map && curproc != NULL) {
 269                                 result = vm_map_growstack(curproc, vaddr);
 270                                 if (result != KERN_SUCCESS)
 271                                         return (KERN_FAILURE);
 272                                 growstack = 0;
 273                                 goto RetryFault;
 274                         }
 275                         return (result);
 276                 }
 277
 278                 /*
 279                  * If we are user-wiring a r/w segment, and it is COW, then
 280                  * we need to do the COW operation.  Note that we don't
 281                  * currently COW RO sections now, because it is NOT desirable
 282                  * to COW .text.  We simply keep .text from ever being COW'ed
 283                  * and take the heat that one cannot debug wired .text sections.
 284                  */
 285                 result = vm_map_lookup(&fs.map, vaddr,
 286                                        VM_PROT_READ|VM_PROT_WRITE|
 287                                         VM_PROT_OVERRIDE_WRITE,
 288                                        &fs.entry, &fs.first_object,
 289                                        &first_pindex, &fs.first_prot,
 290                                        &fs.wired);
 291                 if (result != KERN_SUCCESS)
 292                         return result;
 293
 294                 /*
 295                  * If we don't COW now, on a user wire, the user will never
 296                  * be able to write to the mapping.  If we don't make this
 297                  * restriction, the bookkeeping would be nearly impossible.
 298                  */
 299                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
 300                         fs.entry->max_protection &= ~VM_PROT_WRITE;
 301         }
 302
 303         /*
 304          * fs.map is read-locked
 305          *
 306          * Misc checks.  Save the map generation number to detect races.
 307          */
 308         fs.map_generation = fs.map->timestamp;
 309
 310         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
 311                 panic("vm_fault: fault on nofault entry, addr: %lx",
 312                     (u_long)vaddr);
 313         }
 314
 315         /*
 316          * A system map entry may return a NULL object.  No object means
 317          * no pager means an unrecoverable kernel fault.
 318          */
 319         if (fs.first_object == NULL) {
 320                 panic("vm_fault: unrecoverable fault at %p in entry %p",
 321                         (void *)vaddr, fs.entry);
 322         }
 323
 324         /*
 325          * Make a reference to this object to prevent its disposal while we
 326          * are messing with it.  Once we have the reference, the map is free
 327          * to be diddled.  Since objects reference their shadows (and copies),
 328          * they will stay around as well.
 329          *
 330          * Bump the paging-in-progress count to prevent size changes (e.g.
 331          * truncation operations) during I/O.  This must be done after
 332          * obtaining the vnode lock in order to avoid possible deadlocks.
 333          */
 334         vm_object_reference(fs.first_object);
 335         fs.vp = vnode_pager_lock(fs.first_object);
 336         vm_object_pip_add(fs.first_object, 1);
 337
 338         fs.lookup_still_valid = TRUE;
 339         fs.first_m = NULL;
 340         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
 341
 342         /*
 343          * If the entry is wired we cannot change the page protection.
 344          */
 345         if (fs.wired)
 346                 fault_type = fs.first_prot;
 347
 348         /*
 349          * The page we want is at (first_object, first_pindex), but if the
 350          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
 351          * page table to figure out the actual pindex.
 352          *
 353          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
 354          * ONLY
 355          */
 356         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 357                 result = vm_fault_vpagetable(&fs, &first_pindex,
 358                                              fs.entry->aux.master_pde,
 359                                              fault_type);
 360                 if (result == KERN_TRY_AGAIN)
 361                         goto RetryFault;
 362                 if (result != KERN_SUCCESS)
 363                         return (result);
 364         }
 365
 366         /*
 367          * Now we have the actual (object, pindex), fault in the page.  If
 368          * vm_fault_object() fails it will unlock and deallocate the FS
 369          * data.   If it succeeds everything remains locked and fs->object
 370          * will have an additinal PIP count if it is not equal to
 371          * fs->first_object
 372          *
 373          * vm_fault_object will set fs->prot for the pmap operation.  It is
 374          * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
 375          * page can be safely written.  However, it will force a read-only
 376          * mapping for a read fault if the memory is managed by a virtual
 377          * page table.
 378          */
 379         result = vm_fault_object(&fs, first_pindex, fault_type);
 380
 381         if (result == KERN_TRY_AGAIN)
 382                 goto RetryFault;
 383         if (result != KERN_SUCCESS)
 384                 return (result);
 385
 386         /*
 387          * On success vm_fault_object() does not unlock or deallocate, and fs.m
 388          * will contain a busied page.
 389          *
 390          * Enter the page into the pmap and do pmap-related adjustments.
 391          */
 392         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
 393
 394         /*
 395          * Burst in a few more pages if possible.  The fs.map should still
 396          * be locked.
 397          */
 398         if (fault_flags & VM_FAULT_BURST) {
 399                 if ((fs.fault_flags & VM_FAULT_WIRE_MASK) == 0 &&
 400                     fs.wired == 0) {
 401                         vm_prefault(fs.map->pmap, vaddr, fs.entry, fs.prot);
 402                 }
 403         }
 404         unlock_things(&fs);
 405
 406         vm_page_flag_clear(fs.m, PG_ZERO);
 407         vm_page_flag_set(fs.m, PG_REFERENCED);
 408
 409         /*
 410          * If the page is not wired down, then put it where the pageout daemon
 411          * can find it.
 412          */
 413         if (fs.fault_flags & VM_FAULT_WIRE_MASK) {
 414                 if (fs.wired)
 415                         vm_page_wire(fs.m);
 416                 else
 417                         vm_page_unwire(fs.m, 1);
 418         } else {
 419                 vm_page_activate(fs.m);
 420         }
 421
 422         if (curthread->td_lwp) {
 423                 if (fs.hardfault) {
 424                         curthread->td_lwp->lwp_ru.ru_majflt++;
 425                 } else {
 426                         curthread->td_lwp->lwp_ru.ru_minflt++;
 427                 }
 428         }
 429
 430         /*
 431          * Unlock everything, and return
 432          */
 433         vm_page_wakeup(fs.m);
 434         vm_object_deallocate(fs.first_object);
 435
 436         return (KERN_SUCCESS);
 437 }
 438
 439 /*
 440  * Fault in the specified virtual address in the current process map,
 441  * returning a held VM page or NULL.  See vm_fault_page() for more
 442  * information.
 443  */
 444 vm_page_t
 445 vm_fault_page_quick(vm_offset_t va, vm_prot_t fault_type, int *errorp)
 446 {
 447         struct lwp *lp = curthread->td_lwp;
 448         vm_page_t m;
 449
 450         m = vm_fault_page(&lp->lwp_vmspace->vm_map, va,
 451                           fault_type, VM_FAULT_NORMAL, errorp);
 452         return(m);
 453 }
 454
 455 /*
 456  * Fault in the specified virtual address in the specified map, doing all
 457  * necessary manipulation of the object store and all necessary I/O.  Return
 458  * a held VM page or NULL, and set *errorp.  The related pmap is not
 459  * updated.
 460  *
 461  * The returned page will be properly dirtied if VM_PROT_WRITE was specified,
 462  * and marked PG_REFERENCED as well.
 463  *
 464  * If the page cannot be faulted writable and VM_PROT_WRITE was specified, an
 465  * error will be returned.
 466  */
 467 vm_page_t
 468 vm_fault_page(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
 469               int fault_flags, int *errorp)
 470 {
 471         vm_pindex_t first_pindex;
 472         struct faultstate fs;
 473         int result;
 474         vm_prot_t orig_fault_type = fault_type;
 475
 476         mycpu->gd_cnt.v_vm_faults++;
 477
 478         fs.didlimit = 0;
 479         fs.hardfault = 0;
 480         fs.fault_flags = fault_flags;
 481         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
 482
 483 RetryFault:
 484         /*
 485          * Find the vm_map_entry representing the backing store and resolve
 486          * the top level object and page index.  This may have the side
 487          * effect of executing a copy-on-write on the map entry and/or
 488          * creating a shadow object, but will not COW any actual VM pages.
 489          *
 490          * On success fs.map is left read-locked and various other fields
 491          * are initialized but not otherwise referenced or locked.
 492          *
 493          * NOTE!  vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
 494          * if the map entry is a virtual page table and also writable,
 495          * so we can set the 'A'accessed bit in the virtual page table entry.
 496          */
 497         fs.map = map;
 498         result = vm_map_lookup(&fs.map, vaddr, fault_type,
 499                                &fs.entry, &fs.first_object,
 500                                &first_pindex, &fs.first_prot, &fs.wired);
 501
 502         if (result != KERN_SUCCESS) {
 503                 *errorp = result;
 504                 return (NULL);
 505         }
 506
 507         /*
 508          * fs.map is read-locked
 509          *
 510          * Misc checks.  Save the map generation number to detect races.
 511          */
 512         fs.map_generation = fs.map->timestamp;
 513
 514         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
 515                 panic("vm_fault: fault on nofault entry, addr: %lx",
 516                     (u_long)vaddr);
 517         }
 518
 519         /*
 520          * A system map entry may return a NULL object.  No object means
 521          * no pager means an unrecoverable kernel fault.
 522          */
 523         if (fs.first_object == NULL) {
 524                 panic("vm_fault: unrecoverable fault at %p in entry %p",
 525                         (void *)vaddr, fs.entry);
 526         }
 527
 528         /*
 529          * Make a reference to this object to prevent its disposal while we
 530          * are messing with it.  Once we have the reference, the map is free
 531          * to be diddled.  Since objects reference their shadows (and copies),
 532          * they will stay around as well.
 533          *
 534          * Bump the paging-in-progress count to prevent size changes (e.g.
 535          * truncation operations) during I/O.  This must be done after
 536          * obtaining the vnode lock in order to avoid possible deadlocks.
 537          */
 538         vm_object_reference(fs.first_object);
 539         fs.vp = vnode_pager_lock(fs.first_object);
 540         vm_object_pip_add(fs.first_object, 1);
 541
 542         fs.lookup_still_valid = TRUE;
 543         fs.first_m = NULL;
 544         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
 545
 546         /*
 547          * If the entry is wired we cannot change the page protection.
 548          */
 549         if (fs.wired)
 550                 fault_type = fs.first_prot;
 551
 552         /*
 553          * The page we want is at (first_object, first_pindex), but if the
 554          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
 555          * page table to figure out the actual pindex.
 556          *
 557          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
 558          * ONLY
 559          */
 560         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 561                 result = vm_fault_vpagetable(&fs, &first_pindex,
 562                                              fs.entry->aux.master_pde,
 563                                              fault_type);
 564                 if (result == KERN_TRY_AGAIN)
 565                         goto RetryFault;
 566                 if (result != KERN_SUCCESS) {
 567                         *errorp = result;
 568                         return (NULL);
 569                 }
 570         }
 571
 572         /*
 573          * Now we have the actual (object, pindex), fault in the page.  If
 574          * vm_fault_object() fails it will unlock and deallocate the FS
 575          * data.   If it succeeds everything remains locked and fs->object
 576          * will have an additinal PIP count if it is not equal to
 577          * fs->first_object
 578          */
 579         result = vm_fault_object(&fs, first_pindex, fault_type);
 580
 581         if (result == KERN_TRY_AGAIN)
 582                 goto RetryFault;
 583         if (result != KERN_SUCCESS) {
 584                 *errorp = result;
 585                 return(NULL);
 586         }
 587
 588         if ((orig_fault_type & VM_PROT_WRITE) &&
 589             (fs.prot & VM_PROT_WRITE) == 0) {
 590                 *errorp = KERN_PROTECTION_FAILURE;
 591                 unlock_and_deallocate(&fs);
 592                 return(NULL);
 593         }
 594
 595         /*
 596          * On success vm_fault_object() does not unlock or deallocate, and fs.m
 597          * will contain a busied page.
 598          */
 599         unlock_things(&fs);
 600
 601         /*
 602          * Return a held page.  We are not doing any pmap manipulation so do
 603          * not set PG_MAPPED.  However, adjust the page flags according to
 604          * the fault type because the caller may not use a managed pmapping
 605          * (so we don't want to lose the fact that the page will be dirtied
 606          * if a write fault was specified).
 607          */
 608         vm_page_hold(fs.m);
 609         vm_page_flag_clear(fs.m, PG_ZERO);
 610         if (fault_type & VM_PROT_WRITE)
 611                 vm_page_dirty(fs.m);
 612
 613         /*
 614          * Update the pmap.  We really only have to do this if a COW
 615          * occured to replace the read-only page with the new page.  For
 616          * now just do it unconditionally. XXX
 617          */
 618         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
 619         vm_page_flag_set(fs.m, PG_REFERENCED);
 620
 621         /*
 622          * Unbusy the page by activating it.  It remains held and will not
 623          * be reclaimed.
 624          */
 625         vm_page_activate(fs.m);
 626
 627         if (curthread->td_lwp) {
 628                 if (fs.hardfault) {
 629                         curthread->td_lwp->lwp_ru.ru_majflt++;
 630                 } else {
 631                         curthread->td_lwp->lwp_ru.ru_minflt++;
 632                 }
 633         }
 634
 635         /*
 636          * Unlock everything, and return the held page.
 637          */
 638         vm_page_wakeup(fs.m);
 639         vm_object_deallocate(fs.first_object);
 640
 641         *errorp = 0;
 642         return(fs.m);
 643 }
 644
 645 /*
 646  * Fault in the specified (object,offset), dirty the returned page as
 647  * needed.  If the requested fault_type cannot be done NULL and an
 648  * error is returned.
 649  */
 650 vm_page_t
 651 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
 652                      vm_prot_t fault_type, int fault_flags, int *errorp)
 653 {
 654         int result;
 655         vm_pindex_t first_pindex;
 656         struct faultstate fs;
 657         struct vm_map_entry entry;
 658
 659         bzero(&entry, sizeof(entry));
 660         entry.object.vm_object = object;
 661         entry.maptype = VM_MAPTYPE_NORMAL;
 662         entry.protection = entry.max_protection = fault_type;
 663
 664         fs.didlimit = 0;
 665         fs.hardfault = 0;
 666         fs.fault_flags = fault_flags;
 667         fs.map = NULL;
 668         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
 669
 670 RetryFault:
 671
 672         fs.first_object = object;
 673         first_pindex = OFF_TO_IDX(offset);
 674         fs.entry = &entry;
 675         fs.first_prot = fault_type;
 676         fs.wired = 0;
 677         /*fs.map_generation = 0; unused */
 678
 679         /*
 680          * Make a reference to this object to prevent its disposal while we
 681          * are messing with it.  Once we have the reference, the map is free
 682          * to be diddled.  Since objects reference their shadows (and copies),
 683          * they will stay around as well.
 684          *
 685          * Bump the paging-in-progress count to prevent size changes (e.g.
 686          * truncation operations) during I/O.  This must be done after
 687          * obtaining the vnode lock in order to avoid possible deadlocks.
 688          */
 689         vm_object_reference(fs.first_object);
 690         fs.vp = vnode_pager_lock(fs.first_object);
 691         vm_object_pip_add(fs.first_object, 1);
 692
 693         fs.lookup_still_valid = TRUE;
 694         fs.first_m = NULL;
 695         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
 696
 697 #if 0
 698         /* XXX future - ability to operate on VM object using vpagetable */
 699         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 700                 result = vm_fault_vpagetable(&fs, &first_pindex,
 701                                              fs.entry->aux.master_pde,
 702                                              fault_type);
 703                 if (result == KERN_TRY_AGAIN)
 704                         goto RetryFault;
 705                 if (result != KERN_SUCCESS) {
 706                         *errorp = result;
 707                         return (NULL);
 708                 }
 709         }
 710 #endif
 711
 712         /*
 713          * Now we have the actual (object, pindex), fault in the page.  If
 714          * vm_fault_object() fails it will unlock and deallocate the FS
 715          * data.   If it succeeds everything remains locked and fs->object
 716          * will have an additinal PIP count if it is not equal to
 717          * fs->first_object
 718          */
 719         result = vm_fault_object(&fs, first_pindex, fault_type);
 720
 721         if (result == KERN_TRY_AGAIN)
 722                 goto RetryFault;
 723         if (result != KERN_SUCCESS) {
 724                 *errorp = result;
 725                 return(NULL);
 726         }
 727
 728         if ((fault_type & VM_PROT_WRITE) && (fs.prot & VM_PROT_WRITE) == 0) {
 729                 *errorp = KERN_PROTECTION_FAILURE;
 730                 unlock_and_deallocate(&fs);
 731                 return(NULL);
 732         }
 733
 734         /*
 735          * On success vm_fault_object() does not unlock or deallocate, and fs.m
 736          * will contain a busied page.
 737          */
 738         unlock_things(&fs);
 739
 740         /*
 741          * Return a held page.  We are not doing any pmap manipulation so do
 742          * not set PG_MAPPED.  However, adjust the page flags according to
 743          * the fault type because the caller may not use a managed pmapping
 744          * (so we don't want to lose the fact that the page will be dirtied
 745          * if a write fault was specified).
 746          */
 747         vm_page_hold(fs.m);
 748         vm_page_flag_clear(fs.m, PG_ZERO);
 749         if (fault_type & VM_PROT_WRITE)
 750                 vm_page_dirty(fs.m);
 751
 752         /*
 753          * Indicate that the page was accessed.
 754          */
 755         vm_page_flag_set(fs.m, PG_REFERENCED);
 756
 757         /*
 758          * Unbusy the page by activating it.  It remains held and will not
 759          * be reclaimed.
 760          */
 761         vm_page_activate(fs.m);
 762
 763         if (curthread->td_lwp) {
 764                 if (fs.hardfault) {
 765                         mycpu->gd_cnt.v_vm_faults++;
 766                         curthread->td_lwp->lwp_ru.ru_majflt++;
 767                 } else {
 768                         curthread->td_lwp->lwp_ru.ru_minflt++;
 769                 }
 770         }
 771
 772         /*
 773          * Unlock everything, and return the held page.
 774          */
 775         vm_page_wakeup(fs.m);
 776         vm_object_deallocate(fs.first_object);
 777
 778         *errorp = 0;
 779         return(fs.m);
 780 }
 781
 782 /*
 783  * Translate the virtual page number (first_pindex) that is relative
 784  * to the address space into a logical page number that is relative to the
 785  * backing object.  Use the virtual page table pointed to by (vpte).
 786  *
 787  * This implements an N-level page table.  Any level can terminate the
 788  * scan by setting VPTE_PS.   A linear mapping is accomplished by setting
 789  * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
 790  */
 791 static
 792 int
 793 vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
 794                     vpte_t vpte, int fault_type)
 795 {
 796         struct lwbuf *lwb;
 797         int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
 798         int result = KERN_SUCCESS;
 799         vpte_t *ptep;
 800
 801         for (;;) {
 802                 /*
 803                  * We cannot proceed if the vpte is not valid, not readable
 804                  * for a read fault, or not writable for a write fault.
 805                  */
 806                 if ((vpte & VPTE_V) == 0) {
 807                         unlock_and_deallocate(fs);
 808                         return (KERN_FAILURE);
 809                 }
 810                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_R) == 0) {
 811                         unlock_and_deallocate(fs);
 812                         return (KERN_FAILURE);
 813                 }
 814                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_W) == 0) {
 815                         unlock_and_deallocate(fs);
 816                         return (KERN_FAILURE);
 817                 }
 818                 if ((vpte & VPTE_PS) || vshift == 0)
 819                         break;
 820                 KKASSERT(vshift >= VPTE_PAGE_BITS);
 821
 822                 /*
 823                  * Get the page table page.  Nominally we only read the page
 824                  * table, but since we are actively setting VPTE_M and VPTE_A,
 825                  * tell vm_fault_object() that we are writing it.
 826                  *
 827                  * There is currently no real need to optimize this.
 828                  */
 829                 result = vm_fault_object(fs, (vpte & VPTE_FRAME) >> PAGE_SHIFT,
 830                                          VM_PROT_READ|VM_PROT_WRITE);
 831                 if (result != KERN_SUCCESS)
 832                         return (result);
 833
 834                 /*
 835                  * Process the returned fs.m and look up the page table
 836                  * entry in the page table page.
 837                  */
 838                 vshift -= VPTE_PAGE_BITS;
 839                 lwb = lwbuf_alloc(fs->m);
 840                 ptep = ((vpte_t *)lwbuf_kva(lwb) +
 841                         ((*pindex >> vshift) & VPTE_PAGE_MASK));
 842                 vpte = *ptep;
 843
 844                 /*
 845                  * Page table write-back.  If the vpte is valid for the
 846                  * requested operation, do a write-back to the page table.
 847                  *
 848                  * XXX VPTE_M is not set properly for page directory pages.
 849                  * It doesn't get set in the page directory if the page table
 850                  * is modified during a read access.
 851                  */
 852                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_V) &&
 853                     (vpte & VPTE_W)) {
 854                         if ((vpte & (VPTE_M|VPTE_A)) != (VPTE_M|VPTE_A)) {
 855                                 atomic_set_long(ptep, VPTE_M | VPTE_A);
 856                                 vm_page_dirty(fs->m);
 857                         }
 858                 }
 859                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_V) &&
 860                     (vpte & VPTE_R)) {
 861                         if ((vpte & VPTE_A) == 0) {
 862                                 atomic_set_long(ptep, VPTE_A);
 863                                 vm_page_dirty(fs->m);
 864                         }
 865                 }
 866                 lwbuf_free(lwb);
 867                 vm_page_flag_set(fs->m, PG_REFERENCED);
 868                 vm_page_activate(fs->m);
 869                 vm_page_wakeup(fs->m);
 870                 cleanup_successful_fault(fs);
 871         }
 872         /*
 873          * Combine remaining address bits with the vpte.
 874          */
 875         /* JG how many bits from each? */
 876         *pindex = ((vpte & VPTE_FRAME) >> PAGE_SHIFT) +
 877                   (*pindex & ((1L << vshift) - 1));
 878         return (KERN_SUCCESS);
 879 }
 880
 881
 882 /*
 883  * Do all operations required to fault-in (fs.first_object, pindex).  Run
 884  * through the shadow chain as necessary and do required COW or virtual
 885  * copy operations.  The caller has already fully resolved the vm_map_entry
 886  * and, if appropriate, has created a copy-on-write layer.  All we need to
 887  * do is iterate the object chain.
 888  *
 889  * On failure (fs) is unlocked and deallocated and the caller may return or
 890  * retry depending on the failure code.  On success (fs) is NOT unlocked or
 891  * deallocated, fs.m will contained a resolved, busied page, and fs.object
 892  * will have an additional PIP count if it is not equal to fs.first_object.
 893  */
 894 static
 895 int
 896 vm_fault_object(struct faultstate *fs,
 897                 vm_pindex_t first_pindex, vm_prot_t fault_type)
 898 {
 899         vm_object_t next_object;
 900         vm_pindex_t pindex;
 901
 902         fs->prot = fs->first_prot;
 903         fs->object = fs->first_object;
 904         pindex = first_pindex;
 905
 906         /*
 907          * If a read fault occurs we try to make the page writable if
 908          * possible.  There are three cases where we cannot make the
 909          * page mapping writable:
 910          *
 911          * (1) The mapping is read-only or the VM object is read-only,
 912          *     fs->prot above will simply not have VM_PROT_WRITE set.
 913          *
 914          * (2) If the mapping is a virtual page table we need to be able
 915          *     to detect writes so we can set VPTE_M in the virtual page
 916          *     table.
 917          *
 918          * (3) If the VM page is read-only or copy-on-write, upgrading would
 919          *     just result in an unnecessary COW fault.
 920          *
 921          * VM_PROT_VPAGED is set if faulting via a virtual page table and
 922          * causes adjustments to the 'M'odify bit to also turn off write
 923          * access to force a re-fault.
 924          */
 925         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 926                 if ((fault_type & VM_PROT_WRITE) == 0)
 927                         fs->prot &= ~VM_PROT_WRITE;
 928         }
 929
 930         for (;;) {
 931                 /*
 932                  * If the object is dead, we stop here
 933                  */
 934                 if (fs->object->flags & OBJ_DEAD) {
 935                         unlock_and_deallocate(fs);
 936                         return (KERN_PROTECTION_FAILURE);
 937                 }
 938
 939                 /*
 940                  * See if page is resident.  spl protection is required
 941                  * to avoid an interrupt unbusy/free race against our
 942                  * lookup.  We must hold the protection through a page
 943                  * allocation or busy.
 944                  */
 945                 crit_enter();
 946                 fs->m = vm_page_lookup(fs->object, pindex);
 947                 if (fs->m != NULL) {
 948                         int queue;
 949                         /*
 950                          * Wait/Retry if the page is busy.  We have to do this
 951                          * if the page is busy via either PG_BUSY or
 952                          * vm_page_t->busy because the vm_pager may be using
 953                          * vm_page_t->busy for pageouts ( and even pageins if
 954                          * it is the vnode pager ), and we could end up trying
 955                          * to pagein and pageout the same page simultaneously.
 956                          *
 957                          * We can theoretically allow the busy case on a read
 958                          * fault if the page is marked valid, but since such
 959                          * pages are typically already pmap'd, putting that
 960                          * special case in might be more effort then it is
 961                          * worth.  We cannot under any circumstances mess
 962                          * around with a vm_page_t->busy page except, perhaps,
 963                          * to pmap it.
 964                          */
 965                         if ((fs->m->flags & PG_BUSY) || fs->m->busy) {
 966                                 unlock_things(fs);
 967                                 vm_page_sleep_busy(fs->m, TRUE, "vmpfw");
 968                                 mycpu->gd_cnt.v_intrans++;
 969                                 vm_object_deallocate(fs->first_object);
 970                                 fs->first_object = NULL;
 971                                 crit_exit();
 972                                 return (KERN_TRY_AGAIN);
 973                         }
 974
 975                         /*
 976                          * If reactivating a page from PQ_CACHE we may have
 977                          * to rate-limit.
 978                          */
 979                         queue = fs->m->queue;
 980                         vm_page_unqueue_nowakeup(fs->m);
 981
 982                         if ((queue - fs->m->pc) == PQ_CACHE &&
 983                             vm_page_count_severe()) {
 984                                 vm_page_activate(fs->m);
 985                                 unlock_and_deallocate(fs);
 986                                 vm_waitpfault();
 987                                 crit_exit();
 988                                 return (KERN_TRY_AGAIN);
 989                         }
 990
 991                         /*
 992                          * Mark page busy for other processes, and the
 993                          * pagedaemon.  If it still isn't completely valid
 994                          * (readable), or if a read-ahead-mark is set on
 995                          * the VM page, jump to readrest, else we found the
 996                          * page and can return.
 997                          *
 998                          * We can release the spl once we have marked the
 999                          * page busy.
1000                          */
1001                         vm_page_busy(fs->m);
1002                         crit_exit();
1003
1004                         if (fs->m->object != &kernel_object) {
1005                                 if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
1006                                     VM_PAGE_BITS_ALL) {
1007                                         goto readrest;
1008                                 }
1009                                 if (fs->m->flags & PG_RAM) {
1010                                         if (debug_cluster)
1011                                                 kprintf("R");
1012                                         vm_page_flag_clear(fs->m, PG_RAM);
1013                                         goto readrest;
1014                                 }
1015                         }
1016                         break; /* break to PAGE HAS BEEN FOUND */
1017                 }
1018
1019                 /*
1020                  * Page is not resident, If this is the search termination
1021                  * or the pager might contain the page, allocate a new page.
1022                  *
1023                  * NOTE: We are still in a critical section.
1024                  */
1025                 if (TRYPAGER(fs) || fs->object == fs->first_object) {
1026                         /*
1027                          * If the page is beyond the object size we fail
1028                          */
1029                         if (pindex >= fs->object->size) {
1030                                 crit_exit();
1031                                 unlock_and_deallocate(fs);
1032                                 return (KERN_PROTECTION_FAILURE);
1033                         }
1034
1035                         /*
1036                          * Ratelimit.
1037                          */
1038                         if (fs->didlimit == 0 && curproc != NULL) {
1039                                 int limticks;
1040
1041                                 limticks = vm_fault_ratelimit(curproc->p_vmspace);
1042                                 if (limticks) {
1043                                         crit_exit();
1044                                         unlock_and_deallocate(fs);
1045                                         tsleep(curproc, 0, "vmrate", limticks);
1046                                         fs->didlimit = 1;
1047                                         return (KERN_TRY_AGAIN);
1048                                 }
1049                         }
1050
1051                         /*
1052                          * Allocate a new page for this object/offset pair.
1053                          */
1054                         fs->m = NULL;
1055                         if (!vm_page_count_severe()) {
1056                                 fs->m = vm_page_alloc(fs->object, pindex,
1057                                     (fs->vp || fs->object->backing_object) ? VM_ALLOC_NORMAL : VM_ALLOC_NORMAL | VM_ALLOC_ZERO);
1058                         }
1059                         if (fs->m == NULL) {
1060                                 crit_exit();
1061                                 unlock_and_deallocate(fs);
1062                                 vm_waitpfault();
1063                                 return (KERN_TRY_AGAIN);
1064                         }
1065                 }
1066                 crit_exit();
1067
1068 readrest:
1069                 /*
1070                  * We have found an invalid or partially valid page, a
1071                  * page with a read-ahead mark which might be partially or
1072                  * fully valid (and maybe dirty too), or we have allocated
1073                  * a new page.
1074                  *
1075                  * Attempt to fault-in the page if there is a chance that the
1076                  * pager has it, and potentially fault in additional pages
1077                  * at the same time.
1078                  *
1079                  * We are NOT in splvm here and if TRYPAGER is true then
1080                  * fs.m will be non-NULL and will be PG_BUSY for us.
1081                  */
1082                 if (TRYPAGER(fs)) {
1083                         int rv;
1084                         int seqaccess;
1085                         u_char behavior = vm_map_entry_behavior(fs->entry);
1086
1087                         if (behavior == MAP_ENTRY_BEHAV_RANDOM)
1088                                 seqaccess = 0;
1089                         else
1090                                 seqaccess = -1;
1091
1092                         /*
1093                          * If sequential access is detected then attempt
1094                          * to deactivate/cache pages behind the scan to
1095                          * prevent resource hogging.
1096                          *
1097                          * Use of PG_RAM to detect sequential access
1098                          * also simulates multi-zone sequential access
1099                          * detection for free.
1100                          *
1101                          * NOTE: Partially valid dirty pages cannot be
1102                          *       deactivated without causing NFS picemeal
1103                          *       writes to barf.
1104                          */
1105                         if ((fs->first_object->type != OBJT_DEVICE) &&
1106                             (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
1107                                 (behavior != MAP_ENTRY_BEHAV_RANDOM &&
1108                                  (fs->m->flags & PG_RAM)))
1109                         ) {
1110                                 vm_pindex_t scan_pindex;
1111                                 int scan_count = 16;
1112
1113                                 if (first_pindex < 16) {
1114                                         scan_pindex = 0;
1115                                         scan_count = 0;
1116                                 } else {
1117                                         scan_pindex = first_pindex - 16;
1118                                         if (scan_pindex < 16)
1119                                                 scan_count = scan_pindex;
1120                                         else
1121                                                 scan_count = 16;
1122                                 }
1123
1124                                 crit_enter();
1125                                 while (scan_count) {
1126                                         vm_page_t mt;
1127
1128                                         mt = vm_page_lookup(fs->first_object,
1129                                                             scan_pindex);
1130                                         if (mt == NULL ||
1131                                             (mt->valid != VM_PAGE_BITS_ALL)) {
1132                                                 break;
1133                                         }
1134                                         if (mt->busy ||
1135                                             (mt->flags & (PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED)) ||
1136                                             mt->hold_count ||
1137                                             mt->wire_count)  {
1138                                                 goto skip;
1139                                         }
1140                                         if (mt->dirty == 0)
1141                                                 vm_page_test_dirty(mt);
1142                                         if (mt->dirty) {
1143                                                 vm_page_busy(mt);
1144                                                 vm_page_protect(mt,
1145                                                                 VM_PROT_NONE);
1146                                                 vm_page_deactivate(mt);
1147                                                 vm_page_wakeup(mt);
1148                                         } else {
1149                                                 vm_page_cache(mt);
1150                                         }
1151 skip:
1152                                         --scan_count;
1153                                         --scan_pindex;
1154                                 }
1155                                 crit_exit();
1156
1157                                 seqaccess = 1;
1158                         }
1159
1160                         /*
1161                          * Avoid deadlocking against the map when doing I/O.
1162                          * fs.object and the page is PG_BUSY'd.
1163                          */
1164                         unlock_map(fs);
1165
1166                         /*
1167                          * Acquire the page data.  We still hold a ref on
1168                          * fs.object and the page has been PG_BUSY's.
1169                          *
1170                          * The pager may replace the page (for example, in
1171                          * order to enter a fictitious page into the
1172                          * object).  If it does so it is responsible for
1173                          * cleaning up the passed page and properly setting
1174                          * the new page PG_BUSY.
1175                          *
1176                          * If we got here through a PG_RAM read-ahead
1177                          * mark the page may be partially dirty and thus
1178                          * not freeable.  Don't bother checking to see
1179                          * if the pager has the page because we can't free
1180                          * it anyway.  We have to depend on the get_page
1181                          * operation filling in any gaps whether there is
1182                          * backing store or not.
1183                          */
1184                         rv = vm_pager_get_page(fs->object, &fs->m, seqaccess);
1185
1186                         if (rv == VM_PAGER_OK) {
1187                                 /*
1188                                  * Relookup in case pager changed page. Pager
1189                                  * is responsible for disposition of old page
1190                                  * if moved.
1191                                  *
1192                                  * XXX other code segments do relookups too.
1193                                  * It's a bad abstraction that needs to be
1194                                  * fixed/removed.
1195                                  */
1196                                 fs->m = vm_page_lookup(fs->object, pindex);
1197                                 if (fs->m == NULL) {
1198                                         unlock_and_deallocate(fs);
1199                                         return (KERN_TRY_AGAIN);
1200                                 }
1201
1202                                 ++fs->hardfault;
1203                                 break; /* break to PAGE HAS BEEN FOUND */
1204                         }
1205
1206                         /*
1207                          * Remove the bogus page (which does not exist at this
1208                          * object/offset); before doing so, we must get back
1209                          * our object lock to preserve our invariant.
1210                          *
1211                          * Also wake up any other process that may want to bring
1212                          * in this page.
1213                          *
1214                          * If this is the top-level object, we must leave the
1215                          * busy page to prevent another process from rushing
1216                          * past us, and inserting the page in that object at
1217                          * the same time that we are.
1218                          */
1219                         if (rv == VM_PAGER_ERROR) {
1220                                 if (curproc)
1221                                         kprintf("vm_fault: pager read error, pid %d (%s)\n", curproc->p_pid, curproc->p_comm);
1222                                 else
1223                                         kprintf("vm_fault: pager read error, thread %p (%s)\n", curthread, curproc->p_comm);
1224                         }
1225
1226                         /*
1227                          * Data outside the range of the pager or an I/O error
1228                          *
1229                          * The page may have been wired during the pagein,
1230                          * e.g. by the buffer cache, and cannot simply be
1231                          * freed.  Call vnode_pager_freepage() to deal with it.
1232                          */
1233                         /*
1234                          * XXX - the check for kernel_map is a kludge to work
1235                          * around having the machine panic on a kernel space
1236                          * fault w/ I/O error.
1237                          */
1238                         if (((fs->map != &kernel_map) &&
1239                             (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) {
1240                                 vnode_pager_freepage(fs->m);
1241                                 fs->m = NULL;
1242                                 unlock_and_deallocate(fs);
1243                                 if (rv == VM_PAGER_ERROR)
1244                                         return (KERN_FAILURE);
1245                                 else
1246                                         return (KERN_PROTECTION_FAILURE);
1247                                 /* NOT REACHED */
1248                         }
1249                         if (fs->object != fs->first_object) {
1250                                 vnode_pager_freepage(fs->m);
1251                                 fs->m = NULL;
1252                                 /*
1253                                  * XXX - we cannot just fall out at this
1254                                  * point, m has been freed and is invalid!
1255                                  */
1256                         }
1257                 }
1258
1259                 /*
1260                  * We get here if the object has a default pager (or unwiring)
1261                  * or the pager doesn't have the page.
1262                  */
1263                 if (fs->object == fs->first_object)
1264                         fs->first_m = fs->m;
1265
1266                 /*
1267                  * Move on to the next object.  Lock the next object before
1268                  * unlocking the current one.
1269                  */
1270                 pindex += OFF_TO_IDX(fs->object->backing_object_offset);
1271                 next_object = fs->object->backing_object;
1272                 if (next_object == NULL) {
1273                         /*
1274                          * If there's no object left, fill the page in the top
1275                          * object with zeros.
1276                          */
1277                         if (fs->object != fs->first_object) {
1278                                 vm_object_pip_wakeup(fs->object);
1279
1280                                 fs->object = fs->first_object;
1281                                 pindex = first_pindex;
1282                                 fs->m = fs->first_m;
1283                         }
1284                         fs->first_m = NULL;
1285
1286                         /*
1287                          * Zero the page if necessary and mark it valid.
1288                          */
1289                         if ((fs->m->flags & PG_ZERO) == 0) {
1290                                 vm_page_zero_fill(fs->m);
1291                         } else {
1292                                 mycpu->gd_cnt.v_ozfod++;
1293                         }
1294                         mycpu->gd_cnt.v_zfod++;
1295                         fs->m->valid = VM_PAGE_BITS_ALL;
1296                         break;  /* break to PAGE HAS BEEN FOUND */
1297                 } else {
1298                         if (fs->object != fs->first_object) {
1299                                 vm_object_pip_wakeup(fs->object);
1300                         }
1301                         KASSERT(fs->object != next_object, ("object loop %p", next_object));
1302                         fs->object = next_object;
1303                         vm_object_pip_add(fs->object, 1);
1304                 }
1305         }
1306
1307         /*
1308          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
1309          * is held.]
1310          *
1311          * If the page is being written, but isn't already owned by the
1312          * top-level object, we have to copy it into a new page owned by the
1313          * top-level object.
1314          */
1315         KASSERT((fs->m->flags & PG_BUSY) != 0,
1316                 ("vm_fault: not busy after main loop"));
1317
1318         if (fs->object != fs->first_object) {
1319                 /*
1320                  * We only really need to copy if we want to write it.
1321                  */
1322                 if (fault_type & VM_PROT_WRITE) {
1323                         /*
1324                          * This allows pages to be virtually copied from a
1325                          * backing_object into the first_object, where the
1326                          * backing object has no other refs to it, and cannot
1327                          * gain any more refs.  Instead of a bcopy, we just
1328                          * move the page from the backing object to the
1329                          * first object.  Note that we must mark the page
1330                          * dirty in the first object so that it will go out
1331                          * to swap when needed.
1332                          */
1333                         if (
1334                                 /*
1335                                  * Map, if present, has not changed
1336                                  */
1337                                 (fs->map == NULL ||
1338                                 fs->map_generation == fs->map->timestamp) &&
1339                                 /*
1340                                  * Only one shadow object
1341                                  */
1342                                 (fs->object->shadow_count == 1) &&
1343                                 /*
1344                                  * No COW refs, except us
1345                                  */
1346                                 (fs->object->ref_count == 1) &&
1347                                 /*
1348                                  * No one else can look this object up
1349                                  */
1350                                 (fs->object->handle == NULL) &&
1351                                 /*
1352                                  * No other ways to look the object up
1353                                  */
1354                                 ((fs->object->type == OBJT_DEFAULT) ||
1355                                  (fs->object->type == OBJT_SWAP)) &&
1356                                 /*
1357                                  * We don't chase down the shadow chain
1358                                  */
1359                                 (fs->object == fs->first_object->backing_object) &&
1360
1361                                 /*
1362                                  * grab the lock if we need to
1363                                  */
1364                                 (fs->lookup_still_valid ||
1365                                  fs->map == NULL ||
1366                                  lockmgr(&fs->map->lock, LK_EXCLUSIVE|LK_NOWAIT) == 0)
1367                             ) {
1368
1369                                 fs->lookup_still_valid = 1;
1370                                 /*
1371                                  * get rid of the unnecessary page
1372                                  */
1373                                 vm_page_protect(fs->first_m, VM_PROT_NONE);
1374                                 vm_page_free(fs->first_m);
1375                                 fs->first_m = NULL;
1376
1377                                 /*
1378                                  * grab the page and put it into the
1379                                  * process'es object.  The page is
1380                                  * automatically made dirty.
1381                                  */
1382                                 vm_page_rename(fs->m, fs->first_object, first_pindex);
1383                                 fs->first_m = fs->m;
1384                                 vm_page_busy(fs->first_m);
1385                                 fs->m = NULL;
1386                                 mycpu->gd_cnt.v_cow_optim++;
1387                         } else {
1388                                 /*
1389                                  * Oh, well, lets copy it.
1390                                  */
1391                                 vm_page_copy(fs->m, fs->first_m);
1392                                 vm_page_event(fs->m, VMEVENT_COW);
1393                         }
1394
1395                         if (fs->m) {
1396                                 /*
1397                                  * We no longer need the old page or object.
1398                                  */
1399                                 release_page(fs);
1400                         }
1401
1402                         /*
1403                          * fs->object != fs->first_object due to above
1404                          * conditional
1405                          */
1406                         vm_object_pip_wakeup(fs->object);
1407
1408                         /*
1409                          * Only use the new page below...
1410                          */
1411
1412                         mycpu->gd_cnt.v_cow_faults++;
1413                         fs->m = fs->first_m;
1414                         fs->object = fs->first_object;
1415                         pindex = first_pindex;
1416                 } else {
1417                         /*
1418                          * If it wasn't a write fault avoid having to copy
1419                          * the page by mapping it read-only.
1420                          */
1421                         fs->prot &= ~VM_PROT_WRITE;
1422                 }
1423         }
1424
1425         /*
1426          * We may have had to unlock a map to do I/O.  If we did then
1427          * lookup_still_valid will be FALSE.  If the map generation count
1428          * also changed then all sorts of things could have happened while
1429          * we were doing the I/O and we need to retry.
1430          */
1431
1432         if (!fs->lookup_still_valid &&
1433             fs->map != NULL &&
1434             (fs->map->timestamp != fs->map_generation)) {
1435                 release_page(fs);
1436                 unlock_and_deallocate(fs);
1437                 return (KERN_TRY_AGAIN);
1438         }
1439
1440         /*
1441          * If the fault is a write, we know that this page is being
1442          * written NOW so dirty it explicitly to save on pmap_is_modified()
1443          * calls later.
1444          *
1445          * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
1446          * if the page is already dirty to prevent data written with
1447          * the expectation of being synced from not being synced.
1448          * Likewise if this entry does not request NOSYNC then make
1449          * sure the page isn't marked NOSYNC.  Applications sharing
1450          * data should use the same flags to avoid ping ponging.
1451          *
1452          * Also tell the backing pager, if any, that it should remove
1453          * any swap backing since the page is now dirty.
1454          */
1455         if (fs->prot & VM_PROT_WRITE) {
1456                 vm_object_set_writeable_dirty(fs->m->object);
1457                 if (fs->entry->eflags & MAP_ENTRY_NOSYNC) {
1458                         if (fs->m->dirty == 0)
1459                                 vm_page_flag_set(fs->m, PG_NOSYNC);
1460                 } else {
1461                         vm_page_flag_clear(fs->m, PG_NOSYNC);
1462                 }
1463                 if (fs->fault_flags & VM_FAULT_DIRTY) {
1464                         crit_enter();
1465                         vm_page_dirty(fs->m);
1466                         swap_pager_unswapped(fs->m);
1467                         crit_exit();
1468                 }
1469         }
1470
1471         /*
1472          * Page had better still be busy.  We are still locked up and
1473          * fs->object will have another PIP reference if it is not equal
1474          * to fs->first_object.
1475          */
1476         KASSERT(fs->m->flags & PG_BUSY,
1477                 ("vm_fault: page %p not busy!", fs->m));
1478
1479         /*
1480          * Sanity check: page must be completely valid or it is not fit to
1481          * map into user space.  vm_pager_get_pages() ensures this.
1482          */
1483         if (fs->m->valid != VM_PAGE_BITS_ALL) {
1484                 vm_page_zero_invalid(fs->m, TRUE);
1485                 kprintf("Warning: page %p partially invalid on fault\n", fs->m);
1486         }
1487
1488         return (KERN_SUCCESS);
1489 }
1490
1491 /*
1492  * Wire down a range of virtual addresses in a map.  The entry in question
1493  * should be marked in-transition and the map must be locked.  We must
1494  * release the map temporarily while faulting-in the page to avoid a
1495  * deadlock.  Note that the entry may be clipped while we are blocked but
1496  * will never be freed.
1497  */
1498 int
1499 vm_fault_wire(vm_map_t map, vm_map_entry_t entry, boolean_t user_wire)
1500 {
1501         boolean_t fictitious;
1502         vm_offset_t start;
1503         vm_offset_t end;
1504         vm_offset_t va;
1505         vm_paddr_t pa;
1506         pmap_t pmap;
1507         int rv;
1508
1509         pmap = vm_map_pmap(map);
1510         start = entry->start;
1511         end = entry->end;
1512         fictitious = entry->object.vm_object &&
1513                         (entry->object.vm_object->type == OBJT_DEVICE);
1514
1515         vm_map_unlock(map);
1516         map->timestamp++;
1517
1518         /*
1519          * We simulate a fault to get the page and enter it in the physical
1520          * map.
1521          */
1522         for (va = start; va < end; va += PAGE_SIZE) {
1523                 if (user_wire) {
1524                         rv = vm_fault(map, va, VM_PROT_READ,
1525                                         VM_FAULT_USER_WIRE);
1526                 } else {
1527                         rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
1528                                         VM_FAULT_CHANGE_WIRING);
1529                 }
1530                 if (rv) {
1531                         while (va > start) {
1532                                 va -= PAGE_SIZE;
1533                                 if ((pa = pmap_extract(pmap, va)) == 0)
1534                                         continue;
1535                                 pmap_change_wiring(pmap, va, FALSE);
1536                                 if (!fictitious)
1537                                         vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
1538                         }
1539                         vm_map_lock(map);
1540                         return (rv);
1541                 }
1542         }
1543         vm_map_lock(map);
1544         return (KERN_SUCCESS);
1545 }
1546
1547 /*
1548  * Unwire a range of virtual addresses in a map.  The map should be
1549  * locked.
1550  */
1551 void
1552 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
1553 {
1554         boolean_t fictitious;
1555         vm_offset_t start;
1556         vm_offset_t end;
1557         vm_offset_t va;
1558         vm_paddr_t pa;
1559         pmap_t pmap;
1560
1561         pmap = vm_map_pmap(map);
1562         start = entry->start;
1563         end = entry->end;
1564         fictitious = entry->object.vm_object &&
1565                         (entry->object.vm_object->type == OBJT_DEVICE);
1566
1567         /*
1568          * Since the pages are wired down, we must be able to get their
1569          * mappings from the physical map system.
1570          */
1571         for (va = start; va < end; va += PAGE_SIZE) {
1572                 pa = pmap_extract(pmap, va);
1573                 if (pa != 0) {
1574                         pmap_change_wiring(pmap, va, FALSE);
1575                         if (!fictitious)
1576                                 vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
1577                 }
1578         }
1579 }
1580
1581 /*
1582  * Reduce the rate at which memory is allocated to a process based
1583  * on the perceived load on the VM system. As the load increases
1584  * the allocation burst rate goes down and the delay increases.
1585  *
1586  * Rate limiting does not apply when faulting active or inactive
1587  * pages.  When faulting 'cache' pages, rate limiting only applies
1588  * if the system currently has a severe page deficit.
1589  *
1590  * XXX vm_pagesupply should be increased when a page is freed.
1591  *
1592  * We sleep up to 1/10 of a second.
1593  */
1594 static int
1595 vm_fault_ratelimit(struct vmspace *vmspace)
1596 {
1597         if (vm_load_enable == 0)
1598                 return(0);
1599         if (vmspace->vm_pagesupply > 0) {
1600                 --vmspace->vm_pagesupply;
1601                 return(0);
1602         }
1603 #ifdef INVARIANTS
1604         if (vm_load_debug) {
1605                 kprintf("load %-4d give %d pgs, wait %d, pid %-5d (%s)\n",
1606                         vm_load,
1607                         (1000 - vm_load ) / 10, vm_load * hz / 10000,
1608                         curproc->p_pid, curproc->p_comm);
1609         }
1610 #endif
1611         vmspace->vm_pagesupply = (1000 - vm_load) / 10;
1612         return(vm_load * hz / 10000);
1613 }
1614
1615 /*
1616  *      Routine:
1617  *              vm_fault_copy_entry
1618  *      Function:
1619  *              Copy all of the pages from a wired-down map entry to another.
1620  *
1621  *      In/out conditions:
1622  *              The source and destination maps must be locked for write.
1623  *              The source map entry must be wired down (or be a sharing map
1624  *              entry corresponding to a main map entry that is wired down).
1625  */
1626
1627 void
1628 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1629     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
1630 {
1631         vm_object_t dst_object;
1632         vm_object_t src_object;
1633         vm_ooffset_t dst_offset;
1634         vm_ooffset_t src_offset;
1635         vm_prot_t prot;
1636         vm_offset_t vaddr;
1637         vm_page_t dst_m;
1638         vm_page_t src_m;
1639
1640 #ifdef  lint
1641         src_map++;
1642 #endif  /* lint */
1643
1644         src_object = src_entry->object.vm_object;
1645         src_offset = src_entry->offset;
1646
1647         /*
1648          * Create the top-level object for the destination entry. (Doesn't
1649          * actually shadow anything - we copy the pages directly.)
1650          */
1651         vm_map_entry_allocate_object(dst_entry);
1652         dst_object = dst_entry->object.vm_object;
1653
1654         prot = dst_entry->max_protection;
1655
1656         /*
1657          * Loop through all of the pages in the entry's range, copying each
1658          * one from the source object (it should be there) to the destination
1659          * object.
1660          */
1661         for (vaddr = dst_entry->start, dst_offset = 0;
1662             vaddr < dst_entry->end;
1663             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
1664
1665                 /*
1666                  * Allocate a page in the destination object
1667                  */
1668                 do {
1669                         dst_m = vm_page_alloc(dst_object,
1670                                 OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
1671                         if (dst_m == NULL) {
1672                                 vm_wait(0);
1673                         }
1674                 } while (dst_m == NULL);
1675
1676                 /*
1677                  * Find the page in the source object, and copy it in.
1678                  * (Because the source is wired down, the page will be in
1679                  * memory.)
1680                  */
1681                 src_m = vm_page_lookup(src_object,
1682                         OFF_TO_IDX(dst_offset + src_offset));
1683                 if (src_m == NULL)
1684                         panic("vm_fault_copy_wired: page missing");
1685
1686                 vm_page_copy(src_m, dst_m);
1687                 vm_page_event(src_m, VMEVENT_COW);
1688
1689                 /*
1690                  * Enter it in the pmap...
1691                  */
1692
1693                 vm_page_flag_clear(dst_m, PG_ZERO);
1694                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
1695
1696                 /*
1697                  * Mark it no longer busy, and put it on the active list.
1698                  */
1699                 vm_page_activate(dst_m);
1700                 vm_page_wakeup(dst_m);
1701         }
1702 }
1703
1704 #if 0
1705
1706 /*
1707  * This routine checks around the requested page for other pages that
1708  * might be able to be faulted in.  This routine brackets the viable
1709  * pages for the pages to be paged in.
1710  *
1711  * Inputs:
1712  *      m, rbehind, rahead
1713  *
1714  * Outputs:
1715  *  marray (array of vm_page_t), reqpage (index of requested page)
1716  *
1717  * Return value:
1718  *  number of pages in marray
1719  */
1720 static int
1721 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
1722                           vm_page_t *marray, int *reqpage)
1723 {
1724         int i,j;
1725         vm_object_t object;
1726         vm_pindex_t pindex, startpindex, endpindex, tpindex;
1727         vm_page_t rtm;
1728         int cbehind, cahead;
1729
1730         object = m->object;
1731         pindex = m->pindex;
1732
1733         /*
1734          * we don't fault-ahead for device pager
1735          */
1736         if (object->type == OBJT_DEVICE) {
1737                 *reqpage = 0;
1738                 marray[0] = m;
1739                 return 1;
1740         }
1741
1742         /*
1743          * if the requested page is not available, then give up now
1744          */
1745         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
1746                 *reqpage = 0;   /* not used by caller, fix compiler warn */
1747                 return 0;
1748         }
1749
1750         if ((cbehind == 0) && (cahead == 0)) {
1751                 *reqpage = 0;
1752                 marray[0] = m;
1753                 return 1;
1754         }
1755
1756         if (rahead > cahead) {
1757                 rahead = cahead;
1758         }
1759
1760         if (rbehind > cbehind) {
1761                 rbehind = cbehind;
1762         }
1763
1764         /*
1765          * Do not do any readahead if we have insufficient free memory.
1766          *
1767          * XXX code was broken disabled before and has instability
1768          * with this conditonal fixed, so shortcut for now.
1769          */
1770         if (burst_fault == 0 || vm_page_count_severe()) {
1771                 marray[0] = m;
1772                 *reqpage = 0;
1773                 return 1;
1774         }
1775
1776         /*
1777          * scan backward for the read behind pages -- in memory
1778          *
1779          * Assume that if the page is not found an interrupt will not
1780          * create it.  Theoretically interrupts can only remove (busy)
1781          * pages, not create new associations.
1782          */
1783         if (pindex > 0) {
1784                 if (rbehind > pindex) {
1785                         rbehind = pindex;
1786                         startpindex = 0;
1787                 } else {
1788                         startpindex = pindex - rbehind;
1789                 }
1790
1791                 crit_enter();
1792                 for (tpindex = pindex; tpindex > startpindex; --tpindex) {
1793                         if (vm_page_lookup(object, tpindex - 1))
1794                                 break;
1795                 }
1796
1797                 i = 0;
1798                 while (tpindex < pindex) {
1799                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM);
1800                         if (rtm == NULL) {
1801                                 crit_exit();
1802                                 for (j = 0; j < i; j++) {
1803                                         vm_page_free(marray[j]);
1804                                 }
1805                                 marray[0] = m;
1806                                 *reqpage = 0;
1807                                 return 1;
1808                         }
1809                         marray[i] = rtm;
1810                         ++i;
1811                         ++tpindex;
1812                 }
1813                 crit_exit();
1814         } else {
1815                 i = 0;
1816         }
1817
1818         /*
1819          * Assign requested page
1820          */
1821         marray[i] = m;
1822         *reqpage = i;
1823         ++i;
1824
1825         /*
1826          * Scan forwards for read-ahead pages
1827          */
1828         tpindex = pindex + 1;
1829         endpindex = tpindex + rahead;
1830         if (endpindex > object->size)
1831                 endpindex = object->size;
1832
1833         crit_enter();
1834         while (tpindex < endpindex) {
1835                 if (vm_page_lookup(object, tpindex))
1836                         break;
1837                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM);
1838                 if (rtm == NULL)
1839                         break;
1840                 marray[i] = rtm;
1841                 ++i;
1842                 ++tpindex;
1843         }
1844         crit_exit();
1845
1846         return (i);
1847 }
1848
1849 #endif
1850
1851 /*
1852  * vm_prefault() provides a quick way of clustering pagefaults into a
1853  * processes address space.  It is a "cousin" of pmap_object_init_pt,
1854  * except it runs at page fault time instead of mmap time.
1855  *
1856  * This code used to be per-platform pmap_prefault().  It is now
1857  * machine-independent and enhanced to also pre-fault zero-fill pages
1858  * (see vm.fast_fault) as well as make them writable, which greatly
1859  * reduces the number of page faults programs incur.
1860  *
1861  * Application performance when pre-faulting zero-fill pages is heavily
1862  * dependent on the application.  Very tiny applications like /bin/echo
1863  * lose a little performance while applications of any appreciable size
1864  * gain performance.  Prefaulting multiple pages also reduces SMP
1865  * congestion and can improve SMP performance significantly.
1866  *
1867  * NOTE!  prot may allow writing but this only applies to the top level
1868  *        object.  If we wind up mapping a page extracted from a backing
1869  *        object we have to make sure it is read-only.
1870  *
1871  * NOTE!  The caller has already handled any COW operations on the
1872  *        vm_map_entry via the normal fault code.  Do NOT call this
1873  *        shortcut unless the normal fault code has run on this entry.
1874  */
1875 #define PFBAK 4
1876 #define PFFOR 4
1877 #define PAGEORDER_SIZE (PFBAK+PFFOR)
1878
1879 static int vm_prefault_pageorder[] = {
1880         -PAGE_SIZE, PAGE_SIZE,
1881         -2 * PAGE_SIZE, 2 * PAGE_SIZE,
1882         -3 * PAGE_SIZE, 3 * PAGE_SIZE,
1883         -4 * PAGE_SIZE, 4 * PAGE_SIZE
1884 };
1885
1886 static void
1887 vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot)
1888 {
1889         struct lwp *lp;
1890         vm_page_t m;
1891         vm_offset_t starta;
1892         vm_offset_t addr;
1893         vm_pindex_t index;
1894         vm_pindex_t pindex;
1895         vm_object_t object;
1896         int pprot;
1897         int i;
1898
1899         /*
1900          * We do not currently prefault mappings that use virtual page
1901          * tables.  We do not prefault foreign pmaps.
1902          */
1903         if (entry->maptype == VM_MAPTYPE_VPAGETABLE)
1904                 return;
1905         lp = curthread->td_lwp;
1906         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
1907                 return;
1908
1909         object = entry->object.vm_object;
1910
1911         starta = addra - PFBAK * PAGE_SIZE;
1912         if (starta < entry->start)
1913                 starta = entry->start;
1914         else if (starta > addra)
1915                 starta = 0;
1916
1917         /*
1918          * critical section protection is required to maintain the
1919          * page/object association, interrupts can free pages and remove
1920          * them from their objects.
1921          */
1922         crit_enter();
1923         for (i = 0; i < PAGEORDER_SIZE; i++) {
1924                 vm_object_t lobject;
1925                 int allocated = 0;
1926
1927                 addr = addra + vm_prefault_pageorder[i];
1928                 if (addr > addra + (PFFOR * PAGE_SIZE))
1929                         addr = 0;
1930
1931                 if (addr < starta || addr >= entry->end)
1932                         continue;
1933
1934                 if (pmap_prefault_ok(pmap, addr) == 0)
1935                         continue;
1936
1937                 /*
1938                  * Follow the VM object chain to obtain the page to be mapped
1939                  * into the pmap.
1940                  *
1941                  * If we reach the terminal object without finding a page
1942                  * and we determine it would be advantageous, then allocate
1943                  * a zero-fill page for the base object.  The base object
1944                  * is guaranteed to be OBJT_DEFAULT for this case.
1945                  *
1946                  * In order to not have to check the pager via *haspage*()
1947                  * we stop if any non-default object is encountered.  e.g.
1948                  * a vnode or swap object would stop the loop.
1949                  */
1950                 index = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
1951                 lobject = object;
1952                 pindex = index;
1953                 pprot = prot;
1954
1955                 while ((m = vm_page_lookup(lobject, pindex)) == NULL) {
1956                         if (lobject->type != OBJT_DEFAULT)
1957                                 break;
1958                         if (lobject->backing_object == NULL) {
1959                                 if (vm_fast_fault == 0)
1960                                         break;
1961                                 if (vm_prefault_pageorder[i] < 0 ||
1962                                     (prot & VM_PROT_WRITE) == 0 ||
1963                                     vm_page_count_min(0)) {
1964                                         break;
1965                                 }
1966                                 /* note: allocate from base object */
1967                                 m = vm_page_alloc(object, index,
1968                                               VM_ALLOC_NORMAL | VM_ALLOC_ZERO);
1969
1970                                 if ((m->flags & PG_ZERO) == 0) {
1971                                         vm_page_zero_fill(m);
1972                                 } else {
1973                                         vm_page_flag_clear(m, PG_ZERO);
1974                                         mycpu->gd_cnt.v_ozfod++;
1975                                 }
1976                                 mycpu->gd_cnt.v_zfod++;
1977                                 m->valid = VM_PAGE_BITS_ALL;
1978                                 allocated = 1;
1979                                 pprot = prot;
1980                                 /* lobject = object .. not needed */
1981                                 break;
1982                         }
1983                         if (lobject->backing_object_offset & PAGE_MASK)
1984                                 break;
1985                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
1986                         lobject = lobject->backing_object;
1987                         pprot &= ~VM_PROT_WRITE;
1988                 }
1989                 /*
1990                  * NOTE: lobject now invalid (if we did a zero-fill we didn't
1991                  *       bother assigning lobject = object).
1992                  *
1993                  * Give-up if the page is not available.
1994                  */
1995                 if (m == NULL)
1996                         break;
1997
1998                 /*
1999                  * Do not conditionalize on PG_RAM.  If pages are present in
2000                  * the VM system we assume optimal caching.  If caching is
2001                  * not optimal the I/O gravy train will be restarted when we
2002                  * hit an unavailable page.  We do not want to try to restart
2003                  * the gravy train now because we really don't know how much
2004                  * of the object has been cached.  The cost for restarting
2005                  * the gravy train should be low (since accesses will likely
2006                  * be I/O bound anyway).
2007                  *
2008                  * The object must be marked dirty if we are mapping a
2009                  * writable page.
2010                  */
2011                 if (pprot & VM_PROT_WRITE)
2012                         vm_object_set_writeable_dirty(m->object);
2013
2014                 /*
2015                  * Enter the page into the pmap if appropriate.  If we had
2016                  * allocated the page we have to place it on a queue.  If not
2017                  * we just have to make sure it isn't on the cache queue
2018                  * (pages on the cache queue are not allowed to be mapped).
2019                  */
2020                 if (allocated) {
2021                         pmap_enter(pmap, addr, m, pprot, 0);
2022                         vm_page_deactivate(m);
2023                         vm_page_wakeup(m);
2024                 } else if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2025                     (m->busy == 0) &&
2026                     (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
2027
2028                         if ((m->queue - m->pc) == PQ_CACHE) {
2029                                 vm_page_deactivate(m);
2030                         }
2031                         vm_page_busy(m);
2032                         pmap_enter(pmap, addr, m, pprot, 0);
2033                         vm_page_wakeup(m);
2034                 }
2035         }
2036         crit_exit();
2037 }