sys/vm/vm_fault.c

   1 /*
   2  * Copyright (c) 2003-2014 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * ---
  35  *
  36  * Copyright (c) 1991, 1993
  37  *      The Regents of the University of California.  All rights reserved.
  38  * Copyright (c) 1994 John S. Dyson
  39  * All rights reserved.
  40  * Copyright (c) 1994 David Greenman
  41  * All rights reserved.
  42  *
  43  *
  44  * This code is derived from software contributed to Berkeley by
  45  * The Mach Operating System project at Carnegie-Mellon University.
  46  *
  47  * Redistribution and use in source and binary forms, with or without
  48  * modification, are permitted provided that the following conditions
  49  * are met:
  50  * 1. Redistributions of source code must retain the above copyright
  51  *    notice, this list of conditions and the following disclaimer.
  52  * 2. Redistributions in binary form must reproduce the above copyright
  53  *    notice, this list of conditions and the following disclaimer in the
  54  *    documentation and/or other materials provided with the distribution.
  55  * 3. Neither the name of the University nor the names of its contributors
  56  *    may be used to endorse or promote products derived from this software
  57  *    without specific prior written permission.
  58  *
  59  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  60  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  61  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  62  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  63  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  64  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  65  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  66  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  67  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  68  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  69  * SUCH DAMAGE.
  70  *
  71  * ---
  72  *
  73  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
  74  * All rights reserved.
  75  *
  76  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
  77  *
  78  * Permission to use, copy, modify and distribute this software and
  79  * its documentation is hereby granted, provided that both the copyright
  80  * notice and this permission notice appear in all copies of the
  81  * software, derivative works or modified versions, and any portions
  82  * thereof, and that both notices appear in supporting documentation.
  83  *
  84  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  85  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  86  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  87  *
  88  * Carnegie Mellon requests users of this software to return to
  89  *
  90  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  91  *  School of Computer Science
  92  *  Carnegie Mellon University
  93  *  Pittsburgh PA 15213-3890
  94  *
  95  * any improvements or extensions that they make and grant Carnegie the
  96  * rights to redistribute these changes.
  97  */
  98
  99 /*
 100  *      Page fault handling module.
 101  */
 102
 103 #include <sys/param.h>
 104 #include <sys/systm.h>
 105 #include <sys/kernel.h>
 106 #include <sys/proc.h>
 107 #include <sys/vnode.h>
 108 #include <sys/resourcevar.h>
 109 #include <sys/vmmeter.h>
 110 #include <sys/vkernel.h>
 111 #include <sys/lock.h>
 112 #include <sys/sysctl.h>
 113
 114 #include <cpu/lwbuf.h>
 115
 116 #include <vm/vm.h>
 117 #include <vm/vm_param.h>
 118 #include <vm/pmap.h>
 119 #include <vm/vm_map.h>
 120 #include <vm/vm_object.h>
 121 #include <vm/vm_page.h>
 122 #include <vm/vm_pageout.h>
 123 #include <vm/vm_kern.h>
 124 #include <vm/vm_pager.h>
 125 #include <vm/vnode_pager.h>
 126 #include <vm/vm_extern.h>
 127
 128 #include <vm/vm_page2.h>
 129
 130 struct faultstate {
 131         vm_page_t m;
 132         vm_object_t object;
 133         vm_pindex_t pindex;
 134         vm_prot_t prot;
 135         vm_page_t first_m;
 136         vm_object_t first_object;
 137         vm_prot_t first_prot;
 138         vm_map_t map;
 139         vm_map_entry_t entry;
 140         int lookup_still_valid;
 141         int hardfault;
 142         int fault_flags;
 143         int map_generation;
 144         int shared;
 145         int msoftonly;
 146         int first_shared;
 147         int wflags;
 148         struct vnode *vp;
 149 };
 150
 151 __read_mostly static int debug_fault = 0;
 152 SYSCTL_INT(_vm, OID_AUTO, debug_fault, CTLFLAG_RW, &debug_fault, 0, "");
 153 __read_mostly static int debug_cluster = 0;
 154 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
 155 __read_mostly static int virtual_copy_enable = 1;
 156 SYSCTL_INT(_vm, OID_AUTO, virtual_copy_enable, CTLFLAG_RW,
 157                 &virtual_copy_enable, 0, "");
 158 __read_mostly int vm_shared_fault = 1;
 159 TUNABLE_INT("vm.shared_fault", &vm_shared_fault);
 160 SYSCTL_INT(_vm, OID_AUTO, shared_fault, CTLFLAG_RW,
 161                 &vm_shared_fault, 0, "Allow shared token on vm_object");
 162 __read_mostly static int vm_fault_quick_enable = 0;
 163 TUNABLE_INT("vm.fault_quick", &vm_fault_quick_enable);
 164 SYSCTL_INT(_vm, OID_AUTO, fault_quick, CTLFLAG_RW,
 165                 &vm_fault_quick_enable, 0, "Allow fast vm_fault shortcut");
 166 #ifdef VM_FAULT_QUICK_DEBUG
 167 static long vm_fault_quick_success_count = 0;
 168 SYSCTL_LONG(_vm, OID_AUTO, fault_quick_success_count, CTLFLAG_RW,
 169                 &vm_fault_quick_success_count, 0, "");
 170 static long vm_fault_quick_failure_count1 = 0;
 171 SYSCTL_LONG(_vm, OID_AUTO, fault_quick_failure_count1, CTLFLAG_RW,
 172                 &vm_fault_quick_failure_count1, 0, "");
 173 static long vm_fault_quick_failure_count2 = 0;
 174 SYSCTL_LONG(_vm, OID_AUTO, fault_quick_failure_count2, CTLFLAG_RW,
 175                 &vm_fault_quick_failure_count2, 0, "");
 176 static long vm_fault_quick_failure_count3 = 0;
 177 SYSCTL_LONG(_vm, OID_AUTO, fault_quick_failure_count3, CTLFLAG_RW,
 178                 &vm_fault_quick_failure_count3, 0, "");
 179 static long vm_fault_quick_failure_count4 = 0;
 180 SYSCTL_LONG(_vm, OID_AUTO, fault_quick_failure_count4, CTLFLAG_RW,
 181                 &vm_fault_quick_failure_count4, 0, "");
 182 #endif
 183
 184 static int vm_fault_quick(struct faultstate *fs, vm_pindex_t first_pindex,
 185                         vm_prot_t fault_type);
 186 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t, int);
 187 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *,
 188                         vpte_t, int, int);
 189 #if 0
 190 static int vm_fault_additional_pages (vm_page_t, int, int, vm_page_t *, int *);
 191 #endif
 192 static void vm_set_nosync(vm_page_t m, vm_map_entry_t entry);
 193 static void vm_prefault(pmap_t pmap, vm_offset_t addra,
 194                         vm_map_entry_t entry, int prot, int fault_flags);
 195 static void vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
 196                         vm_map_entry_t entry, int prot, int fault_flags);
 197
 198 static __inline void
 199 release_page(struct faultstate *fs)
 200 {
 201         vm_page_deactivate(fs->m);
 202         vm_page_wakeup(fs->m);
 203         fs->m = NULL;
 204 }
 205
 206 /*
 207  * NOTE: Once unlocked any cached fs->entry becomes invalid, any reuse
 208  *       requires relocking and then checking the timestamp.
 209  *
 210  * NOTE: vm_map_lock_read() does not bump fs->map->timestamp so we do
 211  *       not have to update fs->map_generation here.
 212  *
 213  * NOTE: This function can fail due to a deadlock against the caller's
 214  *       holding of a vm_page BUSY.
 215  */
 216 static __inline int
 217 relock_map(struct faultstate *fs)
 218 {
 219         int error;
 220
 221         if (fs->lookup_still_valid == FALSE && fs->map) {
 222                 error = vm_map_lock_read_to(fs->map);
 223                 if (error == 0)
 224                         fs->lookup_still_valid = TRUE;
 225         } else {
 226                 error = 0;
 227         }
 228         return error;
 229 }
 230
 231 static __inline void
 232 unlock_map(struct faultstate *fs)
 233 {
 234         if (fs->lookup_still_valid && fs->map) {
 235                 vm_map_lookup_done(fs->map, fs->entry, 0);
 236                 fs->lookup_still_valid = FALSE;
 237         }
 238 }
 239
 240 /*
 241  * Clean up after a successful call to vm_fault_object() so another call
 242  * to vm_fault_object() can be made.
 243  */
 244 static void
 245 _cleanup_successful_fault(struct faultstate *fs, int relock)
 246 {
 247         /*
 248          * We allocated a junk page for a COW operation that did
 249          * not occur, the page must be freed.
 250          */
 251         if (fs->object != fs->first_object) {
 252                 KKASSERT(fs->first_shared == 0);
 253                 vm_page_free(fs->first_m);
 254                 vm_object_pip_wakeup(fs->object);
 255                 fs->first_m = NULL;
 256         }
 257
 258         /*
 259          * Reset fs->object.
 260          */
 261         fs->object = fs->first_object;
 262         if (relock && fs->lookup_still_valid == FALSE) {
 263                 if (fs->map)
 264                         vm_map_lock_read(fs->map);
 265                 fs->lookup_still_valid = TRUE;
 266         }
 267 }
 268
 269 static void
 270 _unlock_things(struct faultstate *fs, int dealloc)
 271 {
 272         _cleanup_successful_fault(fs, 0);
 273         if (dealloc) {
 274                 /*vm_object_deallocate(fs->first_object);*/
 275                 /*fs->first_object = NULL; drop used later on */
 276         }
 277         unlock_map(fs);
 278         if (fs->vp != NULL) {
 279                 vput(fs->vp);
 280                 fs->vp = NULL;
 281         }
 282 }
 283
 284 #define unlock_things(fs) _unlock_things(fs, 0)
 285 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
 286 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
 287
 288 /*
 289  * Virtual copy tests.   Used by the fault code to determine if a
 290  * page can be moved from an orphan vm_object into its shadow
 291  * instead of copying its contents.
 292  */
 293 static __inline int
 294 virtual_copy_test(struct faultstate *fs)
 295 {
 296         /*
 297          * Must be holding exclusive locks
 298          */
 299         if (fs->first_shared || fs->shared || virtual_copy_enable == 0)
 300                 return 0;
 301
 302         /*
 303          * Map, if present, has not changed
 304          */
 305         if (fs->map && fs->map_generation != fs->map->timestamp)
 306                 return 0;
 307
 308         /*
 309          * No refs, except us
 310          */
 311         if (fs->object->ref_count != 1)
 312                 return 0;
 313
 314         /*
 315          * No one else can look this object up
 316          */
 317         if (fs->object->handle != NULL)
 318                 return 0;
 319
 320         /*
 321          * No other ways to look the object up
 322          */
 323         if (fs->object->type != OBJT_DEFAULT &&
 324             fs->object->type != OBJT_SWAP)
 325                 return 0;
 326
 327         /*
 328          * We don't chase down the shadow chain
 329          */
 330         if (fs->object != fs->first_object->backing_object)
 331                 return 0;
 332
 333         return 1;
 334 }
 335
 336 static __inline int
 337 virtual_copy_ok(struct faultstate *fs)
 338 {
 339         if (virtual_copy_test(fs)) {
 340                 /*
 341                  * Grab the lock and re-test changeable items.
 342                  */
 343                 if (fs->lookup_still_valid == FALSE && fs->map) {
 344                         if (lockmgr(&fs->map->lock, LK_EXCLUSIVE|LK_NOWAIT))
 345                                 return 0;
 346                         fs->lookup_still_valid = TRUE;
 347                         if (virtual_copy_test(fs)) {
 348                                 fs->map_generation = ++fs->map->timestamp;
 349                                 return 1;
 350                         }
 351                         fs->lookup_still_valid = FALSE;
 352                         lockmgr(&fs->map->lock, LK_RELEASE);
 353                 }
 354         }
 355         return 0;
 356 }
 357
 358 /*
 359  * TRYPAGER
 360  *
 361  * Determine if the pager for the current object *might* contain the page.
 362  *
 363  * We only need to try the pager if this is not a default object (default
 364  * objects are zero-fill and have no real pager), and if we are not taking
 365  * a wiring fault or if the FS entry is wired.
 366  */
 367 #define TRYPAGER(fs)    \
 368                 (fs->object->type != OBJT_DEFAULT &&                    \
 369                 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) ||       \
 370                  (fs->wflags & FW_WIRED)))
 371
 372 /*
 373  * vm_fault:
 374  *
 375  * Handle a page fault occuring at the given address, requiring the given
 376  * permissions, in the map specified.  If successful, the page is inserted
 377  * into the associated physical map.
 378  *
 379  * NOTE: The given address should be truncated to the proper page address.
 380  *
 381  * KERN_SUCCESS is returned if the page fault is handled; otherwise,
 382  * a standard error specifying why the fault is fatal is returned.
 383  *
 384  * The map in question must be referenced, and remains so.
 385  * The caller may hold no locks.
 386  * No other requirements.
 387  */
 388 int
 389 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
 390 {
 391         int result;
 392         vm_pindex_t first_pindex;
 393         struct faultstate fs;
 394         struct lwp *lp;
 395         struct proc *p;
 396         thread_t td;
 397         struct vm_map_ilock ilock;
 398         int didilock;
 399         int didhold;
 400         int growstack;
 401         int retry = 0;
 402         int inherit_prot;
 403
 404         inherit_prot = fault_type & VM_PROT_NOSYNC;
 405         fs.hardfault = 0;
 406         fs.fault_flags = fault_flags;
 407         fs.vp = NULL;
 408         fs.shared = vm_shared_fault;
 409         fs.first_shared = vm_shared_fault;
 410         growstack = 1;
 411
 412         /*
 413          * vm_map interactions
 414          */
 415         td = curthread;
 416         if ((lp = td->td_lwp) != NULL)
 417                 lp->lwp_flags |= LWP_PAGING;
 418
 419 RetryFault:
 420         /*
 421          * vm_fault_quick() can shortcut us.
 422          */
 423         fs.msoftonly = 0;
 424         didhold = 0;
 425
 426         /*
 427          * Find the vm_map_entry representing the backing store and resolve
 428          * the top level object and page index.  This may have the side
 429          * effect of executing a copy-on-write on the map entry,
 430          * creating a shadow object, or splitting an anonymous entry for
 431          * performance, but will not COW any actual VM pages.
 432          *
 433          * On success fs.map is left read-locked and various other fields
 434          * are initialized but not otherwise referenced or locked.
 435          *
 436          * NOTE!  vm_map_lookup will try to upgrade the fault_type to
 437          *        VM_FAULT_WRITE if the map entry is a virtual page table
 438          *        and also writable, so we can set the 'A'accessed bit in
 439          *        the virtual page table entry.
 440          */
 441         fs.map = map;
 442         result = vm_map_lookup(&fs.map, vaddr, fault_type,
 443                                &fs.entry, &fs.first_object,
 444                                &first_pindex, &fs.first_prot, &fs.wflags);
 445
 446         /*
 447          * If the lookup failed or the map protections are incompatible,
 448          * the fault generally fails.
 449          *
 450          * The failure could be due to TDF_NOFAULT if vm_map_lookup()
 451          * tried to do a COW fault.
 452          *
 453          * If the caller is trying to do a user wiring we have more work
 454          * to do.
 455          */
 456         if (result != KERN_SUCCESS) {
 457                 if (result == KERN_FAILURE_NOFAULT) {
 458                         result = KERN_FAILURE;
 459                         goto done;
 460                 }
 461                 if (result != KERN_PROTECTION_FAILURE ||
 462                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
 463                 {
 464                         if (result == KERN_INVALID_ADDRESS && growstack &&
 465                             map != &kernel_map && curproc != NULL) {
 466                                 result = vm_map_growstack(map, vaddr);
 467                                 if (result == KERN_SUCCESS) {
 468                                         growstack = 0;
 469                                         ++retry;
 470                                         goto RetryFault;
 471                                 }
 472                                 result = KERN_FAILURE;
 473                         }
 474                         goto done;
 475                 }
 476
 477                 /*
 478                  * If we are user-wiring a r/w segment, and it is COW, then
 479                  * we need to do the COW operation.  Note that we don't
 480                  * currently COW RO sections now, because it is NOT desirable
 481                  * to COW .text.  We simply keep .text from ever being COW'ed
 482                  * and take the heat that one cannot debug wired .text sections.
 483                  *
 484                  * XXX Try to allow the above by specifying OVERRIDE_WRITE.
 485                  */
 486                 result = vm_map_lookup(&fs.map, vaddr,
 487                                        VM_PROT_READ|VM_PROT_WRITE|
 488                                         VM_PROT_OVERRIDE_WRITE,
 489                                        &fs.entry, &fs.first_object,
 490                                        &first_pindex, &fs.first_prot,
 491                                        &fs.wflags);
 492                 if (result != KERN_SUCCESS) {
 493                         /* could also be KERN_FAILURE_NOFAULT */
 494                         result = KERN_FAILURE;
 495                         goto done;
 496                 }
 497
 498                 /*
 499                  * If we don't COW now, on a user wire, the user will never
 500                  * be able to write to the mapping.  If we don't make this
 501                  * restriction, the bookkeeping would be nearly impossible.
 502                  *
 503                  * XXX We have a shared lock, this will have a MP race but
 504                  * I don't see how it can hurt anything.
 505                  */
 506                 if ((fs.entry->protection & VM_PROT_WRITE) == 0) {
 507                         atomic_clear_char(&fs.entry->max_protection,
 508                                           VM_PROT_WRITE);
 509                 }
 510         }
 511
 512         /*
 513          * fs.map is read-locked
 514          *
 515          * Misc checks.  Save the map generation number to detect races.
 516          */
 517         fs.map_generation = fs.map->timestamp;
 518         fs.lookup_still_valid = TRUE;
 519         fs.first_m = NULL;
 520         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
 521         fs.prot = fs.first_prot;        /* default (used by uksmap) */
 522
 523         if (fs.entry->eflags & (MAP_ENTRY_NOFAULT | MAP_ENTRY_KSTACK)) {
 524                 if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
 525                         panic("vm_fault: fault on nofault entry, addr: %p",
 526                               (void *)vaddr);
 527                 }
 528                 if ((fs.entry->eflags & MAP_ENTRY_KSTACK) &&
 529                     vaddr >= fs.entry->start &&
 530                     vaddr < fs.entry->start + PAGE_SIZE) {
 531                         panic("vm_fault: fault on stack guard, addr: %p",
 532                               (void *)vaddr);
 533                 }
 534         }
 535
 536         /*
 537          * A user-kernel shared map has no VM object and bypasses
 538          * everything.  We execute the uksmap function with a temporary
 539          * fictitious vm_page.  The address is directly mapped with no
 540          * management.
 541          */
 542         if (fs.entry->maptype == VM_MAPTYPE_UKSMAP) {
 543                 struct vm_page fakem;
 544
 545                 bzero(&fakem, sizeof(fakem));
 546                 fakem.pindex = first_pindex;
 547                 fakem.flags = PG_FICTITIOUS | PG_UNMANAGED;
 548                 fakem.busy_count = PBUSY_LOCKED;
 549                 fakem.valid = VM_PAGE_BITS_ALL;
 550                 fakem.pat_mode = VM_MEMATTR_DEFAULT;
 551                 if (fs.entry->object.uksmap(fs.entry->aux.dev, &fakem)) {
 552                         result = KERN_FAILURE;
 553                         unlock_things(&fs);
 554                         goto done2;
 555                 }
 556                 pmap_enter(fs.map->pmap, vaddr, &fakem, fs.prot | inherit_prot,
 557                            (fs.wflags & FW_WIRED), fs.entry);
 558                 goto done_success;
 559         }
 560
 561         /*
 562          * A system map entry may return a NULL object.  No object means
 563          * no pager means an unrecoverable kernel fault.
 564          */
 565         if (fs.first_object == NULL) {
 566                 panic("vm_fault: unrecoverable fault at %p in entry %p",
 567                         (void *)vaddr, fs.entry);
 568         }
 569
 570         /*
 571          * Fail here if not a trivial anonymous page fault and TDF_NOFAULT
 572          * is set.
 573          *
 574          * Unfortunately a deadlock can occur if we are forced to page-in
 575          * from swap, but diving all the way into the vm_pager_get_page()
 576          * function to find out is too much.  Just check the object type.
 577          *
 578          * The deadlock is a CAM deadlock on a busy VM page when trying
 579          * to finish an I/O if another process gets stuck in
 580          * vop_helper_read_shortcut() due to a swap fault.
 581          */
 582         if ((td->td_flags & TDF_NOFAULT) &&
 583             (retry ||
 584              fs.first_object->type == OBJT_VNODE ||
 585              fs.first_object->type == OBJT_SWAP ||
 586              fs.first_object->backing_object)) {
 587                 result = KERN_FAILURE;
 588                 unlock_things(&fs);
 589                 goto done2;
 590         }
 591
 592         /*
 593          * If the entry is wired we cannot change the page protection.
 594          */
 595         if (fs.wflags & FW_WIRED)
 596                 fault_type = fs.first_prot;
 597
 598         /*
 599          * We generally want to avoid unnecessary exclusive modes on backing
 600          * and terminal objects because this can seriously interfere with
 601          * heavily fork()'d processes (particularly /bin/sh scripts).
 602          *
 603          * However, we also want to avoid unnecessary retries due to needed
 604          * shared->exclusive promotion for common faults.  Exclusive mode is
 605          * always needed if any page insertion, rename, or free occurs in an
 606          * object (and also indirectly if any I/O is done).
 607          *
 608          * The main issue here is going to be fs.first_shared.  If the
 609          * first_object has a backing object which isn't shadowed and the
 610          * process is single-threaded we might as well use an exclusive
 611          * lock/chain right off the bat.
 612          */
 613 #if 0
 614         /* WORK IN PROGRESS, CODE REMOVED */
 615         if (fs.first_shared && fs.first_object->backing_object &&
 616             LIST_EMPTY(&fs.first_object->shadow_head) &&
 617             td->td_proc && td->td_proc->p_nthreads == 1) {
 618                 fs.first_shared = 0;
 619         }
 620 #endif
 621
 622         /*
 623          * VM_FAULT_UNSWAP - swap_pager_unswapped() needs an exclusive object
 624          * VM_FAULT_DIRTY  - may require swap_pager_unswapped() later, but
 625          *                   we can try shared first.
 626          */
 627         if (fault_flags & VM_FAULT_UNSWAP) {
 628                 fs.first_shared = 0;
 629         }
 630
 631         /*
 632          * Try to shortcut the entire mess and run the fault lockless.
 633          */
 634         if (vm_fault_quick_enable &&
 635             vm_fault_quick(&fs, first_pindex, fault_type) == KERN_SUCCESS) {
 636                 didilock = 0;
 637                 fault_flags &= ~VM_FAULT_BURST;
 638                 goto success;
 639         }
 640
 641         /*
 642          * Obtain a top-level object lock, shared or exclusive depending
 643          * on fs.first_shared.  If a shared lock winds up being insufficient
 644          * we will retry with an exclusive lock.
 645          *
 646          * The vnode pager lock is always shared.
 647          */
 648         if (fs.first_shared)
 649                 vm_object_hold_shared(fs.first_object);
 650         else
 651                 vm_object_hold(fs.first_object);
 652         if (fs.vp == NULL)
 653                 fs.vp = vnode_pager_lock(fs.first_object);
 654         didhold = 1;
 655
 656         /*
 657          * The page we want is at (first_object, first_pindex), but if the
 658          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
 659          * page table to figure out the actual pindex.
 660          *
 661          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
 662          * ONLY
 663          */
 664         didilock = 0;
 665         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 666                 vm_map_interlock(fs.map, &ilock, vaddr, vaddr + PAGE_SIZE);
 667                 didilock = 1;
 668                 result = vm_fault_vpagetable(&fs, &first_pindex,
 669                                              fs.entry->aux.master_pde,
 670                                              fault_type, 1);
 671                 if (result == KERN_TRY_AGAIN) {
 672                         vm_map_deinterlock(fs.map, &ilock);
 673                         vm_object_drop(fs.first_object);
 674                         ++retry;
 675                         goto RetryFault;
 676                 }
 677                 if (result != KERN_SUCCESS) {
 678                         vm_map_deinterlock(fs.map, &ilock);
 679                         goto done;
 680                 }
 681         }
 682
 683         /*
 684          * Now we have the actual (object, pindex), fault in the page.  If
 685          * vm_fault_object() fails it will unlock and deallocate the FS
 686          * data.   If it succeeds everything remains locked and fs->object
 687          * will have an additional PIP count if it is not equal to
 688          * fs->first_object
 689          *
 690          * vm_fault_object will set fs->prot for the pmap operation.  It is
 691          * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
 692          * page can be safely written.  However, it will force a read-only
 693          * mapping for a read fault if the memory is managed by a virtual
 694          * page table.
 695          *
 696          * If the fault code uses the shared object lock shortcut
 697          * we must not try to burst (we can't allocate VM pages).
 698          */
 699         result = vm_fault_object(&fs, first_pindex, fault_type, 1);
 700
 701         if (debug_fault > 0) {
 702                 --debug_fault;
 703                 kprintf("VM_FAULT result %d addr=%jx type=%02x flags=%02x "
 704                         "fs.m=%p fs.prot=%02x fs.wflags=%02x fs.entry=%p\n",
 705                         result, (intmax_t)vaddr, fault_type, fault_flags,
 706                         fs.m, fs.prot, fs.wflags, fs.entry);
 707         }
 708
 709         if (result == KERN_TRY_AGAIN) {
 710                 if (didilock)
 711                         vm_map_deinterlock(fs.map, &ilock);
 712                 vm_object_drop(fs.first_object);
 713                 ++retry;
 714                 goto RetryFault;
 715         }
 716         if (result != KERN_SUCCESS) {
 717                 if (didilock)
 718                         vm_map_deinterlock(fs.map, &ilock);
 719                 goto done;
 720         }
 721
 722 success:
 723         /*
 724          * On success vm_fault_object() does not unlock or deallocate, and fs.m
 725          * will contain a busied page.
 726          *
 727          * Enter the page into the pmap and do pmap-related adjustments.
 728          *
 729          * WARNING! Soft-busied fs.m's can only be manipulated in limited
 730          *          ways.
 731          */
 732         KKASSERT(fs.lookup_still_valid == TRUE);
 733         vm_page_flag_set(fs.m, PG_REFERENCED);
 734         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot | inherit_prot,
 735                    fs.wflags & FW_WIRED, fs.entry);
 736
 737         if (didilock)
 738                 vm_map_deinterlock(fs.map, &ilock);
 739
 740         /*
 741          * If the page is not wired down, then put it where the pageout daemon
 742          * can find it.
 743          *
 744          * NOTE: We cannot safely wire, unwire, or adjust queues for a
 745          *       soft-busied page.
 746          */
 747         if (fs.msoftonly) {
 748                 KKASSERT(fs.m->busy_count & PBUSY_MASK);
 749                 KKASSERT((fs.fault_flags & VM_FAULT_WIRE_MASK) == 0);
 750                 vm_page_sbusy_drop(fs.m);
 751         } else {
 752                 if (fs.fault_flags & VM_FAULT_WIRE_MASK) {
 753                         if (fs.wflags & FW_WIRED)
 754                                 vm_page_wire(fs.m);
 755                         else
 756                                 vm_page_unwire(fs.m, 1);
 757                 } else {
 758                         vm_page_activate(fs.m);
 759                 }
 760                 KKASSERT(fs.m->busy_count & PBUSY_LOCKED);
 761                 vm_page_wakeup(fs.m);
 762         }
 763
 764         /*
 765          * Burst in a few more pages if possible.  The fs.map should still
 766          * be locked.  To avoid interlocking against a vnode->getblk
 767          * operation we had to be sure to unbusy our primary vm_page above
 768          * first.
 769          *
 770          * A normal burst can continue down backing store, only execute
 771          * if we are holding an exclusive lock, otherwise the exclusive
 772          * locks the burst code gets might cause excessive SMP collisions.
 773          *
 774          * A quick burst can be utilized when there is no backing object
 775          * (i.e. a shared file mmap).
 776          */
 777         if ((fault_flags & VM_FAULT_BURST) &&
 778             (fs.fault_flags & VM_FAULT_WIRE_MASK) == 0 &&
 779             (fs.wflags & FW_WIRED) == 0) {
 780                 if (fs.first_shared == 0 && fs.shared == 0) {
 781                         vm_prefault(fs.map->pmap, vaddr,
 782                                     fs.entry, fs.prot, fault_flags);
 783                 } else {
 784                         vm_prefault_quick(fs.map->pmap, vaddr,
 785                                           fs.entry, fs.prot, fault_flags);
 786                 }
 787         }
 788
 789 done_success:
 790         mycpu->gd_cnt.v_vm_faults++;
 791         if (td->td_lwp)
 792                 ++td->td_lwp->lwp_ru.ru_minflt;
 793
 794         /*
 795          * Unlock everything, and return
 796          */
 797         unlock_things(&fs);
 798
 799         if (td->td_lwp) {
 800                 if (fs.hardfault) {
 801                         td->td_lwp->lwp_ru.ru_majflt++;
 802                 } else {
 803                         td->td_lwp->lwp_ru.ru_minflt++;
 804                 }
 805         }
 806
 807         /*vm_object_deallocate(fs.first_object);*/
 808         /*fs.m = NULL; */
 809         /*fs.first_object = NULL; must still drop later */
 810
 811         result = KERN_SUCCESS;
 812 done:
 813         if (fs.first_object && didhold)
 814                 vm_object_drop(fs.first_object);
 815 done2:
 816         if (lp)
 817                 lp->lwp_flags &= ~LWP_PAGING;
 818
 819 #if !defined(NO_SWAPPING)
 820         /*
 821          * Check the process RSS limit and force deactivation and
 822          * (asynchronous) paging if necessary.  This is a complex operation,
 823          * only do it for direct user-mode faults, for now.
 824          *
 825          * To reduce overhead implement approximately a ~16MB hysteresis.
 826          */
 827         p = td->td_proc;
 828         if ((fault_flags & VM_FAULT_USERMODE) && lp &&
 829             p->p_limit && map->pmap && vm_pageout_memuse_mode >= 1 &&
 830             map != &kernel_map) {
 831                 vm_pindex_t limit;
 832                 vm_pindex_t size;
 833
 834                 limit = OFF_TO_IDX(qmin(p->p_rlimit[RLIMIT_RSS].rlim_cur,
 835                                         p->p_rlimit[RLIMIT_RSS].rlim_max));
 836                 size = pmap_resident_tlnw_count(map->pmap);
 837                 if (limit >= 0 && size > 4096 && size - 4096 >= limit) {
 838                         vm_pageout_map_deactivate_pages(map, limit);
 839                 }
 840         }
 841 #endif
 842
 843         return (result);
 844 }
 845
 846 /*
 847  * Attempt a lockless vm_fault() shortcut.  The stars have to align for this
 848  * to work.  But if it does we can get our page only soft-busied and not
 849  * have to touch the vm_object or vnode locks at all.
 850  */
 851 static
 852 int
 853 vm_fault_quick(struct faultstate *fs, vm_pindex_t first_pindex,
 854                vm_prot_t fault_type)
 855 {
 856         vm_page_t m;
 857         vm_object_t obj;        /* NOT LOCKED */
 858
 859         /*
 860          * Don't waste time if the object is only being used by one vm_map.
 861          */
 862         obj = fs->first_object;
 863         if (obj->flags & OBJ_ONEMAPPING)
 864                 return KERN_FAILURE;
 865
 866         /*
 867          * This will try to wire/unwire a page, which can't be done with
 868          * a soft-busied page.
 869          */
 870         if (fs->fault_flags & VM_FAULT_WIRE_MASK)
 871                 return KERN_FAILURE;
 872
 873         /*
 874          * Ick, can't handle this
 875          */
 876         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
 877 #ifdef VM_FAULT_QUICK_DEBUG
 878                 ++vm_fault_quick_failure_count1;
 879 #endif
 880                 return KERN_FAILURE;
 881         }
 882
 883         /*
 884          * Ok, try to get the vm_page quickly via the hash table.  The
 885          * page will be soft-busied on success (NOT hard-busied).
 886          */
 887         m = vm_page_hash_get(obj, first_pindex);
 888         if (m == NULL) {
 889 #ifdef VM_FAULT_QUICK_DEBUG
 890                 ++vm_fault_quick_failure_count2;
 891 #endif
 892                 return KERN_FAILURE;
 893         }
 894         if ((obj->flags & OBJ_DEAD) ||
 895             m->valid != VM_PAGE_BITS_ALL ||
 896             m->queue - m->pc == PQ_CACHE ||
 897             (m->flags & PG_SWAPPED)) {
 898                 vm_page_sbusy_drop(m);
 899 #ifdef VM_FAULT_QUICK_DEBUG
 900                 ++vm_fault_quick_failure_count3;
 901 #endif
 902                 return KERN_FAILURE;
 903         }
 904
 905         /*
 906          * The page is already fully valid, ACTIVE, and is not PG_SWAPPED.
 907          *
 908          * Don't map the page writable when emulating the dirty bit, a
 909          * fault must be taken for proper emulation (vkernel).
 910          */
 911         if (curthread->td_lwp && curthread->td_lwp->lwp_vmspace &&
 912             pmap_emulate_ad_bits(&curthread->td_lwp->lwp_vmspace->vm_pmap)) {
 913                 if ((fault_type & VM_PROT_WRITE) == 0)
 914                         fs->prot &= ~VM_PROT_WRITE;
 915         }
 916
 917         /*
 918          * If this is a write fault the object and the page must already
 919          * be writable.  Since we don't hold an object lock and only a
 920          * soft-busy on the page, we cannot manipulate the object or
 921          * the page state (other than the page queue).
 922          */
 923         if (fs->prot & VM_PROT_WRITE) {
 924                 if ((obj->flags & (OBJ_WRITEABLE | OBJ_MIGHTBEDIRTY)) !=
 925                     (OBJ_WRITEABLE | OBJ_MIGHTBEDIRTY) ||
 926                     m->dirty != VM_PAGE_BITS_ALL) {
 927                         vm_page_sbusy_drop(m);
 928 #ifdef VM_FAULT_QUICK_DEBUG
 929                         ++vm_fault_quick_failure_count4;
 930 #endif
 931                         return KERN_FAILURE;
 932                 }
 933                 vm_set_nosync(m, fs->entry);
 934         }
 935
 936         /*
 937          * Even though we are only soft-busied we can still move pages
 938          * around in the normal queue(s).  The soft-busy prevents the
 939          * page from being removed from the object, etc (normal operation).
 940          */
 941         vm_page_activate(m);
 942         fs->m = m;
 943         fs->msoftonly = 1;
 944 #ifdef VM_FAULT_QUICK_DEBUG
 945         ++vm_fault_quick_success_count;
 946 #endif
 947
 948         return KERN_SUCCESS;
 949 }
 950
 951 /*
 952  * Fault in the specified virtual address in the current process map,
 953  * returning a held VM page or NULL.  See vm_fault_page() for more
 954  * information.
 955  *
 956  * No requirements.
 957  */
 958 vm_page_t
 959 vm_fault_page_quick(vm_offset_t va, vm_prot_t fault_type,
 960                     int *errorp, int *busyp)
 961 {
 962         struct lwp *lp = curthread->td_lwp;
 963         vm_page_t m;
 964
 965         m = vm_fault_page(&lp->lwp_vmspace->vm_map, va,
 966                           fault_type, VM_FAULT_NORMAL,
 967                           errorp, busyp);
 968         return(m);
 969 }
 970
 971 /*
 972  * Fault in the specified virtual address in the specified map, doing all
 973  * necessary manipulation of the object store and all necessary I/O.  Return
 974  * a held VM page or NULL, and set *errorp.  The related pmap is not
 975  * updated.
 976  *
 977  * If busyp is not NULL then *busyp will be set to TRUE if this routine
 978  * decides to return a busied page (aka VM_PROT_WRITE), or FALSE if it
 979  * does not (VM_PROT_WRITE not specified or busyp is NULL).  If busyp is
 980  * NULL the returned page is only held.
 981  *
 982  * If the caller has no intention of writing to the page's contents, busyp
 983  * can be passed as NULL along with VM_PROT_WRITE to force a COW operation
 984  * without busying the page.
 985  *
 986  * The returned page will also be marked PG_REFERENCED.
 987  *
 988  * If the page cannot be faulted writable and VM_PROT_WRITE was specified, an
 989  * error will be returned.
 990  *
 991  * No requirements.
 992  */
 993 vm_page_t
 994 vm_fault_page(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
 995               int fault_flags, int *errorp, int *busyp)
 996 {
 997         vm_pindex_t first_pindex;
 998         struct faultstate fs;
 999         int result;
1000         int retry;
1001         int growstack;
1002         int didcow;
1003         vm_prot_t orig_fault_type = fault_type;
1004
1005         retry = 0;
1006         didcow = 0;
1007         fs.hardfault = 0;
1008         fs.fault_flags = fault_flags;
1009         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
1010
1011         /*
1012          * Dive the pmap (concurrency possible).  If we find the
1013          * appropriate page we can terminate early and quickly.
1014          *
1015          * This works great for normal programs but will always return
1016          * NULL for host lookups of vkernel maps in VMM mode.
1017          *
1018          * NOTE: pmap_fault_page_quick() might not busy the page.  If
1019          *       VM_PROT_WRITE is set in fault_type and pmap_fault_page_quick()
1020          *       returns non-NULL, it will safely dirty the returned vm_page_t
1021          *       for us.  We cannot safely dirty it here (it might not be
1022          *       busy).
1023          */
1024         fs.m = pmap_fault_page_quick(map->pmap, vaddr, fault_type, busyp);
1025         if (fs.m) {
1026                 *errorp = 0;
1027                 return(fs.m);
1028         }
1029
1030         /*
1031          * Otherwise take a concurrency hit and do a formal page
1032          * fault.
1033          */
1034         fs.vp = NULL;
1035         fs.shared = vm_shared_fault;
1036         fs.first_shared = vm_shared_fault;
1037         fs.msoftonly = 0;
1038         growstack = 1;
1039
1040         /*
1041          * VM_FAULT_UNSWAP - swap_pager_unswapped() needs an exclusive object
1042          * VM_FAULT_DIRTY  - may require swap_pager_unswapped() later, but
1043          *                   we can try shared first.
1044          */
1045         if (fault_flags & VM_FAULT_UNSWAP) {
1046                 fs.first_shared = 0;
1047         }
1048
1049 RetryFault:
1050         /*
1051          * Find the vm_map_entry representing the backing store and resolve
1052          * the top level object and page index.  This may have the side
1053          * effect of executing a copy-on-write on the map entry and/or
1054          * creating a shadow object, but will not COW any actual VM pages.
1055          *
1056          * On success fs.map is left read-locked and various other fields
1057          * are initialized but not otherwise referenced or locked.
1058          *
1059          * NOTE!  vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
1060          *        if the map entry is a virtual page table and also writable,
1061          *        so we can set the 'A'accessed bit in the virtual page table
1062          *        entry.
1063          */
1064         fs.map = map;
1065         result = vm_map_lookup(&fs.map, vaddr, fault_type,
1066                                &fs.entry, &fs.first_object,
1067                                &first_pindex, &fs.first_prot, &fs.wflags);
1068
1069         if (result != KERN_SUCCESS) {
1070                 if (result == KERN_FAILURE_NOFAULT) {
1071                         *errorp = KERN_FAILURE;
1072                         fs.m = NULL;
1073                         goto done;
1074                 }
1075                 if (result != KERN_PROTECTION_FAILURE ||
1076                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
1077                 {
1078                         if (result == KERN_INVALID_ADDRESS && growstack &&
1079                             map != &kernel_map && curproc != NULL) {
1080                                 result = vm_map_growstack(map, vaddr);
1081                                 if (result == KERN_SUCCESS) {
1082                                         growstack = 0;
1083                                         ++retry;
1084                                         goto RetryFault;
1085                                 }
1086                                 result = KERN_FAILURE;
1087                         }
1088                         fs.m = NULL;
1089                         *errorp = result;
1090                         goto done;
1091                 }
1092
1093                 /*
1094                  * If we are user-wiring a r/w segment, and it is COW, then
1095                  * we need to do the COW operation.  Note that we don't
1096                  * currently COW RO sections now, because it is NOT desirable
1097                  * to COW .text.  We simply keep .text from ever being COW'ed
1098                  * and take the heat that one cannot debug wired .text sections.
1099                  */
1100                 result = vm_map_lookup(&fs.map, vaddr,
1101                                        VM_PROT_READ|VM_PROT_WRITE|
1102                                         VM_PROT_OVERRIDE_WRITE,
1103                                        &fs.entry, &fs.first_object,
1104                                        &first_pindex, &fs.first_prot,
1105                                        &fs.wflags);
1106                 if (result != KERN_SUCCESS) {
1107                         /* could also be KERN_FAILURE_NOFAULT */
1108                         *errorp = KERN_FAILURE;
1109                         fs.m = NULL;
1110                         goto done;
1111                 }
1112
1113                 /*
1114                  * If we don't COW now, on a user wire, the user will never
1115                  * be able to write to the mapping.  If we don't make this
1116                  * restriction, the bookkeeping would be nearly impossible.
1117                  *
1118                  * XXX We have a shared lock, this will have a MP race but
1119                  * I don't see how it can hurt anything.
1120                  */
1121                 if ((fs.entry->protection & VM_PROT_WRITE) == 0) {
1122                         atomic_clear_char(&fs.entry->max_protection,
1123                                           VM_PROT_WRITE);
1124                 }
1125         }
1126
1127         /*
1128          * fs.map is read-locked
1129          *
1130          * Misc checks.  Save the map generation number to detect races.
1131          */
1132         fs.map_generation = fs.map->timestamp;
1133         fs.lookup_still_valid = TRUE;
1134         fs.first_m = NULL;
1135         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
1136
1137         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
1138                 panic("vm_fault: fault on nofault entry, addr: %lx",
1139                     (u_long)vaddr);
1140         }
1141
1142         /*
1143          * A user-kernel shared map has no VM object and bypasses
1144          * everything.  We execute the uksmap function with a temporary
1145          * fictitious vm_page.  The address is directly mapped with no
1146          * management.
1147          */
1148         if (fs.entry->maptype == VM_MAPTYPE_UKSMAP) {
1149                 struct vm_page fakem;
1150
1151                 bzero(&fakem, sizeof(fakem));
1152                 fakem.pindex = first_pindex;
1153                 fakem.flags = PG_FICTITIOUS | PG_UNMANAGED;
1154                 fakem.busy_count = PBUSY_LOCKED;
1155                 fakem.valid = VM_PAGE_BITS_ALL;
1156                 fakem.pat_mode = VM_MEMATTR_DEFAULT;
1157                 if (fs.entry->object.uksmap(fs.entry->aux.dev, &fakem)) {
1158                         *errorp = KERN_FAILURE;
1159                         fs.m = NULL;
1160                         unlock_things(&fs);
1161                         goto done2;
1162                 }
1163                 fs.m = PHYS_TO_VM_PAGE(fakem.phys_addr);
1164                 vm_page_hold(fs.m);
1165                 if (busyp)
1166                         *busyp = 0;     /* don't need to busy R or W */
1167                 unlock_things(&fs);
1168                 *errorp = 0;
1169                 goto done;
1170         }
1171
1172
1173         /*
1174          * A system map entry may return a NULL object.  No object means
1175          * no pager means an unrecoverable kernel fault.
1176          */
1177         if (fs.first_object == NULL) {
1178                 panic("vm_fault: unrecoverable fault at %p in entry %p",
1179                         (void *)vaddr, fs.entry);
1180         }
1181
1182         /*
1183          * Fail here if not a trivial anonymous page fault and TDF_NOFAULT
1184          * is set.
1185          *
1186          * Unfortunately a deadlock can occur if we are forced to page-in
1187          * from swap, but diving all the way into the vm_pager_get_page()
1188          * function to find out is too much.  Just check the object type.
1189          */
1190         if ((curthread->td_flags & TDF_NOFAULT) &&
1191             (retry ||
1192              fs.first_object->type == OBJT_VNODE ||
1193              fs.first_object->type == OBJT_SWAP ||
1194              fs.first_object->backing_object)) {
1195                 *errorp = KERN_FAILURE;
1196                 unlock_things(&fs);
1197                 fs.m = NULL;
1198                 goto done2;
1199         }
1200
1201         /*
1202          * If the entry is wired we cannot change the page protection.
1203          */
1204         if (fs.wflags & FW_WIRED)
1205                 fault_type = fs.first_prot;
1206
1207         /*
1208          * Make a reference to this object to prevent its disposal while we
1209          * are messing with it.  Once we have the reference, the map is free
1210          * to be diddled.  Since objects reference their shadows (and copies),
1211          * they will stay around as well.
1212          *
1213          * The reference should also prevent an unexpected collapse of the
1214          * parent that might move pages from the current object into the
1215          * parent unexpectedly, resulting in corruption.
1216          *
1217          * Bump the paging-in-progress count to prevent size changes (e.g.
1218          * truncation operations) during I/O.  This must be done after
1219          * obtaining the vnode lock in order to avoid possible deadlocks.
1220          */
1221         if (fs.first_shared)
1222                 vm_object_hold_shared(fs.first_object);
1223         else
1224                 vm_object_hold(fs.first_object);
1225         if (fs.vp == NULL)
1226                 fs.vp = vnode_pager_lock(fs.first_object);      /* shared */
1227
1228         /*
1229          * The page we want is at (first_object, first_pindex), but if the
1230          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
1231          * page table to figure out the actual pindex.
1232          *
1233          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
1234          * ONLY
1235          */
1236         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1237                 result = vm_fault_vpagetable(&fs, &first_pindex,
1238                                              fs.entry->aux.master_pde,
1239                                              fault_type, 1);
1240                 if (result == KERN_TRY_AGAIN) {
1241                         vm_object_drop(fs.first_object);
1242                         ++retry;
1243                         goto RetryFault;
1244                 }
1245                 if (result != KERN_SUCCESS) {
1246                         *errorp = result;
1247                         fs.m = NULL;
1248                         goto done;
1249                 }
1250         }
1251
1252         /*
1253          * Now we have the actual (object, pindex), fault in the page.  If
1254          * vm_fault_object() fails it will unlock and deallocate the FS
1255          * data.   If it succeeds everything remains locked and fs->object
1256          * will have an additinal PIP count if it is not equal to
1257          * fs->first_object
1258          */
1259         fs.m = NULL;
1260         result = vm_fault_object(&fs, first_pindex, fault_type, 1);
1261
1262         if (result == KERN_TRY_AGAIN) {
1263                 vm_object_drop(fs.first_object);
1264                 ++retry;
1265                 didcow |= fs.wflags & FW_DIDCOW;
1266                 goto RetryFault;
1267         }
1268         if (result != KERN_SUCCESS) {
1269                 *errorp = result;
1270                 fs.m = NULL;
1271                 goto done;
1272         }
1273
1274         if ((orig_fault_type & VM_PROT_WRITE) &&
1275             (fs.prot & VM_PROT_WRITE) == 0) {
1276                 *errorp = KERN_PROTECTION_FAILURE;
1277                 unlock_and_deallocate(&fs);
1278                 fs.m = NULL;
1279                 goto done;
1280         }
1281
1282         /*
1283          * Generally speaking we don't want to update the pmap because
1284          * this routine can be called many times for situations that do
1285          * not require updating the pmap, not to mention the page might
1286          * already be in the pmap.
1287          *
1288          * However, if our vm_map_lookup() results in a COW, we need to
1289          * at least remove the pte from the pmap to guarantee proper
1290          * visibility of modifications made to the process.  For example,
1291          * modifications made by vkernel uiocopy/related routines and
1292          * modifications made by ptrace().
1293          */
1294         vm_page_flag_set(fs.m, PG_REFERENCED);
1295 #if 0
1296         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot,
1297                    fs.wflags & FW_WIRED, NULL);
1298         mycpu->gd_cnt.v_vm_faults++;
1299         if (curthread->td_lwp)
1300                 ++curthread->td_lwp->lwp_ru.ru_minflt;
1301 #endif
1302         if ((fs.wflags | didcow) | FW_DIDCOW) {
1303                 pmap_remove(fs.map->pmap,
1304                             vaddr & ~PAGE_MASK,
1305                             (vaddr & ~PAGE_MASK) + PAGE_SIZE);
1306         }
1307
1308         /*
1309          * On success vm_fault_object() does not unlock or deallocate, and fs.m
1310          * will contain a busied page.  So we must unlock here after having
1311          * messed with the pmap.
1312          */
1313         unlock_things(&fs);
1314
1315         /*
1316          * Return a held page.  We are not doing any pmap manipulation so do
1317          * not set PG_MAPPED.  However, adjust the page flags according to
1318          * the fault type because the caller may not use a managed pmapping
1319          * (so we don't want to lose the fact that the page will be dirtied
1320          * if a write fault was specified).
1321          */
1322         if (fault_type & VM_PROT_WRITE)
1323                 vm_page_dirty(fs.m);
1324         vm_page_activate(fs.m);
1325
1326         if (curthread->td_lwp) {
1327                 if (fs.hardfault) {
1328                         curthread->td_lwp->lwp_ru.ru_majflt++;
1329                 } else {
1330                         curthread->td_lwp->lwp_ru.ru_minflt++;
1331                 }
1332         }
1333
1334         /*
1335          * Unlock everything, and return the held or busied page.
1336          */
1337         if (busyp) {
1338                 if (fault_type & VM_PROT_WRITE) {
1339                         vm_page_dirty(fs.m);
1340                         *busyp = 1;
1341                 } else {
1342                         *busyp = 0;
1343                         vm_page_hold(fs.m);
1344                         vm_page_wakeup(fs.m);
1345                 }
1346         } else {
1347                 vm_page_hold(fs.m);
1348                 vm_page_wakeup(fs.m);
1349         }
1350         /*vm_object_deallocate(fs.first_object);*/
1351         /*fs.first_object = NULL; */
1352         *errorp = 0;
1353
1354 done:
1355         if (fs.first_object)
1356                 vm_object_drop(fs.first_object);
1357 done2:
1358         return(fs.m);
1359 }
1360
1361 /*
1362  * Fault in the specified (object,offset), dirty the returned page as
1363  * needed.  If the requested fault_type cannot be done NULL and an
1364  * error is returned.
1365  *
1366  * A held (but not busied) page is returned.
1367  *
1368  * The passed in object must be held as specified by the shared
1369  * argument.
1370  */
1371 vm_page_t
1372 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
1373                      vm_prot_t fault_type, int fault_flags,
1374                      int *sharedp, int *errorp)
1375 {
1376         int result;
1377         vm_pindex_t first_pindex;
1378         struct faultstate fs;
1379         struct vm_map_entry entry;
1380
1381         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1382         bzero(&entry, sizeof(entry));
1383         entry.object.vm_object = object;
1384         entry.maptype = VM_MAPTYPE_NORMAL;
1385         entry.protection = entry.max_protection = fault_type;
1386
1387         fs.hardfault = 0;
1388         fs.fault_flags = fault_flags;
1389         fs.map = NULL;
1390         fs.shared = vm_shared_fault;
1391         fs.first_shared = *sharedp;
1392         fs.msoftonly = 0;
1393         fs.vp = NULL;
1394         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
1395
1396         /*
1397          * VM_FAULT_UNSWAP - swap_pager_unswapped() needs an exclusive object
1398          * VM_FAULT_DIRTY  - may require swap_pager_unswapped() later, but
1399          *                   we can try shared first.
1400          */
1401         if (fs.first_shared && (fault_flags & VM_FAULT_UNSWAP)) {
1402                 fs.first_shared = 0;
1403                 vm_object_upgrade(object);
1404         }
1405
1406         /*
1407          * Retry loop as needed (typically for shared->exclusive transitions)
1408          */
1409 RetryFault:
1410         *sharedp = fs.first_shared;
1411         first_pindex = OFF_TO_IDX(offset);
1412         fs.first_object = object;
1413         fs.entry = &entry;
1414         fs.first_prot = fault_type;
1415         fs.wflags = 0;
1416         /*fs.map_generation = 0; unused */
1417
1418         /*
1419          * Make a reference to this object to prevent its disposal while we
1420          * are messing with it.  Once we have the reference, the map is free
1421          * to be diddled.  Since objects reference their shadows (and copies),
1422          * they will stay around as well.
1423          *
1424          * The reference should also prevent an unexpected collapse of the
1425          * parent that might move pages from the current object into the
1426          * parent unexpectedly, resulting in corruption.
1427          *
1428          * Bump the paging-in-progress count to prevent size changes (e.g.
1429          * truncation operations) during I/O.  This must be done after
1430          * obtaining the vnode lock in order to avoid possible deadlocks.
1431          */
1432         if (fs.vp == NULL)
1433                 fs.vp = vnode_pager_lock(fs.first_object);
1434
1435         fs.lookup_still_valid = TRUE;
1436         fs.first_m = NULL;
1437         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
1438
1439 #if 0
1440         /* XXX future - ability to operate on VM object using vpagetable */
1441         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1442                 result = vm_fault_vpagetable(&fs, &first_pindex,
1443                                              fs.entry->aux.master_pde,
1444                                              fault_type, 0);
1445                 if (result == KERN_TRY_AGAIN) {
1446                         if (fs.first_shared == 0 && *sharedp)
1447                                 vm_object_upgrade(object);
1448                         goto RetryFault;
1449                 }
1450                 if (result != KERN_SUCCESS) {
1451                         *errorp = result;
1452                         return (NULL);
1453                 }
1454         }
1455 #endif
1456
1457         /*
1458          * Now we have the actual (object, pindex), fault in the page.  If
1459          * vm_fault_object() fails it will unlock and deallocate the FS
1460          * data.   If it succeeds everything remains locked and fs->object
1461          * will have an additinal PIP count if it is not equal to
1462          * fs->first_object
1463          *
1464          * On KERN_TRY_AGAIN vm_fault_object() leaves fs.first_object intact.
1465          * We may have to upgrade its lock to handle the requested fault.
1466          */
1467         result = vm_fault_object(&fs, first_pindex, fault_type, 0);
1468
1469         if (result == KERN_TRY_AGAIN) {
1470                 if (fs.first_shared == 0 && *sharedp)
1471                         vm_object_upgrade(object);
1472                 goto RetryFault;
1473         }
1474         if (result != KERN_SUCCESS) {
1475                 *errorp = result;
1476                 return(NULL);
1477         }
1478
1479         if ((fault_type & VM_PROT_WRITE) && (fs.prot & VM_PROT_WRITE) == 0) {
1480                 *errorp = KERN_PROTECTION_FAILURE;
1481                 unlock_and_deallocate(&fs);
1482                 return(NULL);
1483         }
1484
1485         /*
1486          * On success vm_fault_object() does not unlock or deallocate, so we
1487          * do it here.  Note that the returned fs.m will be busied.
1488          */
1489         unlock_things(&fs);
1490
1491         /*
1492          * Return a held page.  We are not doing any pmap manipulation so do
1493          * not set PG_MAPPED.  However, adjust the page flags according to
1494          * the fault type because the caller may not use a managed pmapping
1495          * (so we don't want to lose the fact that the page will be dirtied
1496          * if a write fault was specified).
1497          */
1498         vm_page_hold(fs.m);
1499         vm_page_activate(fs.m);
1500         if ((fault_type & VM_PROT_WRITE) || (fault_flags & VM_FAULT_DIRTY))
1501                 vm_page_dirty(fs.m);
1502         if (fault_flags & VM_FAULT_UNSWAP)
1503                 swap_pager_unswapped(fs.m);
1504
1505         /*
1506          * Indicate that the page was accessed.
1507          */
1508         vm_page_flag_set(fs.m, PG_REFERENCED);
1509
1510         if (curthread->td_lwp) {
1511                 if (fs.hardfault) {
1512                         curthread->td_lwp->lwp_ru.ru_majflt++;
1513                 } else {
1514                         curthread->td_lwp->lwp_ru.ru_minflt++;
1515                 }
1516         }
1517
1518         /*
1519          * Unlock everything, and return the held page.
1520          */
1521         vm_page_wakeup(fs.m);
1522         /*vm_object_deallocate(fs.first_object);*/
1523         /*fs.first_object = NULL; */
1524
1525         *errorp = 0;
1526         return(fs.m);
1527 }
1528
1529 /*
1530  * Translate the virtual page number (first_pindex) that is relative
1531  * to the address space into a logical page number that is relative to the
1532  * backing object.  Use the virtual page table pointed to by (vpte).
1533  *
1534  * Possibly downgrade the protection based on the vpte bits.
1535  *
1536  * This implements an N-level page table.  Any level can terminate the
1537  * scan by setting VPTE_PS.   A linear mapping is accomplished by setting
1538  * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
1539  */
1540 static
1541 int
1542 vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
1543                     vpte_t vpte, int fault_type, int allow_nofault)
1544 {
1545         struct lwbuf *lwb;
1546         struct lwbuf lwb_cache;
1547         int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
1548         int result;
1549         vpte_t *ptep;
1550
1551         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
1552         for (;;) {
1553                 /*
1554                  * We cannot proceed if the vpte is not valid, not readable
1555                  * for a read fault, not writable for a write fault, or
1556                  * not executable for an instruction execution fault.
1557                  */
1558                 if ((vpte & VPTE_V) == 0) {
1559                         unlock_and_deallocate(fs);
1560                         return (KERN_FAILURE);
1561                 }
1562                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_RW) == 0) {
1563                         unlock_and_deallocate(fs);
1564                         return (KERN_FAILURE);
1565                 }
1566                 if ((fault_type & VM_PROT_EXECUTE) && (vpte & VPTE_NX)) {
1567                         unlock_and_deallocate(fs);
1568                         return (KERN_FAILURE);
1569                 }
1570                 if ((vpte & VPTE_PS) || vshift == 0)
1571                         break;
1572
1573                 /*
1574                  * Get the page table page.  Nominally we only read the page
1575                  * table, but since we are actively setting VPTE_M and VPTE_A,
1576                  * tell vm_fault_object() that we are writing it.
1577                  *
1578                  * There is currently no real need to optimize this.
1579                  */
1580                 result = vm_fault_object(fs, (vpte & VPTE_FRAME) >> PAGE_SHIFT,
1581                                          VM_PROT_READ|VM_PROT_WRITE,
1582                                          allow_nofault);
1583                 if (result != KERN_SUCCESS)
1584                         return (result);
1585
1586                 /*
1587                  * Process the returned fs.m and look up the page table
1588                  * entry in the page table page.
1589                  */
1590                 vshift -= VPTE_PAGE_BITS;
1591                 lwb = lwbuf_alloc(fs->m, &lwb_cache);
1592                 ptep = ((vpte_t *)lwbuf_kva(lwb) +
1593                         ((*pindex >> vshift) & VPTE_PAGE_MASK));
1594                 vm_page_activate(fs->m);
1595
1596                 /*
1597                  * Page table write-back - entire operation including
1598                  * validation of the pte must be atomic to avoid races
1599                  * against the vkernel changing the pte.
1600                  *
1601                  * If the vpte is valid for the* requested operation, do
1602                  * a write-back to the page table.
1603                  *
1604                  * XXX VPTE_M is not set properly for page directory pages.
1605                  * It doesn't get set in the page directory if the page table
1606                  * is modified during a read access.
1607                  */
1608                 for (;;) {
1609                         vpte_t nvpte;
1610
1611                         /*
1612                          * Reload for the cmpset, but make sure the pte is
1613                          * still valid.
1614                          */
1615                         vpte = *ptep;
1616                         cpu_ccfence();
1617                         nvpte = vpte;
1618
1619                         if ((vpte & VPTE_V) == 0)
1620                                 break;
1621
1622                         if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_RW))
1623                                 nvpte |= VPTE_M | VPTE_A;
1624                         if (fault_type & (VM_PROT_READ | VM_PROT_EXECUTE))
1625                                 nvpte |= VPTE_A;
1626                         if (vpte == nvpte)
1627                                 break;
1628                         if (atomic_cmpset_long(ptep, vpte, nvpte)) {
1629                                 vm_page_dirty(fs->m);
1630                                 break;
1631                         }
1632                 }
1633                 lwbuf_free(lwb);
1634                 vm_page_flag_set(fs->m, PG_REFERENCED);
1635                 vm_page_wakeup(fs->m);
1636                 fs->m = NULL;
1637                 cleanup_successful_fault(fs);
1638         }
1639
1640         /*
1641          * When the vkernel sets VPTE_RW it expects the real kernel to
1642          * reflect VPTE_M back when the page is modified via the mapping.
1643          * In order to accomplish this the real kernel must map the page
1644          * read-only for read faults and use write faults to reflect VPTE_M
1645          * back.
1646          *
1647          * Once VPTE_M has been set, the real kernel's pte allows writing.
1648          * If the vkernel clears VPTE_M the vkernel must be sure to
1649          * MADV_INVAL the real kernel's mappings to force the real kernel
1650          * to re-fault on the next write so oit can set VPTE_M again.
1651          */
1652         if ((fault_type & VM_PROT_WRITE) == 0 &&
1653             (vpte & (VPTE_RW | VPTE_M)) != (VPTE_RW | VPTE_M)) {
1654                 fs->first_prot &= ~VM_PROT_WRITE;
1655         }
1656
1657         /*
1658          * Disable EXECUTE perms if NX bit is set.
1659          */
1660         if (vpte & VPTE_NX)
1661                 fs->first_prot &= ~VM_PROT_EXECUTE;
1662
1663         /*
1664          * Combine remaining address bits with the vpte.
1665          */
1666         *pindex = ((vpte & VPTE_FRAME) >> PAGE_SHIFT) +
1667                   (*pindex & ((1L << vshift) - 1));
1668         return (KERN_SUCCESS);
1669 }
1670
1671
1672 /*
1673  * This is the core of the vm_fault code.
1674  *
1675  * Do all operations required to fault-in (fs.first_object, pindex).  Run
1676  * through the shadow chain as necessary and do required COW or virtual
1677  * copy operations.  The caller has already fully resolved the vm_map_entry
1678  * and, if appropriate, has created a copy-on-write layer.  All we need to
1679  * do is iterate the object chain.
1680  *
1681  * On failure (fs) is unlocked and deallocated and the caller may return or
1682  * retry depending on the failure code.  On success (fs) is NOT unlocked or
1683  * deallocated, fs.m will contained a resolved, busied page, and fs.object
1684  * will have an additional PIP count if it is not equal to fs.first_object.
1685  *
1686  * If locks based on fs->first_shared or fs->shared are insufficient,
1687  * clear the appropriate field(s) and return RETRY.  COWs require that
1688  * first_shared be 0, while page allocations (or frees) require that
1689  * shared be 0.  Renames require that both be 0.
1690  *
1691  * NOTE! fs->[first_]shared might be set with VM_FAULT_DIRTY also set.
1692  *       we will have to retry with it exclusive if the vm_page is
1693  *       PG_SWAPPED.
1694  *
1695  * fs->first_object must be held on call.
1696  */
1697 static
1698 int
1699 vm_fault_object(struct faultstate *fs, vm_pindex_t first_pindex,
1700                 vm_prot_t fault_type, int allow_nofault)
1701 {
1702         vm_object_t next_object;
1703         vm_pindex_t pindex;
1704         int error;
1705
1706         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
1707         fs->prot = fs->first_prot;
1708         fs->object = fs->first_object;
1709         pindex = first_pindex;
1710
1711         vm_object_chain_acquire(fs->first_object, fs->shared);
1712         vm_object_pip_add(fs->first_object, 1);
1713
1714         /*
1715          * If a read fault occurs we try to upgrade the page protection
1716          * and make it also writable if possible.  There are three cases
1717          * where we cannot make the page mapping writable:
1718          *
1719          * (1) The mapping is read-only or the VM object is read-only,
1720          *     fs->prot above will simply not have VM_PROT_WRITE set.
1721          *
1722          * (2) If the mapping is a virtual page table fs->first_prot will
1723          *     have already been properly adjusted by vm_fault_vpagetable().
1724          *     to detect writes so we can set VPTE_M in the virtual page
1725          *     table.  Used by vkernels.
1726          *
1727          * (3) If the VM page is read-only or copy-on-write, upgrading would
1728          *     just result in an unnecessary COW fault.
1729          *
1730          * (4) If the pmap specifically requests A/M bit emulation, downgrade
1731          *     here.
1732          */
1733 #if 0
1734         /* see vpagetable code */
1735         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1736                 if ((fault_type & VM_PROT_WRITE) == 0)
1737                         fs->prot &= ~VM_PROT_WRITE;
1738         }
1739 #endif
1740
1741         if (curthread->td_lwp && curthread->td_lwp->lwp_vmspace &&
1742             pmap_emulate_ad_bits(&curthread->td_lwp->lwp_vmspace->vm_pmap)) {
1743                 if ((fault_type & VM_PROT_WRITE) == 0)
1744                         fs->prot &= ~VM_PROT_WRITE;
1745         }
1746
1747         /* vm_object_hold(fs->object); implied b/c object == first_object */
1748
1749         for (;;) {
1750                 /*
1751                  * The entire backing chain from first_object to object
1752                  * inclusive is chainlocked.
1753                  *
1754                  * If the object is dead, we stop here
1755                  */
1756                 if (fs->object->flags & OBJ_DEAD) {
1757                         vm_object_pip_wakeup(fs->first_object);
1758                         vm_object_chain_release_all(fs->first_object,
1759                                                     fs->object);
1760                         if (fs->object != fs->first_object)
1761                                 vm_object_drop(fs->object);
1762                         unlock_and_deallocate(fs);
1763                         return (KERN_PROTECTION_FAILURE);
1764                 }
1765
1766                 /*
1767                  * See if the page is resident.  Wait/Retry if the page is
1768                  * busy (lots of stuff may have changed so we can't continue
1769                  * in that case).
1770                  *
1771                  * We can theoretically allow the soft-busy case on a read
1772                  * fault if the page is marked valid, but since such
1773                  * pages are typically already pmap'd, putting that
1774                  * special case in might be more effort then it is
1775                  * worth.  We cannot under any circumstances mess
1776                  * around with a vm_page_t->busy page except, perhaps,
1777                  * to pmap it.
1778                  */
1779                 fs->m = vm_page_lookup_busy_try(fs->object, pindex,
1780                                                 TRUE, &error);
1781                 if (error) {
1782                         vm_object_pip_wakeup(fs->first_object);
1783                         vm_object_chain_release_all(fs->first_object,
1784                                                     fs->object);
1785                         if (fs->object != fs->first_object)
1786                                 vm_object_drop(fs->object);
1787                         unlock_things(fs);
1788                         vm_page_sleep_busy(fs->m, TRUE, "vmpfw");
1789                         mycpu->gd_cnt.v_intrans++;
1790                         /*vm_object_deallocate(fs->first_object);*/
1791                         /*fs->first_object = NULL;*/
1792                         fs->m = NULL;
1793                         return (KERN_TRY_AGAIN);
1794                 }
1795                 if (fs->m) {
1796                         /*
1797                          * The page is busied for us.
1798                          *
1799                          * If reactivating a page from PQ_CACHE we may have
1800                          * to rate-limit.
1801                          */
1802                         int queue = fs->m->queue;
1803                         vm_page_unqueue_nowakeup(fs->m);
1804
1805                         if ((queue - fs->m->pc) == PQ_CACHE &&
1806                             vm_page_count_severe()) {
1807                                 vm_page_activate(fs->m);
1808                                 vm_page_wakeup(fs->m);
1809                                 fs->m = NULL;
1810                                 vm_object_pip_wakeup(fs->first_object);
1811                                 vm_object_chain_release_all(fs->first_object,
1812                                                             fs->object);
1813                                 if (fs->object != fs->first_object)
1814                                         vm_object_drop(fs->object);
1815                                 unlock_and_deallocate(fs);
1816                                 if (allow_nofault == 0 ||
1817                                     (curthread->td_flags & TDF_NOFAULT) == 0) {
1818                                         thread_t td;
1819
1820                                         vm_wait_pfault();
1821                                         td = curthread;
1822                                         if (td->td_proc && (td->td_proc->p_flags & P_LOWMEMKILL))
1823                                                 return (KERN_PROTECTION_FAILURE);
1824                                 }
1825                                 return (KERN_TRY_AGAIN);
1826                         }
1827
1828                         /*
1829                          * If it still isn't completely valid (readable),
1830                          * or if a read-ahead-mark is set on the VM page,
1831                          * jump to readrest, else we found the page and
1832                          * can return.
1833                          *
1834                          * We can release the spl once we have marked the
1835                          * page busy.
1836                          */
1837                         if (fs->m->object != &kernel_object) {
1838                                 if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
1839                                     VM_PAGE_BITS_ALL) {
1840                                         goto readrest;
1841                                 }
1842                                 if (fs->m->flags & PG_RAM) {
1843                                         if (debug_cluster)
1844                                                 kprintf("R");
1845                                         vm_page_flag_clear(fs->m, PG_RAM);
1846                                         goto readrest;
1847                                 }
1848                         }
1849                         break; /* break to PAGE HAS BEEN FOUND */
1850                 }
1851
1852                 /*
1853                  * Page is not resident, If this is the search termination
1854                  * or the pager might contain the page, allocate a new page.
1855                  */
1856                 if (TRYPAGER(fs) || fs->object == fs->first_object) {
1857                         /*
1858                          * Allocating, must be exclusive.
1859                          */
1860                         if (fs->object == fs->first_object &&
1861                             fs->first_shared) {
1862                                 fs->first_shared = 0;
1863                                 vm_object_pip_wakeup(fs->first_object);
1864                                 vm_object_chain_release_all(fs->first_object,
1865                                                             fs->object);
1866                                 if (fs->object != fs->first_object)
1867                                         vm_object_drop(fs->object);
1868                                 unlock_and_deallocate(fs);
1869                                 return (KERN_TRY_AGAIN);
1870                         }
1871                         if (fs->object != fs->first_object &&
1872                             fs->shared) {
1873                                 fs->first_shared = 0;
1874                                 fs->shared = 0;
1875                                 vm_object_pip_wakeup(fs->first_object);
1876                                 vm_object_chain_release_all(fs->first_object,
1877                                                             fs->object);
1878                                 if (fs->object != fs->first_object)
1879                                         vm_object_drop(fs->object);
1880                                 unlock_and_deallocate(fs);
1881                                 return (KERN_TRY_AGAIN);
1882                         }
1883
1884                         /*
1885                          * If the page is beyond the object size we fail
1886                          */
1887                         if (pindex >= fs->object->size) {
1888                                 vm_object_pip_wakeup(fs->first_object);
1889                                 vm_object_chain_release_all(fs->first_object,
1890                                                             fs->object);
1891                                 if (fs->object != fs->first_object)
1892                                         vm_object_drop(fs->object);
1893                                 unlock_and_deallocate(fs);
1894                                 return (KERN_PROTECTION_FAILURE);
1895                         }
1896
1897                         /*
1898                          * Allocate a new page for this object/offset pair.
1899                          *
1900                          * It is possible for the allocation to race, so
1901                          * handle the case.
1902                          */
1903                         fs->m = NULL;
1904                         if (!vm_page_count_severe()) {
1905                                 fs->m = vm_page_alloc(fs->object, pindex,
1906                                     ((fs->vp || fs->object->backing_object) ?
1907                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL :
1908                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL |
1909                                         VM_ALLOC_USE_GD | VM_ALLOC_ZERO));
1910                         }
1911                         if (fs->m == NULL) {
1912                                 vm_object_pip_wakeup(fs->first_object);
1913                                 vm_object_chain_release_all(fs->first_object,
1914                                                             fs->object);
1915                                 if (fs->object != fs->first_object)
1916                                         vm_object_drop(fs->object);
1917                                 unlock_and_deallocate(fs);
1918                                 if (allow_nofault == 0 ||
1919                                     (curthread->td_flags & TDF_NOFAULT) == 0) {
1920                                         thread_t td;
1921
1922                                         vm_wait_pfault();
1923                                         td = curthread;
1924                                         if (td->td_proc && (td->td_proc->p_flags & P_LOWMEMKILL))
1925                                                 return (KERN_PROTECTION_FAILURE);
1926                                 }
1927                                 return (KERN_TRY_AGAIN);
1928                         }
1929
1930                         /*
1931                          * Fall through to readrest.  We have a new page which
1932                          * will have to be paged (since m->valid will be 0).
1933                          */
1934                 }
1935
1936 readrest:
1937                 /*
1938                  * We have found an invalid or partially valid page, a
1939                  * page with a read-ahead mark which might be partially or
1940                  * fully valid (and maybe dirty too), or we have allocated
1941                  * a new page.
1942                  *
1943                  * Attempt to fault-in the page if there is a chance that the
1944                  * pager has it, and potentially fault in additional pages
1945                  * at the same time.
1946                  *
1947                  * If TRYPAGER is true then fs.m will be non-NULL and busied
1948                  * for us.
1949                  */
1950                 if (TRYPAGER(fs)) {
1951                         int rv;
1952                         int seqaccess;
1953                         u_char behavior = vm_map_entry_behavior(fs->entry);
1954
1955                         if (behavior == MAP_ENTRY_BEHAV_RANDOM)
1956                                 seqaccess = 0;
1957                         else
1958                                 seqaccess = -1;
1959
1960                         /*
1961                          * Doing I/O may synchronously insert additional
1962                          * pages so we can't be shared at this point either.
1963                          *
1964                          * NOTE: We can't free fs->m here in the allocated
1965                          *       case (fs->object != fs->first_object) as
1966                          *       this would require an exclusively locked
1967                          *       VM object.
1968                          */
1969                         if (fs->object == fs->first_object &&
1970                             fs->first_shared) {
1971                                 vm_page_deactivate(fs->m);
1972                                 vm_page_wakeup(fs->m);
1973                                 fs->m = NULL;
1974                                 fs->first_shared = 0;
1975                                 vm_object_pip_wakeup(fs->first_object);
1976                                 vm_object_chain_release_all(fs->first_object,
1977                                                             fs->object);
1978                                 if (fs->object != fs->first_object)
1979                                         vm_object_drop(fs->object);
1980                                 unlock_and_deallocate(fs);
1981                                 return (KERN_TRY_AGAIN);
1982                         }
1983                         if (fs->object != fs->first_object &&
1984                             fs->shared) {
1985                                 vm_page_deactivate(fs->m);
1986                                 vm_page_wakeup(fs->m);
1987                                 fs->m = NULL;
1988                                 fs->first_shared = 0;
1989                                 fs->shared = 0;
1990                                 vm_object_pip_wakeup(fs->first_object);
1991                                 vm_object_chain_release_all(fs->first_object,
1992                                                             fs->object);
1993                                 if (fs->object != fs->first_object)
1994                                         vm_object_drop(fs->object);
1995                                 unlock_and_deallocate(fs);
1996                                 return (KERN_TRY_AGAIN);
1997                         }
1998
1999                         /*
2000                          * Avoid deadlocking against the map when doing I/O.
2001                          * fs.object and the page is BUSY'd.
2002                          *
2003                          * NOTE: Once unlocked, fs->entry can become stale
2004                          *       so this will NULL it out.
2005                          *
2006                          * NOTE: fs->entry is invalid until we relock the
2007                          *       map and verify that the timestamp has not
2008                          *       changed.
2009                          */
2010                         unlock_map(fs);
2011
2012                         /*
2013                          * Acquire the page data.  We still hold a ref on
2014                          * fs.object and the page has been BUSY's.
2015                          *
2016                          * The pager may replace the page (for example, in
2017                          * order to enter a fictitious page into the
2018                          * object).  If it does so it is responsible for
2019                          * cleaning up the passed page and properly setting
2020                          * the new page BUSY.
2021                          *
2022                          * If we got here through a PG_RAM read-ahead
2023                          * mark the page may be partially dirty and thus
2024                          * not freeable.  Don't bother checking to see
2025                          * if the pager has the page because we can't free
2026                          * it anyway.  We have to depend on the get_page
2027                          * operation filling in any gaps whether there is
2028                          * backing store or not.
2029                          */
2030                         rv = vm_pager_get_page(fs->object, &fs->m, seqaccess);
2031
2032                         if (rv == VM_PAGER_OK) {
2033                                 /*
2034                                  * Relookup in case pager changed page. Pager
2035                                  * is responsible for disposition of old page
2036                                  * if moved.
2037                                  *
2038                                  * XXX other code segments do relookups too.
2039                                  * It's a bad abstraction that needs to be
2040                                  * fixed/removed.
2041                                  */
2042                                 fs->m = vm_page_lookup(fs->object, pindex);
2043                                 if (fs->m == NULL) {
2044                                         vm_object_pip_wakeup(fs->first_object);
2045                                         vm_object_chain_release_all(
2046                                                 fs->first_object, fs->object);
2047                                         if (fs->object != fs->first_object)
2048                                                 vm_object_drop(fs->object);
2049                                         unlock_and_deallocate(fs);
2050                                         return (KERN_TRY_AGAIN);
2051                                 }
2052                                 ++fs->hardfault;
2053                                 break; /* break to PAGE HAS BEEN FOUND */
2054                         }
2055
2056                         /*
2057                          * Remove the bogus page (which does not exist at this
2058                          * object/offset); before doing so, we must get back
2059                          * our object lock to preserve our invariant.
2060                          *
2061                          * Also wake up any other process that may want to bring
2062                          * in this page.
2063                          *
2064                          * If this is the top-level object, we must leave the
2065                          * busy page to prevent another process from rushing
2066                          * past us, and inserting the page in that object at
2067                          * the same time that we are.
2068                          */
2069                         if (rv == VM_PAGER_ERROR) {
2070                                 if (curproc) {
2071                                         kprintf("vm_fault: pager read error, "
2072                                                 "pid %d (%s)\n",
2073                                                 curproc->p_pid,
2074                                                 curproc->p_comm);
2075                                 } else {
2076                                         kprintf("vm_fault: pager read error, "
2077                                                 "thread %p (%s)\n",
2078                                                 curthread,
2079                                                 curthread->td_comm);
2080                                 }
2081                         }
2082
2083                         /*
2084                          * Data outside the range of the pager or an I/O error
2085                          *
2086                          * The page may have been wired during the pagein,
2087                          * e.g. by the buffer cache, and cannot simply be
2088                          * freed.  Call vnode_pager_freepage() to deal with it.
2089                          *
2090                          * Also note that we cannot free the page if we are
2091                          * holding the related object shared. XXX not sure
2092                          * what to do in that case.
2093                          */
2094                         if (fs->object != fs->first_object) {
2095                                 /*
2096                                  * Scrap the page.  Check to see if the
2097                                  * vm_pager_get_page() call has already
2098                                  * dealt with it.
2099                                  */
2100                                 if (fs->m) {
2101                                         vnode_pager_freepage(fs->m);
2102                                         fs->m = NULL;
2103                                 }
2104
2105                                 /*
2106                                  * XXX - we cannot just fall out at this
2107                                  * point, m has been freed and is invalid!
2108                                  */
2109                         }
2110                         /*
2111                          * XXX - the check for kernel_map is a kludge to work
2112                          * around having the machine panic on a kernel space
2113                          * fault w/ I/O error.
2114                          */
2115                         if (((fs->map != &kernel_map) &&
2116                             (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) {
2117                                 if (fs->m) {
2118                                         if (fs->first_shared) {
2119                                                 vm_page_deactivate(fs->m);
2120                                                 vm_page_wakeup(fs->m);
2121                                         } else {
2122                                                 vnode_pager_freepage(fs->m);
2123                                         }
2124                                         fs->m = NULL;
2125                                 }
2126                                 vm_object_pip_wakeup(fs->first_object);
2127                                 vm_object_chain_release_all(fs->first_object,
2128                                                             fs->object);
2129                                 if (fs->object != fs->first_object)
2130                                         vm_object_drop(fs->object);
2131                                 unlock_and_deallocate(fs);
2132                                 if (rv == VM_PAGER_ERROR)
2133                                         return (KERN_FAILURE);
2134                                 else
2135                                         return (KERN_PROTECTION_FAILURE);
2136                                 /* NOT REACHED */
2137                         }
2138                 }
2139
2140                 /*
2141                  * We get here if the object has a default pager (or unwiring)
2142                  * or the pager doesn't have the page.
2143                  *
2144                  * fs->first_m will be used for the COW unless we find a
2145                  * deeper page to be mapped read-only, in which case the
2146                  * unlock*(fs) will free first_m.
2147                  */
2148                 if (fs->object == fs->first_object)
2149                         fs->first_m = fs->m;
2150
2151                 /*
2152                  * Move on to the next object.  The chain lock should prevent
2153                  * the backing_object from getting ripped out from under us.
2154                  *
2155                  * The object lock for the next object is governed by
2156                  * fs->shared.
2157                  */
2158                 if ((next_object = fs->object->backing_object) != NULL) {
2159                         if (fs->shared)
2160                                 vm_object_hold_shared(next_object);
2161                         else
2162                                 vm_object_hold(next_object);
2163                         vm_object_chain_acquire(next_object, fs->shared);
2164                         KKASSERT(next_object == fs->object->backing_object);
2165                         pindex += OFF_TO_IDX(fs->object->backing_object_offset);
2166                 }
2167
2168                 if (next_object == NULL) {
2169                         /*
2170                          * If there's no object left, fill the page in the top
2171                          * object with zeros.
2172                          */
2173                         if (fs->object != fs->first_object) {
2174 #if 0
2175                                 if (fs->first_object->backing_object !=
2176                                     fs->object) {
2177                                         vm_object_hold(fs->first_object->backing_object);
2178                                 }
2179 #endif
2180                                 vm_object_chain_release_all(
2181                                         fs->first_object->backing_object,
2182                                         fs->object);
2183 #if 0
2184                                 if (fs->first_object->backing_object !=
2185                                     fs->object) {
2186                                         vm_object_drop(fs->first_object->backing_object);
2187                                 }
2188 #endif
2189                                 vm_object_pip_wakeup(fs->object);
2190                                 vm_object_drop(fs->object);
2191                                 fs->object = fs->first_object;
2192                                 pindex = first_pindex;
2193                                 fs->m = fs->first_m;
2194                         }
2195                         fs->first_m = NULL;
2196
2197                         /*
2198                          * Zero the page and mark it valid.
2199                          */
2200                         vm_page_zero_fill(fs->m);
2201                         mycpu->gd_cnt.v_zfod++;
2202                         fs->m->valid = VM_PAGE_BITS_ALL;
2203                         break;  /* break to PAGE HAS BEEN FOUND */
2204                 }
2205                 if (fs->object != fs->first_object) {
2206                         vm_object_pip_wakeup(fs->object);
2207                         vm_object_lock_swap();
2208                         vm_object_drop(fs->object);
2209                 }
2210                 KASSERT(fs->object != next_object,
2211                         ("object loop %p", next_object));
2212                 fs->object = next_object;
2213                 vm_object_pip_add(fs->object, 1);
2214         }
2215
2216         /*
2217          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
2218          * is held.]
2219          *
2220          * object still held.
2221          * vm_map may not be locked (determined by fs->lookup_still_valid)
2222          *
2223          * local shared variable may be different from fs->shared.
2224          *
2225          * If the page is being written, but isn't already owned by the
2226          * top-level object, we have to copy it into a new page owned by the
2227          * top-level object.
2228          */
2229         KASSERT((fs->m->busy_count & PBUSY_LOCKED) != 0,
2230                 ("vm_fault: not busy after main loop"));
2231
2232         if (fs->object != fs->first_object) {
2233                 /*
2234                  * We only really need to copy if we want to write it.
2235                  */
2236                 if (fault_type & VM_PROT_WRITE) {
2237                         /*
2238                          * This allows pages to be virtually copied from a
2239                          * backing_object into the first_object, where the
2240                          * backing object has no other refs to it, and cannot
2241                          * gain any more refs.  Instead of a bcopy, we just
2242                          * move the page from the backing object to the
2243                          * first object.  Note that we must mark the page
2244                          * dirty in the first object so that it will go out
2245                          * to swap when needed.
2246                          */
2247                         if (virtual_copy_ok(fs)) {
2248                                 /*
2249                                  * (first_m) and (m) are both busied.  We have
2250                                  * move (m) into (first_m)'s object/pindex
2251                                  * in an atomic fashion, then free (first_m).
2252                                  *
2253                                  * first_object is held so second remove
2254                                  * followed by the rename should wind
2255                                  * up being atomic.  vm_page_free() might
2256                                  * block so we don't do it until after the
2257                                  * rename.
2258                                  */
2259                                 vm_page_protect(fs->first_m, VM_PROT_NONE);
2260                                 vm_page_remove(fs->first_m);
2261                                 vm_page_rename(fs->m, fs->first_object,
2262                                                first_pindex);
2263                                 vm_page_free(fs->first_m);
2264                                 fs->first_m = fs->m;
2265                                 fs->m = NULL;
2266                                 mycpu->gd_cnt.v_cow_optim++;
2267                         } else {
2268                                 /*
2269                                  * Oh, well, lets copy it.
2270                                  *
2271                                  * Why are we unmapping the original page
2272                                  * here?  Well, in short, not all accessors
2273                                  * of user memory go through the pmap.  The
2274                                  * procfs code doesn't have access user memory
2275                                  * via a local pmap, so vm_fault_page*()
2276                                  * can't call pmap_enter().  And the umtx*()
2277                                  * code may modify the COW'd page via a DMAP
2278                                  * or kernel mapping and not via the pmap,
2279                                  * leaving the original page still mapped
2280                                  * read-only into the pmap.
2281                                  *
2282                                  * So we have to remove the page from at
2283                                  * least the current pmap if it is in it.
2284                                  *
2285                                  * We used to just remove it from all pmaps
2286                                  * but that creates inefficiencies on SMP,
2287                                  * particularly for COW program & library
2288                                  * mappings that are concurrently exec'd.
2289                                  * Only remove the page from the current
2290                                  * pmap.
2291                                  */
2292                                 KKASSERT(fs->first_shared == 0);
2293                                 vm_page_copy(fs->m, fs->first_m);
2294                                 /*vm_page_protect(fs->m, VM_PROT_NONE);*/
2295                                 pmap_remove_specific(
2296                                     &curthread->td_lwp->lwp_vmspace->vm_pmap,
2297                                     fs->m);
2298                         }
2299
2300                         /*
2301                          * We no longer need the old page or object.
2302                          */
2303                         if (fs->m)
2304                                 release_page(fs);
2305
2306                         /*
2307                          * We intend to revert to first_object, undo the
2308                          * chain lock through to that.
2309                          */
2310 #if 0
2311                         if (fs->first_object->backing_object != fs->object)
2312                                 vm_object_hold(fs->first_object->backing_object);
2313 #endif
2314                         vm_object_chain_release_all(
2315                                         fs->first_object->backing_object,
2316                                         fs->object);
2317 #if 0
2318                         if (fs->first_object->backing_object != fs->object)
2319                                 vm_object_drop(fs->first_object->backing_object);
2320 #endif
2321
2322                         /*
2323                          * fs->object != fs->first_object due to above
2324                          * conditional
2325                          */
2326                         vm_object_pip_wakeup(fs->object);
2327                         vm_object_drop(fs->object);
2328
2329                         /*
2330                          * Only use the new page below...
2331                          */
2332                         mycpu->gd_cnt.v_cow_faults++;
2333                         fs->m = fs->first_m;
2334                         fs->object = fs->first_object;
2335                         pindex = first_pindex;
2336                 } else {
2337                         /*
2338                          * If it wasn't a write fault avoid having to copy
2339                          * the page by mapping it read-only.
2340                          */
2341                         fs->prot &= ~VM_PROT_WRITE;
2342                 }
2343         }
2344
2345         /*
2346          * Relock the map if necessary, then check the generation count.
2347          * relock_map() will update fs->timestamp to account for the
2348          * relocking if necessary.
2349          *
2350          * If the count has changed after relocking then all sorts of
2351          * crap may have happened and we have to retry.
2352          *
2353          * NOTE: The relock_map() can fail due to a deadlock against
2354          *       the vm_page we are holding BUSY.
2355          */
2356         if (fs->lookup_still_valid == FALSE && fs->map) {
2357                 if (relock_map(fs) ||
2358                     fs->map->timestamp != fs->map_generation) {
2359                         release_page(fs);
2360                         vm_object_pip_wakeup(fs->first_object);
2361                         vm_object_chain_release_all(fs->first_object,
2362                                                     fs->object);
2363                         if (fs->object != fs->first_object)
2364                                 vm_object_drop(fs->object);
2365                         unlock_and_deallocate(fs);
2366                         return (KERN_TRY_AGAIN);
2367                 }
2368         }
2369
2370         /*
2371          * If the fault is a write, we know that this page is being
2372          * written NOW so dirty it explicitly to save on pmap_is_modified()
2373          * calls later.
2374          *
2375          * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
2376          * if the page is already dirty to prevent data written with
2377          * the expectation of being synced from not being synced.
2378          * Likewise if this entry does not request NOSYNC then make
2379          * sure the page isn't marked NOSYNC.  Applications sharing
2380          * data should use the same flags to avoid ping ponging.
2381          *
2382          * Also tell the backing pager, if any, that it should remove
2383          * any swap backing since the page is now dirty.
2384          */
2385         vm_page_activate(fs->m);
2386         if (fs->prot & VM_PROT_WRITE) {
2387                 vm_object_set_writeable_dirty(fs->m->object);
2388                 vm_set_nosync(fs->m, fs->entry);
2389                 if (fs->fault_flags & VM_FAULT_DIRTY) {
2390                         vm_page_dirty(fs->m);
2391                         if (fs->m->flags & PG_SWAPPED) {
2392                                 /*
2393                                  * If the page is swapped out we have to call
2394                                  * swap_pager_unswapped() which requires an
2395                                  * exclusive object lock.  If we are shared,
2396                                  * we must clear the shared flag and retry.
2397                                  */
2398                                 if ((fs->object == fs->first_object &&
2399                                      fs->first_shared) ||
2400                                     (fs->object != fs->first_object &&
2401                                      fs->shared)) {
2402                                         vm_page_wakeup(fs->m);
2403                                         fs->m = NULL;
2404                                         if (fs->object == fs->first_object)
2405                                                 fs->first_shared = 0;
2406                                         else
2407                                                 fs->shared = 0;
2408                                         vm_object_pip_wakeup(fs->first_object);
2409                                         vm_object_chain_release_all(
2410                                                 fs->first_object, fs->object);
2411                                         if (fs->object != fs->first_object)
2412                                                 vm_object_drop(fs->object);
2413                                         unlock_and_deallocate(fs);
2414                                         return (KERN_TRY_AGAIN);
2415                                 }
2416                                 swap_pager_unswapped(fs->m);
2417                         }
2418                 }
2419         }
2420
2421         vm_object_pip_wakeup(fs->first_object);
2422         vm_object_chain_release_all(fs->first_object, fs->object);
2423         if (fs->object != fs->first_object)
2424                 vm_object_drop(fs->object);
2425
2426         /*
2427          * Page had better still be busy.  We are still locked up and
2428          * fs->object will have another PIP reference if it is not equal
2429          * to fs->first_object.
2430          */
2431         KASSERT(fs->m->busy_count & PBUSY_LOCKED,
2432                 ("vm_fault: page %p not busy!", fs->m));
2433
2434         /*
2435          * Sanity check: page must be completely valid or it is not fit to
2436          * map into user space.  vm_pager_get_pages() ensures this.
2437          */
2438         if (fs->m->valid != VM_PAGE_BITS_ALL) {
2439                 vm_page_zero_invalid(fs->m, TRUE);
2440                 kprintf("Warning: page %p partially invalid on fault\n", fs->m);
2441         }
2442
2443         return (KERN_SUCCESS);
2444 }
2445
2446 /*
2447  * Wire down a range of virtual addresses in a map.  The entry in question
2448  * should be marked in-transition and the map must be locked.  We must
2449  * release the map temporarily while faulting-in the page to avoid a
2450  * deadlock.  Note that the entry may be clipped while we are blocked but
2451  * will never be freed.
2452  *
2453  * No requirements.
2454  */
2455 int
2456 vm_fault_wire(vm_map_t map, vm_map_entry_t entry,
2457               boolean_t user_wire, int kmflags)
2458 {
2459         boolean_t fictitious;
2460         vm_offset_t start;
2461         vm_offset_t end;
2462         vm_offset_t va;
2463         pmap_t pmap;
2464         int rv;
2465         int wire_prot;
2466         int fault_flags;
2467         vm_page_t m;
2468
2469         if (user_wire) {
2470                 wire_prot = VM_PROT_READ;
2471                 fault_flags = VM_FAULT_USER_WIRE;
2472         } else {
2473                 wire_prot = VM_PROT_READ | VM_PROT_WRITE;
2474                 fault_flags = VM_FAULT_CHANGE_WIRING;
2475         }
2476         if (kmflags & KM_NOTLBSYNC)
2477                 wire_prot |= VM_PROT_NOSYNC;
2478
2479         pmap = vm_map_pmap(map);
2480         start = entry->start;
2481         end = entry->end;
2482
2483         switch(entry->maptype) {
2484         case VM_MAPTYPE_NORMAL:
2485         case VM_MAPTYPE_VPAGETABLE:
2486                 fictitious = entry->object.vm_object &&
2487                             ((entry->object.vm_object->type == OBJT_DEVICE) ||
2488                              (entry->object.vm_object->type == OBJT_MGTDEVICE));
2489                 break;
2490         case VM_MAPTYPE_UKSMAP:
2491                 fictitious = TRUE;
2492                 break;
2493         default:
2494                 fictitious = FALSE;
2495                 break;
2496         }
2497
2498         if (entry->eflags & MAP_ENTRY_KSTACK)
2499                 start += PAGE_SIZE;
2500         map->timestamp++;
2501         vm_map_unlock(map);
2502
2503         /*
2504          * We simulate a fault to get the page and enter it in the physical
2505          * map.
2506          */
2507         for (va = start; va < end; va += PAGE_SIZE) {
2508                 rv = vm_fault(map, va, wire_prot, fault_flags);
2509                 if (rv) {
2510                         while (va > start) {
2511                                 va -= PAGE_SIZE;
2512                                 m = pmap_unwire(pmap, va);
2513                                 if (m && !fictitious) {
2514                                         vm_page_busy_wait(m, FALSE, "vmwrpg");
2515                                         vm_page_unwire(m, 1);
2516                                         vm_page_wakeup(m);
2517                                 }
2518                         }
2519                         goto done;
2520                 }
2521         }
2522         rv = KERN_SUCCESS;
2523 done:
2524         vm_map_lock(map);
2525
2526         return (rv);
2527 }
2528
2529 /*
2530  * Unwire a range of virtual addresses in a map.  The map should be
2531  * locked.
2532  */
2533 void
2534 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
2535 {
2536         boolean_t fictitious;
2537         vm_offset_t start;
2538         vm_offset_t end;
2539         vm_offset_t va;
2540         pmap_t pmap;
2541         vm_page_t m;
2542
2543         pmap = vm_map_pmap(map);
2544         start = entry->start;
2545         end = entry->end;
2546         fictitious = entry->object.vm_object &&
2547                         ((entry->object.vm_object->type == OBJT_DEVICE) ||
2548                          (entry->object.vm_object->type == OBJT_MGTDEVICE));
2549         if (entry->eflags & MAP_ENTRY_KSTACK)
2550                 start += PAGE_SIZE;
2551
2552         /*
2553          * Since the pages are wired down, we must be able to get their
2554          * mappings from the physical map system.
2555          */
2556         for (va = start; va < end; va += PAGE_SIZE) {
2557                 m = pmap_unwire(pmap, va);
2558                 if (m && !fictitious) {
2559                         vm_page_busy_wait(m, FALSE, "vmwrpg");
2560                         vm_page_unwire(m, 1);
2561                         vm_page_wakeup(m);
2562                 }
2563         }
2564 }
2565
2566 /*
2567  * Copy all of the pages from a wired-down map entry to another.
2568  *
2569  * The source and destination maps must be locked for write.
2570  * The source and destination maps token must be held
2571  * The source map entry must be wired down (or be a sharing map
2572  * entry corresponding to a main map entry that is wired down).
2573  *
2574  * No other requirements.
2575  *
2576  * XXX do segment optimization
2577  */
2578 void
2579 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
2580                     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
2581 {
2582         vm_object_t dst_object;
2583         vm_object_t src_object;
2584         vm_ooffset_t dst_offset;
2585         vm_ooffset_t src_offset;
2586         vm_prot_t prot;
2587         vm_offset_t vaddr;
2588         vm_page_t dst_m;
2589         vm_page_t src_m;
2590
2591         src_object = src_entry->object.vm_object;
2592         src_offset = src_entry->offset;
2593
2594         /*
2595          * Create the top-level object for the destination entry. (Doesn't
2596          * actually shadow anything - we copy the pages directly.)
2597          */
2598         vm_map_entry_allocate_object(dst_entry);
2599         dst_object = dst_entry->object.vm_object;
2600
2601         prot = dst_entry->max_protection;
2602
2603         /*
2604          * Loop through all of the pages in the entry's range, copying each
2605          * one from the source object (it should be there) to the destination
2606          * object.
2607          */
2608         vm_object_hold(src_object);
2609         vm_object_hold(dst_object);
2610
2611         for (vaddr = dst_entry->start, dst_offset = 0;
2612              vaddr < dst_entry->end;
2613              vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
2614
2615                 /*
2616                  * Allocate a page in the destination object
2617                  */
2618                 do {
2619                         dst_m = vm_page_alloc(dst_object,
2620                                               OFF_TO_IDX(dst_offset),
2621                                               VM_ALLOC_NORMAL);
2622                         if (dst_m == NULL) {
2623                                 vm_wait(0);
2624                         }
2625                 } while (dst_m == NULL);
2626
2627                 /*
2628                  * Find the page in the source object, and copy it in.
2629                  * (Because the source is wired down, the page will be in
2630                  * memory.)
2631                  */
2632                 src_m = vm_page_lookup(src_object,
2633                                        OFF_TO_IDX(dst_offset + src_offset));
2634                 if (src_m == NULL)
2635                         panic("vm_fault_copy_wired: page missing");
2636
2637                 vm_page_copy(src_m, dst_m);
2638
2639                 /*
2640                  * Enter it in the pmap...
2641                  */
2642                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE, dst_entry);
2643
2644                 /*
2645                  * Mark it no longer busy, and put it on the active list.
2646                  */
2647                 vm_page_activate(dst_m);
2648                 vm_page_wakeup(dst_m);
2649         }
2650         vm_object_drop(dst_object);
2651         vm_object_drop(src_object);
2652 }
2653
2654 #if 0
2655
2656 /*
2657  * This routine checks around the requested page for other pages that
2658  * might be able to be faulted in.  This routine brackets the viable
2659  * pages for the pages to be paged in.
2660  *
2661  * Inputs:
2662  *      m, rbehind, rahead
2663  *
2664  * Outputs:
2665  *  marray (array of vm_page_t), reqpage (index of requested page)
2666  *
2667  * Return value:
2668  *  number of pages in marray
2669  */
2670 static int
2671 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
2672                           vm_page_t *marray, int *reqpage)
2673 {
2674         int i,j;
2675         vm_object_t object;
2676         vm_pindex_t pindex, startpindex, endpindex, tpindex;
2677         vm_page_t rtm;
2678         int cbehind, cahead;
2679
2680         object = m->object;
2681         pindex = m->pindex;
2682
2683         /*
2684          * we don't fault-ahead for device pager
2685          */
2686         if ((object->type == OBJT_DEVICE) ||
2687             (object->type == OBJT_MGTDEVICE)) {
2688                 *reqpage = 0;
2689                 marray[0] = m;
2690                 return 1;
2691         }
2692
2693         /*
2694          * if the requested page is not available, then give up now
2695          */
2696         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
2697                 *reqpage = 0;   /* not used by caller, fix compiler warn */
2698                 return 0;
2699         }
2700
2701         if ((cbehind == 0) && (cahead == 0)) {
2702                 *reqpage = 0;
2703                 marray[0] = m;
2704                 return 1;
2705         }
2706
2707         if (rahead > cahead) {
2708                 rahead = cahead;
2709         }
2710
2711         if (rbehind > cbehind) {
2712                 rbehind = cbehind;
2713         }
2714
2715         /*
2716          * Do not do any readahead if we have insufficient free memory.
2717          *
2718          * XXX code was broken disabled before and has instability
2719          * with this conditonal fixed, so shortcut for now.
2720          */
2721         if (burst_fault == 0 || vm_page_count_severe()) {
2722                 marray[0] = m;
2723                 *reqpage = 0;
2724                 return 1;
2725         }
2726
2727         /*
2728          * scan backward for the read behind pages -- in memory
2729          *
2730          * Assume that if the page is not found an interrupt will not
2731          * create it.  Theoretically interrupts can only remove (busy)
2732          * pages, not create new associations.
2733          */
2734         if (pindex > 0) {
2735                 if (rbehind > pindex) {
2736                         rbehind = pindex;
2737                         startpindex = 0;
2738                 } else {
2739                         startpindex = pindex - rbehind;
2740                 }
2741
2742                 vm_object_hold(object);
2743                 for (tpindex = pindex; tpindex > startpindex; --tpindex) {
2744                         if (vm_page_lookup(object, tpindex - 1))
2745                                 break;
2746                 }
2747
2748                 i = 0;
2749                 while (tpindex < pindex) {
2750                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2751                                                              VM_ALLOC_NULL_OK);
2752                         if (rtm == NULL) {
2753                                 for (j = 0; j < i; j++) {
2754                                         vm_page_free(marray[j]);
2755                                 }
2756                                 vm_object_drop(object);
2757                                 marray[0] = m;
2758                                 *reqpage = 0;
2759                                 return 1;
2760                         }
2761                         marray[i] = rtm;
2762                         ++i;
2763                         ++tpindex;
2764                 }
2765                 vm_object_drop(object);
2766         } else {
2767                 i = 0;
2768         }
2769
2770         /*
2771          * Assign requested page
2772          */
2773         marray[i] = m;
2774         *reqpage = i;
2775         ++i;
2776
2777         /*
2778          * Scan forwards for read-ahead pages
2779          */
2780         tpindex = pindex + 1;
2781         endpindex = tpindex + rahead;
2782         if (endpindex > object->size)
2783                 endpindex = object->size;
2784
2785         vm_object_hold(object);
2786         while (tpindex < endpindex) {
2787                 if (vm_page_lookup(object, tpindex))
2788                         break;
2789                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2790                                                      VM_ALLOC_NULL_OK);
2791                 if (rtm == NULL)
2792                         break;
2793                 marray[i] = rtm;
2794                 ++i;
2795                 ++tpindex;
2796         }
2797         vm_object_drop(object);
2798
2799         return (i);
2800 }
2801
2802 #endif
2803
2804 /*
2805  * vm_prefault() provides a quick way of clustering pagefaults into a
2806  * processes address space.  It is a "cousin" of pmap_object_init_pt,
2807  * except it runs at page fault time instead of mmap time.
2808  *
2809  * vm.fast_fault        Enables pre-faulting zero-fill pages
2810  *
2811  * vm.prefault_pages    Number of pages (1/2 negative, 1/2 positive) to
2812  *                      prefault.  Scan stops in either direction when
2813  *                      a page is found to already exist.
2814  *
2815  * This code used to be per-platform pmap_prefault().  It is now
2816  * machine-independent and enhanced to also pre-fault zero-fill pages
2817  * (see vm.fast_fault) as well as make them writable, which greatly
2818  * reduces the number of page faults programs incur.
2819  *
2820  * Application performance when pre-faulting zero-fill pages is heavily
2821  * dependent on the application.  Very tiny applications like /bin/echo
2822  * lose a little performance while applications of any appreciable size
2823  * gain performance.  Prefaulting multiple pages also reduces SMP
2824  * congestion and can improve SMP performance significantly.
2825  *
2826  * NOTE!  prot may allow writing but this only applies to the top level
2827  *        object.  If we wind up mapping a page extracted from a backing
2828  *        object we have to make sure it is read-only.
2829  *
2830  * NOTE!  The caller has already handled any COW operations on the
2831  *        vm_map_entry via the normal fault code.  Do NOT call this
2832  *        shortcut unless the normal fault code has run on this entry.
2833  *
2834  * The related map must be locked.
2835  * No other requirements.
2836  */
2837 __read_mostly static int vm_prefault_pages = 8;
2838 SYSCTL_INT(_vm, OID_AUTO, prefault_pages, CTLFLAG_RW, &vm_prefault_pages, 0,
2839            "Maximum number of pages to pre-fault");
2840 __read_mostly static int vm_fast_fault = 1;
2841 SYSCTL_INT(_vm, OID_AUTO, fast_fault, CTLFLAG_RW, &vm_fast_fault, 0,
2842            "Burst fault zero-fill regions");
2843
2844 /*
2845  * Set PG_NOSYNC if the map entry indicates so, but only if the page
2846  * is not already dirty by other means.  This will prevent passive
2847  * filesystem syncing as well as 'sync' from writing out the page.
2848  */
2849 static void
2850 vm_set_nosync(vm_page_t m, vm_map_entry_t entry)
2851 {
2852         if (entry->eflags & MAP_ENTRY_NOSYNC) {
2853                 if (m->dirty == 0)
2854                         vm_page_flag_set(m, PG_NOSYNC);
2855         } else {
2856                 vm_page_flag_clear(m, PG_NOSYNC);
2857         }
2858 }
2859
2860 static void
2861 vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot,
2862             int fault_flags)
2863 {
2864         struct lwp *lp;
2865         vm_page_t m;
2866         vm_offset_t addr;
2867         vm_pindex_t index;
2868         vm_pindex_t pindex;
2869         vm_object_t object;
2870         int pprot;
2871         int i;
2872         int noneg;
2873         int nopos;
2874         int maxpages;
2875
2876         /*
2877          * Get stable max count value, disabled if set to 0
2878          */
2879         maxpages = vm_prefault_pages;
2880         cpu_ccfence();
2881         if (maxpages <= 0)
2882                 return;
2883
2884         /*
2885          * We do not currently prefault mappings that use virtual page
2886          * tables.  We do not prefault foreign pmaps.
2887          */
2888         if (entry->maptype != VM_MAPTYPE_NORMAL)
2889                 return;
2890         lp = curthread->td_lwp;
2891         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2892                 return;
2893
2894         /*
2895          * Limit pre-fault count to 1024 pages.
2896          */
2897         if (maxpages > 1024)
2898                 maxpages = 1024;
2899
2900         object = entry->object.vm_object;
2901         KKASSERT(object != NULL);
2902         KKASSERT(object == entry->object.vm_object);
2903
2904         /*
2905          * NOTE: VM_FAULT_DIRTY allowed later so must hold object exclusively
2906          *       now (or do something more complex XXX).
2907          */
2908         vm_object_hold(object);
2909         vm_object_chain_acquire(object, 0);
2910
2911         noneg = 0;
2912         nopos = 0;
2913         for (i = 0; i < maxpages; ++i) {
2914                 vm_object_t lobject;
2915                 vm_object_t nobject;
2916                 int allocated = 0;
2917                 int error;
2918
2919                 /*
2920                  * This can eat a lot of time on a heavily contended
2921                  * machine so yield on the tick if needed.
2922                  */
2923                 if ((i & 7) == 7)
2924                         lwkt_yield();
2925
2926                 /*
2927                  * Calculate the page to pre-fault, stopping the scan in
2928                  * each direction separately if the limit is reached.
2929                  */
2930                 if (i & 1) {
2931                         if (noneg)
2932                                 continue;
2933                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2934                 } else {
2935                         if (nopos)
2936                                 continue;
2937                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2938                 }
2939                 if (addr < entry->start) {
2940                         noneg = 1;
2941                         if (noneg && nopos)
2942                                 break;
2943                         continue;
2944                 }
2945                 if (addr >= entry->end) {
2946                         nopos = 1;
2947                         if (noneg && nopos)
2948                                 break;
2949                         continue;
2950                 }
2951
2952                 /*
2953                  * Skip pages already mapped, and stop scanning in that
2954                  * direction.  When the scan terminates in both directions
2955                  * we are done.
2956                  */
2957                 if (pmap_prefault_ok(pmap, addr) == 0) {
2958                         if (i & 1)
2959                                 noneg = 1;
2960                         else
2961                                 nopos = 1;
2962                         if (noneg && nopos)
2963                                 break;
2964                         continue;
2965                 }
2966
2967                 /*
2968                  * Follow the VM object chain to obtain the page to be mapped
2969                  * into the pmap.
2970                  *
2971                  * If we reach the terminal object without finding a page
2972                  * and we determine it would be advantageous, then allocate
2973                  * a zero-fill page for the base object.  The base object
2974                  * is guaranteed to be OBJT_DEFAULT for this case.
2975                  *
2976                  * In order to not have to check the pager via *haspage*()
2977                  * we stop if any non-default object is encountered.  e.g.
2978                  * a vnode or swap object would stop the loop.
2979                  */
2980                 index = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2981                 lobject = object;
2982                 pindex = index;
2983                 pprot = prot;
2984
2985                 KKASSERT(lobject == entry->object.vm_object);
2986                 /*vm_object_hold(lobject); implied */
2987
2988                 while ((m = vm_page_lookup_busy_try(lobject, pindex,
2989                                                     TRUE, &error)) == NULL) {
2990                         if (lobject->type != OBJT_DEFAULT)
2991                                 break;
2992                         if (lobject->backing_object == NULL) {
2993                                 if (vm_fast_fault == 0)
2994                                         break;
2995                                 if ((prot & VM_PROT_WRITE) == 0 ||
2996                                     vm_page_count_min(0)) {
2997                                         break;
2998                                 }
2999
3000                                 /*
3001                                  * NOTE: Allocated from base object
3002                                  */
3003                                 m = vm_page_alloc(object, index,
3004                                                   VM_ALLOC_NORMAL |
3005                                                   VM_ALLOC_ZERO |
3006                                                   VM_ALLOC_USE_GD |
3007                                                   VM_ALLOC_NULL_OK);
3008                                 if (m == NULL)
3009                                         break;
3010                                 allocated = 1;
3011                                 pprot = prot;
3012                                 /* lobject = object .. not needed */
3013                                 break;
3014                         }
3015                         if (lobject->backing_object_offset & PAGE_MASK)
3016                                 break;
3017                         nobject = lobject->backing_object;
3018                         vm_object_hold(nobject);
3019                         KKASSERT(nobject == lobject->backing_object);
3020                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
3021                         if (lobject != object) {
3022                                 vm_object_lock_swap();
3023                                 vm_object_drop(lobject);
3024                         }
3025                         lobject = nobject;
3026                         pprot &= ~VM_PROT_WRITE;
3027                         vm_object_chain_acquire(lobject, 0);
3028                 }
3029
3030                 /*
3031                  * NOTE: A non-NULL (m) will be associated with lobject if
3032                  *       it was found there, otherwise it is probably a
3033                  *       zero-fill page associated with the base object.
3034                  *
3035                  * Give-up if no page is available.
3036                  */
3037                 if (m == NULL) {
3038                         if (lobject != object) {
3039 #if 0
3040                                 if (object->backing_object != lobject)
3041                                         vm_object_hold(object->backing_object);
3042 #endif
3043                                 vm_object_chain_release_all(
3044                                         object->backing_object, lobject);
3045 #if 0
3046                                 if (object->backing_object != lobject)
3047                                         vm_object_drop(object->backing_object);
3048 #endif
3049                                 vm_object_drop(lobject);
3050                         }
3051                         break;
3052                 }
3053
3054                 /*
3055                  * The object must be marked dirty if we are mapping a
3056                  * writable page.  m->object is either lobject or object,
3057                  * both of which are still held.  Do this before we
3058                  * potentially drop the object.
3059                  */
3060                 if (pprot & VM_PROT_WRITE)
3061                         vm_object_set_writeable_dirty(m->object);
3062
3063                 /*
3064                  * Do not conditionalize on PG_RAM.  If pages are present in
3065                  * the VM system we assume optimal caching.  If caching is
3066                  * not optimal the I/O gravy train will be restarted when we
3067                  * hit an unavailable page.  We do not want to try to restart
3068                  * the gravy train now because we really don't know how much
3069                  * of the object has been cached.  The cost for restarting
3070                  * the gravy train should be low (since accesses will likely
3071                  * be I/O bound anyway).
3072                  */
3073                 if (lobject != object) {
3074 #if 0
3075                         if (object->backing_object != lobject)
3076                                 vm_object_hold(object->backing_object);
3077 #endif
3078                         vm_object_chain_release_all(object->backing_object,
3079                                                     lobject);
3080 #if 0
3081                         if (object->backing_object != lobject)
3082                                 vm_object_drop(object->backing_object);
3083 #endif
3084                         vm_object_drop(lobject);
3085                 }
3086
3087                 /*
3088                  * Enter the page into the pmap if appropriate.  If we had
3089                  * allocated the page we have to place it on a queue.  If not
3090                  * we just have to make sure it isn't on the cache queue
3091                  * (pages on the cache queue are not allowed to be mapped).
3092                  */
3093                 if (allocated) {
3094                         /*
3095                          * Page must be zerod.
3096                          */
3097                         vm_page_zero_fill(m);
3098                         mycpu->gd_cnt.v_zfod++;
3099                         m->valid = VM_PAGE_BITS_ALL;
3100
3101                         /*
3102                          * Handle dirty page case
3103                          */
3104                         if (pprot & VM_PROT_WRITE)
3105                                 vm_set_nosync(m, entry);
3106                         pmap_enter(pmap, addr, m, pprot, 0, entry);
3107                         mycpu->gd_cnt.v_vm_faults++;
3108                         if (curthread->td_lwp)
3109                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
3110                         vm_page_deactivate(m);
3111                         if (pprot & VM_PROT_WRITE) {
3112                                 /*vm_object_set_writeable_dirty(m->object);*/
3113                                 vm_set_nosync(m, entry);
3114                                 if (fault_flags & VM_FAULT_DIRTY) {
3115                                         vm_page_dirty(m);
3116                                         /*XXX*/
3117                                         swap_pager_unswapped(m);
3118                                 }
3119                         }
3120                         vm_page_wakeup(m);
3121                 } else if (error) {
3122                         /* couldn't busy page, no wakeup */
3123                 } else if (
3124                     ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
3125                     (m->flags & PG_FICTITIOUS) == 0) {
3126                         /*
3127                          * A fully valid page not undergoing soft I/O can
3128                          * be immediately entered into the pmap.
3129                          */
3130                         if ((m->queue - m->pc) == PQ_CACHE)
3131                                 vm_page_deactivate(m);
3132                         if (pprot & VM_PROT_WRITE) {
3133                                 /*vm_object_set_writeable_dirty(m->object);*/
3134                                 vm_set_nosync(m, entry);
3135                                 if (fault_flags & VM_FAULT_DIRTY) {
3136                                         vm_page_dirty(m);
3137                                         /*XXX*/
3138                                         swap_pager_unswapped(m);
3139                                 }
3140                         }
3141                         if (pprot & VM_PROT_WRITE)
3142                                 vm_set_nosync(m, entry);
3143                         pmap_enter(pmap, addr, m, pprot, 0, entry);
3144                         mycpu->gd_cnt.v_vm_faults++;
3145                         if (curthread->td_lwp)
3146                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
3147                         vm_page_wakeup(m);
3148                 } else {
3149                         vm_page_wakeup(m);
3150                 }
3151         }
3152         vm_object_chain_release(object);
3153         vm_object_drop(object);
3154 }
3155
3156 /*
3157  * Object can be held shared
3158  */
3159 static void
3160 vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
3161                   vm_map_entry_t entry, int prot, int fault_flags)
3162 {
3163         struct lwp *lp;
3164         vm_page_t m;
3165         vm_offset_t addr;
3166         vm_pindex_t pindex;
3167         vm_object_t object;
3168         int i;
3169         int noneg;
3170         int nopos;
3171         int maxpages;
3172
3173         /*
3174          * Get stable max count value, disabled if set to 0
3175          */
3176         maxpages = vm_prefault_pages;
3177         cpu_ccfence();
3178         if (maxpages <= 0)
3179                 return;
3180
3181         /*
3182          * We do not currently prefault mappings that use virtual page
3183          * tables.  We do not prefault foreign pmaps.
3184          */
3185         if (entry->maptype != VM_MAPTYPE_NORMAL)
3186                 return;
3187         lp = curthread->td_lwp;
3188         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
3189                 return;
3190         object = entry->object.vm_object;
3191         if (object->backing_object != NULL)
3192                 return;
3193         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
3194
3195         /*
3196          * Limit pre-fault count to 1024 pages.
3197          */
3198         if (maxpages > 1024)
3199                 maxpages = 1024;
3200
3201         noneg = 0;
3202         nopos = 0;
3203         for (i = 0; i < maxpages; ++i) {
3204                 int error;
3205
3206                 /*
3207                  * Calculate the page to pre-fault, stopping the scan in
3208                  * each direction separately if the limit is reached.
3209                  */
3210                 if (i & 1) {
3211                         if (noneg)
3212                                 continue;
3213                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
3214                 } else {
3215                         if (nopos)
3216                                 continue;
3217                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
3218                 }
3219                 if (addr < entry->start) {
3220                         noneg = 1;
3221                         if (noneg && nopos)
3222                                 break;
3223                         continue;
3224                 }
3225                 if (addr >= entry->end) {
3226                         nopos = 1;
3227                         if (noneg && nopos)
3228                                 break;
3229                         continue;
3230                 }
3231
3232                 /*
3233                  * Follow the VM object chain to obtain the page to be mapped
3234                  * into the pmap.  This version of the prefault code only
3235                  * works with terminal objects.
3236                  *
3237                  * The page must already exist.  If we encounter a problem
3238                  * we stop here.
3239                  *
3240                  * WARNING!  We cannot call swap_pager_unswapped() or insert
3241                  *           a new vm_page with a shared token.
3242                  */
3243                 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
3244
3245                 /*
3246                  * Skip pages already mapped, and stop scanning in that
3247                  * direction.  When the scan terminates in both directions
3248                  * we are done.
3249                  */
3250                 if (pmap_prefault_ok(pmap, addr) == 0) {
3251                         if (i & 1)
3252                                 noneg = 1;
3253                         else
3254                                 nopos = 1;
3255                         if (noneg && nopos)
3256                                 break;
3257                         continue;
3258                 }
3259
3260                 /*
3261                  * Shortcut the read-only mapping case using the far more
3262                  * efficient vm_page_lookup_sbusy_try() function.  This
3263                  * allows us to acquire the page soft-busied only which
3264                  * is especially nice for concurrent execs of the same
3265                  * program.
3266                  *
3267                  * The lookup function also validates page suitability
3268                  * (all valid bits set, and not fictitious).
3269                  *
3270                  * If the page is in PQ_CACHE we have to fall-through
3271                  * and hard-busy it so we can move it out of PQ_CACHE.
3272                  */
3273                 if ((prot & VM_PROT_WRITE) == 0) {
3274                         m = vm_page_lookup_sbusy_try(object, pindex,
3275                                                      0, PAGE_SIZE);
3276                         if (m == NULL)
3277                                 break;
3278                         if ((m->queue - m->pc) != PQ_CACHE) {
3279                                 pmap_enter(pmap, addr, m, prot, 0, entry);
3280                                 mycpu->gd_cnt.v_vm_faults++;
3281                                 if (curthread->td_lwp)
3282                                         ++curthread->td_lwp->lwp_ru.ru_minflt;
3283                                 vm_page_sbusy_drop(m);
3284                                 continue;
3285                         }
3286                         vm_page_sbusy_drop(m);
3287                 }
3288
3289                 /*
3290                  * Fallback to normal vm_page lookup code.  This code
3291                  * hard-busies the page.  Not only that, but the page
3292                  * can remain in that state for a significant period
3293                  * time due to pmap_enter()'s overhead.
3294                  */
3295                 m = vm_page_lookup_busy_try(object, pindex, TRUE, &error);
3296                 if (m == NULL || error)
3297                         break;
3298
3299                 /*
3300                  * Stop if the page cannot be trivially entered into the
3301                  * pmap.
3302                  */
3303                 if (((m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) ||
3304                     (m->flags & PG_FICTITIOUS) ||
3305                     ((m->flags & PG_SWAPPED) &&
3306                      (prot & VM_PROT_WRITE) &&
3307                      (fault_flags & VM_FAULT_DIRTY))) {
3308                         vm_page_wakeup(m);
3309                         break;
3310                 }
3311
3312                 /*
3313                  * Enter the page into the pmap.  The object might be held
3314                  * shared so we can't do any (serious) modifying operation
3315                  * on it.
3316                  */
3317                 if ((m->queue - m->pc) == PQ_CACHE)
3318                         vm_page_deactivate(m);
3319                 if (prot & VM_PROT_WRITE) {
3320                         vm_object_set_writeable_dirty(m->object);
3321                         vm_set_nosync(m, entry);
3322                         if (fault_flags & VM_FAULT_DIRTY) {
3323                                 vm_page_dirty(m);
3324                                 /* can't happeen due to conditional above */
3325                                 /* swap_pager_unswapped(m); */
3326                         }
3327                 }
3328                 pmap_enter(pmap, addr, m, prot, 0, entry);
3329                 mycpu->gd_cnt.v_vm_faults++;
3330                 if (curthread->td_lwp)
3331                         ++curthread->td_lwp->lwp_ru.ru_minflt;
3332                 vm_page_wakeup(m);
3333         }
3334 }