kernel - Fix VM deadlock in optimized file read path
[dragonfly.git] / sys / vm / vm_fault.c
1 /*
2  * Copyright (c) 2003-2014 The DragonFly Project.  All rights reserved.
3  *
4  * This code is derived from software contributed to The DragonFly Project
5  * by Matthew Dillon <dillon@backplane.com>
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in
15  *    the documentation and/or other materials provided with the
16  *    distribution.
17  * 3. Neither the name of The DragonFly Project nor the names of its
18  *    contributors may be used to endorse or promote products derived
19  *    from this software without specific, prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32  * SUCH DAMAGE.
33  *
34  * ---
35  *
36  * Copyright (c) 1991, 1993
37  *      The Regents of the University of California.  All rights reserved.
38  * Copyright (c) 1994 John S. Dyson
39  * All rights reserved.
40  * Copyright (c) 1994 David Greenman
41  * All rights reserved.
42  *
43  *
44  * This code is derived from software contributed to Berkeley by
45  * The Mach Operating System project at Carnegie-Mellon University.
46  *
47  * Redistribution and use in source and binary forms, with or without
48  * modification, are permitted provided that the following conditions
49  * are met:
50  * 1. Redistributions of source code must retain the above copyright
51  *    notice, this list of conditions and the following disclaimer.
52  * 2. Redistributions in binary form must reproduce the above copyright
53  *    notice, this list of conditions and the following disclaimer in the
54  *    documentation and/or other materials provided with the distribution.
55  * 3. Neither the name of the University nor the names of its contributors
56  *    may be used to endorse or promote products derived from this software
57  *    without specific prior written permission.
58  *
59  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
60  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
61  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
62  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
63  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
64  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
65  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69  * SUCH DAMAGE.
70  *
71  * ---
72  *
73  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
74  * All rights reserved.
75  *
76  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
77  *
78  * Permission to use, copy, modify and distribute this software and
79  * its documentation is hereby granted, provided that both the copyright
80  * notice and this permission notice appear in all copies of the
81  * software, derivative works or modified versions, and any portions
82  * thereof, and that both notices appear in supporting documentation.
83  *
84  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
85  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
86  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
87  *
88  * Carnegie Mellon requests users of this software to return to
89  *
90  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
91  *  School of Computer Science
92  *  Carnegie Mellon University
93  *  Pittsburgh PA 15213-3890
94  *
95  * any improvements or extensions that they make and grant Carnegie the
96  * rights to redistribute these changes.
97  */
98
99 /*
100  *      Page fault handling module.
101  */
102
103 #include <sys/param.h>
104 #include <sys/systm.h>
105 #include <sys/kernel.h>
106 #include <sys/proc.h>
107 #include <sys/vnode.h>
108 #include <sys/resourcevar.h>
109 #include <sys/vmmeter.h>
110 #include <sys/vkernel.h>
111 #include <sys/lock.h>
112 #include <sys/sysctl.h>
113
114 #include <cpu/lwbuf.h>
115
116 #include <vm/vm.h>
117 #include <vm/vm_param.h>
118 #include <vm/pmap.h>
119 #include <vm/vm_map.h>
120 #include <vm/vm_object.h>
121 #include <vm/vm_page.h>
122 #include <vm/vm_pageout.h>
123 #include <vm/vm_kern.h>
124 #include <vm/vm_pager.h>
125 #include <vm/vnode_pager.h>
126 #include <vm/vm_extern.h>
127
128 #include <sys/thread2.h>
129 #include <vm/vm_page2.h>
130
131 struct faultstate {
132         vm_page_t m;
133         vm_object_t object;
134         vm_pindex_t pindex;
135         vm_prot_t prot;
136         vm_page_t first_m;
137         vm_object_t first_object;
138         vm_prot_t first_prot;
139         vm_map_t map;
140         vm_map_entry_t entry;
141         int lookup_still_valid;
142         int hardfault;
143         int fault_flags;
144         int map_generation;
145         int shared;
146         int first_shared;
147         boolean_t wired;
148         struct vnode *vp;
149 };
150
151 static int debug_fault = 0;
152 SYSCTL_INT(_vm, OID_AUTO, debug_fault, CTLFLAG_RW, &debug_fault, 0, "");
153 static int debug_cluster = 0;
154 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
155 int vm_shared_fault = 1;
156 TUNABLE_INT("vm.shared_fault", &vm_shared_fault);
157 SYSCTL_INT(_vm, OID_AUTO, shared_fault, CTLFLAG_RW, &vm_shared_fault, 0,
158            "Allow shared token on vm_object");
159 static long vm_shared_hit = 0;
160 SYSCTL_LONG(_vm, OID_AUTO, shared_hit, CTLFLAG_RW, &vm_shared_hit, 0,
161            "Successful shared faults");
162 static long vm_shared_count = 0;
163 SYSCTL_LONG(_vm, OID_AUTO, shared_count, CTLFLAG_RW, &vm_shared_count, 0,
164            "Shared fault attempts");
165 static long vm_shared_miss = 0;
166 SYSCTL_LONG(_vm, OID_AUTO, shared_miss, CTLFLAG_RW, &vm_shared_miss, 0,
167            "Unsuccessful shared faults");
168
169 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t, int);
170 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *,
171                         vpte_t, int, int);
172 #if 0
173 static int vm_fault_additional_pages (vm_page_t, int, int, vm_page_t *, int *);
174 #endif
175 static void vm_set_nosync(vm_page_t m, vm_map_entry_t entry);
176 static void vm_prefault(pmap_t pmap, vm_offset_t addra,
177                         vm_map_entry_t entry, int prot, int fault_flags);
178 static void vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
179                         vm_map_entry_t entry, int prot, int fault_flags);
180
181 static __inline void
182 release_page(struct faultstate *fs)
183 {
184         vm_page_deactivate(fs->m);
185         vm_page_wakeup(fs->m);
186         fs->m = NULL;
187 }
188
189 /*
190  * NOTE: Once unlocked any cached fs->entry becomes invalid, any reuse
191  *       requires relocking and then checking the timestamp.
192  *
193  * NOTE: vm_map_lock_read() does not bump fs->map->timestamp so we do
194  *       not have to update fs->map_generation here.
195  *
196  * NOTE: This function can fail due to a deadlock against the caller's
197  *       holding of a vm_page BUSY.
198  */
199 static __inline int
200 relock_map(struct faultstate *fs)
201 {
202         int error;
203
204         if (fs->lookup_still_valid == FALSE && fs->map) {
205                 error = vm_map_lock_read_to(fs->map);
206                 if (error == 0)
207                         fs->lookup_still_valid = TRUE;
208         } else {
209                 error = 0;
210         }
211         return error;
212 }
213
214 static __inline void
215 unlock_map(struct faultstate *fs)
216 {
217         if (fs->lookup_still_valid && fs->map) {
218                 vm_map_lookup_done(fs->map, fs->entry, 0);
219                 fs->lookup_still_valid = FALSE;
220         }
221 }
222
223 /*
224  * Clean up after a successful call to vm_fault_object() so another call
225  * to vm_fault_object() can be made.
226  */
227 static void
228 _cleanup_successful_fault(struct faultstate *fs, int relock)
229 {
230         /*
231          * We allocated a junk page for a COW operation that did
232          * not occur, the page must be freed.
233          */
234         if (fs->object != fs->first_object) {
235                 KKASSERT(fs->first_shared == 0);
236                 vm_page_free(fs->first_m);
237                 vm_object_pip_wakeup(fs->object);
238                 fs->first_m = NULL;
239         }
240
241         /*
242          * Reset fs->object.
243          */
244         fs->object = fs->first_object;
245         if (relock && fs->lookup_still_valid == FALSE) {
246                 if (fs->map)
247                         vm_map_lock_read(fs->map);
248                 fs->lookup_still_valid = TRUE;
249         }
250 }
251
252 static void
253 _unlock_things(struct faultstate *fs, int dealloc)
254 {
255         _cleanup_successful_fault(fs, 0);
256         if (dealloc) {
257                 /*vm_object_deallocate(fs->first_object);*/
258                 /*fs->first_object = NULL; drop used later on */
259         }
260         unlock_map(fs); 
261         if (fs->vp != NULL) { 
262                 vput(fs->vp);
263                 fs->vp = NULL;
264         }
265 }
266
267 #define unlock_things(fs) _unlock_things(fs, 0)
268 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
269 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
270
271 /*
272  * TRYPAGER 
273  *
274  * Determine if the pager for the current object *might* contain the page.
275  *
276  * We only need to try the pager if this is not a default object (default
277  * objects are zero-fill and have no real pager), and if we are not taking
278  * a wiring fault or if the FS entry is wired.
279  */
280 #define TRYPAGER(fs)    \
281                 (fs->object->type != OBJT_DEFAULT && \
282                 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) || fs->wired))
283
284 /*
285  * vm_fault:
286  *
287  * Handle a page fault occuring at the given address, requiring the given
288  * permissions, in the map specified.  If successful, the page is inserted
289  * into the associated physical map.
290  *
291  * NOTE: The given address should be truncated to the proper page address.
292  *
293  * KERN_SUCCESS is returned if the page fault is handled; otherwise,
294  * a standard error specifying why the fault is fatal is returned.
295  *
296  * The map in question must be referenced, and remains so.
297  * The caller may hold no locks.
298  * No other requirements.
299  */
300 int
301 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
302 {
303         int result;
304         vm_pindex_t first_pindex;
305         struct faultstate fs;
306         struct lwp *lp;
307         int growstack;
308         int retry = 0;
309         int inherit_prot;
310
311         inherit_prot = fault_type & VM_PROT_NOSYNC;
312         vm_page_pcpu_cache();
313         fs.hardfault = 0;
314         fs.fault_flags = fault_flags;
315         fs.vp = NULL;
316         fs.shared = vm_shared_fault;
317         fs.first_shared = vm_shared_fault;
318         growstack = 1;
319         if (vm_shared_fault)
320                 ++vm_shared_count;
321
322         /*
323          * vm_map interactions
324          */
325         if ((lp = curthread->td_lwp) != NULL)
326                 lp->lwp_flags |= LWP_PAGING;
327         lwkt_gettoken(&map->token);
328
329 RetryFault:
330         /*
331          * Find the vm_map_entry representing the backing store and resolve
332          * the top level object and page index.  This may have the side
333          * effect of executing a copy-on-write on the map entry and/or
334          * creating a shadow object, but will not COW any actual VM pages.
335          *
336          * On success fs.map is left read-locked and various other fields 
337          * are initialized but not otherwise referenced or locked.
338          *
339          * NOTE!  vm_map_lookup will try to upgrade the fault_type to
340          * VM_FAULT_WRITE if the map entry is a virtual page table and also
341          * writable, so we can set the 'A'accessed bit in the virtual page
342          * table entry.
343          */
344         fs.map = map;
345         result = vm_map_lookup(&fs.map, vaddr, fault_type,
346                                &fs.entry, &fs.first_object,
347                                &first_pindex, &fs.first_prot, &fs.wired);
348
349         /*
350          * If the lookup failed or the map protections are incompatible,
351          * the fault generally fails.
352          *
353          * The failure could be due to TDF_NOFAULT if vm_map_lookup()
354          * tried to do a COW fault.
355          *
356          * If the caller is trying to do a user wiring we have more work
357          * to do.
358          */
359         if (result != KERN_SUCCESS) {
360                 if (result == KERN_FAILURE_NOFAULT) {
361                         result = KERN_FAILURE;
362                         goto done;
363                 }
364                 if (result != KERN_PROTECTION_FAILURE ||
365                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
366                 {
367                         if (result == KERN_INVALID_ADDRESS && growstack &&
368                             map != &kernel_map && curproc != NULL) {
369                                 result = vm_map_growstack(curproc, vaddr);
370                                 if (result == KERN_SUCCESS) {
371                                         growstack = 0;
372                                         ++retry;
373                                         goto RetryFault;
374                                 }
375                                 result = KERN_FAILURE;
376                         }
377                         goto done;
378                 }
379
380                 /*
381                  * If we are user-wiring a r/w segment, and it is COW, then
382                  * we need to do the COW operation.  Note that we don't
383                  * currently COW RO sections now, because it is NOT desirable
384                  * to COW .text.  We simply keep .text from ever being COW'ed
385                  * and take the heat that one cannot debug wired .text sections.
386                  */
387                 result = vm_map_lookup(&fs.map, vaddr,
388                                        VM_PROT_READ|VM_PROT_WRITE|
389                                         VM_PROT_OVERRIDE_WRITE,
390                                        &fs.entry, &fs.first_object,
391                                        &first_pindex, &fs.first_prot,
392                                        &fs.wired);
393                 if (result != KERN_SUCCESS) {
394                         /* could also be KERN_FAILURE_NOFAULT */
395                         result = KERN_FAILURE;
396                         goto done;
397                 }
398
399                 /*
400                  * If we don't COW now, on a user wire, the user will never
401                  * be able to write to the mapping.  If we don't make this
402                  * restriction, the bookkeeping would be nearly impossible.
403                  *
404                  * XXX We have a shared lock, this will have a MP race but
405                  * I don't see how it can hurt anything.
406                  */
407                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
408                         fs.entry->max_protection &= ~VM_PROT_WRITE;
409         }
410
411         /*
412          * fs.map is read-locked
413          *
414          * Misc checks.  Save the map generation number to detect races.
415          */
416         fs.map_generation = fs.map->timestamp;
417         fs.lookup_still_valid = TRUE;
418         fs.first_m = NULL;
419         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
420         fs.prot = fs.first_prot;        /* default (used by uksmap) */
421
422         if (fs.entry->eflags & (MAP_ENTRY_NOFAULT | MAP_ENTRY_KSTACK)) {
423                 if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
424                         panic("vm_fault: fault on nofault entry, addr: %p",
425                               (void *)vaddr);
426                 }
427                 if ((fs.entry->eflags & MAP_ENTRY_KSTACK) &&
428                     vaddr >= fs.entry->start &&
429                     vaddr < fs.entry->start + PAGE_SIZE) {
430                         panic("vm_fault: fault on stack guard, addr: %p",
431                               (void *)vaddr);
432                 }
433         }
434
435         /*
436          * A user-kernel shared map has no VM object and bypasses
437          * everything.  We execute the uksmap function with a temporary
438          * fictitious vm_page.  The address is directly mapped with no
439          * management.
440          */
441         if (fs.entry->maptype == VM_MAPTYPE_UKSMAP) {
442                 struct vm_page fakem;
443
444                 bzero(&fakem, sizeof(fakem));
445                 fakem.pindex = first_pindex;
446                 fakem.flags = PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED;
447                 fakem.valid = VM_PAGE_BITS_ALL;
448                 fakem.pat_mode = VM_MEMATTR_DEFAULT;
449                 if (fs.entry->object.uksmap(fs.entry->aux.dev, &fakem)) {
450                         result = KERN_FAILURE;
451                         unlock_things(&fs);
452                         goto done2;
453                 }
454                 pmap_enter(fs.map->pmap, vaddr, &fakem, fs.prot | inherit_prot,
455                            fs.wired, fs.entry);
456                 goto done_success;
457         }
458
459         /*
460          * A system map entry may return a NULL object.  No object means
461          * no pager means an unrecoverable kernel fault.
462          */
463         if (fs.first_object == NULL) {
464                 panic("vm_fault: unrecoverable fault at %p in entry %p",
465                         (void *)vaddr, fs.entry);
466         }
467
468         /*
469          * Fail here if not a trivial anonymous page fault and TDF_NOFAULT
470          * is set.
471          */
472         if ((curthread->td_flags & TDF_NOFAULT) &&
473             (retry ||
474              fs.first_object->type == OBJT_VNODE ||
475              fs.first_object->backing_object)) {
476                 result = KERN_FAILURE;
477                 unlock_things(&fs);
478                 goto done2;
479         }
480
481         /*
482          * If the entry is wired we cannot change the page protection.
483          */
484         if (fs.wired)
485                 fault_type = fs.first_prot;
486
487         /*
488          * We generally want to avoid unnecessary exclusive modes on backing
489          * and terminal objects because this can seriously interfere with
490          * heavily fork()'d processes (particularly /bin/sh scripts).
491          *
492          * However, we also want to avoid unnecessary retries due to needed
493          * shared->exclusive promotion for common faults.  Exclusive mode is
494          * always needed if any page insertion, rename, or free occurs in an
495          * object (and also indirectly if any I/O is done).
496          *
497          * The main issue here is going to be fs.first_shared.  If the
498          * first_object has a backing object which isn't shadowed and the
499          * process is single-threaded we might as well use an exclusive
500          * lock/chain right off the bat.
501          */
502         if (fs.first_shared && fs.first_object->backing_object &&
503             LIST_EMPTY(&fs.first_object->shadow_head) &&
504             curthread->td_proc && curthread->td_proc->p_nthreads == 1) {
505                 fs.first_shared = 0;
506         }
507
508         /*
509          * swap_pager_unswapped() needs an exclusive object
510          */
511         if (fault_flags & (VM_FAULT_UNSWAP | VM_FAULT_DIRTY)) {
512                 fs.first_shared = 0;
513         }
514
515         /*
516          * Obtain a top-level object lock, shared or exclusive depending
517          * on fs.first_shared.  If a shared lock winds up being insufficient
518          * we will retry with an exclusive lock.
519          *
520          * The vnode pager lock is always shared.
521          */
522         if (fs.first_shared)
523                 vm_object_hold_shared(fs.first_object);
524         else
525                 vm_object_hold(fs.first_object);
526         if (fs.vp == NULL)
527                 fs.vp = vnode_pager_lock(fs.first_object);
528
529         /*
530          * The page we want is at (first_object, first_pindex), but if the
531          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
532          * page table to figure out the actual pindex.
533          *
534          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
535          * ONLY
536          */
537         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
538                 result = vm_fault_vpagetable(&fs, &first_pindex,
539                                              fs.entry->aux.master_pde,
540                                              fault_type, 1);
541                 if (result == KERN_TRY_AGAIN) {
542                         vm_object_drop(fs.first_object);
543                         ++retry;
544                         goto RetryFault;
545                 }
546                 if (result != KERN_SUCCESS)
547                         goto done;
548         }
549
550         /*
551          * Now we have the actual (object, pindex), fault in the page.  If
552          * vm_fault_object() fails it will unlock and deallocate the FS
553          * data.   If it succeeds everything remains locked and fs->object
554          * will have an additional PIP count if it is not equal to
555          * fs->first_object
556          *
557          * vm_fault_object will set fs->prot for the pmap operation.  It is
558          * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
559          * page can be safely written.  However, it will force a read-only
560          * mapping for a read fault if the memory is managed by a virtual
561          * page table.
562          *
563          * If the fault code uses the shared object lock shortcut
564          * we must not try to burst (we can't allocate VM pages).
565          */
566         result = vm_fault_object(&fs, first_pindex, fault_type, 1);
567
568         if (debug_fault > 0) {
569                 --debug_fault;
570                 kprintf("VM_FAULT result %d addr=%jx type=%02x flags=%02x "
571                         "fs.m=%p fs.prot=%02x fs.wired=%02x fs.entry=%p\n",
572                         result, (intmax_t)vaddr, fault_type, fault_flags,
573                         fs.m, fs.prot, fs.wired, fs.entry);
574         }
575
576         if (result == KERN_TRY_AGAIN) {
577                 vm_object_drop(fs.first_object);
578                 ++retry;
579                 goto RetryFault;
580         }
581         if (result != KERN_SUCCESS)
582                 goto done;
583
584         /*
585          * On success vm_fault_object() does not unlock or deallocate, and fs.m
586          * will contain a busied page.
587          *
588          * Enter the page into the pmap and do pmap-related adjustments.
589          */
590         KKASSERT(fs.lookup_still_valid == TRUE);
591         vm_page_flag_set(fs.m, PG_REFERENCED);
592         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot | inherit_prot,
593                    fs.wired, fs.entry);
594
595         /*KKASSERT(fs.m->queue == PQ_NONE); page-in op may deactivate page */
596         KKASSERT(fs.m->flags & PG_BUSY);
597
598         /*
599          * If the page is not wired down, then put it where the pageout daemon
600          * can find it.
601          */
602         if (fs.fault_flags & VM_FAULT_WIRE_MASK) {
603                 if (fs.wired)
604                         vm_page_wire(fs.m);
605                 else
606                         vm_page_unwire(fs.m, 1);
607         } else {
608                 vm_page_activate(fs.m);
609         }
610         vm_page_wakeup(fs.m);
611
612         /*
613          * Burst in a few more pages if possible.  The fs.map should still
614          * be locked.  To avoid interlocking against a vnode->getblk
615          * operation we had to be sure to unbusy our primary vm_page above
616          * first.
617          *
618          * A normal burst can continue down backing store, only execute
619          * if we are holding an exclusive lock, otherwise the exclusive
620          * locks the burst code gets might cause excessive SMP collisions.
621          *
622          * A quick burst can be utilized when there is no backing object
623          * (i.e. a shared file mmap).
624          */
625         if ((fault_flags & VM_FAULT_BURST) &&
626             (fs.fault_flags & VM_FAULT_WIRE_MASK) == 0 &&
627             fs.wired == 0) {
628                 if (fs.first_shared == 0 && fs.shared == 0) {
629                         vm_prefault(fs.map->pmap, vaddr,
630                                     fs.entry, fs.prot, fault_flags);
631                 } else {
632                         vm_prefault_quick(fs.map->pmap, vaddr,
633                                           fs.entry, fs.prot, fault_flags);
634                 }
635         }
636
637 done_success:
638         mycpu->gd_cnt.v_vm_faults++;
639         if (curthread->td_lwp)
640                 ++curthread->td_lwp->lwp_ru.ru_minflt;
641
642         /*
643          * Unlock everything, and return
644          */
645         unlock_things(&fs);
646
647         if (curthread->td_lwp) {
648                 if (fs.hardfault) {
649                         curthread->td_lwp->lwp_ru.ru_majflt++;
650                 } else {
651                         curthread->td_lwp->lwp_ru.ru_minflt++;
652                 }
653         }
654
655         /*vm_object_deallocate(fs.first_object);*/
656         /*fs.m = NULL; */
657         /*fs.first_object = NULL; must still drop later */
658
659         result = KERN_SUCCESS;
660 done:
661         if (fs.first_object)
662                 vm_object_drop(fs.first_object);
663 done2:
664         lwkt_reltoken(&map->token);
665         if (lp)
666                 lp->lwp_flags &= ~LWP_PAGING;
667         if (vm_shared_fault && fs.shared == 0)
668                 ++vm_shared_miss;
669         return (result);
670 }
671
672 /*
673  * Fault in the specified virtual address in the current process map, 
674  * returning a held VM page or NULL.  See vm_fault_page() for more 
675  * information.
676  *
677  * No requirements.
678  */
679 vm_page_t
680 vm_fault_page_quick(vm_offset_t va, vm_prot_t fault_type, int *errorp)
681 {
682         struct lwp *lp = curthread->td_lwp;
683         vm_page_t m;
684
685         m = vm_fault_page(&lp->lwp_vmspace->vm_map, va, 
686                           fault_type, VM_FAULT_NORMAL, errorp);
687         return(m);
688 }
689
690 /*
691  * Fault in the specified virtual address in the specified map, doing all
692  * necessary manipulation of the object store and all necessary I/O.  Return
693  * a held VM page or NULL, and set *errorp.  The related pmap is not
694  * updated.
695  *
696  * The returned page will be properly dirtied if VM_PROT_WRITE was specified,
697  * and marked PG_REFERENCED as well.
698  *
699  * If the page cannot be faulted writable and VM_PROT_WRITE was specified, an
700  * error will be returned.
701  *
702  * No requirements.
703  */
704 vm_page_t
705 vm_fault_page(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
706               int fault_flags, int *errorp)
707 {
708         vm_pindex_t first_pindex;
709         struct faultstate fs;
710         int result;
711         int retry = 0;
712         vm_prot_t orig_fault_type = fault_type;
713
714         fs.hardfault = 0;
715         fs.fault_flags = fault_flags;
716         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
717
718         /*
719          * Dive the pmap (concurrency possible).  If we find the
720          * appropriate page we can terminate early and quickly.
721          */
722         fs.m = pmap_fault_page_quick(map->pmap, vaddr, fault_type);
723         if (fs.m) {
724                 *errorp = 0;
725                 return(fs.m);
726         }
727
728         /*
729          * Otherwise take a concurrency hit and do a formal page
730          * fault.
731          */
732         fs.shared = vm_shared_fault;
733         fs.first_shared = vm_shared_fault;
734         fs.vp = NULL;
735         lwkt_gettoken(&map->token);
736
737         /*
738          * swap_pager_unswapped() needs an exclusive object
739          */
740         if (fault_flags & (VM_FAULT_UNSWAP | VM_FAULT_DIRTY)) {
741                 fs.first_shared = 0;
742         }
743
744 RetryFault:
745         /*
746          * Find the vm_map_entry representing the backing store and resolve
747          * the top level object and page index.  This may have the side
748          * effect of executing a copy-on-write on the map entry and/or
749          * creating a shadow object, but will not COW any actual VM pages.
750          *
751          * On success fs.map is left read-locked and various other fields 
752          * are initialized but not otherwise referenced or locked.
753          *
754          * NOTE!  vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
755          * if the map entry is a virtual page table and also writable,
756          * so we can set the 'A'accessed bit in the virtual page table entry.
757          */
758         fs.map = map;
759         result = vm_map_lookup(&fs.map, vaddr, fault_type,
760                                &fs.entry, &fs.first_object,
761                                &first_pindex, &fs.first_prot, &fs.wired);
762
763         if (result != KERN_SUCCESS) {
764                 *errorp = result;
765                 fs.m = NULL;
766                 goto done;
767         }
768
769         /*
770          * fs.map is read-locked
771          *
772          * Misc checks.  Save the map generation number to detect races.
773          */
774         fs.map_generation = fs.map->timestamp;
775         fs.lookup_still_valid = TRUE;
776         fs.first_m = NULL;
777         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
778
779         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
780                 panic("vm_fault: fault on nofault entry, addr: %lx",
781                     (u_long)vaddr);
782         }
783
784         /*
785          * A user-kernel shared map has no VM object and bypasses
786          * everything.  We execute the uksmap function with a temporary
787          * fictitious vm_page.  The address is directly mapped with no
788          * management.
789          */
790         if (fs.entry->maptype == VM_MAPTYPE_UKSMAP) {
791                 struct vm_page fakem;
792
793                 bzero(&fakem, sizeof(fakem));
794                 fakem.pindex = first_pindex;
795                 fakem.flags = PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED;
796                 fakem.valid = VM_PAGE_BITS_ALL;
797                 fakem.pat_mode = VM_MEMATTR_DEFAULT;
798                 if (fs.entry->object.uksmap(fs.entry->aux.dev, &fakem)) {
799                         *errorp = KERN_FAILURE;
800                         fs.m = NULL;
801                         unlock_things(&fs);
802                         goto done2;
803                 }
804                 fs.m = PHYS_TO_VM_PAGE(fakem.phys_addr);
805                 vm_page_hold(fs.m);
806
807                 unlock_things(&fs);
808                 *errorp = 0;
809                 goto done;
810         }
811
812
813         /*
814          * A system map entry may return a NULL object.  No object means
815          * no pager means an unrecoverable kernel fault.
816          */
817         if (fs.first_object == NULL) {
818                 panic("vm_fault: unrecoverable fault at %p in entry %p",
819                         (void *)vaddr, fs.entry);
820         }
821
822         /*
823          * Fail here if not a trivial anonymous page fault and TDF_NOFAULT
824          * is set.
825          */
826         if ((curthread->td_flags & TDF_NOFAULT) &&
827             (retry ||
828              fs.first_object->type == OBJT_VNODE ||
829              fs.first_object->backing_object)) {
830                 *errorp = KERN_FAILURE;
831                 unlock_things(&fs);
832                 goto done2;
833         }
834
835         /*
836          * If the entry is wired we cannot change the page protection.
837          */
838         if (fs.wired)
839                 fault_type = fs.first_prot;
840
841         /*
842          * Make a reference to this object to prevent its disposal while we
843          * are messing with it.  Once we have the reference, the map is free
844          * to be diddled.  Since objects reference their shadows (and copies),
845          * they will stay around as well.
846          *
847          * The reference should also prevent an unexpected collapse of the
848          * parent that might move pages from the current object into the
849          * parent unexpectedly, resulting in corruption.
850          *
851          * Bump the paging-in-progress count to prevent size changes (e.g.
852          * truncation operations) during I/O.  This must be done after
853          * obtaining the vnode lock in order to avoid possible deadlocks.
854          */
855         if (fs.first_shared)
856                 vm_object_hold_shared(fs.first_object);
857         else
858                 vm_object_hold(fs.first_object);
859         if (fs.vp == NULL)
860                 fs.vp = vnode_pager_lock(fs.first_object);      /* shared */
861
862         /*
863          * The page we want is at (first_object, first_pindex), but if the
864          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
865          * page table to figure out the actual pindex.
866          *
867          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
868          * ONLY
869          */
870         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
871                 result = vm_fault_vpagetable(&fs, &first_pindex,
872                                              fs.entry->aux.master_pde,
873                                              fault_type, 1);
874                 if (result == KERN_TRY_AGAIN) {
875                         vm_object_drop(fs.first_object);
876                         ++retry;
877                         goto RetryFault;
878                 }
879                 if (result != KERN_SUCCESS) {
880                         *errorp = result;
881                         fs.m = NULL;
882                         goto done;
883                 }
884         }
885
886         /*
887          * Now we have the actual (object, pindex), fault in the page.  If
888          * vm_fault_object() fails it will unlock and deallocate the FS
889          * data.   If it succeeds everything remains locked and fs->object
890          * will have an additinal PIP count if it is not equal to
891          * fs->first_object
892          */
893         fs.m = NULL;
894         result = vm_fault_object(&fs, first_pindex, fault_type, 1);
895
896         if (result == KERN_TRY_AGAIN) {
897                 vm_object_drop(fs.first_object);
898                 ++retry;
899                 goto RetryFault;
900         }
901         if (result != KERN_SUCCESS) {
902                 *errorp = result;
903                 fs.m = NULL;
904                 goto done;
905         }
906
907         if ((orig_fault_type & VM_PROT_WRITE) &&
908             (fs.prot & VM_PROT_WRITE) == 0) {
909                 *errorp = KERN_PROTECTION_FAILURE;
910                 unlock_and_deallocate(&fs);
911                 fs.m = NULL;
912                 goto done;
913         }
914
915         /*
916          * DO NOT UPDATE THE PMAP!!!  This function may be called for
917          * a pmap unrelated to the current process pmap, in which case
918          * the current cpu core will not be listed in the pmap's pm_active
919          * mask.  Thus invalidation interlocks will fail to work properly.
920          *
921          * (for example, 'ps' uses procfs to read program arguments from
922          * each process's stack).
923          *
924          * In addition to the above this function will be called to acquire
925          * a page that might already be faulted in, re-faulting it
926          * continuously is a waste of time.
927          *
928          * XXX could this have been the cause of our random seg-fault
929          *     issues?  procfs accesses user stacks.
930          */
931         vm_page_flag_set(fs.m, PG_REFERENCED);
932 #if 0
933         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired, NULL);
934         mycpu->gd_cnt.v_vm_faults++;
935         if (curthread->td_lwp)
936                 ++curthread->td_lwp->lwp_ru.ru_minflt;
937 #endif
938
939         /*
940          * On success vm_fault_object() does not unlock or deallocate, and fs.m
941          * will contain a busied page.  So we must unlock here after having
942          * messed with the pmap.
943          */
944         unlock_things(&fs);
945
946         /*
947          * Return a held page.  We are not doing any pmap manipulation so do
948          * not set PG_MAPPED.  However, adjust the page flags according to
949          * the fault type because the caller may not use a managed pmapping
950          * (so we don't want to lose the fact that the page will be dirtied
951          * if a write fault was specified).
952          */
953         vm_page_hold(fs.m);
954         vm_page_activate(fs.m);
955         if (fault_type & VM_PROT_WRITE)
956                 vm_page_dirty(fs.m);
957
958         if (curthread->td_lwp) {
959                 if (fs.hardfault) {
960                         curthread->td_lwp->lwp_ru.ru_majflt++;
961                 } else {
962                         curthread->td_lwp->lwp_ru.ru_minflt++;
963                 }
964         }
965
966         /*
967          * Unlock everything, and return the held page.
968          */
969         vm_page_wakeup(fs.m);
970         /*vm_object_deallocate(fs.first_object);*/
971         /*fs.first_object = NULL; */
972         *errorp = 0;
973
974 done:
975         if (fs.first_object)
976                 vm_object_drop(fs.first_object);
977 done2:
978         lwkt_reltoken(&map->token);
979         return(fs.m);
980 }
981
982 /*
983  * Fault in the specified (object,offset), dirty the returned page as
984  * needed.  If the requested fault_type cannot be done NULL and an
985  * error is returned.
986  *
987  * A held (but not busied) page is returned.
988  *
989  * The passed in object must be held as specified by the shared
990  * argument.
991  */
992 vm_page_t
993 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
994                      vm_prot_t fault_type, int fault_flags,
995                      int *sharedp, int *errorp)
996 {
997         int result;
998         vm_pindex_t first_pindex;
999         struct faultstate fs;
1000         struct vm_map_entry entry;
1001
1002         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1003         bzero(&entry, sizeof(entry));
1004         entry.object.vm_object = object;
1005         entry.maptype = VM_MAPTYPE_NORMAL;
1006         entry.protection = entry.max_protection = fault_type;
1007
1008         fs.hardfault = 0;
1009         fs.fault_flags = fault_flags;
1010         fs.map = NULL;
1011         fs.shared = vm_shared_fault;
1012         fs.first_shared = *sharedp;
1013         fs.vp = NULL;
1014         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
1015
1016         /*
1017          * Might require swap block adjustments
1018          */
1019         if (fs.first_shared && (fault_flags & (VM_FAULT_UNSWAP | VM_FAULT_DIRTY))) {
1020                 fs.first_shared = 0;
1021                 vm_object_upgrade(object);
1022         }
1023
1024         /*
1025          * Retry loop as needed (typically for shared->exclusive transitions)
1026          */
1027 RetryFault:
1028         *sharedp = fs.first_shared;
1029         first_pindex = OFF_TO_IDX(offset);
1030         fs.first_object = object;
1031         fs.entry = &entry;
1032         fs.first_prot = fault_type;
1033         fs.wired = 0;
1034         /*fs.map_generation = 0; unused */
1035
1036         /*
1037          * Make a reference to this object to prevent its disposal while we
1038          * are messing with it.  Once we have the reference, the map is free
1039          * to be diddled.  Since objects reference their shadows (and copies),
1040          * they will stay around as well.
1041          *
1042          * The reference should also prevent an unexpected collapse of the
1043          * parent that might move pages from the current object into the
1044          * parent unexpectedly, resulting in corruption.
1045          *
1046          * Bump the paging-in-progress count to prevent size changes (e.g.
1047          * truncation operations) during I/O.  This must be done after
1048          * obtaining the vnode lock in order to avoid possible deadlocks.
1049          */
1050         if (fs.vp == NULL)
1051                 fs.vp = vnode_pager_lock(fs.first_object);
1052
1053         fs.lookup_still_valid = TRUE;
1054         fs.first_m = NULL;
1055         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
1056
1057 #if 0
1058         /* XXX future - ability to operate on VM object using vpagetable */
1059         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1060                 result = vm_fault_vpagetable(&fs, &first_pindex,
1061                                              fs.entry->aux.master_pde,
1062                                              fault_type, 0);
1063                 if (result == KERN_TRY_AGAIN) {
1064                         if (fs.first_shared == 0 && *sharedp)
1065                                 vm_object_upgrade(object);
1066                         goto RetryFault;
1067                 }
1068                 if (result != KERN_SUCCESS) {
1069                         *errorp = result;
1070                         return (NULL);
1071                 }
1072         }
1073 #endif
1074
1075         /*
1076          * Now we have the actual (object, pindex), fault in the page.  If
1077          * vm_fault_object() fails it will unlock and deallocate the FS
1078          * data.   If it succeeds everything remains locked and fs->object
1079          * will have an additinal PIP count if it is not equal to
1080          * fs->first_object
1081          *
1082          * On KERN_TRY_AGAIN vm_fault_object() leaves fs.first_object intact.
1083          * We may have to upgrade its lock to handle the requested fault.
1084          */
1085         result = vm_fault_object(&fs, first_pindex, fault_type, 0);
1086
1087         if (result == KERN_TRY_AGAIN) {
1088                 if (fs.first_shared == 0 && *sharedp)
1089                         vm_object_upgrade(object);
1090                 goto RetryFault;
1091         }
1092         if (result != KERN_SUCCESS) {
1093                 *errorp = result;
1094                 return(NULL);
1095         }
1096
1097         if ((fault_type & VM_PROT_WRITE) && (fs.prot & VM_PROT_WRITE) == 0) {
1098                 *errorp = KERN_PROTECTION_FAILURE;
1099                 unlock_and_deallocate(&fs);
1100                 return(NULL);
1101         }
1102
1103         /*
1104          * On success vm_fault_object() does not unlock or deallocate, so we
1105          * do it here.  Note that the returned fs.m will be busied.
1106          */
1107         unlock_things(&fs);
1108
1109         /*
1110          * Return a held page.  We are not doing any pmap manipulation so do
1111          * not set PG_MAPPED.  However, adjust the page flags according to
1112          * the fault type because the caller may not use a managed pmapping
1113          * (so we don't want to lose the fact that the page will be dirtied
1114          * if a write fault was specified).
1115          */
1116         vm_page_hold(fs.m);
1117         vm_page_activate(fs.m);
1118         if ((fault_type & VM_PROT_WRITE) || (fault_flags & VM_FAULT_DIRTY))
1119                 vm_page_dirty(fs.m);
1120         if (fault_flags & VM_FAULT_UNSWAP)
1121                 swap_pager_unswapped(fs.m);
1122
1123         /*
1124          * Indicate that the page was accessed.
1125          */
1126         vm_page_flag_set(fs.m, PG_REFERENCED);
1127
1128         if (curthread->td_lwp) {
1129                 if (fs.hardfault) {
1130                         curthread->td_lwp->lwp_ru.ru_majflt++;
1131                 } else {
1132                         curthread->td_lwp->lwp_ru.ru_minflt++;
1133                 }
1134         }
1135
1136         /*
1137          * Unlock everything, and return the held page.
1138          */
1139         vm_page_wakeup(fs.m);
1140         /*vm_object_deallocate(fs.first_object);*/
1141         /*fs.first_object = NULL; */
1142
1143         *errorp = 0;
1144         return(fs.m);
1145 }
1146
1147 /*
1148  * Translate the virtual page number (first_pindex) that is relative
1149  * to the address space into a logical page number that is relative to the
1150  * backing object.  Use the virtual page table pointed to by (vpte).
1151  *
1152  * This implements an N-level page table.  Any level can terminate the
1153  * scan by setting VPTE_PS.   A linear mapping is accomplished by setting
1154  * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
1155  */
1156 static
1157 int
1158 vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
1159                     vpte_t vpte, int fault_type, int allow_nofault)
1160 {
1161         struct lwbuf *lwb;
1162         struct lwbuf lwb_cache;
1163         int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
1164         int result = KERN_SUCCESS;
1165         vpte_t *ptep;
1166
1167         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
1168         for (;;) {
1169                 /*
1170                  * We cannot proceed if the vpte is not valid, not readable
1171                  * for a read fault, or not writable for a write fault.
1172                  */
1173                 if ((vpte & VPTE_V) == 0) {
1174                         unlock_and_deallocate(fs);
1175                         return (KERN_FAILURE);
1176                 }
1177                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_RW) == 0) {
1178                         unlock_and_deallocate(fs);
1179                         return (KERN_FAILURE);
1180                 }
1181                 if ((vpte & VPTE_PS) || vshift == 0)
1182                         break;
1183                 KKASSERT(vshift >= VPTE_PAGE_BITS);
1184
1185                 /*
1186                  * Get the page table page.  Nominally we only read the page
1187                  * table, but since we are actively setting VPTE_M and VPTE_A,
1188                  * tell vm_fault_object() that we are writing it. 
1189                  *
1190                  * There is currently no real need to optimize this.
1191                  */
1192                 result = vm_fault_object(fs, (vpte & VPTE_FRAME) >> PAGE_SHIFT,
1193                                          VM_PROT_READ|VM_PROT_WRITE,
1194                                          allow_nofault);
1195                 if (result != KERN_SUCCESS)
1196                         return (result);
1197
1198                 /*
1199                  * Process the returned fs.m and look up the page table
1200                  * entry in the page table page.
1201                  */
1202                 vshift -= VPTE_PAGE_BITS;
1203                 lwb = lwbuf_alloc(fs->m, &lwb_cache);
1204                 ptep = ((vpte_t *)lwbuf_kva(lwb) +
1205                         ((*pindex >> vshift) & VPTE_PAGE_MASK));
1206                 vpte = *ptep;
1207
1208                 /*
1209                  * Page table write-back.  If the vpte is valid for the
1210                  * requested operation, do a write-back to the page table.
1211                  *
1212                  * XXX VPTE_M is not set properly for page directory pages.
1213                  * It doesn't get set in the page directory if the page table
1214                  * is modified during a read access.
1215                  */
1216                 vm_page_activate(fs->m);
1217                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_V) &&
1218                     (vpte & VPTE_RW)) {
1219                         if ((vpte & (VPTE_M|VPTE_A)) != (VPTE_M|VPTE_A)) {
1220                                 atomic_set_long(ptep, VPTE_M | VPTE_A);
1221                                 vm_page_dirty(fs->m);
1222                         }
1223                 }
1224                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_V)) {
1225                         if ((vpte & VPTE_A) == 0) {
1226                                 atomic_set_long(ptep, VPTE_A);
1227                                 vm_page_dirty(fs->m);
1228                         }
1229                 }
1230                 lwbuf_free(lwb);
1231                 vm_page_flag_set(fs->m, PG_REFERENCED);
1232                 vm_page_wakeup(fs->m);
1233                 fs->m = NULL;
1234                 cleanup_successful_fault(fs);
1235         }
1236         /*
1237          * Combine remaining address bits with the vpte.
1238          */
1239         /* JG how many bits from each? */
1240         *pindex = ((vpte & VPTE_FRAME) >> PAGE_SHIFT) +
1241                   (*pindex & ((1L << vshift) - 1));
1242         return (KERN_SUCCESS);
1243 }
1244
1245
1246 /*
1247  * This is the core of the vm_fault code.
1248  *
1249  * Do all operations required to fault-in (fs.first_object, pindex).  Run
1250  * through the shadow chain as necessary and do required COW or virtual
1251  * copy operations.  The caller has already fully resolved the vm_map_entry
1252  * and, if appropriate, has created a copy-on-write layer.  All we need to
1253  * do is iterate the object chain.
1254  *
1255  * On failure (fs) is unlocked and deallocated and the caller may return or
1256  * retry depending on the failure code.  On success (fs) is NOT unlocked or
1257  * deallocated, fs.m will contained a resolved, busied page, and fs.object
1258  * will have an additional PIP count if it is not equal to fs.first_object.
1259  *
1260  * If locks based on fs->first_shared or fs->shared are insufficient,
1261  * clear the appropriate field(s) and return RETRY.  COWs require that
1262  * first_shared be 0, while page allocations (or frees) require that
1263  * shared be 0.  Renames require that both be 0.
1264  *
1265  * fs->first_object must be held on call.
1266  */
1267 static
1268 int
1269 vm_fault_object(struct faultstate *fs, vm_pindex_t first_pindex,
1270                 vm_prot_t fault_type, int allow_nofault)
1271 {
1272         vm_object_t next_object;
1273         vm_pindex_t pindex;
1274         int error;
1275
1276         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
1277         fs->prot = fs->first_prot;
1278         fs->object = fs->first_object;
1279         pindex = first_pindex;
1280
1281         vm_object_chain_acquire(fs->first_object, fs->shared);
1282         vm_object_pip_add(fs->first_object, 1);
1283
1284         /* 
1285          * If a read fault occurs we try to make the page writable if
1286          * possible.  There are three cases where we cannot make the
1287          * page mapping writable:
1288          *
1289          * (1) The mapping is read-only or the VM object is read-only,
1290          *     fs->prot above will simply not have VM_PROT_WRITE set.
1291          *
1292          * (2) If the mapping is a virtual page table we need to be able
1293          *     to detect writes so we can set VPTE_M in the virtual page
1294          *     table.
1295          *
1296          * (3) If the VM page is read-only or copy-on-write, upgrading would
1297          *     just result in an unnecessary COW fault.
1298          *
1299          * VM_PROT_VPAGED is set if faulting via a virtual page table and
1300          * causes adjustments to the 'M'odify bit to also turn off write
1301          * access to force a re-fault.
1302          */
1303         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1304                 if ((fault_type & VM_PROT_WRITE) == 0)
1305                         fs->prot &= ~VM_PROT_WRITE;
1306         }
1307
1308         if (curthread->td_lwp && curthread->td_lwp->lwp_vmspace &&
1309             pmap_emulate_ad_bits(&curthread->td_lwp->lwp_vmspace->vm_pmap)) {
1310                 if ((fault_type & VM_PROT_WRITE) == 0)
1311                         fs->prot &= ~VM_PROT_WRITE;
1312         }
1313
1314         /* vm_object_hold(fs->object); implied b/c object == first_object */
1315
1316         for (;;) {
1317                 /*
1318                  * The entire backing chain from first_object to object
1319                  * inclusive is chainlocked.
1320                  *
1321                  * If the object is dead, we stop here
1322                  */
1323                 if (fs->object->flags & OBJ_DEAD) {
1324                         vm_object_pip_wakeup(fs->first_object);
1325                         vm_object_chain_release_all(fs->first_object,
1326                                                     fs->object);
1327                         if (fs->object != fs->first_object)
1328                                 vm_object_drop(fs->object);
1329                         unlock_and_deallocate(fs);
1330                         return (KERN_PROTECTION_FAILURE);
1331                 }
1332
1333                 /*
1334                  * See if the page is resident.  Wait/Retry if the page is
1335                  * busy (lots of stuff may have changed so we can't continue
1336                  * in that case).
1337                  *
1338                  * We can theoretically allow the soft-busy case on a read
1339                  * fault if the page is marked valid, but since such
1340                  * pages are typically already pmap'd, putting that
1341                  * special case in might be more effort then it is
1342                  * worth.  We cannot under any circumstances mess
1343                  * around with a vm_page_t->busy page except, perhaps,
1344                  * to pmap it.
1345                  */
1346                 fs->m = vm_page_lookup_busy_try(fs->object, pindex,
1347                                                 TRUE, &error);
1348                 if (error) {
1349                         vm_object_pip_wakeup(fs->first_object);
1350                         vm_object_chain_release_all(fs->first_object,
1351                                                     fs->object);
1352                         if (fs->object != fs->first_object)
1353                                 vm_object_drop(fs->object);
1354                         unlock_things(fs);
1355                         vm_page_sleep_busy(fs->m, TRUE, "vmpfw");
1356                         mycpu->gd_cnt.v_intrans++;
1357                         /*vm_object_deallocate(fs->first_object);*/
1358                         /*fs->first_object = NULL;*/
1359                         fs->m = NULL;
1360                         return (KERN_TRY_AGAIN);
1361                 }
1362                 if (fs->m) {
1363                         /*
1364                          * The page is busied for us.
1365                          *
1366                          * If reactivating a page from PQ_CACHE we may have
1367                          * to rate-limit.
1368                          */
1369                         int queue = fs->m->queue;
1370                         vm_page_unqueue_nowakeup(fs->m);
1371
1372                         if ((queue - fs->m->pc) == PQ_CACHE && 
1373                             vm_page_count_severe()) {
1374                                 vm_page_activate(fs->m);
1375                                 vm_page_wakeup(fs->m);
1376                                 fs->m = NULL;
1377                                 vm_object_pip_wakeup(fs->first_object);
1378                                 vm_object_chain_release_all(fs->first_object,
1379                                                             fs->object);
1380                                 if (fs->object != fs->first_object)
1381                                         vm_object_drop(fs->object);
1382                                 unlock_and_deallocate(fs);
1383                                 if (allow_nofault == 0 ||
1384                                     (curthread->td_flags & TDF_NOFAULT) == 0) {
1385                                         vm_wait_pfault();
1386                                 }
1387                                 return (KERN_TRY_AGAIN);
1388                         }
1389
1390                         /*
1391                          * If it still isn't completely valid (readable),
1392                          * or if a read-ahead-mark is set on the VM page,
1393                          * jump to readrest, else we found the page and
1394                          * can return.
1395                          *
1396                          * We can release the spl once we have marked the
1397                          * page busy.
1398                          */
1399                         if (fs->m->object != &kernel_object) {
1400                                 if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
1401                                     VM_PAGE_BITS_ALL) {
1402                                         goto readrest;
1403                                 }
1404                                 if (fs->m->flags & PG_RAM) {
1405                                         if (debug_cluster)
1406                                                 kprintf("R");
1407                                         vm_page_flag_clear(fs->m, PG_RAM);
1408                                         goto readrest;
1409                                 }
1410                         }
1411                         break; /* break to PAGE HAS BEEN FOUND */
1412                 }
1413
1414                 /*
1415                  * Page is not resident, If this is the search termination
1416                  * or the pager might contain the page, allocate a new page.
1417                  */
1418                 if (TRYPAGER(fs) || fs->object == fs->first_object) {
1419                         /*
1420                          * Allocating, must be exclusive.
1421                          */
1422                         if (fs->object == fs->first_object &&
1423                             fs->first_shared) {
1424                                 fs->first_shared = 0;
1425                                 vm_object_pip_wakeup(fs->first_object);
1426                                 vm_object_chain_release_all(fs->first_object,
1427                                                             fs->object);
1428                                 if (fs->object != fs->first_object)
1429                                         vm_object_drop(fs->object);
1430                                 unlock_and_deallocate(fs);
1431                                 return (KERN_TRY_AGAIN);
1432                         }
1433                         if (fs->object != fs->first_object &&
1434                             fs->shared) {
1435                                 fs->first_shared = 0;
1436                                 fs->shared = 0;
1437                                 vm_object_pip_wakeup(fs->first_object);
1438                                 vm_object_chain_release_all(fs->first_object,
1439                                                             fs->object);
1440                                 if (fs->object != fs->first_object)
1441                                         vm_object_drop(fs->object);
1442                                 unlock_and_deallocate(fs);
1443                                 return (KERN_TRY_AGAIN);
1444                         }
1445
1446                         /*
1447                          * If the page is beyond the object size we fail
1448                          */
1449                         if (pindex >= fs->object->size) {
1450                                 vm_object_pip_wakeup(fs->first_object);
1451                                 vm_object_chain_release_all(fs->first_object,
1452                                                             fs->object);
1453                                 if (fs->object != fs->first_object)
1454                                         vm_object_drop(fs->object);
1455                                 unlock_and_deallocate(fs);
1456                                 return (KERN_PROTECTION_FAILURE);
1457                         }
1458
1459                         /*
1460                          * Allocate a new page for this object/offset pair.
1461                          *
1462                          * It is possible for the allocation to race, so
1463                          * handle the case.
1464                          */
1465                         fs->m = NULL;
1466                         if (!vm_page_count_severe()) {
1467                                 fs->m = vm_page_alloc(fs->object, pindex,
1468                                     ((fs->vp || fs->object->backing_object) ?
1469                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL :
1470                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL |
1471                                         VM_ALLOC_USE_GD | VM_ALLOC_ZERO));
1472                         }
1473                         if (fs->m == NULL) {
1474                                 vm_object_pip_wakeup(fs->first_object);
1475                                 vm_object_chain_release_all(fs->first_object,
1476                                                             fs->object);
1477                                 if (fs->object != fs->first_object)
1478                                         vm_object_drop(fs->object);
1479                                 unlock_and_deallocate(fs);
1480                                 if (allow_nofault == 0 ||
1481                                     (curthread->td_flags & TDF_NOFAULT) == 0) {
1482                                         vm_wait_pfault();
1483                                 }
1484                                 return (KERN_TRY_AGAIN);
1485                         }
1486
1487                         /*
1488                          * Fall through to readrest.  We have a new page which
1489                          * will have to be paged (since m->valid will be 0).
1490                          */
1491                 }
1492
1493 readrest:
1494                 /*
1495                  * We have found an invalid or partially valid page, a
1496                  * page with a read-ahead mark which might be partially or
1497                  * fully valid (and maybe dirty too), or we have allocated
1498                  * a new page.
1499                  *
1500                  * Attempt to fault-in the page if there is a chance that the
1501                  * pager has it, and potentially fault in additional pages
1502                  * at the same time.
1503                  *
1504                  * If TRYPAGER is true then fs.m will be non-NULL and busied
1505                  * for us.
1506                  */
1507                 if (TRYPAGER(fs)) {
1508                         int rv;
1509                         int seqaccess;
1510                         u_char behavior = vm_map_entry_behavior(fs->entry);
1511
1512                         if (behavior == MAP_ENTRY_BEHAV_RANDOM)
1513                                 seqaccess = 0;
1514                         else
1515                                 seqaccess = -1;
1516
1517                         /*
1518                          * Doing I/O may synchronously insert additional
1519                          * pages so we can't be shared at this point either.
1520                          *
1521                          * NOTE: We can't free fs->m here in the allocated
1522                          *       case (fs->object != fs->first_object) as
1523                          *       this would require an exclusively locked
1524                          *       VM object.
1525                          */
1526                         if (fs->object == fs->first_object &&
1527                             fs->first_shared) {
1528                                 vm_page_deactivate(fs->m);
1529                                 vm_page_wakeup(fs->m);
1530                                 fs->m = NULL;
1531                                 fs->first_shared = 0;
1532                                 vm_object_pip_wakeup(fs->first_object);
1533                                 vm_object_chain_release_all(fs->first_object,
1534                                                             fs->object);
1535                                 if (fs->object != fs->first_object)
1536                                         vm_object_drop(fs->object);
1537                                 unlock_and_deallocate(fs);
1538                                 return (KERN_TRY_AGAIN);
1539                         }
1540                         if (fs->object != fs->first_object &&
1541                             fs->shared) {
1542                                 vm_page_deactivate(fs->m);
1543                                 vm_page_wakeup(fs->m);
1544                                 fs->m = NULL;
1545                                 fs->first_shared = 0;
1546                                 fs->shared = 0;
1547                                 vm_object_pip_wakeup(fs->first_object);
1548                                 vm_object_chain_release_all(fs->first_object,
1549                                                             fs->object);
1550                                 if (fs->object != fs->first_object)
1551                                         vm_object_drop(fs->object);
1552                                 unlock_and_deallocate(fs);
1553                                 return (KERN_TRY_AGAIN);
1554                         }
1555
1556                         /*
1557                          * Avoid deadlocking against the map when doing I/O.
1558                          * fs.object and the page is PG_BUSY'd.
1559                          *
1560                          * NOTE: Once unlocked, fs->entry can become stale
1561                          *       so this will NULL it out.
1562                          *
1563                          * NOTE: fs->entry is invalid until we relock the
1564                          *       map and verify that the timestamp has not
1565                          *       changed.
1566                          */
1567                         unlock_map(fs);
1568
1569                         /*
1570                          * Acquire the page data.  We still hold a ref on
1571                          * fs.object and the page has been PG_BUSY's.
1572                          *
1573                          * The pager may replace the page (for example, in
1574                          * order to enter a fictitious page into the
1575                          * object).  If it does so it is responsible for
1576                          * cleaning up the passed page and properly setting
1577                          * the new page PG_BUSY.
1578                          *
1579                          * If we got here through a PG_RAM read-ahead
1580                          * mark the page may be partially dirty and thus
1581                          * not freeable.  Don't bother checking to see
1582                          * if the pager has the page because we can't free
1583                          * it anyway.  We have to depend on the get_page
1584                          * operation filling in any gaps whether there is
1585                          * backing store or not.
1586                          */
1587                         rv = vm_pager_get_page(fs->object, &fs->m, seqaccess);
1588
1589                         if (rv == VM_PAGER_OK) {
1590                                 /*
1591                                  * Relookup in case pager changed page. Pager
1592                                  * is responsible for disposition of old page
1593                                  * if moved.
1594                                  *
1595                                  * XXX other code segments do relookups too.
1596                                  * It's a bad abstraction that needs to be
1597                                  * fixed/removed.
1598                                  */
1599                                 fs->m = vm_page_lookup(fs->object, pindex);
1600                                 if (fs->m == NULL) {
1601                                         vm_object_pip_wakeup(fs->first_object);
1602                                         vm_object_chain_release_all(
1603                                                 fs->first_object, fs->object);
1604                                         if (fs->object != fs->first_object)
1605                                                 vm_object_drop(fs->object);
1606                                         unlock_and_deallocate(fs);
1607                                         return (KERN_TRY_AGAIN);
1608                                 }
1609                                 ++fs->hardfault;
1610                                 break; /* break to PAGE HAS BEEN FOUND */
1611                         }
1612
1613                         /*
1614                          * Remove the bogus page (which does not exist at this
1615                          * object/offset); before doing so, we must get back
1616                          * our object lock to preserve our invariant.
1617                          *
1618                          * Also wake up any other process that may want to bring
1619                          * in this page.
1620                          *
1621                          * If this is the top-level object, we must leave the
1622                          * busy page to prevent another process from rushing
1623                          * past us, and inserting the page in that object at
1624                          * the same time that we are.
1625                          */
1626                         if (rv == VM_PAGER_ERROR) {
1627                                 if (curproc) {
1628                                         kprintf("vm_fault: pager read error, "
1629                                                 "pid %d (%s)\n",
1630                                                 curproc->p_pid,
1631                                                 curproc->p_comm);
1632                                 } else {
1633                                         kprintf("vm_fault: pager read error, "
1634                                                 "thread %p (%s)\n",
1635                                                 curthread,
1636                                                 curproc->p_comm);
1637                                 }
1638                         }
1639
1640                         /*
1641                          * Data outside the range of the pager or an I/O error
1642                          *
1643                          * The page may have been wired during the pagein,
1644                          * e.g. by the buffer cache, and cannot simply be
1645                          * freed.  Call vnode_pager_freepage() to deal with it.
1646                          *
1647                          * Also note that we cannot free the page if we are
1648                          * holding the related object shared. XXX not sure
1649                          * what to do in that case.
1650                          */
1651                         if (fs->object != fs->first_object) {
1652                                 vnode_pager_freepage(fs->m);
1653                                 fs->m = NULL;
1654                                 /*
1655                                  * XXX - we cannot just fall out at this
1656                                  * point, m has been freed and is invalid!
1657                                  */
1658                         }
1659                         /*
1660                          * XXX - the check for kernel_map is a kludge to work
1661                          * around having the machine panic on a kernel space
1662                          * fault w/ I/O error.
1663                          */
1664                         if (((fs->map != &kernel_map) &&
1665                             (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) {
1666                                 if (fs->m) {
1667                                         if (fs->first_shared) {
1668                                                 vm_page_deactivate(fs->m);
1669                                                 vm_page_wakeup(fs->m);
1670                                         } else {
1671                                                 vnode_pager_freepage(fs->m);
1672                                         }
1673                                         fs->m = NULL;
1674                                 }
1675                                 vm_object_pip_wakeup(fs->first_object);
1676                                 vm_object_chain_release_all(fs->first_object,
1677                                                             fs->object);
1678                                 if (fs->object != fs->first_object)
1679                                         vm_object_drop(fs->object);
1680                                 unlock_and_deallocate(fs);
1681                                 if (rv == VM_PAGER_ERROR)
1682                                         return (KERN_FAILURE);
1683                                 else
1684                                         return (KERN_PROTECTION_FAILURE);
1685                                 /* NOT REACHED */
1686                         }
1687                 }
1688
1689                 /*
1690                  * We get here if the object has a default pager (or unwiring) 
1691                  * or the pager doesn't have the page.
1692                  *
1693                  * fs->first_m will be used for the COW unless we find a
1694                  * deeper page to be mapped read-only, in which case the
1695                  * unlock*(fs) will free first_m.
1696                  */
1697                 if (fs->object == fs->first_object)
1698                         fs->first_m = fs->m;
1699
1700                 /*
1701                  * Move on to the next object.  The chain lock should prevent
1702                  * the backing_object from getting ripped out from under us.
1703                  *
1704                  * The object lock for the next object is governed by
1705                  * fs->shared.
1706                  */
1707                 if ((next_object = fs->object->backing_object) != NULL) {
1708                         if (fs->shared)
1709                                 vm_object_hold_shared(next_object);
1710                         else
1711                                 vm_object_hold(next_object);
1712                         vm_object_chain_acquire(next_object, fs->shared);
1713                         KKASSERT(next_object == fs->object->backing_object);
1714                         pindex += OFF_TO_IDX(fs->object->backing_object_offset);
1715                 }
1716
1717                 if (next_object == NULL) {
1718                         /*
1719                          * If there's no object left, fill the page in the top
1720                          * object with zeros.
1721                          */
1722                         if (fs->object != fs->first_object) {
1723 #if 0
1724                                 if (fs->first_object->backing_object !=
1725                                     fs->object) {
1726                                         vm_object_hold(fs->first_object->backing_object);
1727                                 }
1728 #endif
1729                                 vm_object_chain_release_all(
1730                                         fs->first_object->backing_object,
1731                                         fs->object);
1732 #if 0
1733                                 if (fs->first_object->backing_object !=
1734                                     fs->object) {
1735                                         vm_object_drop(fs->first_object->backing_object);
1736                                 }
1737 #endif
1738                                 vm_object_pip_wakeup(fs->object);
1739                                 vm_object_drop(fs->object);
1740                                 fs->object = fs->first_object;
1741                                 pindex = first_pindex;
1742                                 fs->m = fs->first_m;
1743                         }
1744                         fs->first_m = NULL;
1745
1746                         /*
1747                          * Zero the page if necessary and mark it valid.
1748                          */
1749                         if ((fs->m->flags & PG_ZERO) == 0) {
1750                                 vm_page_zero_fill(fs->m);
1751                         } else {
1752 #ifdef PMAP_DEBUG
1753                                 pmap_page_assertzero(VM_PAGE_TO_PHYS(fs->m));
1754 #endif
1755                                 vm_page_flag_clear(fs->m, PG_ZERO);
1756                                 mycpu->gd_cnt.v_ozfod++;
1757                         }
1758                         mycpu->gd_cnt.v_zfod++;
1759                         fs->m->valid = VM_PAGE_BITS_ALL;
1760                         break;  /* break to PAGE HAS BEEN FOUND */
1761                 }
1762                 if (fs->object != fs->first_object) {
1763                         vm_object_pip_wakeup(fs->object);
1764                         vm_object_lock_swap();
1765                         vm_object_drop(fs->object);
1766                 }
1767                 KASSERT(fs->object != next_object,
1768                         ("object loop %p", next_object));
1769                 fs->object = next_object;
1770                 vm_object_pip_add(fs->object, 1);
1771         }
1772
1773         /*
1774          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
1775          * is held.]
1776          *
1777          * object still held.
1778          *
1779          * local shared variable may be different from fs->shared.
1780          *
1781          * If the page is being written, but isn't already owned by the
1782          * top-level object, we have to copy it into a new page owned by the
1783          * top-level object.
1784          */
1785         KASSERT((fs->m->flags & PG_BUSY) != 0,
1786                 ("vm_fault: not busy after main loop"));
1787
1788         if (fs->object != fs->first_object) {
1789                 /*
1790                  * We only really need to copy if we want to write it.
1791                  */
1792                 if (fault_type & VM_PROT_WRITE) {
1793                         /*
1794                          * This allows pages to be virtually copied from a 
1795                          * backing_object into the first_object, where the 
1796                          * backing object has no other refs to it, and cannot
1797                          * gain any more refs.  Instead of a bcopy, we just 
1798                          * move the page from the backing object to the 
1799                          * first object.  Note that we must mark the page 
1800                          * dirty in the first object so that it will go out 
1801                          * to swap when needed.
1802                          */
1803                         if (
1804                                 /*
1805                                  * Must be holding exclusive locks
1806                                  */
1807                                 fs->first_shared == 0 &&
1808                                 fs->shared == 0 &&
1809                                 /*
1810                                  * Map, if present, has not changed
1811                                  */
1812                                 (fs->map == NULL ||
1813                                 fs->map_generation == fs->map->timestamp) &&
1814                                 /*
1815                                  * Only one shadow object
1816                                  */
1817                                 (fs->object->shadow_count == 1) &&
1818                                 /*
1819                                  * No COW refs, except us
1820                                  */
1821                                 (fs->object->ref_count == 1) &&
1822                                 /*
1823                                  * No one else can look this object up
1824                                  */
1825                                 (fs->object->handle == NULL) &&
1826                                 /*
1827                                  * No other ways to look the object up
1828                                  */
1829                                 ((fs->object->type == OBJT_DEFAULT) ||
1830                                  (fs->object->type == OBJT_SWAP)) &&
1831                                 /*
1832                                  * We don't chase down the shadow chain
1833                                  */
1834                                 (fs->object == fs->first_object->backing_object) &&
1835
1836                                 /*
1837                                  * grab the lock if we need to
1838                                  */
1839                                 (fs->lookup_still_valid ||
1840                                  fs->map == NULL ||
1841                                  lockmgr(&fs->map->lock, LK_EXCLUSIVE|LK_NOWAIT) == 0)
1842                             ) {
1843                                 /*
1844                                  * (first_m) and (m) are both busied.  We have
1845                                  * move (m) into (first_m)'s object/pindex
1846                                  * in an atomic fashion, then free (first_m).
1847                                  *
1848                                  * first_object is held so second remove
1849                                  * followed by the rename should wind
1850                                  * up being atomic.  vm_page_free() might
1851                                  * block so we don't do it until after the
1852                                  * rename.
1853                                  */
1854                                 fs->lookup_still_valid = 1;
1855                                 vm_page_protect(fs->first_m, VM_PROT_NONE);
1856                                 vm_page_remove(fs->first_m);
1857                                 vm_page_rename(fs->m, fs->first_object,
1858                                                first_pindex);
1859                                 vm_page_free(fs->first_m);
1860                                 fs->first_m = fs->m;
1861                                 fs->m = NULL;
1862                                 mycpu->gd_cnt.v_cow_optim++;
1863                         } else {
1864                                 /*
1865                                  * Oh, well, lets copy it.
1866                                  *
1867                                  * Why are we unmapping the original page
1868                                  * here?  Well, in short, not all accessors
1869                                  * of user memory go through the pmap.  The
1870                                  * procfs code doesn't have access user memory
1871                                  * via a local pmap, so vm_fault_page*()
1872                                  * can't call pmap_enter().  And the umtx*()
1873                                  * code may modify the COW'd page via a DMAP
1874                                  * or kernel mapping and not via the pmap,
1875                                  * leaving the original page still mapped
1876                                  * read-only into the pmap.
1877                                  *
1878                                  * So we have to remove the page from at
1879                                  * least the current pmap if it is in it.
1880                                  * Just remove it from all pmaps.
1881                                  */
1882                                 KKASSERT(fs->first_shared == 0);
1883                                 vm_page_copy(fs->m, fs->first_m);
1884                                 vm_page_protect(fs->m, VM_PROT_NONE);
1885                                 vm_page_event(fs->m, VMEVENT_COW);
1886                         }
1887
1888                         /*
1889                          * We no longer need the old page or object.
1890                          */
1891                         if (fs->m)
1892                                 release_page(fs);
1893
1894                         /*
1895                          * We intend to revert to first_object, undo the
1896                          * chain lock through to that.
1897                          */
1898 #if 0
1899                         if (fs->first_object->backing_object != fs->object)
1900                                 vm_object_hold(fs->first_object->backing_object);
1901 #endif
1902                         vm_object_chain_release_all(
1903                                         fs->first_object->backing_object,
1904                                         fs->object);
1905 #if 0
1906                         if (fs->first_object->backing_object != fs->object)
1907                                 vm_object_drop(fs->first_object->backing_object);
1908 #endif
1909
1910                         /*
1911                          * fs->object != fs->first_object due to above 
1912                          * conditional
1913                          */
1914                         vm_object_pip_wakeup(fs->object);
1915                         vm_object_drop(fs->object);
1916
1917                         /*
1918                          * Only use the new page below...
1919                          */
1920                         mycpu->gd_cnt.v_cow_faults++;
1921                         fs->m = fs->first_m;
1922                         fs->object = fs->first_object;
1923                         pindex = first_pindex;
1924                 } else {
1925                         /*
1926                          * If it wasn't a write fault avoid having to copy
1927                          * the page by mapping it read-only.
1928                          */
1929                         fs->prot &= ~VM_PROT_WRITE;
1930                 }
1931         }
1932
1933         /*
1934          * Relock the map if necessary, then check the generation count.
1935          * relock_map() will update fs->timestamp to account for the
1936          * relocking if necessary.
1937          *
1938          * If the count has changed after relocking then all sorts of
1939          * crap may have happened and we have to retry.
1940          *
1941          * NOTE: The relock_map() can fail due to a deadlock against
1942          *       the vm_page we are holding BUSY.
1943          */
1944         if (fs->lookup_still_valid == FALSE && fs->map) {
1945                 if (relock_map(fs) ||
1946                     fs->map->timestamp != fs->map_generation) {
1947                         release_page(fs);
1948                         vm_object_pip_wakeup(fs->first_object);
1949                         vm_object_chain_release_all(fs->first_object,
1950                                                     fs->object);
1951                         if (fs->object != fs->first_object)
1952                                 vm_object_drop(fs->object);
1953                         unlock_and_deallocate(fs);
1954                         return (KERN_TRY_AGAIN);
1955                 }
1956         }
1957
1958         /*
1959          * If the fault is a write, we know that this page is being
1960          * written NOW so dirty it explicitly to save on pmap_is_modified()
1961          * calls later.
1962          *
1963          * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
1964          * if the page is already dirty to prevent data written with
1965          * the expectation of being synced from not being synced.
1966          * Likewise if this entry does not request NOSYNC then make
1967          * sure the page isn't marked NOSYNC.  Applications sharing
1968          * data should use the same flags to avoid ping ponging.
1969          *
1970          * Also tell the backing pager, if any, that it should remove
1971          * any swap backing since the page is now dirty.
1972          */
1973         vm_page_activate(fs->m);
1974         if (fs->prot & VM_PROT_WRITE) {
1975                 vm_object_set_writeable_dirty(fs->m->object);
1976                 vm_set_nosync(fs->m, fs->entry);
1977                 if (fs->fault_flags & VM_FAULT_DIRTY) {
1978                         vm_page_dirty(fs->m);
1979                         swap_pager_unswapped(fs->m);
1980                 }
1981         }
1982
1983         vm_object_pip_wakeup(fs->first_object);
1984         vm_object_chain_release_all(fs->first_object, fs->object);
1985         if (fs->object != fs->first_object)
1986                 vm_object_drop(fs->object);
1987
1988         /*
1989          * Page had better still be busy.  We are still locked up and 
1990          * fs->object will have another PIP reference if it is not equal
1991          * to fs->first_object.
1992          */
1993         KASSERT(fs->m->flags & PG_BUSY,
1994                 ("vm_fault: page %p not busy!", fs->m));
1995
1996         /*
1997          * Sanity check: page must be completely valid or it is not fit to
1998          * map into user space.  vm_pager_get_pages() ensures this.
1999          */
2000         if (fs->m->valid != VM_PAGE_BITS_ALL) {
2001                 vm_page_zero_invalid(fs->m, TRUE);
2002                 kprintf("Warning: page %p partially invalid on fault\n", fs->m);
2003         }
2004         vm_page_flag_clear(fs->m, PG_ZERO);
2005
2006         return (KERN_SUCCESS);
2007 }
2008
2009 /*
2010  * Hold each of the physical pages that are mapped by the specified range of
2011  * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid
2012  * and allow the specified types of access, "prot".  If all of the implied
2013  * pages are successfully held, then the number of held pages is returned
2014  * together with pointers to those pages in the array "ma".  However, if any
2015  * of the pages cannot be held, -1 is returned.
2016  */
2017 int
2018 vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
2019     vm_prot_t prot, vm_page_t *ma, int max_count)
2020 {
2021         vm_offset_t start, end;
2022         int i, npages, error;
2023
2024         start = trunc_page(addr);
2025         end = round_page(addr + len);
2026
2027         npages = howmany(end - start, PAGE_SIZE);
2028
2029         if (npages > max_count)
2030                 return -1;
2031
2032         for (i = 0; i < npages; i++) {
2033                 // XXX error handling
2034                 ma[i] = vm_fault_page_quick(start + (i * PAGE_SIZE),
2035                         prot,
2036                         &error);
2037         }
2038
2039         return npages;
2040 }
2041
2042 /*
2043  * Wire down a range of virtual addresses in a map.  The entry in question
2044  * should be marked in-transition and the map must be locked.  We must
2045  * release the map temporarily while faulting-in the page to avoid a
2046  * deadlock.  Note that the entry may be clipped while we are blocked but
2047  * will never be freed.
2048  *
2049  * No requirements.
2050  */
2051 int
2052 vm_fault_wire(vm_map_t map, vm_map_entry_t entry,
2053               boolean_t user_wire, int kmflags)
2054 {
2055         boolean_t fictitious;
2056         vm_offset_t start;
2057         vm_offset_t end;
2058         vm_offset_t va;
2059         vm_paddr_t pa;
2060         vm_page_t m;
2061         pmap_t pmap;
2062         int rv;
2063         int wire_prot;
2064         int fault_flags;
2065
2066         lwkt_gettoken(&map->token);
2067
2068         if (user_wire) {
2069                 wire_prot = VM_PROT_READ;
2070                 fault_flags = VM_FAULT_USER_WIRE;
2071         } else {
2072                 wire_prot = VM_PROT_READ | VM_PROT_WRITE;
2073                 fault_flags = VM_FAULT_CHANGE_WIRING;
2074         }
2075         if (kmflags & KM_NOTLBSYNC)
2076                 wire_prot |= VM_PROT_NOSYNC;
2077
2078         pmap = vm_map_pmap(map);
2079         start = entry->start;
2080         end = entry->end;
2081         switch(entry->maptype) {
2082         case VM_MAPTYPE_NORMAL:
2083         case VM_MAPTYPE_VPAGETABLE:
2084                 fictitious = entry->object.vm_object &&
2085                             ((entry->object.vm_object->type == OBJT_DEVICE) ||
2086                              (entry->object.vm_object->type == OBJT_MGTDEVICE));
2087                 break;
2088         case VM_MAPTYPE_UKSMAP:
2089                 fictitious = TRUE;
2090                 break;
2091         default:
2092                 fictitious = FALSE;
2093                 break;
2094         }
2095
2096         if (entry->eflags & MAP_ENTRY_KSTACK)
2097                 start += PAGE_SIZE;
2098         map->timestamp++;
2099         vm_map_unlock(map);
2100
2101         /*
2102          * We simulate a fault to get the page and enter it in the physical
2103          * map.
2104          */
2105         for (va = start; va < end; va += PAGE_SIZE) {
2106                 rv = vm_fault(map, va, wire_prot, fault_flags);
2107                 if (rv) {
2108                         while (va > start) {
2109                                 va -= PAGE_SIZE;
2110                                 if ((pa = pmap_extract(pmap, va)) == 0)
2111                                         continue;
2112                                 pmap_change_wiring(pmap, va, FALSE, entry);
2113                                 if (!fictitious) {
2114                                         m = PHYS_TO_VM_PAGE(pa);
2115                                         vm_page_busy_wait(m, FALSE, "vmwrpg");
2116                                         vm_page_unwire(m, 1);
2117                                         vm_page_wakeup(m);
2118                                 }
2119                         }
2120                         goto done;
2121                 }
2122         }
2123         rv = KERN_SUCCESS;
2124 done:
2125         vm_map_lock(map);
2126         lwkt_reltoken(&map->token);
2127         return (rv);
2128 }
2129
2130 /*
2131  * Unwire a range of virtual addresses in a map.  The map should be
2132  * locked.
2133  */
2134 void
2135 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
2136 {
2137         boolean_t fictitious;
2138         vm_offset_t start;
2139         vm_offset_t end;
2140         vm_offset_t va;
2141         vm_paddr_t pa;
2142         vm_page_t m;
2143         pmap_t pmap;
2144
2145         lwkt_gettoken(&map->token);
2146
2147         pmap = vm_map_pmap(map);
2148         start = entry->start;
2149         end = entry->end;
2150         fictitious = entry->object.vm_object &&
2151                         ((entry->object.vm_object->type == OBJT_DEVICE) ||
2152                          (entry->object.vm_object->type == OBJT_MGTDEVICE));
2153         if (entry->eflags & MAP_ENTRY_KSTACK)
2154                 start += PAGE_SIZE;
2155
2156         /*
2157          * Since the pages are wired down, we must be able to get their
2158          * mappings from the physical map system.
2159          */
2160         for (va = start; va < end; va += PAGE_SIZE) {
2161                 pa = pmap_extract(pmap, va);
2162                 if (pa != 0) {
2163                         pmap_change_wiring(pmap, va, FALSE, entry);
2164                         if (!fictitious) {
2165                                 m = PHYS_TO_VM_PAGE(pa);
2166                                 vm_page_busy_wait(m, FALSE, "vmwupg");
2167                                 vm_page_unwire(m, 1);
2168                                 vm_page_wakeup(m);
2169                         }
2170                 }
2171         }
2172         lwkt_reltoken(&map->token);
2173 }
2174
2175 /*
2176  * Copy all of the pages from a wired-down map entry to another.
2177  *
2178  * The source and destination maps must be locked for write.
2179  * The source and destination maps token must be held
2180  * The source map entry must be wired down (or be a sharing map
2181  * entry corresponding to a main map entry that is wired down).
2182  *
2183  * No other requirements.
2184  *
2185  * XXX do segment optimization
2186  */
2187 void
2188 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
2189                     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
2190 {
2191         vm_object_t dst_object;
2192         vm_object_t src_object;
2193         vm_ooffset_t dst_offset;
2194         vm_ooffset_t src_offset;
2195         vm_prot_t prot;
2196         vm_offset_t vaddr;
2197         vm_page_t dst_m;
2198         vm_page_t src_m;
2199
2200         src_object = src_entry->object.vm_object;
2201         src_offset = src_entry->offset;
2202
2203         /*
2204          * Create the top-level object for the destination entry. (Doesn't
2205          * actually shadow anything - we copy the pages directly.)
2206          */
2207         vm_map_entry_allocate_object(dst_entry);
2208         dst_object = dst_entry->object.vm_object;
2209
2210         prot = dst_entry->max_protection;
2211
2212         /*
2213          * Loop through all of the pages in the entry's range, copying each
2214          * one from the source object (it should be there) to the destination
2215          * object.
2216          */
2217         vm_object_hold(src_object);
2218         vm_object_hold(dst_object);
2219         for (vaddr = dst_entry->start, dst_offset = 0;
2220             vaddr < dst_entry->end;
2221             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
2222
2223                 /*
2224                  * Allocate a page in the destination object
2225                  */
2226                 do {
2227                         dst_m = vm_page_alloc(dst_object,
2228                                               OFF_TO_IDX(dst_offset),
2229                                               VM_ALLOC_NORMAL);
2230                         if (dst_m == NULL) {
2231                                 vm_wait(0);
2232                         }
2233                 } while (dst_m == NULL);
2234
2235                 /*
2236                  * Find the page in the source object, and copy it in.
2237                  * (Because the source is wired down, the page will be in
2238                  * memory.)
2239                  */
2240                 src_m = vm_page_lookup(src_object,
2241                                        OFF_TO_IDX(dst_offset + src_offset));
2242                 if (src_m == NULL)
2243                         panic("vm_fault_copy_wired: page missing");
2244
2245                 vm_page_copy(src_m, dst_m);
2246                 vm_page_event(src_m, VMEVENT_COW);
2247
2248                 /*
2249                  * Enter it in the pmap...
2250                  */
2251
2252                 vm_page_flag_clear(dst_m, PG_ZERO);
2253                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE, dst_entry);
2254
2255                 /*
2256                  * Mark it no longer busy, and put it on the active list.
2257                  */
2258                 vm_page_activate(dst_m);
2259                 vm_page_wakeup(dst_m);
2260         }
2261         vm_object_drop(dst_object);
2262         vm_object_drop(src_object);
2263 }
2264
2265 #if 0
2266
2267 /*
2268  * This routine checks around the requested page for other pages that
2269  * might be able to be faulted in.  This routine brackets the viable
2270  * pages for the pages to be paged in.
2271  *
2272  * Inputs:
2273  *      m, rbehind, rahead
2274  *
2275  * Outputs:
2276  *  marray (array of vm_page_t), reqpage (index of requested page)
2277  *
2278  * Return value:
2279  *  number of pages in marray
2280  */
2281 static int
2282 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
2283                           vm_page_t *marray, int *reqpage)
2284 {
2285         int i,j;
2286         vm_object_t object;
2287         vm_pindex_t pindex, startpindex, endpindex, tpindex;
2288         vm_page_t rtm;
2289         int cbehind, cahead;
2290
2291         object = m->object;
2292         pindex = m->pindex;
2293
2294         /*
2295          * we don't fault-ahead for device pager
2296          */
2297         if ((object->type == OBJT_DEVICE) ||
2298             (object->type == OBJT_MGTDEVICE)) {
2299                 *reqpage = 0;
2300                 marray[0] = m;
2301                 return 1;
2302         }
2303
2304         /*
2305          * if the requested page is not available, then give up now
2306          */
2307         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
2308                 *reqpage = 0;   /* not used by caller, fix compiler warn */
2309                 return 0;
2310         }
2311
2312         if ((cbehind == 0) && (cahead == 0)) {
2313                 *reqpage = 0;
2314                 marray[0] = m;
2315                 return 1;
2316         }
2317
2318         if (rahead > cahead) {
2319                 rahead = cahead;
2320         }
2321
2322         if (rbehind > cbehind) {
2323                 rbehind = cbehind;
2324         }
2325
2326         /*
2327          * Do not do any readahead if we have insufficient free memory.
2328          *
2329          * XXX code was broken disabled before and has instability
2330          * with this conditonal fixed, so shortcut for now.
2331          */
2332         if (burst_fault == 0 || vm_page_count_severe()) {
2333                 marray[0] = m;
2334                 *reqpage = 0;
2335                 return 1;
2336         }
2337
2338         /*
2339          * scan backward for the read behind pages -- in memory 
2340          *
2341          * Assume that if the page is not found an interrupt will not
2342          * create it.  Theoretically interrupts can only remove (busy)
2343          * pages, not create new associations.
2344          */
2345         if (pindex > 0) {
2346                 if (rbehind > pindex) {
2347                         rbehind = pindex;
2348                         startpindex = 0;
2349                 } else {
2350                         startpindex = pindex - rbehind;
2351                 }
2352
2353                 vm_object_hold(object);
2354                 for (tpindex = pindex; tpindex > startpindex; --tpindex) {
2355                         if (vm_page_lookup(object, tpindex - 1))
2356                                 break;
2357                 }
2358
2359                 i = 0;
2360                 while (tpindex < pindex) {
2361                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2362                                                              VM_ALLOC_NULL_OK);
2363                         if (rtm == NULL) {
2364                                 for (j = 0; j < i; j++) {
2365                                         vm_page_free(marray[j]);
2366                                 }
2367                                 vm_object_drop(object);
2368                                 marray[0] = m;
2369                                 *reqpage = 0;
2370                                 return 1;
2371                         }
2372                         marray[i] = rtm;
2373                         ++i;
2374                         ++tpindex;
2375                 }
2376                 vm_object_drop(object);
2377         } else {
2378                 i = 0;
2379         }
2380
2381         /*
2382          * Assign requested page
2383          */
2384         marray[i] = m;
2385         *reqpage = i;
2386         ++i;
2387
2388         /*
2389          * Scan forwards for read-ahead pages
2390          */
2391         tpindex = pindex + 1;
2392         endpindex = tpindex + rahead;
2393         if (endpindex > object->size)
2394                 endpindex = object->size;
2395
2396         vm_object_hold(object);
2397         while (tpindex < endpindex) {
2398                 if (vm_page_lookup(object, tpindex))
2399                         break;
2400                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2401                                                      VM_ALLOC_NULL_OK);
2402                 if (rtm == NULL)
2403                         break;
2404                 marray[i] = rtm;
2405                 ++i;
2406                 ++tpindex;
2407         }
2408         vm_object_drop(object);
2409
2410         return (i);
2411 }
2412
2413 #endif
2414
2415 /*
2416  * vm_prefault() provides a quick way of clustering pagefaults into a
2417  * processes address space.  It is a "cousin" of pmap_object_init_pt,
2418  * except it runs at page fault time instead of mmap time.
2419  *
2420  * vm.fast_fault        Enables pre-faulting zero-fill pages
2421  *
2422  * vm.prefault_pages    Number of pages (1/2 negative, 1/2 positive) to
2423  *                      prefault.  Scan stops in either direction when
2424  *                      a page is found to already exist.
2425  *
2426  * This code used to be per-platform pmap_prefault().  It is now
2427  * machine-independent and enhanced to also pre-fault zero-fill pages
2428  * (see vm.fast_fault) as well as make them writable, which greatly
2429  * reduces the number of page faults programs incur.
2430  *
2431  * Application performance when pre-faulting zero-fill pages is heavily
2432  * dependent on the application.  Very tiny applications like /bin/echo
2433  * lose a little performance while applications of any appreciable size
2434  * gain performance.  Prefaulting multiple pages also reduces SMP
2435  * congestion and can improve SMP performance significantly.
2436  *
2437  * NOTE!  prot may allow writing but this only applies to the top level
2438  *        object.  If we wind up mapping a page extracted from a backing
2439  *        object we have to make sure it is read-only.
2440  *
2441  * NOTE!  The caller has already handled any COW operations on the
2442  *        vm_map_entry via the normal fault code.  Do NOT call this
2443  *        shortcut unless the normal fault code has run on this entry.
2444  *
2445  * The related map must be locked.
2446  * No other requirements.
2447  */
2448 static int vm_prefault_pages = 8;
2449 SYSCTL_INT(_vm, OID_AUTO, prefault_pages, CTLFLAG_RW, &vm_prefault_pages, 0,
2450            "Maximum number of pages to pre-fault");
2451 static int vm_fast_fault = 1;
2452 SYSCTL_INT(_vm, OID_AUTO, fast_fault, CTLFLAG_RW, &vm_fast_fault, 0,
2453            "Burst fault zero-fill regions");
2454
2455 /*
2456  * Set PG_NOSYNC if the map entry indicates so, but only if the page
2457  * is not already dirty by other means.  This will prevent passive
2458  * filesystem syncing as well as 'sync' from writing out the page.
2459  */
2460 static void
2461 vm_set_nosync(vm_page_t m, vm_map_entry_t entry)
2462 {
2463         if (entry->eflags & MAP_ENTRY_NOSYNC) {
2464                 if (m->dirty == 0)
2465                         vm_page_flag_set(m, PG_NOSYNC);
2466         } else {
2467                 vm_page_flag_clear(m, PG_NOSYNC);
2468         }
2469 }
2470
2471 static void
2472 vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot,
2473             int fault_flags)
2474 {
2475         struct lwp *lp;
2476         vm_page_t m;
2477         vm_offset_t addr;
2478         vm_pindex_t index;
2479         vm_pindex_t pindex;
2480         vm_object_t object;
2481         int pprot;
2482         int i;
2483         int noneg;
2484         int nopos;
2485         int maxpages;
2486
2487         /*
2488          * Get stable max count value, disabled if set to 0
2489          */
2490         maxpages = vm_prefault_pages;
2491         cpu_ccfence();
2492         if (maxpages <= 0)
2493                 return;
2494
2495         /*
2496          * We do not currently prefault mappings that use virtual page
2497          * tables.  We do not prefault foreign pmaps.
2498          */
2499         if (entry->maptype != VM_MAPTYPE_NORMAL)
2500                 return;
2501         lp = curthread->td_lwp;
2502         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2503                 return;
2504
2505         /*
2506          * Limit pre-fault count to 1024 pages.
2507          */
2508         if (maxpages > 1024)
2509                 maxpages = 1024;
2510
2511         object = entry->object.vm_object;
2512         KKASSERT(object != NULL);
2513         KKASSERT(object == entry->object.vm_object);
2514         vm_object_hold(object);
2515         vm_object_chain_acquire(object, 0);
2516
2517         noneg = 0;
2518         nopos = 0;
2519         for (i = 0; i < maxpages; ++i) {
2520                 vm_object_t lobject;
2521                 vm_object_t nobject;
2522                 int allocated = 0;
2523                 int error;
2524
2525                 /*
2526                  * This can eat a lot of time on a heavily contended
2527                  * machine so yield on the tick if needed.
2528                  */
2529                 if ((i & 7) == 7)
2530                         lwkt_yield();
2531
2532                 /*
2533                  * Calculate the page to pre-fault, stopping the scan in
2534                  * each direction separately if the limit is reached.
2535                  */
2536                 if (i & 1) {
2537                         if (noneg)
2538                                 continue;
2539                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2540                 } else {
2541                         if (nopos)
2542                                 continue;
2543                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2544                 }
2545                 if (addr < entry->start) {
2546                         noneg = 1;
2547                         if (noneg && nopos)
2548                                 break;
2549                         continue;
2550                 }
2551                 if (addr >= entry->end) {
2552                         nopos = 1;
2553                         if (noneg && nopos)
2554                                 break;
2555                         continue;
2556                 }
2557
2558                 /*
2559                  * Skip pages already mapped, and stop scanning in that
2560                  * direction.  When the scan terminates in both directions
2561                  * we are done.
2562                  */
2563                 if (pmap_prefault_ok(pmap, addr) == 0) {
2564                         if (i & 1)
2565                                 noneg = 1;
2566                         else
2567                                 nopos = 1;
2568                         if (noneg && nopos)
2569                                 break;
2570                         continue;
2571                 }
2572
2573                 /*
2574                  * Follow the VM object chain to obtain the page to be mapped
2575                  * into the pmap.
2576                  *
2577                  * If we reach the terminal object without finding a page
2578                  * and we determine it would be advantageous, then allocate
2579                  * a zero-fill page for the base object.  The base object
2580                  * is guaranteed to be OBJT_DEFAULT for this case.
2581                  *
2582                  * In order to not have to check the pager via *haspage*()
2583                  * we stop if any non-default object is encountered.  e.g.
2584                  * a vnode or swap object would stop the loop.
2585                  */
2586                 index = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2587                 lobject = object;
2588                 pindex = index;
2589                 pprot = prot;
2590
2591                 KKASSERT(lobject == entry->object.vm_object);
2592                 /*vm_object_hold(lobject); implied */
2593
2594                 while ((m = vm_page_lookup_busy_try(lobject, pindex,
2595                                                     TRUE, &error)) == NULL) {
2596                         if (lobject->type != OBJT_DEFAULT)
2597                                 break;
2598                         if (lobject->backing_object == NULL) {
2599                                 if (vm_fast_fault == 0)
2600                                         break;
2601                                 if ((prot & VM_PROT_WRITE) == 0 ||
2602                                     vm_page_count_min(0)) {
2603                                         break;
2604                                 }
2605
2606                                 /*
2607                                  * NOTE: Allocated from base object
2608                                  */
2609                                 m = vm_page_alloc(object, index,
2610                                                   VM_ALLOC_NORMAL |
2611                                                   VM_ALLOC_ZERO |
2612                                                   VM_ALLOC_USE_GD |
2613                                                   VM_ALLOC_NULL_OK);
2614                                 if (m == NULL)
2615                                         break;
2616                                 allocated = 1;
2617                                 pprot = prot;
2618                                 /* lobject = object .. not needed */
2619                                 break;
2620                         }
2621                         if (lobject->backing_object_offset & PAGE_MASK)
2622                                 break;
2623                         nobject = lobject->backing_object;
2624                         vm_object_hold(nobject);
2625                         KKASSERT(nobject == lobject->backing_object);
2626                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
2627                         if (lobject != object) {
2628                                 vm_object_lock_swap();
2629                                 vm_object_drop(lobject);
2630                         }
2631                         lobject = nobject;
2632                         pprot &= ~VM_PROT_WRITE;
2633                         vm_object_chain_acquire(lobject, 0);
2634                 }
2635
2636                 /*
2637                  * NOTE: A non-NULL (m) will be associated with lobject if
2638                  *       it was found there, otherwise it is probably a
2639                  *       zero-fill page associated with the base object.
2640                  *
2641                  * Give-up if no page is available.
2642                  */
2643                 if (m == NULL) {
2644                         if (lobject != object) {
2645 #if 0
2646                                 if (object->backing_object != lobject)
2647                                         vm_object_hold(object->backing_object);
2648 #endif
2649                                 vm_object_chain_release_all(
2650                                         object->backing_object, lobject);
2651 #if 0
2652                                 if (object->backing_object != lobject)
2653                                         vm_object_drop(object->backing_object);
2654 #endif
2655                                 vm_object_drop(lobject);
2656                         }
2657                         break;
2658                 }
2659
2660                 /*
2661                  * The object must be marked dirty if we are mapping a
2662                  * writable page.  m->object is either lobject or object,
2663                  * both of which are still held.  Do this before we
2664                  * potentially drop the object.
2665                  */
2666                 if (pprot & VM_PROT_WRITE)
2667                         vm_object_set_writeable_dirty(m->object);
2668
2669                 /*
2670                  * Do not conditionalize on PG_RAM.  If pages are present in
2671                  * the VM system we assume optimal caching.  If caching is
2672                  * not optimal the I/O gravy train will be restarted when we
2673                  * hit an unavailable page.  We do not want to try to restart
2674                  * the gravy train now because we really don't know how much
2675                  * of the object has been cached.  The cost for restarting
2676                  * the gravy train should be low (since accesses will likely
2677                  * be I/O bound anyway).
2678                  */
2679                 if (lobject != object) {
2680 #if 0
2681                         if (object->backing_object != lobject)
2682                                 vm_object_hold(object->backing_object);
2683 #endif
2684                         vm_object_chain_release_all(object->backing_object,
2685                                                     lobject);
2686 #if 0
2687                         if (object->backing_object != lobject)
2688                                 vm_object_drop(object->backing_object);
2689 #endif
2690                         vm_object_drop(lobject);
2691                 }
2692
2693                 /*
2694                  * Enter the page into the pmap if appropriate.  If we had
2695                  * allocated the page we have to place it on a queue.  If not
2696                  * we just have to make sure it isn't on the cache queue
2697                  * (pages on the cache queue are not allowed to be mapped).
2698                  */
2699                 if (allocated) {
2700                         /*
2701                          * Page must be zerod.
2702                          */
2703                         if ((m->flags & PG_ZERO) == 0) {
2704                                 vm_page_zero_fill(m);
2705                         } else {
2706 #ifdef PMAP_DEBUG
2707                                 pmap_page_assertzero(
2708                                                 VM_PAGE_TO_PHYS(m));
2709 #endif
2710                                 vm_page_flag_clear(m, PG_ZERO);
2711                                 mycpu->gd_cnt.v_ozfod++;
2712                         }
2713                         mycpu->gd_cnt.v_zfod++;
2714                         m->valid = VM_PAGE_BITS_ALL;
2715
2716                         /*
2717                          * Handle dirty page case
2718                          */
2719                         if (pprot & VM_PROT_WRITE)
2720                                 vm_set_nosync(m, entry);
2721                         pmap_enter(pmap, addr, m, pprot, 0, entry);
2722                         mycpu->gd_cnt.v_vm_faults++;
2723                         if (curthread->td_lwp)
2724                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2725                         vm_page_deactivate(m);
2726                         if (pprot & VM_PROT_WRITE) {
2727                                 /*vm_object_set_writeable_dirty(m->object);*/
2728                                 vm_set_nosync(m, entry);
2729                                 if (fault_flags & VM_FAULT_DIRTY) {
2730                                         vm_page_dirty(m);
2731                                         /*XXX*/
2732                                         swap_pager_unswapped(m);
2733                                 }
2734                         }
2735                         vm_page_wakeup(m);
2736                 } else if (error) {
2737                         /* couldn't busy page, no wakeup */
2738                 } else if (
2739                     ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2740                     (m->flags & PG_FICTITIOUS) == 0) {
2741                         /*
2742                          * A fully valid page not undergoing soft I/O can
2743                          * be immediately entered into the pmap.
2744                          */
2745                         if ((m->queue - m->pc) == PQ_CACHE)
2746                                 vm_page_deactivate(m);
2747                         if (pprot & VM_PROT_WRITE) {
2748                                 /*vm_object_set_writeable_dirty(m->object);*/
2749                                 vm_set_nosync(m, entry);
2750                                 if (fault_flags & VM_FAULT_DIRTY) {
2751                                         vm_page_dirty(m);
2752                                         /*XXX*/
2753                                         swap_pager_unswapped(m);
2754                                 }
2755                         }
2756                         if (pprot & VM_PROT_WRITE)
2757                                 vm_set_nosync(m, entry);
2758                         pmap_enter(pmap, addr, m, pprot, 0, entry);
2759                         mycpu->gd_cnt.v_vm_faults++;
2760                         if (curthread->td_lwp)
2761                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2762                         vm_page_wakeup(m);
2763                 } else {
2764                         vm_page_wakeup(m);
2765                 }
2766         }
2767         vm_object_chain_release(object);
2768         vm_object_drop(object);
2769 }
2770
2771 /*
2772  * Object can be held shared
2773  */
2774 static void
2775 vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
2776                   vm_map_entry_t entry, int prot, int fault_flags)
2777 {
2778         struct lwp *lp;
2779         vm_page_t m;
2780         vm_offset_t addr;
2781         vm_pindex_t pindex;
2782         vm_object_t object;
2783         int i;
2784         int noneg;
2785         int nopos;
2786         int maxpages;
2787
2788         /*
2789          * Get stable max count value, disabled if set to 0
2790          */
2791         maxpages = vm_prefault_pages;
2792         cpu_ccfence();
2793         if (maxpages <= 0)
2794                 return;
2795
2796         /*
2797          * We do not currently prefault mappings that use virtual page
2798          * tables.  We do not prefault foreign pmaps.
2799          */
2800         if (entry->maptype != VM_MAPTYPE_NORMAL)
2801                 return;
2802         lp = curthread->td_lwp;
2803         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2804                 return;
2805         object = entry->object.vm_object;
2806         if (object->backing_object != NULL)
2807                 return;
2808         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
2809
2810         /*
2811          * Limit pre-fault count to 1024 pages.
2812          */
2813         if (maxpages > 1024)
2814                 maxpages = 1024;
2815
2816         noneg = 0;
2817         nopos = 0;
2818         for (i = 0; i < maxpages; ++i) {
2819                 int error;
2820
2821                 /*
2822                  * Calculate the page to pre-fault, stopping the scan in
2823                  * each direction separately if the limit is reached.
2824                  */
2825                 if (i & 1) {
2826                         if (noneg)
2827                                 continue;
2828                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2829                 } else {
2830                         if (nopos)
2831                                 continue;
2832                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2833                 }
2834                 if (addr < entry->start) {
2835                         noneg = 1;
2836                         if (noneg && nopos)
2837                                 break;
2838                         continue;
2839                 }
2840                 if (addr >= entry->end) {
2841                         nopos = 1;
2842                         if (noneg && nopos)
2843                                 break;
2844                         continue;
2845                 }
2846
2847                 /*
2848                  * Skip pages already mapped, and stop scanning in that
2849                  * direction.  When the scan terminates in both directions
2850                  * we are done.
2851                  */
2852                 if (pmap_prefault_ok(pmap, addr) == 0) {
2853                         if (i & 1)
2854                                 noneg = 1;
2855                         else
2856                                 nopos = 1;
2857                         if (noneg && nopos)
2858                                 break;
2859                         continue;
2860                 }
2861
2862                 /*
2863                  * Follow the VM object chain to obtain the page to be mapped
2864                  * into the pmap.  This version of the prefault code only
2865                  * works with terminal objects.
2866                  *
2867                  * WARNING!  We cannot call swap_pager_unswapped() with a
2868                  *           shared token.
2869                  */
2870                 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2871
2872                 m = vm_page_lookup_busy_try(object, pindex, TRUE, &error);
2873                 if (m == NULL || error)
2874                         continue;
2875
2876                 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2877                     (m->flags & PG_FICTITIOUS) == 0 &&
2878                     ((m->flags & PG_SWAPPED) == 0 ||
2879                      (prot & VM_PROT_WRITE) == 0 ||
2880                      (fault_flags & VM_FAULT_DIRTY) == 0)) {
2881                         /*
2882                          * A fully valid page not undergoing soft I/O can
2883                          * be immediately entered into the pmap.
2884                          */
2885                         if ((m->queue - m->pc) == PQ_CACHE)
2886                                 vm_page_deactivate(m);
2887                         if (prot & VM_PROT_WRITE) {
2888                                 vm_object_set_writeable_dirty(m->object);
2889                                 vm_set_nosync(m, entry);
2890                                 if (fault_flags & VM_FAULT_DIRTY) {
2891                                         vm_page_dirty(m);
2892                                         /*XXX*/
2893                                         swap_pager_unswapped(m);
2894                                 }
2895                         }
2896                         pmap_enter(pmap, addr, m, prot, 0, entry);
2897                         mycpu->gd_cnt.v_vm_faults++;
2898                         if (curthread->td_lwp)
2899                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2900                 }
2901                 vm_page_wakeup(m);
2902         }
2903 }