13ee44417877ca94295de6e1e3bf7651961f38e3
[dragonfly.git] / sys / vm / vm_fault.c
1 /*
2  * (MPSAFE)
3  *
4  * Copyright (c) 1991, 1993
5  *      The Regents of the University of California.  All rights reserved.
6  * Copyright (c) 1994 John S. Dyson
7  * All rights reserved.
8  * Copyright (c) 1994 David Greenman
9  * All rights reserved.
10  *
11  *
12  * This code is derived from software contributed to Berkeley by
13  * The Mach Operating System project at Carnegie-Mellon University.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  * 3. All advertising materials mentioning features or use of this software
24  *    must display the following acknowledgement:
25  *      This product includes software developed by the University of
26  *      California, Berkeley and its contributors.
27  * 4. Neither the name of the University nor the names of its contributors
28  *    may be used to endorse or promote products derived from this software
29  *    without specific prior written permission.
30  *
31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41  * SUCH DAMAGE.
42  *
43  *      from: @(#)vm_fault.c    8.4 (Berkeley) 1/12/94
44  *
45  *
46  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
47  * All rights reserved.
48  *
49  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
50  *
51  * Permission to use, copy, modify and distribute this software and
52  * its documentation is hereby granted, provided that both the copyright
53  * notice and this permission notice appear in all copies of the
54  * software, derivative works or modified versions, and any portions
55  * thereof, and that both notices appear in supporting documentation.
56  *
57  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
58  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
59  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
60  *
61  * Carnegie Mellon requests users of this software to return to
62  *
63  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
64  *  School of Computer Science
65  *  Carnegie Mellon University
66  *  Pittsburgh PA 15213-3890
67  *
68  * any improvements or extensions that they make and grant Carnegie the
69  * rights to redistribute these changes.
70  *
71  * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
72  * $DragonFly: src/sys/vm/vm_fault.c,v 1.47 2008/07/01 02:02:56 dillon Exp $
73  */
74
75 /*
76  *      Page fault handling module.
77  */
78
79 #include <sys/param.h>
80 #include <sys/systm.h>
81 #include <sys/kernel.h>
82 #include <sys/proc.h>
83 #include <sys/vnode.h>
84 #include <sys/resourcevar.h>
85 #include <sys/vmmeter.h>
86 #include <sys/vkernel.h>
87 #include <sys/lock.h>
88 #include <sys/sysctl.h>
89
90 #include <cpu/lwbuf.h>
91
92 #include <vm/vm.h>
93 #include <vm/vm_param.h>
94 #include <vm/pmap.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_object.h>
97 #include <vm/vm_page.h>
98 #include <vm/vm_pageout.h>
99 #include <vm/vm_kern.h>
100 #include <vm/vm_pager.h>
101 #include <vm/vnode_pager.h>
102 #include <vm/vm_extern.h>
103
104 #include <sys/thread2.h>
105 #include <vm/vm_page2.h>
106
107 struct faultstate {
108         vm_page_t m;
109         vm_object_t object;
110         vm_pindex_t pindex;
111         vm_prot_t prot;
112         vm_page_t first_m;
113         vm_object_t first_object;
114         vm_prot_t first_prot;
115         vm_map_t map;
116         vm_map_entry_t entry;
117         int lookup_still_valid;
118         int hardfault;
119         int fault_flags;
120         int map_generation;
121         boolean_t wired;
122         struct vnode *vp;
123 };
124
125 static int debug_cluster = 0;
126 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
127 static int vm_shared_fault = 1;
128 SYSCTL_INT(_vm, OID_AUTO, shared_fault, CTLFLAG_RW, &vm_shared_fault, 0,
129            "Allow shared token on vm_object");
130 static long vm_shared_hit = 0;
131 SYSCTL_LONG(_vm, OID_AUTO, shared_hit, CTLFLAG_RW, &vm_shared_hit, 0,
132            "Successful shared faults");
133 static long vm_shared_miss = 0;
134 SYSCTL_LONG(_vm, OID_AUTO, shared_miss, CTLFLAG_RW, &vm_shared_miss, 0,
135            "Successful shared faults");
136
137 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t);
138 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *, vpte_t, int);
139 #if 0
140 static int vm_fault_additional_pages (vm_page_t, int, int, vm_page_t *, int *);
141 #endif
142 static void vm_set_nosync(vm_page_t m, vm_map_entry_t entry);
143 static void vm_prefault(pmap_t pmap, vm_offset_t addra,
144                         vm_map_entry_t entry, int prot, int fault_flags);
145 static void vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
146                         vm_map_entry_t entry, int prot, int fault_flags);
147
148 static __inline void
149 release_page(struct faultstate *fs)
150 {
151         vm_page_deactivate(fs->m);
152         vm_page_wakeup(fs->m);
153         fs->m = NULL;
154 }
155
156 /*
157  * NOTE: Once unlocked any cached fs->entry becomes invalid, any reuse
158  *       requires relocking and then checking the timestamp.
159  *
160  * NOTE: vm_map_lock_read() does not bump fs->map->timestamp so we do
161  *       not have to update fs->map_generation here.
162  *
163  * NOTE: This function can fail due to a deadlock against the caller's
164  *       holding of a vm_page BUSY.
165  */
166 static __inline int
167 relock_map(struct faultstate *fs)
168 {
169         int error;
170
171         if (fs->lookup_still_valid == FALSE && fs->map) {
172                 error = vm_map_lock_read_to(fs->map);
173                 if (error == 0)
174                         fs->lookup_still_valid = TRUE;
175         } else {
176                 error = 0;
177         }
178         return error;
179 }
180
181 static __inline void
182 unlock_map(struct faultstate *fs)
183 {
184         if (fs->lookup_still_valid && fs->map) {
185                 vm_map_lookup_done(fs->map, fs->entry, 0);
186                 fs->lookup_still_valid = FALSE;
187         }
188 }
189
190 /*
191  * Clean up after a successful call to vm_fault_object() so another call
192  * to vm_fault_object() can be made.
193  */
194 static void
195 _cleanup_successful_fault(struct faultstate *fs, int relock)
196 {
197         if (fs->object != fs->first_object) {
198                 vm_page_free(fs->first_m);
199                 vm_object_pip_wakeup(fs->object);
200                 fs->first_m = NULL;
201         }
202         fs->object = fs->first_object;
203         if (relock && fs->lookup_still_valid == FALSE) {
204                 if (fs->map)
205                         vm_map_lock_read(fs->map);
206                 fs->lookup_still_valid = TRUE;
207         }
208 }
209
210 static void
211 _unlock_things(struct faultstate *fs, int dealloc)
212 {
213         _cleanup_successful_fault(fs, 0);
214         if (dealloc) {
215                 /*vm_object_deallocate(fs->first_object);*/
216                 /*fs->first_object = NULL; drop used later on */
217         }
218         unlock_map(fs); 
219         if (fs->vp != NULL) { 
220                 vput(fs->vp);
221                 fs->vp = NULL;
222         }
223 }
224
225 #define unlock_things(fs) _unlock_things(fs, 0)
226 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
227 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
228
229 /*
230  * TRYPAGER 
231  *
232  * Determine if the pager for the current object *might* contain the page.
233  *
234  * We only need to try the pager if this is not a default object (default
235  * objects are zero-fill and have no real pager), and if we are not taking
236  * a wiring fault or if the FS entry is wired.
237  */
238 #define TRYPAGER(fs)    \
239                 (fs->object->type != OBJT_DEFAULT && \
240                 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) || fs->wired))
241
242 /*
243  * vm_fault:
244  *
245  * Handle a page fault occuring at the given address, requiring the given
246  * permissions, in the map specified.  If successful, the page is inserted
247  * into the associated physical map.
248  *
249  * NOTE: The given address should be truncated to the proper page address.
250  *
251  * KERN_SUCCESS is returned if the page fault is handled; otherwise,
252  * a standard error specifying why the fault is fatal is returned.
253  *
254  * The map in question must be referenced, and remains so.
255  * The caller may hold no locks.
256  * No other requirements.
257  */
258 int
259 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
260 {
261         int result;
262         vm_pindex_t first_pindex;
263         struct faultstate fs;
264         struct lwp *lp;
265         int growstack;
266
267         vm_page_pcpu_cache();
268         fs.hardfault = 0;
269         fs.fault_flags = fault_flags;
270         fs.vp = NULL;
271         growstack = 1;
272
273         if ((lp = curthread->td_lwp) != NULL)
274                 lp->lwp_flag |= LWP_PAGING;
275
276         lwkt_gettoken(&map->token);
277
278 RetryFault:
279         /*
280          * Find the vm_map_entry representing the backing store and resolve
281          * the top level object and page index.  This may have the side
282          * effect of executing a copy-on-write on the map entry and/or
283          * creating a shadow object, but will not COW any actual VM pages.
284          *
285          * On success fs.map is left read-locked and various other fields 
286          * are initialized but not otherwise referenced or locked.
287          *
288          * NOTE!  vm_map_lookup will try to upgrade the fault_type to
289          * VM_FAULT_WRITE if the map entry is a virtual page table and also
290          * writable, so we can set the 'A'accessed bit in the virtual page
291          * table entry.
292          */
293         fs.map = map;
294         result = vm_map_lookup(&fs.map, vaddr, fault_type,
295                                &fs.entry, &fs.first_object,
296                                &first_pindex, &fs.first_prot, &fs.wired);
297
298         /*
299          * If the lookup failed or the map protections are incompatible,
300          * the fault generally fails.  However, if the caller is trying
301          * to do a user wiring we have more work to do.
302          */
303         if (result != KERN_SUCCESS) {
304                 if (result != KERN_PROTECTION_FAILURE ||
305                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
306                 {
307                         if (result == KERN_INVALID_ADDRESS && growstack &&
308                             map != &kernel_map && curproc != NULL) {
309                                 result = vm_map_growstack(curproc, vaddr);
310                                 if (result == KERN_SUCCESS) {
311                                         growstack = 0;
312                                         goto RetryFault;
313                                 }
314                                 result = KERN_FAILURE;
315                         }
316                         goto done;
317                 }
318
319                 /*
320                  * If we are user-wiring a r/w segment, and it is COW, then
321                  * we need to do the COW operation.  Note that we don't
322                  * currently COW RO sections now, because it is NOT desirable
323                  * to COW .text.  We simply keep .text from ever being COW'ed
324                  * and take the heat that one cannot debug wired .text sections.
325                  */
326                 result = vm_map_lookup(&fs.map, vaddr,
327                                        VM_PROT_READ|VM_PROT_WRITE|
328                                         VM_PROT_OVERRIDE_WRITE,
329                                        &fs.entry, &fs.first_object,
330                                        &first_pindex, &fs.first_prot,
331                                        &fs.wired);
332                 if (result != KERN_SUCCESS) {
333                         result = KERN_FAILURE;
334                         goto done;
335                 }
336
337                 /*
338                  * If we don't COW now, on a user wire, the user will never
339                  * be able to write to the mapping.  If we don't make this
340                  * restriction, the bookkeeping would be nearly impossible.
341                  */
342                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
343                         fs.entry->max_protection &= ~VM_PROT_WRITE;
344         }
345
346         /*
347          * fs.map is read-locked
348          *
349          * Misc checks.  Save the map generation number to detect races.
350          */
351         fs.map_generation = fs.map->timestamp;
352
353         if (fs.entry->eflags & (MAP_ENTRY_NOFAULT | MAP_ENTRY_KSTACK)) {
354                 if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
355                         panic("vm_fault: fault on nofault entry, addr: %p",
356                               (void *)vaddr);
357                 }
358                 if ((fs.entry->eflags & MAP_ENTRY_KSTACK) &&
359                     vaddr >= fs.entry->start &&
360                     vaddr < fs.entry->start + PAGE_SIZE) {
361                         panic("vm_fault: fault on stack guard, addr: %p",
362                               (void *)vaddr);
363                 }
364         }
365
366         /*
367          * A system map entry may return a NULL object.  No object means
368          * no pager means an unrecoverable kernel fault.
369          */
370         if (fs.first_object == NULL) {
371                 panic("vm_fault: unrecoverable fault at %p in entry %p",
372                         (void *)vaddr, fs.entry);
373         }
374
375         /*
376          * Attempt to shortcut the fault if the lookup returns a
377          * terminal object and the page is present.  This allows us
378          * to obtain a shared token on the object instead of an exclusive
379          * token, which theoretically should allow concurrent faults.
380          */
381         if (vm_shared_fault &&
382             fs.first_object->backing_object == NULL &&
383             fs.entry->maptype == VM_MAPTYPE_NORMAL) {
384                 int error;
385                 vm_object_hold_shared(fs.first_object);
386                 /*fs.vp = vnode_pager_lock(fs.first_object);*/
387                 fs.m = vm_page_lookup_busy_try(fs.first_object,
388                                                 first_pindex,
389                                                 TRUE, &error);
390                 if (error == 0 && fs.m) {
391                         /*
392                          * Activate the page and figure out if we can
393                          * short-cut a quick mapping.
394                          *
395                          * WARNING!  We cannot call swap_pager_unswapped()
396                          *           with a shared token!
397                          */
398                         vm_page_activate(fs.m);
399                         if (fs.m->valid == VM_PAGE_BITS_ALL &&
400                             ((fs.m->flags & PG_SWAPPED) == 0 ||
401                              (fs.prot & VM_PROT_WRITE) == 0 ||
402                              (fs.fault_flags & VM_FAULT_DIRTY) == 0)) {
403                                 fs.lookup_still_valid = TRUE;
404                                 fs.first_m = NULL;
405                                 fs.object = fs.first_object;
406                                 fs.prot = fs.first_prot;
407                                 if (fs.wired)
408                                         fault_type = fs.first_prot;
409                                 if (fs.prot & VM_PROT_WRITE) {
410                                         vm_object_set_writeable_dirty(
411                                                         fs.m->object);
412                                         vm_set_nosync(fs.m, fs.entry);
413                                         if (fs.fault_flags & VM_FAULT_DIRTY) {
414                                                 vm_page_dirty(fs.m);
415                                                 /*XXX*/
416                                                 swap_pager_unswapped(fs.m);
417                                         }
418                                 }
419                                 result = KERN_SUCCESS;
420                                 fault_flags |= VM_FAULT_BURST_QUICK;
421                                 fault_flags &= ~VM_FAULT_BURST;
422                                 ++vm_shared_hit;
423                                 goto quick;
424                         }
425                         vm_page_wakeup(fs.m);
426                         fs.m = NULL;
427                 }
428                 vm_object_drop(fs.first_object); /* XXX drop on shared tok?*/
429         }
430         ++vm_shared_miss;
431
432         /*
433          * Bump the paging-in-progress count to prevent size changes (e.g.
434          * truncation operations) during I/O.  This must be done after
435          * obtaining the vnode lock in order to avoid possible deadlocks.
436          */
437         vm_object_hold(fs.first_object);
438         if (fs.vp == NULL)
439                 fs.vp = vnode_pager_lock(fs.first_object);
440
441         fs.lookup_still_valid = TRUE;
442         fs.first_m = NULL;
443         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
444
445         /*
446          * If the entry is wired we cannot change the page protection.
447          */
448         if (fs.wired)
449                 fault_type = fs.first_prot;
450
451         /*
452          * The page we want is at (first_object, first_pindex), but if the
453          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
454          * page table to figure out the actual pindex.
455          *
456          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
457          * ONLY
458          */
459         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
460                 result = vm_fault_vpagetable(&fs, &first_pindex,
461                                              fs.entry->aux.master_pde,
462                                              fault_type);
463                 if (result == KERN_TRY_AGAIN) {
464                         vm_object_drop(fs.first_object);
465                         goto RetryFault;
466                 }
467                 if (result != KERN_SUCCESS)
468                         goto done;
469         }
470
471         /*
472          * Now we have the actual (object, pindex), fault in the page.  If
473          * vm_fault_object() fails it will unlock and deallocate the FS
474          * data.   If it succeeds everything remains locked and fs->object
475          * will have an additional PIP count if it is not equal to
476          * fs->first_object
477          *
478          * vm_fault_object will set fs->prot for the pmap operation.  It is
479          * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
480          * page can be safely written.  However, it will force a read-only
481          * mapping for a read fault if the memory is managed by a virtual
482          * page table.
483          */
484         /* BEFORE */
485         result = vm_fault_object(&fs, first_pindex, fault_type);
486
487         if (result == KERN_TRY_AGAIN) {
488                 vm_object_drop(fs.first_object);
489                 goto RetryFault;
490         }
491         if (result != KERN_SUCCESS)
492                 goto done;
493
494 quick:
495         /*
496          * On success vm_fault_object() does not unlock or deallocate, and fs.m
497          * will contain a busied page.
498          *
499          * Enter the page into the pmap and do pmap-related adjustments.
500          */
501         vm_page_flag_set(fs.m, PG_REFERENCED);
502         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
503         mycpu->gd_cnt.v_vm_faults++;
504         if (curthread->td_lwp)
505                 ++curthread->td_lwp->lwp_ru.ru_minflt;
506
507         /*KKASSERT(fs.m->queue == PQ_NONE); page-in op may deactivate page */
508         KKASSERT(fs.m->flags & PG_BUSY);
509
510         /*
511          * If the page is not wired down, then put it where the pageout daemon
512          * can find it.
513          */
514         if (fs.fault_flags & VM_FAULT_WIRE_MASK) {
515                 if (fs.wired)
516                         vm_page_wire(fs.m);
517                 else
518                         vm_page_unwire(fs.m, 1);
519         } else {
520                 vm_page_activate(fs.m);
521         }
522         vm_page_wakeup(fs.m);
523
524         /*
525          * Burst in a few more pages if possible.  The fs.map should still
526          * be locked.  To avoid interlocking against a vnode->getblk
527          * operation we had to be sure to unbusy our primary vm_page above
528          * first.
529          */
530         if (fault_flags & VM_FAULT_BURST) {
531                 if ((fs.fault_flags & VM_FAULT_WIRE_MASK) == 0
532                     && fs.wired == 0) {
533                         vm_prefault(fs.map->pmap, vaddr,
534                                     fs.entry, fs.prot, fault_flags);
535                 }
536         }
537         if (fault_flags & VM_FAULT_BURST_QUICK) {
538                 if ((fs.fault_flags & VM_FAULT_WIRE_MASK) == 0
539                     && fs.wired == 0) {
540                         vm_prefault_quick(fs.map->pmap, vaddr,
541                                           fs.entry, fs.prot, fault_flags);
542                 }
543         }
544
545         /*
546          * Unlock everything, and return
547          */
548         unlock_things(&fs);
549
550         if (curthread->td_lwp) {
551                 if (fs.hardfault) {
552                         curthread->td_lwp->lwp_ru.ru_majflt++;
553                 } else {
554                         curthread->td_lwp->lwp_ru.ru_minflt++;
555                 }
556         }
557
558         /*vm_object_deallocate(fs.first_object);*/
559         /*fs.m = NULL; */
560         /*fs.first_object = NULL; must still drop later */
561
562         result = KERN_SUCCESS;
563 done:
564         if (fs.first_object)
565                 vm_object_drop(fs.first_object);
566         lwkt_reltoken(&map->token);
567         if (lp)
568                 lp->lwp_flag &= ~LWP_PAGING;
569         return (result);
570 }
571
572 /*
573  * Fault in the specified virtual address in the current process map, 
574  * returning a held VM page or NULL.  See vm_fault_page() for more 
575  * information.
576  *
577  * No requirements.
578  */
579 vm_page_t
580 vm_fault_page_quick(vm_offset_t va, vm_prot_t fault_type, int *errorp)
581 {
582         struct lwp *lp = curthread->td_lwp;
583         vm_page_t m;
584
585         m = vm_fault_page(&lp->lwp_vmspace->vm_map, va, 
586                           fault_type, VM_FAULT_NORMAL, errorp);
587         return(m);
588 }
589
590 /*
591  * Fault in the specified virtual address in the specified map, doing all
592  * necessary manipulation of the object store and all necessary I/O.  Return
593  * a held VM page or NULL, and set *errorp.  The related pmap is not
594  * updated.
595  *
596  * The returned page will be properly dirtied if VM_PROT_WRITE was specified,
597  * and marked PG_REFERENCED as well.
598  *
599  * If the page cannot be faulted writable and VM_PROT_WRITE was specified, an
600  * error will be returned.
601  *
602  * No requirements.
603  */
604 vm_page_t
605 vm_fault_page(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
606               int fault_flags, int *errorp)
607 {
608         vm_pindex_t first_pindex;
609         struct faultstate fs;
610         int result;
611         vm_prot_t orig_fault_type = fault_type;
612
613         fs.hardfault = 0;
614         fs.fault_flags = fault_flags;
615         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
616
617         lwkt_gettoken(&map->token);
618
619 RetryFault:
620         /*
621          * Find the vm_map_entry representing the backing store and resolve
622          * the top level object and page index.  This may have the side
623          * effect of executing a copy-on-write on the map entry and/or
624          * creating a shadow object, but will not COW any actual VM pages.
625          *
626          * On success fs.map is left read-locked and various other fields 
627          * are initialized but not otherwise referenced or locked.
628          *
629          * NOTE!  vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
630          * if the map entry is a virtual page table and also writable,
631          * so we can set the 'A'accessed bit in the virtual page table entry.
632          */
633         fs.map = map;
634         result = vm_map_lookup(&fs.map, vaddr, fault_type,
635                                &fs.entry, &fs.first_object,
636                                &first_pindex, &fs.first_prot, &fs.wired);
637
638         if (result != KERN_SUCCESS) {
639                 *errorp = result;
640                 fs.m = NULL;
641                 goto done;
642         }
643
644         /*
645          * fs.map is read-locked
646          *
647          * Misc checks.  Save the map generation number to detect races.
648          */
649         fs.map_generation = fs.map->timestamp;
650
651         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
652                 panic("vm_fault: fault on nofault entry, addr: %lx",
653                     (u_long)vaddr);
654         }
655
656         /*
657          * A system map entry may return a NULL object.  No object means
658          * no pager means an unrecoverable kernel fault.
659          */
660         if (fs.first_object == NULL) {
661                 panic("vm_fault: unrecoverable fault at %p in entry %p",
662                         (void *)vaddr, fs.entry);
663         }
664
665         /*
666          * Make a reference to this object to prevent its disposal while we
667          * are messing with it.  Once we have the reference, the map is free
668          * to be diddled.  Since objects reference their shadows (and copies),
669          * they will stay around as well.
670          *
671          * The reference should also prevent an unexpected collapse of the
672          * parent that might move pages from the current object into the
673          * parent unexpectedly, resulting in corruption.
674          *
675          * Bump the paging-in-progress count to prevent size changes (e.g.
676          * truncation operations) during I/O.  This must be done after
677          * obtaining the vnode lock in order to avoid possible deadlocks.
678          */
679         vm_object_hold(fs.first_object);
680         fs.vp = vnode_pager_lock(fs.first_object);
681
682         fs.lookup_still_valid = TRUE;
683         fs.first_m = NULL;
684         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
685
686         /*
687          * If the entry is wired we cannot change the page protection.
688          */
689         if (fs.wired)
690                 fault_type = fs.first_prot;
691
692         /*
693          * The page we want is at (first_object, first_pindex), but if the
694          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
695          * page table to figure out the actual pindex.
696          *
697          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
698          * ONLY
699          */
700         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
701                 result = vm_fault_vpagetable(&fs, &first_pindex,
702                                              fs.entry->aux.master_pde,
703                                              fault_type);
704                 if (result == KERN_TRY_AGAIN) {
705                         vm_object_drop(fs.first_object);
706                         goto RetryFault;
707                 }
708                 if (result != KERN_SUCCESS) {
709                         *errorp = result;
710                         fs.m = NULL;
711                         goto done;
712                 }
713         }
714
715         /*
716          * Now we have the actual (object, pindex), fault in the page.  If
717          * vm_fault_object() fails it will unlock and deallocate the FS
718          * data.   If it succeeds everything remains locked and fs->object
719          * will have an additinal PIP count if it is not equal to
720          * fs->first_object
721          */
722         result = vm_fault_object(&fs, first_pindex, fault_type);
723
724         if (result == KERN_TRY_AGAIN) {
725                 vm_object_drop(fs.first_object);
726                 goto RetryFault;
727         }
728         if (result != KERN_SUCCESS) {
729                 *errorp = result;
730                 fs.m = NULL;
731                 goto done;
732         }
733
734         if ((orig_fault_type & VM_PROT_WRITE) &&
735             (fs.prot & VM_PROT_WRITE) == 0) {
736                 *errorp = KERN_PROTECTION_FAILURE;
737                 unlock_and_deallocate(&fs);
738                 fs.m = NULL;
739                 goto done;
740         }
741
742         /*
743          * Update the pmap.  We really only have to do this if a COW
744          * occured to replace the read-only page with the new page.  For
745          * now just do it unconditionally. XXX
746          */
747         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
748         vm_page_flag_set(fs.m, PG_REFERENCED);
749         mycpu->gd_cnt.v_vm_faults++;
750         if (curthread->td_lwp)
751                 ++curthread->td_lwp->lwp_ru.ru_minflt;
752
753         /*
754          * On success vm_fault_object() does not unlock or deallocate, and fs.m
755          * will contain a busied page.  So we must unlock here after having
756          * messed with the pmap.
757          */
758         unlock_things(&fs);
759
760         /*
761          * Return a held page.  We are not doing any pmap manipulation so do
762          * not set PG_MAPPED.  However, adjust the page flags according to
763          * the fault type because the caller may not use a managed pmapping
764          * (so we don't want to lose the fact that the page will be dirtied
765          * if a write fault was specified).
766          */
767         vm_page_hold(fs.m);
768         vm_page_activate(fs.m);
769         if (fault_type & VM_PROT_WRITE)
770                 vm_page_dirty(fs.m);
771
772         if (curthread->td_lwp) {
773                 if (fs.hardfault) {
774                         curthread->td_lwp->lwp_ru.ru_majflt++;
775                 } else {
776                         curthread->td_lwp->lwp_ru.ru_minflt++;
777                 }
778         }
779
780         /*
781          * Unlock everything, and return the held page.
782          */
783         vm_page_wakeup(fs.m);
784         /*vm_object_deallocate(fs.first_object);*/
785         /*fs.first_object = NULL; */
786         *errorp = 0;
787
788 done:
789         if (fs.first_object)
790                 vm_object_drop(fs.first_object);
791         lwkt_reltoken(&map->token);
792         return(fs.m);
793 }
794
795 /*
796  * Fault in the specified (object,offset), dirty the returned page as
797  * needed.  If the requested fault_type cannot be done NULL and an
798  * error is returned.
799  *
800  * A held (but not busied) page is returned.
801  *
802  * No requirements.
803  */
804 vm_page_t
805 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
806                      vm_prot_t fault_type, int fault_flags, int *errorp)
807 {
808         int result;
809         vm_pindex_t first_pindex;
810         struct faultstate fs;
811         struct vm_map_entry entry;
812
813         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
814         bzero(&entry, sizeof(entry));
815         entry.object.vm_object = object;
816         entry.maptype = VM_MAPTYPE_NORMAL;
817         entry.protection = entry.max_protection = fault_type;
818
819         fs.hardfault = 0;
820         fs.fault_flags = fault_flags;
821         fs.map = NULL;
822         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
823
824 RetryFault:
825         
826         fs.first_object = object;
827         first_pindex = OFF_TO_IDX(offset);
828         fs.entry = &entry;
829         fs.first_prot = fault_type;
830         fs.wired = 0;
831         /*fs.map_generation = 0; unused */
832
833         /*
834          * Make a reference to this object to prevent its disposal while we
835          * are messing with it.  Once we have the reference, the map is free
836          * to be diddled.  Since objects reference their shadows (and copies),
837          * they will stay around as well.
838          *
839          * The reference should also prevent an unexpected collapse of the
840          * parent that might move pages from the current object into the
841          * parent unexpectedly, resulting in corruption.
842          *
843          * Bump the paging-in-progress count to prevent size changes (e.g.
844          * truncation operations) during I/O.  This must be done after
845          * obtaining the vnode lock in order to avoid possible deadlocks.
846          */
847         fs.vp = vnode_pager_lock(fs.first_object);
848
849         fs.lookup_still_valid = TRUE;
850         fs.first_m = NULL;
851         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
852
853 #if 0
854         /* XXX future - ability to operate on VM object using vpagetable */
855         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
856                 result = vm_fault_vpagetable(&fs, &first_pindex,
857                                              fs.entry->aux.master_pde,
858                                              fault_type);
859                 if (result == KERN_TRY_AGAIN)
860                         goto RetryFault;
861                 if (result != KERN_SUCCESS) {
862                         *errorp = result;
863                         return (NULL);
864                 }
865         }
866 #endif
867
868         /*
869          * Now we have the actual (object, pindex), fault in the page.  If
870          * vm_fault_object() fails it will unlock and deallocate the FS
871          * data.   If it succeeds everything remains locked and fs->object
872          * will have an additinal PIP count if it is not equal to
873          * fs->first_object
874          */
875         result = vm_fault_object(&fs, first_pindex, fault_type);
876
877         if (result == KERN_TRY_AGAIN)
878                 goto RetryFault;
879         if (result != KERN_SUCCESS) {
880                 *errorp = result;
881                 return(NULL);
882         }
883
884         if ((fault_type & VM_PROT_WRITE) && (fs.prot & VM_PROT_WRITE) == 0) {
885                 *errorp = KERN_PROTECTION_FAILURE;
886                 unlock_and_deallocate(&fs);
887                 return(NULL);
888         }
889
890         /*
891          * On success vm_fault_object() does not unlock or deallocate, so we
892          * do it here.  Note that the returned fs.m will be busied.
893          */
894         unlock_things(&fs);
895
896         /*
897          * Return a held page.  We are not doing any pmap manipulation so do
898          * not set PG_MAPPED.  However, adjust the page flags according to
899          * the fault type because the caller may not use a managed pmapping
900          * (so we don't want to lose the fact that the page will be dirtied
901          * if a write fault was specified).
902          */
903         vm_page_hold(fs.m);
904         vm_page_activate(fs.m);
905         if ((fault_type & VM_PROT_WRITE) || (fault_flags & VM_FAULT_DIRTY))
906                 vm_page_dirty(fs.m);
907         if (fault_flags & VM_FAULT_UNSWAP)
908                 swap_pager_unswapped(fs.m);
909
910         /*
911          * Indicate that the page was accessed.
912          */
913         vm_page_flag_set(fs.m, PG_REFERENCED);
914
915         if (curthread->td_lwp) {
916                 if (fs.hardfault) {
917                         curthread->td_lwp->lwp_ru.ru_majflt++;
918                 } else {
919                         curthread->td_lwp->lwp_ru.ru_minflt++;
920                 }
921         }
922
923         /*
924          * Unlock everything, and return the held page.
925          */
926         vm_page_wakeup(fs.m);
927         /*vm_object_deallocate(fs.first_object);*/
928         /*fs.first_object = NULL; */
929
930         *errorp = 0;
931         return(fs.m);
932 }
933
934 /*
935  * Translate the virtual page number (first_pindex) that is relative
936  * to the address space into a logical page number that is relative to the
937  * backing object.  Use the virtual page table pointed to by (vpte).
938  *
939  * This implements an N-level page table.  Any level can terminate the
940  * scan by setting VPTE_PS.   A linear mapping is accomplished by setting
941  * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
942  */
943 static
944 int
945 vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
946                     vpte_t vpte, int fault_type)
947 {
948         struct lwbuf *lwb;
949         struct lwbuf lwb_cache;
950         int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
951         int result = KERN_SUCCESS;
952         vpte_t *ptep;
953
954         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
955         for (;;) {
956                 /*
957                  * We cannot proceed if the vpte is not valid, not readable
958                  * for a read fault, or not writable for a write fault.
959                  */
960                 if ((vpte & VPTE_V) == 0) {
961                         unlock_and_deallocate(fs);
962                         return (KERN_FAILURE);
963                 }
964                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_R) == 0) {
965                         unlock_and_deallocate(fs);
966                         return (KERN_FAILURE);
967                 }
968                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_W) == 0) {
969                         unlock_and_deallocate(fs);
970                         return (KERN_FAILURE);
971                 }
972                 if ((vpte & VPTE_PS) || vshift == 0)
973                         break;
974                 KKASSERT(vshift >= VPTE_PAGE_BITS);
975
976                 /*
977                  * Get the page table page.  Nominally we only read the page
978                  * table, but since we are actively setting VPTE_M and VPTE_A,
979                  * tell vm_fault_object() that we are writing it. 
980                  *
981                  * There is currently no real need to optimize this.
982                  */
983                 result = vm_fault_object(fs, (vpte & VPTE_FRAME) >> PAGE_SHIFT,
984                                          VM_PROT_READ|VM_PROT_WRITE);
985                 if (result != KERN_SUCCESS)
986                         return (result);
987
988                 /*
989                  * Process the returned fs.m and look up the page table
990                  * entry in the page table page.
991                  */
992                 vshift -= VPTE_PAGE_BITS;
993                 lwb = lwbuf_alloc(fs->m, &lwb_cache);
994                 ptep = ((vpte_t *)lwbuf_kva(lwb) +
995                         ((*pindex >> vshift) & VPTE_PAGE_MASK));
996                 vpte = *ptep;
997
998                 /*
999                  * Page table write-back.  If the vpte is valid for the
1000                  * requested operation, do a write-back to the page table.
1001                  *
1002                  * XXX VPTE_M is not set properly for page directory pages.
1003                  * It doesn't get set in the page directory if the page table
1004                  * is modified during a read access.
1005                  */
1006                 vm_page_activate(fs->m);
1007                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_V) &&
1008                     (vpte & VPTE_W)) {
1009                         if ((vpte & (VPTE_M|VPTE_A)) != (VPTE_M|VPTE_A)) {
1010                                 atomic_set_long(ptep, VPTE_M | VPTE_A);
1011                                 vm_page_dirty(fs->m);
1012                         }
1013                 }
1014                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_V) &&
1015                     (vpte & VPTE_R)) {
1016                         if ((vpte & VPTE_A) == 0) {
1017                                 atomic_set_long(ptep, VPTE_A);
1018                                 vm_page_dirty(fs->m);
1019                         }
1020                 }
1021                 lwbuf_free(lwb);
1022                 vm_page_flag_set(fs->m, PG_REFERENCED);
1023                 vm_page_wakeup(fs->m);
1024                 fs->m = NULL;
1025                 cleanup_successful_fault(fs);
1026         }
1027         /*
1028          * Combine remaining address bits with the vpte.
1029          */
1030         /* JG how many bits from each? */
1031         *pindex = ((vpte & VPTE_FRAME) >> PAGE_SHIFT) +
1032                   (*pindex & ((1L << vshift) - 1));
1033         return (KERN_SUCCESS);
1034 }
1035
1036
1037 /*
1038  * This is the core of the vm_fault code.
1039  *
1040  * Do all operations required to fault-in (fs.first_object, pindex).  Run
1041  * through the shadow chain as necessary and do required COW or virtual
1042  * copy operations.  The caller has already fully resolved the vm_map_entry
1043  * and, if appropriate, has created a copy-on-write layer.  All we need to
1044  * do is iterate the object chain.
1045  *
1046  * On failure (fs) is unlocked and deallocated and the caller may return or
1047  * retry depending on the failure code.  On success (fs) is NOT unlocked or
1048  * deallocated, fs.m will contained a resolved, busied page, and fs.object
1049  * will have an additional PIP count if it is not equal to fs.first_object.
1050  *
1051  * fs->first_object must be held on call.
1052  */
1053 static
1054 int
1055 vm_fault_object(struct faultstate *fs,
1056                 vm_pindex_t first_pindex, vm_prot_t fault_type)
1057 {
1058         vm_object_t next_object;
1059         vm_pindex_t pindex;
1060         int error;
1061
1062         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
1063         fs->prot = fs->first_prot;
1064         fs->object = fs->first_object;
1065         pindex = first_pindex;
1066
1067         vm_object_chain_acquire(fs->first_object);
1068         vm_object_pip_add(fs->first_object, 1);
1069
1070         /* 
1071          * If a read fault occurs we try to make the page writable if
1072          * possible.  There are three cases where we cannot make the
1073          * page mapping writable:
1074          *
1075          * (1) The mapping is read-only or the VM object is read-only,
1076          *     fs->prot above will simply not have VM_PROT_WRITE set.
1077          *
1078          * (2) If the mapping is a virtual page table we need to be able
1079          *     to detect writes so we can set VPTE_M in the virtual page
1080          *     table.
1081          *
1082          * (3) If the VM page is read-only or copy-on-write, upgrading would
1083          *     just result in an unnecessary COW fault.
1084          *
1085          * VM_PROT_VPAGED is set if faulting via a virtual page table and
1086          * causes adjustments to the 'M'odify bit to also turn off write
1087          * access to force a re-fault.
1088          */
1089         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1090                 if ((fault_type & VM_PROT_WRITE) == 0)
1091                         fs->prot &= ~VM_PROT_WRITE;
1092         }
1093
1094         /* vm_object_hold(fs->object); implied b/c object == first_object */
1095
1096         for (;;) {
1097                 /*
1098                  * The entire backing chain from first_object to object
1099                  * inclusive is chainlocked.
1100                  *
1101                  * If the object is dead, we stop here
1102                  */
1103                 if (fs->object->flags & OBJ_DEAD) {
1104                         vm_object_pip_wakeup(fs->first_object);
1105                         vm_object_chain_release_all(fs->first_object,
1106                                                     fs->object);
1107                         if (fs->object != fs->first_object)
1108                                 vm_object_drop(fs->object);
1109                         unlock_and_deallocate(fs);
1110                         return (KERN_PROTECTION_FAILURE);
1111                 }
1112
1113                 /*
1114                  * See if the page is resident.  Wait/Retry if the page is
1115                  * busy (lots of stuff may have changed so we can't continue
1116                  * in that case).
1117                  *
1118                  * We can theoretically allow the soft-busy case on a read
1119                  * fault if the page is marked valid, but since such
1120                  * pages are typically already pmap'd, putting that
1121                  * special case in might be more effort then it is
1122                  * worth.  We cannot under any circumstances mess
1123                  * around with a vm_page_t->busy page except, perhaps,
1124                  * to pmap it.
1125                  */
1126                 fs->m = vm_page_lookup_busy_try(fs->object, pindex,
1127                                                 TRUE, &error);
1128                 if (error) {
1129                         vm_object_pip_wakeup(fs->first_object);
1130                         vm_object_chain_release_all(fs->first_object,
1131                                                     fs->object);
1132                         if (fs->object != fs->first_object)
1133                                 vm_object_drop(fs->object);
1134                         unlock_things(fs);
1135                         vm_page_sleep_busy(fs->m, TRUE, "vmpfw");
1136                         mycpu->gd_cnt.v_intrans++;
1137                         /*vm_object_deallocate(fs->first_object);*/
1138                         /*fs->first_object = NULL;*/
1139                         fs->m = NULL;
1140                         return (KERN_TRY_AGAIN);
1141                 }
1142                 if (fs->m) {
1143                         /*
1144                          * The page is busied for us.
1145                          *
1146                          * If reactivating a page from PQ_CACHE we may have
1147                          * to rate-limit.
1148                          */
1149                         int queue = fs->m->queue;
1150                         vm_page_unqueue_nowakeup(fs->m);
1151
1152                         if ((queue - fs->m->pc) == PQ_CACHE && 
1153                             vm_page_count_severe()) {
1154                                 vm_page_activate(fs->m);
1155                                 vm_page_wakeup(fs->m);
1156                                 fs->m = NULL;
1157                                 vm_object_pip_wakeup(fs->first_object);
1158                                 vm_object_chain_release_all(fs->first_object,
1159                                                             fs->object);
1160                                 if (fs->object != fs->first_object)
1161                                         vm_object_drop(fs->object);
1162                                 unlock_and_deallocate(fs);
1163                                 vm_waitpfault();
1164                                 return (KERN_TRY_AGAIN);
1165                         }
1166
1167                         /*
1168                          * If it still isn't completely valid (readable),
1169                          * or if a read-ahead-mark is set on the VM page,
1170                          * jump to readrest, else we found the page and
1171                          * can return.
1172                          *
1173                          * We can release the spl once we have marked the
1174                          * page busy.
1175                          */
1176                         if (fs->m->object != &kernel_object) {
1177                                 if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
1178                                     VM_PAGE_BITS_ALL) {
1179                                         goto readrest;
1180                                 }
1181                                 if (fs->m->flags & PG_RAM) {
1182                                         if (debug_cluster)
1183                                                 kprintf("R");
1184                                         vm_page_flag_clear(fs->m, PG_RAM);
1185                                         goto readrest;
1186                                 }
1187                         }
1188                         break; /* break to PAGE HAS BEEN FOUND */
1189                 }
1190
1191                 /*
1192                  * Page is not resident, If this is the search termination
1193                  * or the pager might contain the page, allocate a new page.
1194                  */
1195                 if (TRYPAGER(fs) || fs->object == fs->first_object) {
1196                         /*
1197                          * If the page is beyond the object size we fail
1198                          */
1199                         if (pindex >= fs->object->size) {
1200                                 vm_object_pip_wakeup(fs->first_object);
1201                                 vm_object_chain_release_all(fs->first_object,
1202                                                             fs->object);
1203                                 if (fs->object != fs->first_object)
1204                                         vm_object_drop(fs->object);
1205                                 unlock_and_deallocate(fs);
1206                                 return (KERN_PROTECTION_FAILURE);
1207                         }
1208
1209                         /*
1210                          * Allocate a new page for this object/offset pair.
1211                          *
1212                          * It is possible for the allocation to race, so
1213                          * handle the case.
1214                          */
1215                         fs->m = NULL;
1216                         if (!vm_page_count_severe()) {
1217                                 fs->m = vm_page_alloc(fs->object, pindex,
1218                                     ((fs->vp || fs->object->backing_object) ?
1219                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL :
1220                                         VM_ALLOC_NULL_OK | VM_ALLOC_NORMAL |
1221                                         VM_ALLOC_USE_GD | VM_ALLOC_ZERO));
1222                         }
1223                         if (fs->m == NULL) {
1224                                 vm_object_pip_wakeup(fs->first_object);
1225                                 vm_object_chain_release_all(fs->first_object,
1226                                                             fs->object);
1227                                 if (fs->object != fs->first_object)
1228                                         vm_object_drop(fs->object);
1229                                 unlock_and_deallocate(fs);
1230                                 vm_waitpfault();
1231                                 return (KERN_TRY_AGAIN);
1232                         }
1233
1234                         /*
1235                          * Fall through to readrest.  We have a new page which
1236                          * will have to be paged (since m->valid will be 0).
1237                          */
1238                 }
1239
1240 readrest:
1241                 /*
1242                  * We have found an invalid or partially valid page, a
1243                  * page with a read-ahead mark which might be partially or
1244                  * fully valid (and maybe dirty too), or we have allocated
1245                  * a new page.
1246                  *
1247                  * Attempt to fault-in the page if there is a chance that the
1248                  * pager has it, and potentially fault in additional pages
1249                  * at the same time.
1250                  *
1251                  * If TRYPAGER is true then fs.m will be non-NULL and busied
1252                  * for us.
1253                  */
1254                 if (TRYPAGER(fs)) {
1255                         int rv;
1256                         int seqaccess;
1257                         u_char behavior = vm_map_entry_behavior(fs->entry);
1258
1259                         if (behavior == MAP_ENTRY_BEHAV_RANDOM)
1260                                 seqaccess = 0;
1261                         else
1262                                 seqaccess = -1;
1263
1264 #if 0
1265                         /*
1266                          * If sequential access is detected then attempt
1267                          * to deactivate/cache pages behind the scan to
1268                          * prevent resource hogging.
1269                          *
1270                          * Use of PG_RAM to detect sequential access
1271                          * also simulates multi-zone sequential access
1272                          * detection for free.
1273                          *
1274                          * NOTE: Partially valid dirty pages cannot be
1275                          *       deactivated without causing NFS picemeal
1276                          *       writes to barf.
1277                          */
1278                         if ((fs->first_object->type != OBJT_DEVICE) &&
1279                             (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
1280                                 (behavior != MAP_ENTRY_BEHAV_RANDOM &&
1281                                  (fs->m->flags & PG_RAM)))
1282                         ) {
1283                                 vm_pindex_t scan_pindex;
1284                                 int scan_count = 16;
1285
1286                                 if (first_pindex < 16) {
1287                                         scan_pindex = 0;
1288                                         scan_count = 0;
1289                                 } else {
1290                                         scan_pindex = first_pindex - 16;
1291                                         if (scan_pindex < 16)
1292                                                 scan_count = scan_pindex;
1293                                         else
1294                                                 scan_count = 16;
1295                                 }
1296
1297                                 while (scan_count) {
1298                                         vm_page_t mt;
1299
1300                                         mt = vm_page_lookup(fs->first_object,
1301                                                             scan_pindex);
1302                                         if (mt == NULL)
1303                                                 break;
1304                                         if (vm_page_busy_try(mt, TRUE))
1305                                                 goto skip;
1306
1307                                         if (mt->valid != VM_PAGE_BITS_ALL) {
1308                                                 vm_page_wakeup(mt);
1309                                                 break;
1310                                         }
1311                                         if ((mt->flags &
1312                                              (PG_FICTITIOUS | PG_UNMANAGED)) ||
1313                                             mt->hold_count ||
1314                                             mt->wire_count)  {
1315                                                 vm_page_wakeup(mt);
1316                                                 goto skip;
1317                                         }
1318                                         if (mt->dirty == 0)
1319                                                 vm_page_test_dirty(mt);
1320                                         if (mt->dirty) {
1321                                                 vm_page_protect(mt,
1322                                                                 VM_PROT_NONE);
1323                                                 vm_page_deactivate(mt);
1324                                                 vm_page_wakeup(mt);
1325                                         } else {
1326                                                 vm_page_cache(mt);
1327                                         }
1328 skip:
1329                                         --scan_count;
1330                                         --scan_pindex;
1331                                 }
1332
1333                                 seqaccess = 1;
1334                         }
1335 #endif
1336
1337                         /*
1338                          * Avoid deadlocking against the map when doing I/O.
1339                          * fs.object and the page is PG_BUSY'd.
1340                          *
1341                          * NOTE: Once unlocked, fs->entry can become stale
1342                          *       so this will NULL it out.
1343                          *
1344                          * NOTE: fs->entry is invalid until we relock the
1345                          *       map and verify that the timestamp has not
1346                          *       changed.
1347                          */
1348                         unlock_map(fs);
1349
1350                         /*
1351                          * Acquire the page data.  We still hold a ref on
1352                          * fs.object and the page has been PG_BUSY's.
1353                          *
1354                          * The pager may replace the page (for example, in
1355                          * order to enter a fictitious page into the
1356                          * object).  If it does so it is responsible for
1357                          * cleaning up the passed page and properly setting
1358                          * the new page PG_BUSY.
1359                          *
1360                          * If we got here through a PG_RAM read-ahead
1361                          * mark the page may be partially dirty and thus
1362                          * not freeable.  Don't bother checking to see
1363                          * if the pager has the page because we can't free
1364                          * it anyway.  We have to depend on the get_page
1365                          * operation filling in any gaps whether there is
1366                          * backing store or not.
1367                          */
1368                         rv = vm_pager_get_page(fs->object, &fs->m, seqaccess);
1369
1370                         if (rv == VM_PAGER_OK) {
1371                                 /*
1372                                  * Relookup in case pager changed page. Pager
1373                                  * is responsible for disposition of old page
1374                                  * if moved.
1375                                  *
1376                                  * XXX other code segments do relookups too.
1377                                  * It's a bad abstraction that needs to be
1378                                  * fixed/removed.
1379                                  */
1380                                 fs->m = vm_page_lookup(fs->object, pindex);
1381                                 if (fs->m == NULL) {
1382                                         vm_object_pip_wakeup(fs->first_object);
1383                                         vm_object_chain_release_all(
1384                                                 fs->first_object, fs->object);
1385                                         if (fs->object != fs->first_object)
1386                                                 vm_object_drop(fs->object);
1387                                         unlock_and_deallocate(fs);
1388                                         return (KERN_TRY_AGAIN);
1389                                 }
1390
1391                                 ++fs->hardfault;
1392                                 break; /* break to PAGE HAS BEEN FOUND */
1393                         }
1394
1395                         /*
1396                          * Remove the bogus page (which does not exist at this
1397                          * object/offset); before doing so, we must get back
1398                          * our object lock to preserve our invariant.
1399                          *
1400                          * Also wake up any other process that may want to bring
1401                          * in this page.
1402                          *
1403                          * If this is the top-level object, we must leave the
1404                          * busy page to prevent another process from rushing
1405                          * past us, and inserting the page in that object at
1406                          * the same time that we are.
1407                          */
1408                         if (rv == VM_PAGER_ERROR) {
1409                                 if (curproc) {
1410                                         kprintf("vm_fault: pager read error, "
1411                                                 "pid %d (%s)\n",
1412                                                 curproc->p_pid,
1413                                                 curproc->p_comm);
1414                                 } else {
1415                                         kprintf("vm_fault: pager read error, "
1416                                                 "thread %p (%s)\n",
1417                                                 curthread,
1418                                                 curproc->p_comm);
1419                                 }
1420                         }
1421
1422                         /*
1423                          * Data outside the range of the pager or an I/O error
1424                          *
1425                          * The page may have been wired during the pagein,
1426                          * e.g. by the buffer cache, and cannot simply be
1427                          * freed.  Call vnode_pager_freepage() to deal with it.
1428                          */
1429                         /*
1430                          * XXX - the check for kernel_map is a kludge to work
1431                          * around having the machine panic on a kernel space
1432                          * fault w/ I/O error.
1433                          */
1434                         if (((fs->map != &kernel_map) &&
1435                             (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) {
1436                                 vnode_pager_freepage(fs->m);
1437                                 fs->m = NULL;
1438                                 vm_object_pip_wakeup(fs->first_object);
1439                                 vm_object_chain_release_all(fs->first_object,
1440                                                             fs->object);
1441                                 if (fs->object != fs->first_object)
1442                                         vm_object_drop(fs->object);
1443                                 unlock_and_deallocate(fs);
1444                                 if (rv == VM_PAGER_ERROR)
1445                                         return (KERN_FAILURE);
1446                                 else
1447                                         return (KERN_PROTECTION_FAILURE);
1448                                 /* NOT REACHED */
1449                         }
1450                         if (fs->object != fs->first_object) {
1451                                 vnode_pager_freepage(fs->m);
1452                                 fs->m = NULL;
1453                                 /*
1454                                  * XXX - we cannot just fall out at this
1455                                  * point, m has been freed and is invalid!
1456                                  */
1457                         }
1458                 }
1459
1460                 /*
1461                  * We get here if the object has a default pager (or unwiring) 
1462                  * or the pager doesn't have the page.
1463                  */
1464                 if (fs->object == fs->first_object)
1465                         fs->first_m = fs->m;
1466
1467                 /*
1468                  * Move on to the next object.  The chain lock should prevent
1469                  * the backing_object from getting ripped out from under us.
1470                  */
1471                 if ((next_object = fs->object->backing_object) != NULL) {
1472                         vm_object_hold(next_object);
1473                         vm_object_chain_acquire(next_object);
1474                         KKASSERT(next_object == fs->object->backing_object);
1475                         pindex += OFF_TO_IDX(fs->object->backing_object_offset);
1476                 }
1477
1478                 if (next_object == NULL) {
1479                         /*
1480                          * If there's no object left, fill the page in the top
1481                          * object with zeros.
1482                          */
1483                         if (fs->object != fs->first_object) {
1484                                 if (fs->first_object->backing_object !=
1485                                     fs->object) {
1486                                         vm_object_hold(fs->first_object->backing_object);
1487                                 }
1488                                 vm_object_chain_release_all(
1489                                         fs->first_object->backing_object,
1490                                         fs->object);
1491                                 if (fs->first_object->backing_object !=
1492                                     fs->object) {
1493                                         vm_object_drop(fs->first_object->backing_object);
1494                                 }
1495                                 vm_object_pip_wakeup(fs->object);
1496                                 vm_object_drop(fs->object);
1497                                 fs->object = fs->first_object;
1498                                 pindex = first_pindex;
1499                                 fs->m = fs->first_m;
1500                         }
1501                         fs->first_m = NULL;
1502
1503                         /*
1504                          * Zero the page if necessary and mark it valid.
1505                          */
1506                         if ((fs->m->flags & PG_ZERO) == 0) {
1507                                 vm_page_zero_fill(fs->m);
1508                         } else {
1509 #ifdef PMAP_DEBUG
1510                                 pmap_page_assertzero(VM_PAGE_TO_PHYS(fs->m));
1511 #endif
1512                                 vm_page_flag_clear(fs->m, PG_ZERO);
1513                                 mycpu->gd_cnt.v_ozfod++;
1514                         }
1515                         mycpu->gd_cnt.v_zfod++;
1516                         fs->m->valid = VM_PAGE_BITS_ALL;
1517                         break;  /* break to PAGE HAS BEEN FOUND */
1518                 }
1519                 if (fs->object != fs->first_object) {
1520                         vm_object_pip_wakeup(fs->object);
1521                         vm_object_lock_swap();
1522                         vm_object_drop(fs->object);
1523                 }
1524                 KASSERT(fs->object != next_object,
1525                         ("object loop %p", next_object));
1526                 fs->object = next_object;
1527                 vm_object_pip_add(fs->object, 1);
1528         }
1529
1530         /*
1531          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
1532          * is held.]
1533          *
1534          * object still held.
1535          *
1536          * If the page is being written, but isn't already owned by the
1537          * top-level object, we have to copy it into a new page owned by the
1538          * top-level object.
1539          */
1540         KASSERT((fs->m->flags & PG_BUSY) != 0,
1541                 ("vm_fault: not busy after main loop"));
1542
1543         if (fs->object != fs->first_object) {
1544                 /*
1545                  * We only really need to copy if we want to write it.
1546                  */
1547                 if (fault_type & VM_PROT_WRITE) {
1548                         /*
1549                          * This allows pages to be virtually copied from a 
1550                          * backing_object into the first_object, where the 
1551                          * backing object has no other refs to it, and cannot
1552                          * gain any more refs.  Instead of a bcopy, we just 
1553                          * move the page from the backing object to the 
1554                          * first object.  Note that we must mark the page 
1555                          * dirty in the first object so that it will go out 
1556                          * to swap when needed.
1557                          */
1558                         if (
1559                                 /*
1560                                  * Map, if present, has not changed
1561                                  */
1562                                 (fs->map == NULL ||
1563                                 fs->map_generation == fs->map->timestamp) &&
1564                                 /*
1565                                  * Only one shadow object
1566                                  */
1567                                 (fs->object->shadow_count == 1) &&
1568                                 /*
1569                                  * No COW refs, except us
1570                                  */
1571                                 (fs->object->ref_count == 1) &&
1572                                 /*
1573                                  * No one else can look this object up
1574                                  */
1575                                 (fs->object->handle == NULL) &&
1576                                 /*
1577                                  * No other ways to look the object up
1578                                  */
1579                                 ((fs->object->type == OBJT_DEFAULT) ||
1580                                  (fs->object->type == OBJT_SWAP)) &&
1581                                 /*
1582                                  * We don't chase down the shadow chain
1583                                  */
1584                                 (fs->object == fs->first_object->backing_object) &&
1585
1586                                 /*
1587                                  * grab the lock if we need to
1588                                  */
1589                                 (fs->lookup_still_valid ||
1590                                  fs->map == NULL ||
1591                                  lockmgr(&fs->map->lock, LK_EXCLUSIVE|LK_NOWAIT) == 0)
1592                             ) {
1593                                 /*
1594                                  * (first_m) and (m) are both busied.  We have
1595                                  * move (m) into (first_m)'s object/pindex
1596                                  * in an atomic fashion, then free (first_m).
1597                                  *
1598                                  * first_object is held so second remove
1599                                  * followed by the rename should wind
1600                                  * up being atomic.  vm_page_free() might
1601                                  * block so we don't do it until after the
1602                                  * rename.
1603                                  */
1604                                 fs->lookup_still_valid = 1;
1605                                 vm_page_protect(fs->first_m, VM_PROT_NONE);
1606                                 vm_page_remove(fs->first_m);
1607                                 vm_page_rename(fs->m, fs->first_object,
1608                                                first_pindex);
1609                                 vm_page_free(fs->first_m);
1610                                 fs->first_m = fs->m;
1611                                 fs->m = NULL;
1612                                 mycpu->gd_cnt.v_cow_optim++;
1613                         } else {
1614                                 /*
1615                                  * Oh, well, lets copy it.
1616                                  */
1617                                 vm_page_copy(fs->m, fs->first_m);
1618                                 vm_page_event(fs->m, VMEVENT_COW);
1619                         }
1620
1621                         if (fs->m) {
1622                                 /*
1623                                  * We no longer need the old page or object.
1624                                  */
1625                                 release_page(fs);
1626                         }
1627
1628                         /*
1629                          * We intend to revert to first_object, undo the
1630                          * chain lock through to that.
1631                          */
1632                         if (fs->first_object->backing_object != fs->object)
1633                                 vm_object_hold(fs->first_object->backing_object);
1634                         vm_object_chain_release_all(
1635                                         fs->first_object->backing_object,
1636                                         fs->object);
1637                         if (fs->first_object->backing_object != fs->object)
1638                                 vm_object_drop(fs->first_object->backing_object);
1639
1640                         /*
1641                          * fs->object != fs->first_object due to above 
1642                          * conditional
1643                          */
1644                         vm_object_pip_wakeup(fs->object);
1645                         vm_object_drop(fs->object);
1646
1647                         /*
1648                          * Only use the new page below...
1649                          */
1650
1651                         mycpu->gd_cnt.v_cow_faults++;
1652                         fs->m = fs->first_m;
1653                         fs->object = fs->first_object;
1654                         pindex = first_pindex;
1655                 } else {
1656                         /*
1657                          * If it wasn't a write fault avoid having to copy
1658                          * the page by mapping it read-only.
1659                          */
1660                         fs->prot &= ~VM_PROT_WRITE;
1661                 }
1662         }
1663
1664         /*
1665          * Relock the map if necessary, then check the generation count.
1666          * relock_map() will update fs->timestamp to account for the
1667          * relocking if necessary.
1668          *
1669          * If the count has changed after relocking then all sorts of
1670          * crap may have happened and we have to retry.
1671          *
1672          * NOTE: The relock_map() can fail due to a deadlock against
1673          *       the vm_page we are holding BUSY.
1674          */
1675         if (fs->lookup_still_valid == FALSE && fs->map) {
1676                 if (relock_map(fs) ||
1677                     fs->map->timestamp != fs->map_generation) {
1678                         release_page(fs);
1679                         vm_object_pip_wakeup(fs->first_object);
1680                         vm_object_chain_release_all(fs->first_object,
1681                                                     fs->object);
1682                         if (fs->object != fs->first_object)
1683                                 vm_object_drop(fs->object);
1684                         unlock_and_deallocate(fs);
1685                         return (KERN_TRY_AGAIN);
1686                 }
1687         }
1688
1689         /*
1690          * If the fault is a write, we know that this page is being
1691          * written NOW so dirty it explicitly to save on pmap_is_modified()
1692          * calls later.
1693          *
1694          * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
1695          * if the page is already dirty to prevent data written with
1696          * the expectation of being synced from not being synced.
1697          * Likewise if this entry does not request NOSYNC then make
1698          * sure the page isn't marked NOSYNC.  Applications sharing
1699          * data should use the same flags to avoid ping ponging.
1700          *
1701          * Also tell the backing pager, if any, that it should remove
1702          * any swap backing since the page is now dirty.
1703          */
1704         vm_page_activate(fs->m);
1705         if (fs->prot & VM_PROT_WRITE) {
1706                 vm_object_set_writeable_dirty(fs->m->object);
1707                 vm_set_nosync(fs->m, fs->entry);
1708                 if (fs->fault_flags & VM_FAULT_DIRTY) {
1709                         vm_page_dirty(fs->m);
1710                         swap_pager_unswapped(fs->m);
1711                 }
1712         }
1713
1714         vm_object_pip_wakeup(fs->first_object);
1715         vm_object_chain_release_all(fs->first_object, fs->object);
1716         if (fs->object != fs->first_object)
1717                 vm_object_drop(fs->object);
1718
1719         /*
1720          * Page had better still be busy.  We are still locked up and 
1721          * fs->object will have another PIP reference if it is not equal
1722          * to fs->first_object.
1723          */
1724         KASSERT(fs->m->flags & PG_BUSY,
1725                 ("vm_fault: page %p not busy!", fs->m));
1726
1727         /*
1728          * Sanity check: page must be completely valid or it is not fit to
1729          * map into user space.  vm_pager_get_pages() ensures this.
1730          */
1731         if (fs->m->valid != VM_PAGE_BITS_ALL) {
1732                 vm_page_zero_invalid(fs->m, TRUE);
1733                 kprintf("Warning: page %p partially invalid on fault\n", fs->m);
1734         }
1735         vm_page_flag_clear(fs->m, PG_ZERO);
1736
1737         return (KERN_SUCCESS);
1738 }
1739
1740 /*
1741  * Wire down a range of virtual addresses in a map.  The entry in question
1742  * should be marked in-transition and the map must be locked.  We must
1743  * release the map temporarily while faulting-in the page to avoid a
1744  * deadlock.  Note that the entry may be clipped while we are blocked but
1745  * will never be freed.
1746  *
1747  * No requirements.
1748  */
1749 int
1750 vm_fault_wire(vm_map_t map, vm_map_entry_t entry, boolean_t user_wire)
1751 {
1752         boolean_t fictitious;
1753         vm_offset_t start;
1754         vm_offset_t end;
1755         vm_offset_t va;
1756         vm_paddr_t pa;
1757         vm_page_t m;
1758         pmap_t pmap;
1759         int rv;
1760
1761         lwkt_gettoken(&map->token);
1762
1763         pmap = vm_map_pmap(map);
1764         start = entry->start;
1765         end = entry->end;
1766         fictitious = entry->object.vm_object &&
1767                         (entry->object.vm_object->type == OBJT_DEVICE);
1768         if (entry->eflags & MAP_ENTRY_KSTACK)
1769                 start += PAGE_SIZE;
1770         map->timestamp++;
1771         vm_map_unlock(map);
1772
1773         /*
1774          * We simulate a fault to get the page and enter it in the physical
1775          * map.
1776          */
1777         for (va = start; va < end; va += PAGE_SIZE) {
1778                 if (user_wire) {
1779                         rv = vm_fault(map, va, VM_PROT_READ, 
1780                                         VM_FAULT_USER_WIRE);
1781                 } else {
1782                         rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
1783                                         VM_FAULT_CHANGE_WIRING);
1784                 }
1785                 if (rv) {
1786                         while (va > start) {
1787                                 va -= PAGE_SIZE;
1788                                 if ((pa = pmap_extract(pmap, va)) == 0)
1789                                         continue;
1790                                 pmap_change_wiring(pmap, va, FALSE);
1791                                 if (!fictitious) {
1792                                         m = PHYS_TO_VM_PAGE(pa);
1793                                         vm_page_busy_wait(m, FALSE, "vmwrpg");
1794                                         vm_page_unwire(m, 1);
1795                                         vm_page_wakeup(m);
1796                                 }
1797                         }
1798                         goto done;
1799                 }
1800         }
1801         rv = KERN_SUCCESS;
1802 done:
1803         vm_map_lock(map);
1804         lwkt_reltoken(&map->token);
1805         return (rv);
1806 }
1807
1808 /*
1809  * Unwire a range of virtual addresses in a map.  The map should be
1810  * locked.
1811  */
1812 void
1813 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
1814 {
1815         boolean_t fictitious;
1816         vm_offset_t start;
1817         vm_offset_t end;
1818         vm_offset_t va;
1819         vm_paddr_t pa;
1820         vm_page_t m;
1821         pmap_t pmap;
1822
1823         lwkt_gettoken(&map->token);
1824
1825         pmap = vm_map_pmap(map);
1826         start = entry->start;
1827         end = entry->end;
1828         fictitious = entry->object.vm_object &&
1829                         (entry->object.vm_object->type == OBJT_DEVICE);
1830         if (entry->eflags & MAP_ENTRY_KSTACK)
1831                 start += PAGE_SIZE;
1832
1833         /*
1834          * Since the pages are wired down, we must be able to get their
1835          * mappings from the physical map system.
1836          */
1837         for (va = start; va < end; va += PAGE_SIZE) {
1838                 pa = pmap_extract(pmap, va);
1839                 if (pa != 0) {
1840                         pmap_change_wiring(pmap, va, FALSE);
1841                         if (!fictitious) {
1842                                 m = PHYS_TO_VM_PAGE(pa);
1843                                 vm_page_busy_wait(m, FALSE, "vmwupg");
1844                                 vm_page_unwire(m, 1);
1845                                 vm_page_wakeup(m);
1846                         }
1847                 }
1848         }
1849         lwkt_reltoken(&map->token);
1850 }
1851
1852 /*
1853  * Copy all of the pages from a wired-down map entry to another.
1854  *
1855  * The source and destination maps must be locked for write.
1856  * The source and destination maps token must be held
1857  * The source map entry must be wired down (or be a sharing map
1858  * entry corresponding to a main map entry that is wired down).
1859  *
1860  * No other requirements.
1861  */
1862 void
1863 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1864                     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
1865 {
1866         vm_object_t dst_object;
1867         vm_object_t src_object;
1868         vm_ooffset_t dst_offset;
1869         vm_ooffset_t src_offset;
1870         vm_prot_t prot;
1871         vm_offset_t vaddr;
1872         vm_page_t dst_m;
1873         vm_page_t src_m;
1874
1875         src_object = src_entry->object.vm_object;
1876         src_offset = src_entry->offset;
1877
1878         /*
1879          * Create the top-level object for the destination entry. (Doesn't
1880          * actually shadow anything - we copy the pages directly.)
1881          */
1882         vm_map_entry_allocate_object(dst_entry);
1883         dst_object = dst_entry->object.vm_object;
1884
1885         prot = dst_entry->max_protection;
1886
1887         /*
1888          * Loop through all of the pages in the entry's range, copying each
1889          * one from the source object (it should be there) to the destination
1890          * object.
1891          */
1892         for (vaddr = dst_entry->start, dst_offset = 0;
1893             vaddr < dst_entry->end;
1894             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
1895
1896                 /*
1897                  * Allocate a page in the destination object
1898                  */
1899                 do {
1900                         dst_m = vm_page_alloc(dst_object,
1901                                               OFF_TO_IDX(dst_offset),
1902                                               VM_ALLOC_NORMAL);
1903                         if (dst_m == NULL) {
1904                                 vm_wait(0);
1905                         }
1906                 } while (dst_m == NULL);
1907
1908                 /*
1909                  * Find the page in the source object, and copy it in.
1910                  * (Because the source is wired down, the page will be in
1911                  * memory.)
1912                  */
1913                 src_m = vm_page_lookup(src_object,
1914                                        OFF_TO_IDX(dst_offset + src_offset));
1915                 if (src_m == NULL)
1916                         panic("vm_fault_copy_wired: page missing");
1917
1918                 vm_page_copy(src_m, dst_m);
1919                 vm_page_event(src_m, VMEVENT_COW);
1920
1921                 /*
1922                  * Enter it in the pmap...
1923                  */
1924
1925                 vm_page_flag_clear(dst_m, PG_ZERO);
1926                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
1927
1928                 /*
1929                  * Mark it no longer busy, and put it on the active list.
1930                  */
1931                 vm_page_activate(dst_m);
1932                 vm_page_wakeup(dst_m);
1933         }
1934 }
1935
1936 #if 0
1937
1938 /*
1939  * This routine checks around the requested page for other pages that
1940  * might be able to be faulted in.  This routine brackets the viable
1941  * pages for the pages to be paged in.
1942  *
1943  * Inputs:
1944  *      m, rbehind, rahead
1945  *
1946  * Outputs:
1947  *  marray (array of vm_page_t), reqpage (index of requested page)
1948  *
1949  * Return value:
1950  *  number of pages in marray
1951  */
1952 static int
1953 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
1954                           vm_page_t *marray, int *reqpage)
1955 {
1956         int i,j;
1957         vm_object_t object;
1958         vm_pindex_t pindex, startpindex, endpindex, tpindex;
1959         vm_page_t rtm;
1960         int cbehind, cahead;
1961
1962         object = m->object;
1963         pindex = m->pindex;
1964
1965         /*
1966          * we don't fault-ahead for device pager
1967          */
1968         if (object->type == OBJT_DEVICE) {
1969                 *reqpage = 0;
1970                 marray[0] = m;
1971                 return 1;
1972         }
1973
1974         /*
1975          * if the requested page is not available, then give up now
1976          */
1977         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
1978                 *reqpage = 0;   /* not used by caller, fix compiler warn */
1979                 return 0;
1980         }
1981
1982         if ((cbehind == 0) && (cahead == 0)) {
1983                 *reqpage = 0;
1984                 marray[0] = m;
1985                 return 1;
1986         }
1987
1988         if (rahead > cahead) {
1989                 rahead = cahead;
1990         }
1991
1992         if (rbehind > cbehind) {
1993                 rbehind = cbehind;
1994         }
1995
1996         /*
1997          * Do not do any readahead if we have insufficient free memory.
1998          *
1999          * XXX code was broken disabled before and has instability
2000          * with this conditonal fixed, so shortcut for now.
2001          */
2002         if (burst_fault == 0 || vm_page_count_severe()) {
2003                 marray[0] = m;
2004                 *reqpage = 0;
2005                 return 1;
2006         }
2007
2008         /*
2009          * scan backward for the read behind pages -- in memory 
2010          *
2011          * Assume that if the page is not found an interrupt will not
2012          * create it.  Theoretically interrupts can only remove (busy)
2013          * pages, not create new associations.
2014          */
2015         if (pindex > 0) {
2016                 if (rbehind > pindex) {
2017                         rbehind = pindex;
2018                         startpindex = 0;
2019                 } else {
2020                         startpindex = pindex - rbehind;
2021                 }
2022
2023                 vm_object_hold(object);
2024                 for (tpindex = pindex; tpindex > startpindex; --tpindex) {
2025                         if (vm_page_lookup(object, tpindex - 1))
2026                                 break;
2027                 }
2028
2029                 i = 0;
2030                 while (tpindex < pindex) {
2031                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2032                                                              VM_ALLOC_NULL_OK);
2033                         if (rtm == NULL) {
2034                                 for (j = 0; j < i; j++) {
2035                                         vm_page_free(marray[j]);
2036                                 }
2037                                 vm_object_drop(object);
2038                                 marray[0] = m;
2039                                 *reqpage = 0;
2040                                 return 1;
2041                         }
2042                         marray[i] = rtm;
2043                         ++i;
2044                         ++tpindex;
2045                 }
2046                 vm_object_drop(object);
2047         } else {
2048                 i = 0;
2049         }
2050
2051         /*
2052          * Assign requested page
2053          */
2054         marray[i] = m;
2055         *reqpage = i;
2056         ++i;
2057
2058         /*
2059          * Scan forwards for read-ahead pages
2060          */
2061         tpindex = pindex + 1;
2062         endpindex = tpindex + rahead;
2063         if (endpindex > object->size)
2064                 endpindex = object->size;
2065
2066         vm_object_hold(object);
2067         while (tpindex < endpindex) {
2068                 if (vm_page_lookup(object, tpindex))
2069                         break;
2070                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM |
2071                                                      VM_ALLOC_NULL_OK);
2072                 if (rtm == NULL)
2073                         break;
2074                 marray[i] = rtm;
2075                 ++i;
2076                 ++tpindex;
2077         }
2078         vm_object_drop(object);
2079
2080         return (i);
2081 }
2082
2083 #endif
2084
2085 /*
2086  * vm_prefault() provides a quick way of clustering pagefaults into a
2087  * processes address space.  It is a "cousin" of pmap_object_init_pt,
2088  * except it runs at page fault time instead of mmap time.
2089  *
2090  * vm.fast_fault        Enables pre-faulting zero-fill pages
2091  *
2092  * vm.prefault_pages    Number of pages (1/2 negative, 1/2 positive) to
2093  *                      prefault.  Scan stops in either direction when
2094  *                      a page is found to already exist.
2095  *
2096  * This code used to be per-platform pmap_prefault().  It is now
2097  * machine-independent and enhanced to also pre-fault zero-fill pages
2098  * (see vm.fast_fault) as well as make them writable, which greatly
2099  * reduces the number of page faults programs incur.
2100  *
2101  * Application performance when pre-faulting zero-fill pages is heavily
2102  * dependent on the application.  Very tiny applications like /bin/echo
2103  * lose a little performance while applications of any appreciable size
2104  * gain performance.  Prefaulting multiple pages also reduces SMP
2105  * congestion and can improve SMP performance significantly.
2106  *
2107  * NOTE!  prot may allow writing but this only applies to the top level
2108  *        object.  If we wind up mapping a page extracted from a backing
2109  *        object we have to make sure it is read-only.
2110  *
2111  * NOTE!  The caller has already handled any COW operations on the
2112  *        vm_map_entry via the normal fault code.  Do NOT call this
2113  *        shortcut unless the normal fault code has run on this entry.
2114  *
2115  * The related map must be locked.
2116  * No other requirements.
2117  */
2118 static int vm_prefault_pages = 8;
2119 SYSCTL_INT(_vm, OID_AUTO, prefault_pages, CTLFLAG_RW, &vm_prefault_pages, 0,
2120            "Maximum number of pages to pre-fault");
2121 static int vm_fast_fault = 1;
2122 SYSCTL_INT(_vm, OID_AUTO, fast_fault, CTLFLAG_RW, &vm_fast_fault, 0,
2123            "Burst fault zero-fill regions");
2124
2125 /*
2126  * Set PG_NOSYNC if the map entry indicates so, but only if the page
2127  * is not already dirty by other means.  This will prevent passive
2128  * filesystem syncing as well as 'sync' from writing out the page.
2129  */
2130 static void
2131 vm_set_nosync(vm_page_t m, vm_map_entry_t entry)
2132 {
2133         if (entry->eflags & MAP_ENTRY_NOSYNC) {
2134                 if (m->dirty == 0)
2135                         vm_page_flag_set(m, PG_NOSYNC);
2136         } else {
2137                 vm_page_flag_clear(m, PG_NOSYNC);
2138         }
2139 }
2140
2141 static void
2142 vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot,
2143             int fault_flags)
2144 {
2145         struct lwp *lp;
2146         vm_page_t m;
2147         vm_offset_t addr;
2148         vm_pindex_t index;
2149         vm_pindex_t pindex;
2150         vm_object_t object;
2151         int pprot;
2152         int i;
2153         int noneg;
2154         int nopos;
2155         int maxpages;
2156
2157         /*
2158          * Get stable max count value, disabled if set to 0
2159          */
2160         maxpages = vm_prefault_pages;
2161         cpu_ccfence();
2162         if (maxpages <= 0)
2163                 return;
2164
2165         /*
2166          * We do not currently prefault mappings that use virtual page
2167          * tables.  We do not prefault foreign pmaps.
2168          */
2169         if (entry->maptype == VM_MAPTYPE_VPAGETABLE)
2170                 return;
2171         lp = curthread->td_lwp;
2172         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2173                 return;
2174
2175         /*
2176          * Limit pre-fault count to 1024 pages.
2177          */
2178         if (maxpages > 1024)
2179                 maxpages = 1024;
2180
2181         object = entry->object.vm_object;
2182         KKASSERT(object != NULL);
2183         KKASSERT(object == entry->object.vm_object);
2184         vm_object_hold(object);
2185         vm_object_chain_acquire(object);
2186
2187         noneg = 0;
2188         nopos = 0;
2189         for (i = 0; i < maxpages; ++i) {
2190                 vm_object_t lobject;
2191                 vm_object_t nobject;
2192                 int allocated = 0;
2193                 int error;
2194
2195                 /*
2196                  * This can eat a lot of time on a heavily contended
2197                  * machine so yield on the tick if needed.
2198                  */
2199                 if ((i & 7) == 7)
2200                         lwkt_yield();
2201
2202                 /*
2203                  * Calculate the page to pre-fault, stopping the scan in
2204                  * each direction separately if the limit is reached.
2205                  */
2206                 if (i & 1) {
2207                         if (noneg)
2208                                 continue;
2209                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2210                 } else {
2211                         if (nopos)
2212                                 continue;
2213                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2214                 }
2215                 if (addr < entry->start) {
2216                         noneg = 1;
2217                         if (noneg && nopos)
2218                                 break;
2219                         continue;
2220                 }
2221                 if (addr >= entry->end) {
2222                         nopos = 1;
2223                         if (noneg && nopos)
2224                                 break;
2225                         continue;
2226                 }
2227
2228                 /*
2229                  * Skip pages already mapped, and stop scanning in that
2230                  * direction.  When the scan terminates in both directions
2231                  * we are done.
2232                  */
2233                 if (pmap_prefault_ok(pmap, addr) == 0) {
2234                         if (i & 1)
2235                                 noneg = 1;
2236                         else
2237                                 nopos = 1;
2238                         if (noneg && nopos)
2239                                 break;
2240                         continue;
2241                 }
2242
2243                 /*
2244                  * Follow the VM object chain to obtain the page to be mapped
2245                  * into the pmap.
2246                  *
2247                  * If we reach the terminal object without finding a page
2248                  * and we determine it would be advantageous, then allocate
2249                  * a zero-fill page for the base object.  The base object
2250                  * is guaranteed to be OBJT_DEFAULT for this case.
2251                  *
2252                  * In order to not have to check the pager via *haspage*()
2253                  * we stop if any non-default object is encountered.  e.g.
2254                  * a vnode or swap object would stop the loop.
2255                  */
2256                 index = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2257                 lobject = object;
2258                 pindex = index;
2259                 pprot = prot;
2260
2261                 KKASSERT(lobject == entry->object.vm_object);
2262                 /*vm_object_hold(lobject); implied */
2263
2264                 while ((m = vm_page_lookup_busy_try(lobject, pindex,
2265                                                     TRUE, &error)) == NULL) {
2266                         if (lobject->type != OBJT_DEFAULT)
2267                                 break;
2268                         if (lobject->backing_object == NULL) {
2269                                 if (vm_fast_fault == 0)
2270                                         break;
2271                                 if ((prot & VM_PROT_WRITE) == 0 ||
2272                                     vm_page_count_min(0)) {
2273                                         break;
2274                                 }
2275
2276                                 /*
2277                                  * NOTE: Allocated from base object
2278                                  */
2279                                 m = vm_page_alloc(object, index,
2280                                                   VM_ALLOC_NORMAL |
2281                                                   VM_ALLOC_ZERO |
2282                                                   VM_ALLOC_USE_GD |
2283                                                   VM_ALLOC_NULL_OK);
2284                                 if (m == NULL)
2285                                         break;
2286                                 allocated = 1;
2287                                 pprot = prot;
2288                                 /* lobject = object .. not needed */
2289                                 break;
2290                         }
2291                         if (lobject->backing_object_offset & PAGE_MASK)
2292                                 break;
2293                         nobject = lobject->backing_object;
2294                         vm_object_hold(nobject);
2295                         KKASSERT(nobject == lobject->backing_object);
2296                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
2297                         if (lobject != object) {
2298                                 vm_object_lock_swap();
2299                                 vm_object_drop(lobject);
2300                         }
2301                         lobject = nobject;
2302                         pprot &= ~VM_PROT_WRITE;
2303                         vm_object_chain_acquire(lobject);
2304                 }
2305
2306                 /*
2307                  * NOTE: A non-NULL (m) will be associated with lobject if
2308                  *       it was found there, otherwise it is probably a
2309                  *       zero-fill page associated with the base object.
2310                  *
2311                  * Give-up if no page is available.
2312                  */
2313                 if (m == NULL) {
2314                         if (lobject != object) {
2315                                 if (object->backing_object != lobject)
2316                                         vm_object_hold(object->backing_object);
2317                                 vm_object_chain_release_all(
2318                                         object->backing_object, lobject);
2319                                 if (object->backing_object != lobject)
2320                                         vm_object_drop(object->backing_object);
2321                                 vm_object_drop(lobject);
2322                         }
2323                         break;
2324                 }
2325
2326                 /*
2327                  * The object must be marked dirty if we are mapping a
2328                  * writable page.  m->object is either lobject or object,
2329                  * both of which are still held.  Do this before we
2330                  * potentially drop the object.
2331                  */
2332                 if (pprot & VM_PROT_WRITE)
2333                         vm_object_set_writeable_dirty(m->object);
2334
2335                 /*
2336                  * Do not conditionalize on PG_RAM.  If pages are present in
2337                  * the VM system we assume optimal caching.  If caching is
2338                  * not optimal the I/O gravy train will be restarted when we
2339                  * hit an unavailable page.  We do not want to try to restart
2340                  * the gravy train now because we really don't know how much
2341                  * of the object has been cached.  The cost for restarting
2342                  * the gravy train should be low (since accesses will likely
2343                  * be I/O bound anyway).
2344                  */
2345                 if (lobject != object) {
2346                         if (object->backing_object != lobject)
2347                                 vm_object_hold(object->backing_object);
2348                         vm_object_chain_release_all(object->backing_object,
2349                                                     lobject);
2350                         if (object->backing_object != lobject)
2351                                 vm_object_drop(object->backing_object);
2352                         vm_object_drop(lobject);
2353                 }
2354
2355                 /*
2356                  * Enter the page into the pmap if appropriate.  If we had
2357                  * allocated the page we have to place it on a queue.  If not
2358                  * we just have to make sure it isn't on the cache queue
2359                  * (pages on the cache queue are not allowed to be mapped).
2360                  */
2361                 if (allocated) {
2362                         /*
2363                          * Page must be zerod.
2364                          */
2365                         if ((m->flags & PG_ZERO) == 0) {
2366                                 vm_page_zero_fill(m);
2367                         } else {
2368 #ifdef PMAP_DEBUG
2369                                 pmap_page_assertzero(
2370                                                 VM_PAGE_TO_PHYS(m));
2371 #endif
2372                                 vm_page_flag_clear(m, PG_ZERO);
2373                                 mycpu->gd_cnt.v_ozfod++;
2374                         }
2375                         mycpu->gd_cnt.v_zfod++;
2376                         m->valid = VM_PAGE_BITS_ALL;
2377
2378                         /*
2379                          * Handle dirty page case
2380                          */
2381                         if (pprot & VM_PROT_WRITE)
2382                                 vm_set_nosync(m, entry);
2383                         pmap_enter(pmap, addr, m, pprot, 0);
2384                         mycpu->gd_cnt.v_vm_faults++;
2385                         if (curthread->td_lwp)
2386                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2387                         vm_page_deactivate(m);
2388                         if (pprot & VM_PROT_WRITE) {
2389                                 /*vm_object_set_writeable_dirty(m->object);*/
2390                                 vm_set_nosync(m, entry);
2391                                 if (fault_flags & VM_FAULT_DIRTY) {
2392                                         vm_page_dirty(m);
2393                                         /*XXX*/
2394                                         swap_pager_unswapped(m);
2395                                 }
2396                         }
2397                         vm_page_wakeup(m);
2398                 } else if (error) {
2399                         /* couldn't busy page, no wakeup */
2400                 } else if (
2401                     ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2402                     (m->flags & PG_FICTITIOUS) == 0) {
2403                         /*
2404                          * A fully valid page not undergoing soft I/O can
2405                          * be immediately entered into the pmap.
2406                          */
2407                         if ((m->queue - m->pc) == PQ_CACHE)
2408                                 vm_page_deactivate(m);
2409                         if (pprot & VM_PROT_WRITE) {
2410                                 /*vm_object_set_writeable_dirty(m->object);*/
2411                                 vm_set_nosync(m, entry);
2412                                 if (fault_flags & VM_FAULT_DIRTY) {
2413                                         vm_page_dirty(m);
2414                                         /*XXX*/
2415                                         swap_pager_unswapped(m);
2416                                 }
2417                         }
2418                         if (pprot & VM_PROT_WRITE)
2419                                 vm_set_nosync(m, entry);
2420                         pmap_enter(pmap, addr, m, pprot, 0);
2421                         mycpu->gd_cnt.v_vm_faults++;
2422                         if (curthread->td_lwp)
2423                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2424                         vm_page_wakeup(m);
2425                 } else {
2426                         vm_page_wakeup(m);
2427                 }
2428         }
2429         vm_object_chain_release(object);
2430         vm_object_drop(object);
2431 }
2432
2433 static void
2434 vm_prefault_quick(pmap_t pmap, vm_offset_t addra,
2435                   vm_map_entry_t entry, int prot, int fault_flags)
2436 {
2437         struct lwp *lp;
2438         vm_page_t m;
2439         vm_offset_t addr;
2440         vm_pindex_t pindex;
2441         vm_object_t object;
2442         int i;
2443         int noneg;
2444         int nopos;
2445         int maxpages;
2446
2447         /*
2448          * Get stable max count value, disabled if set to 0
2449          */
2450         maxpages = vm_prefault_pages;
2451         cpu_ccfence();
2452         if (maxpages <= 0)
2453                 return;
2454
2455         /*
2456          * We do not currently prefault mappings that use virtual page
2457          * tables.  We do not prefault foreign pmaps.
2458          */
2459         if (entry->maptype == VM_MAPTYPE_VPAGETABLE)
2460                 return;
2461         lp = curthread->td_lwp;
2462         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2463                 return;
2464
2465         /*
2466          * Limit pre-fault count to 1024 pages.
2467          */
2468         if (maxpages > 1024)
2469                 maxpages = 1024;
2470
2471         object = entry->object.vm_object;
2472         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
2473         KKASSERT(object->backing_object == NULL);
2474
2475         noneg = 0;
2476         nopos = 0;
2477         for (i = 0; i < maxpages; ++i) {
2478                 int error;
2479
2480                 /*
2481                  * Calculate the page to pre-fault, stopping the scan in
2482                  * each direction separately if the limit is reached.
2483                  */
2484                 if (i & 1) {
2485                         if (noneg)
2486                                 continue;
2487                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2488                 } else {
2489                         if (nopos)
2490                                 continue;
2491                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2492                 }
2493                 if (addr < entry->start) {
2494                         noneg = 1;
2495                         if (noneg && nopos)
2496                                 break;
2497                         continue;
2498                 }
2499                 if (addr >= entry->end) {
2500                         nopos = 1;
2501                         if (noneg && nopos)
2502                                 break;
2503                         continue;
2504                 }
2505
2506                 /*
2507                  * Skip pages already mapped, and stop scanning in that
2508                  * direction.  When the scan terminates in both directions
2509                  * we are done.
2510                  */
2511                 if (pmap_prefault_ok(pmap, addr) == 0) {
2512                         if (i & 1)
2513                                 noneg = 1;
2514                         else
2515                                 nopos = 1;
2516                         if (noneg && nopos)
2517                                 break;
2518                         continue;
2519                 }
2520
2521                 /*
2522                  * Follow the VM object chain to obtain the page to be mapped
2523                  * into the pmap.  This version of the prefault code only
2524                  * works with terminal objects.
2525                  *
2526                  * WARNING!  We cannot call swap_pager_unswapped() with a
2527                  *           shared token.
2528                  */
2529                 pindex = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2530
2531                 m = vm_page_lookup_busy_try(object, pindex, TRUE, &error);
2532                 if (m == NULL || error)
2533                         continue;
2534
2535                 if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2536                     (m->flags & PG_FICTITIOUS) == 0 &&
2537                     ((m->flags & PG_SWAPPED) == 0 ||
2538                      (prot & VM_PROT_WRITE) == 0 ||
2539                      (fault_flags & VM_FAULT_DIRTY) == 0)) {
2540                         /*
2541                          * A fully valid page not undergoing soft I/O can
2542                          * be immediately entered into the pmap.
2543                          */
2544                         if ((m->queue - m->pc) == PQ_CACHE)
2545                                 vm_page_deactivate(m);
2546                         if (prot & VM_PROT_WRITE) {
2547                                 vm_object_set_writeable_dirty(m->object);
2548                                 vm_set_nosync(m, entry);
2549                                 if (fault_flags & VM_FAULT_DIRTY) {
2550                                         vm_page_dirty(m);
2551                                         /*XXX*/
2552                                         swap_pager_unswapped(m);
2553                                 }
2554                         }
2555                         pmap_enter(pmap, addr, m, prot, 0);
2556                         mycpu->gd_cnt.v_vm_faults++;
2557                         if (curthread->td_lwp)
2558                                 ++curthread->td_lwp->lwp_ru.ru_minflt;
2559                 }
2560                 vm_page_wakeup(m);
2561         }
2562 }