kernel - Adjust pagezero/pagecopy assembly and re-enable VM_ALLOC_ZERO
[dragonfly.git] / sys / vm / vm_fault.c
1 /*
2  * (MPSAFE)
3  *
4  * Copyright (c) 1991, 1993
5  *      The Regents of the University of California.  All rights reserved.
6  * Copyright (c) 1994 John S. Dyson
7  * All rights reserved.
8  * Copyright (c) 1994 David Greenman
9  * All rights reserved.
10  *
11  *
12  * This code is derived from software contributed to Berkeley by
13  * The Mach Operating System project at Carnegie-Mellon University.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  * 3. All advertising materials mentioning features or use of this software
24  *    must display the following acknowledgement:
25  *      This product includes software developed by the University of
26  *      California, Berkeley and its contributors.
27  * 4. Neither the name of the University nor the names of its contributors
28  *    may be used to endorse or promote products derived from this software
29  *    without specific prior written permission.
30  *
31  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
32  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
33  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
35  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
36  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
37  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
38  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
39  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
40  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
41  * SUCH DAMAGE.
42  *
43  *      from: @(#)vm_fault.c    8.4 (Berkeley) 1/12/94
44  *
45  *
46  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
47  * All rights reserved.
48  *
49  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
50  *
51  * Permission to use, copy, modify and distribute this software and
52  * its documentation is hereby granted, provided that both the copyright
53  * notice and this permission notice appear in all copies of the
54  * software, derivative works or modified versions, and any portions
55  * thereof, and that both notices appear in supporting documentation.
56  *
57  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
58  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
59  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
60  *
61  * Carnegie Mellon requests users of this software to return to
62  *
63  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
64  *  School of Computer Science
65  *  Carnegie Mellon University
66  *  Pittsburgh PA 15213-3890
67  *
68  * any improvements or extensions that they make and grant Carnegie the
69  * rights to redistribute these changes.
70  *
71  * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
72  * $DragonFly: src/sys/vm/vm_fault.c,v 1.47 2008/07/01 02:02:56 dillon Exp $
73  */
74
75 /*
76  *      Page fault handling module.
77  */
78
79 #include <sys/param.h>
80 #include <sys/systm.h>
81 #include <sys/kernel.h>
82 #include <sys/proc.h>
83 #include <sys/vnode.h>
84 #include <sys/resourcevar.h>
85 #include <sys/vmmeter.h>
86 #include <sys/vkernel.h>
87 #include <sys/lock.h>
88 #include <sys/sysctl.h>
89
90 #include <cpu/lwbuf.h>
91
92 #include <vm/vm.h>
93 #include <vm/vm_param.h>
94 #include <vm/pmap.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_object.h>
97 #include <vm/vm_page.h>
98 #include <vm/vm_pageout.h>
99 #include <vm/vm_kern.h>
100 #include <vm/vm_pager.h>
101 #include <vm/vnode_pager.h>
102 #include <vm/vm_extern.h>
103
104 #include <sys/thread2.h>
105 #include <vm/vm_page2.h>
106
107 struct faultstate {
108         vm_page_t m;
109         vm_object_t object;
110         vm_pindex_t pindex;
111         vm_prot_t prot;
112         vm_page_t first_m;
113         vm_object_t first_object;
114         vm_prot_t first_prot;
115         vm_map_t map;
116         vm_map_entry_t entry;
117         int lookup_still_valid;
118         int didlimit;
119         int hardfault;
120         int fault_flags;
121         int map_generation;
122         boolean_t wired;
123         struct vnode *vp;
124 };
125
126 static int debug_cluster = 0;
127 SYSCTL_INT(_vm, OID_AUTO, debug_cluster, CTLFLAG_RW, &debug_cluster, 0, "");
128
129 static int vm_fault_object(struct faultstate *, vm_pindex_t, vm_prot_t);
130 static int vm_fault_vpagetable(struct faultstate *, vm_pindex_t *, vpte_t, int);
131 #if 0
132 static int vm_fault_additional_pages (vm_page_t, int, int, vm_page_t *, int *);
133 #endif
134 static int vm_fault_ratelimit(struct vmspace *);
135 static void vm_set_nosync(vm_page_t m, vm_map_entry_t entry);
136 static void vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry,
137                         int prot);
138
139 static __inline void
140 release_page(struct faultstate *fs)
141 {
142         vm_page_deactivate(fs->m);
143         vm_page_wakeup(fs->m);
144         fs->m = NULL;
145 }
146
147 /*
148  * NOTE: Once unlocked any cached fs->entry becomes invalid, any reuse
149  *       requires relocking and then checking the timestamp.
150  *
151  * NOTE: vm_map_lock_read() does not bump fs->map->timestamp so we do
152  *       not have to update fs->map_generation here.
153  *
154  * NOTE: This function can fail due to a deadlock against the caller's
155  *       holding of a vm_page BUSY.
156  */
157 static __inline int
158 relock_map(struct faultstate *fs)
159 {
160         int error;
161
162         if (fs->lookup_still_valid == FALSE && fs->map) {
163                 error = vm_map_lock_read_to(fs->map);
164                 if (error == 0)
165                         fs->lookup_still_valid = TRUE;
166         } else {
167                 error = 0;
168         }
169         return error;
170 }
171
172 static __inline void
173 unlock_map(struct faultstate *fs)
174 {
175         if (fs->lookup_still_valid && fs->map) {
176                 vm_map_lookup_done(fs->map, fs->entry, 0);
177                 fs->lookup_still_valid = FALSE;
178         }
179 }
180
181 /*
182  * Clean up after a successful call to vm_fault_object() so another call
183  * to vm_fault_object() can be made.
184  */
185 static void
186 _cleanup_successful_fault(struct faultstate *fs, int relock)
187 {
188         if (fs->object != fs->first_object) {
189                 vm_page_free(fs->first_m);
190                 vm_object_pip_wakeup(fs->object);
191                 fs->first_m = NULL;
192         }
193         fs->object = fs->first_object;
194         if (relock && fs->lookup_still_valid == FALSE) {
195                 if (fs->map)
196                         vm_map_lock_read(fs->map);
197                 fs->lookup_still_valid = TRUE;
198         }
199 }
200
201 static void
202 _unlock_things(struct faultstate *fs, int dealloc)
203 {
204         _cleanup_successful_fault(fs, 0);
205         if (dealloc) {
206                 /*vm_object_deallocate(fs->first_object);*/
207                 /*fs->first_object = NULL; drop used later on */
208         }
209         unlock_map(fs); 
210         if (fs->vp != NULL) { 
211                 vput(fs->vp);
212                 fs->vp = NULL;
213         }
214 }
215
216 #define unlock_things(fs) _unlock_things(fs, 0)
217 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
218 #define cleanup_successful_fault(fs) _cleanup_successful_fault(fs, 1)
219
220 /*
221  * TRYPAGER 
222  *
223  * Determine if the pager for the current object *might* contain the page.
224  *
225  * We only need to try the pager if this is not a default object (default
226  * objects are zero-fill and have no real pager), and if we are not taking
227  * a wiring fault or if the FS entry is wired.
228  */
229 #define TRYPAGER(fs)    \
230                 (fs->object->type != OBJT_DEFAULT && \
231                 (((fs->fault_flags & VM_FAULT_WIRE_MASK) == 0) || fs->wired))
232
233 /*
234  * vm_fault:
235  *
236  * Handle a page fault occuring at the given address, requiring the given
237  * permissions, in the map specified.  If successful, the page is inserted
238  * into the associated physical map.
239  *
240  * NOTE: The given address should be truncated to the proper page address.
241  *
242  * KERN_SUCCESS is returned if the page fault is handled; otherwise,
243  * a standard error specifying why the fault is fatal is returned.
244  *
245  * The map in question must be referenced, and remains so.
246  * The caller may hold no locks.
247  * No other requirements.
248  */
249 int
250 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
251 {
252         int result;
253         vm_pindex_t first_pindex;
254         struct faultstate fs;
255         int growstack;
256
257         mycpu->gd_cnt.v_vm_faults++;
258
259         fs.didlimit = 0;
260         fs.hardfault = 0;
261         fs.fault_flags = fault_flags;
262         growstack = 1;
263
264         lwkt_gettoken(&map->token);
265
266 RetryFault:
267         /*
268          * Find the vm_map_entry representing the backing store and resolve
269          * the top level object and page index.  This may have the side
270          * effect of executing a copy-on-write on the map entry and/or
271          * creating a shadow object, but will not COW any actual VM pages.
272          *
273          * On success fs.map is left read-locked and various other fields 
274          * are initialized but not otherwise referenced or locked.
275          *
276          * NOTE!  vm_map_lookup will try to upgrade the fault_type to
277          * VM_FAULT_WRITE if the map entry is a virtual page table and also
278          * writable, so we can set the 'A'accessed bit in the virtual page
279          * table entry.
280          */
281         fs.map = map;
282         result = vm_map_lookup(&fs.map, vaddr, fault_type,
283                                &fs.entry, &fs.first_object,
284                                &first_pindex, &fs.first_prot, &fs.wired);
285
286         /*
287          * If the lookup failed or the map protections are incompatible,
288          * the fault generally fails.  However, if the caller is trying
289          * to do a user wiring we have more work to do.
290          */
291         if (result != KERN_SUCCESS) {
292                 if (result != KERN_PROTECTION_FAILURE ||
293                     (fs.fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)
294                 {
295                         if (result == KERN_INVALID_ADDRESS && growstack &&
296                             map != &kernel_map && curproc != NULL) {
297                                 result = vm_map_growstack(curproc, vaddr);
298                                 if (result == KERN_SUCCESS) {
299                                         growstack = 0;
300                                         goto RetryFault;
301                                 }
302                                 result = KERN_FAILURE;
303                         }
304                         goto done;
305                 }
306
307                 /*
308                  * If we are user-wiring a r/w segment, and it is COW, then
309                  * we need to do the COW operation.  Note that we don't
310                  * currently COW RO sections now, because it is NOT desirable
311                  * to COW .text.  We simply keep .text from ever being COW'ed
312                  * and take the heat that one cannot debug wired .text sections.
313                  */
314                 result = vm_map_lookup(&fs.map, vaddr,
315                                        VM_PROT_READ|VM_PROT_WRITE|
316                                         VM_PROT_OVERRIDE_WRITE,
317                                        &fs.entry, &fs.first_object,
318                                        &first_pindex, &fs.first_prot,
319                                        &fs.wired);
320                 if (result != KERN_SUCCESS) {
321                         result = KERN_FAILURE;
322                         goto done;
323                 }
324
325                 /*
326                  * If we don't COW now, on a user wire, the user will never
327                  * be able to write to the mapping.  If we don't make this
328                  * restriction, the bookkeeping would be nearly impossible.
329                  */
330                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
331                         fs.entry->max_protection &= ~VM_PROT_WRITE;
332         }
333
334         /*
335          * fs.map is read-locked
336          *
337          * Misc checks.  Save the map generation number to detect races.
338          */
339         fs.map_generation = fs.map->timestamp;
340
341         if (fs.entry->eflags & (MAP_ENTRY_NOFAULT | MAP_ENTRY_KSTACK)) {
342                 if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
343                         panic("vm_fault: fault on nofault entry, addr: %p",
344                               (void *)vaddr);
345                 }
346                 if ((fs.entry->eflags & MAP_ENTRY_KSTACK) &&
347                     vaddr >= fs.entry->start &&
348                     vaddr < fs.entry->start + PAGE_SIZE) {
349                         panic("vm_fault: fault on stack guard, addr: %p",
350                               (void *)vaddr);
351                 }
352         }
353
354         /*
355          * A system map entry may return a NULL object.  No object means
356          * no pager means an unrecoverable kernel fault.
357          */
358         if (fs.first_object == NULL) {
359                 panic("vm_fault: unrecoverable fault at %p in entry %p",
360                         (void *)vaddr, fs.entry);
361         }
362
363         /*
364          * Bump the paging-in-progress count to prevent size changes (e.g.
365          * truncation operations) during I/O.  This must be done after
366          * obtaining the vnode lock in order to avoid possible deadlocks.
367          */
368         vm_object_hold(fs.first_object);
369         fs.vp = vnode_pager_lock(fs.first_object);
370
371         fs.lookup_still_valid = TRUE;
372         fs.first_m = NULL;
373         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
374
375         /*
376          * If the entry is wired we cannot change the page protection.
377          */
378         if (fs.wired)
379                 fault_type = fs.first_prot;
380
381         /*
382          * The page we want is at (first_object, first_pindex), but if the
383          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
384          * page table to figure out the actual pindex.
385          *
386          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
387          * ONLY
388          */
389         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
390                 result = vm_fault_vpagetable(&fs, &first_pindex,
391                                              fs.entry->aux.master_pde,
392                                              fault_type);
393                 if (result == KERN_TRY_AGAIN) {
394                         vm_object_drop(fs.first_object);
395                         goto RetryFault;
396                 }
397                 if (result != KERN_SUCCESS)
398                         goto done;
399         }
400
401         /*
402          * Now we have the actual (object, pindex), fault in the page.  If
403          * vm_fault_object() fails it will unlock and deallocate the FS
404          * data.   If it succeeds everything remains locked and fs->object
405          * will have an additional PIP count if it is not equal to
406          * fs->first_object
407          *
408          * vm_fault_object will set fs->prot for the pmap operation.  It is
409          * allowed to set VM_PROT_WRITE if fault_type == VM_PROT_READ if the
410          * page can be safely written.  However, it will force a read-only
411          * mapping for a read fault if the memory is managed by a virtual
412          * page table.
413          */
414         /* BEFORE */
415         result = vm_fault_object(&fs, first_pindex, fault_type);
416
417         if (result == KERN_TRY_AGAIN) {
418                 vm_object_drop(fs.first_object);
419                 goto RetryFault;
420         }
421         if (result != KERN_SUCCESS)
422                 goto done;
423
424         /*
425          * On success vm_fault_object() does not unlock or deallocate, and fs.m
426          * will contain a busied page.
427          *
428          * Enter the page into the pmap and do pmap-related adjustments.
429          */
430         vm_page_flag_set(fs.m, PG_REFERENCED);
431         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
432
433         /*
434          * Burst in a few more pages if possible.  The fs.map should still
435          * be locked.
436          */
437         if (fault_flags & VM_FAULT_BURST) {
438                 if ((fs.fault_flags & VM_FAULT_WIRE_MASK) == 0 &&
439                     fs.wired == 0) {
440                         vm_prefault(fs.map->pmap, vaddr, fs.entry, fs.prot);
441                 }
442         }
443         unlock_things(&fs);
444
445         /*KKASSERT(fs.m->queue == PQ_NONE); page-in op may deactivate page */
446         KKASSERT(fs.m->flags & PG_BUSY);
447
448         /*
449          * If the page is not wired down, then put it where the pageout daemon
450          * can find it.
451          */
452
453         if (fs.fault_flags & VM_FAULT_WIRE_MASK) {
454                 if (fs.wired)
455                         vm_page_wire(fs.m);
456                 else
457                         vm_page_unwire(fs.m, 1);
458         } else {
459                 vm_page_activate(fs.m);
460         }
461
462         if (curthread->td_lwp) {
463                 if (fs.hardfault) {
464                         curthread->td_lwp->lwp_ru.ru_majflt++;
465                 } else {
466                         curthread->td_lwp->lwp_ru.ru_minflt++;
467                 }
468         }
469
470         /*
471          * Unlock everything, and return
472          */
473         vm_page_wakeup(fs.m);
474         /*vm_object_deallocate(fs.first_object);*/
475         /*fs.m = NULL; */
476         /*fs.first_object = NULL; must still drop later */
477
478         result = KERN_SUCCESS;
479 done:
480         if (fs.first_object)
481                 vm_object_drop(fs.first_object);
482         lwkt_reltoken(&map->token);
483         return (result);
484 }
485
486 /*
487  * Fault in the specified virtual address in the current process map, 
488  * returning a held VM page or NULL.  See vm_fault_page() for more 
489  * information.
490  *
491  * No requirements.
492  */
493 vm_page_t
494 vm_fault_page_quick(vm_offset_t va, vm_prot_t fault_type, int *errorp)
495 {
496         struct lwp *lp = curthread->td_lwp;
497         vm_page_t m;
498
499         m = vm_fault_page(&lp->lwp_vmspace->vm_map, va, 
500                           fault_type, VM_FAULT_NORMAL, errorp);
501         return(m);
502 }
503
504 /*
505  * Fault in the specified virtual address in the specified map, doing all
506  * necessary manipulation of the object store and all necessary I/O.  Return
507  * a held VM page or NULL, and set *errorp.  The related pmap is not
508  * updated.
509  *
510  * The returned page will be properly dirtied if VM_PROT_WRITE was specified,
511  * and marked PG_REFERENCED as well.
512  *
513  * If the page cannot be faulted writable and VM_PROT_WRITE was specified, an
514  * error will be returned.
515  *
516  * No requirements.
517  */
518 vm_page_t
519 vm_fault_page(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
520               int fault_flags, int *errorp)
521 {
522         vm_pindex_t first_pindex;
523         struct faultstate fs;
524         int result;
525         vm_prot_t orig_fault_type = fault_type;
526
527         mycpu->gd_cnt.v_vm_faults++;
528
529         fs.didlimit = 0;
530         fs.hardfault = 0;
531         fs.fault_flags = fault_flags;
532         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
533
534         lwkt_gettoken(&map->token);
535
536 RetryFault:
537         /*
538          * Find the vm_map_entry representing the backing store and resolve
539          * the top level object and page index.  This may have the side
540          * effect of executing a copy-on-write on the map entry and/or
541          * creating a shadow object, but will not COW any actual VM pages.
542          *
543          * On success fs.map is left read-locked and various other fields 
544          * are initialized but not otherwise referenced or locked.
545          *
546          * NOTE!  vm_map_lookup will upgrade the fault_type to VM_FAULT_WRITE
547          * if the map entry is a virtual page table and also writable,
548          * so we can set the 'A'accessed bit in the virtual page table entry.
549          */
550         fs.map = map;
551         result = vm_map_lookup(&fs.map, vaddr, fault_type,
552                                &fs.entry, &fs.first_object,
553                                &first_pindex, &fs.first_prot, &fs.wired);
554
555         if (result != KERN_SUCCESS) {
556                 *errorp = result;
557                 fs.m = NULL;
558                 goto done;
559         }
560
561         /*
562          * fs.map is read-locked
563          *
564          * Misc checks.  Save the map generation number to detect races.
565          */
566         fs.map_generation = fs.map->timestamp;
567
568         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
569                 panic("vm_fault: fault on nofault entry, addr: %lx",
570                     (u_long)vaddr);
571         }
572
573         /*
574          * A system map entry may return a NULL object.  No object means
575          * no pager means an unrecoverable kernel fault.
576          */
577         if (fs.first_object == NULL) {
578                 panic("vm_fault: unrecoverable fault at %p in entry %p",
579                         (void *)vaddr, fs.entry);
580         }
581
582         /*
583          * Make a reference to this object to prevent its disposal while we
584          * are messing with it.  Once we have the reference, the map is free
585          * to be diddled.  Since objects reference their shadows (and copies),
586          * they will stay around as well.
587          *
588          * The reference should also prevent an unexpected collapse of the
589          * parent that might move pages from the current object into the
590          * parent unexpectedly, resulting in corruption.
591          *
592          * Bump the paging-in-progress count to prevent size changes (e.g.
593          * truncation operations) during I/O.  This must be done after
594          * obtaining the vnode lock in order to avoid possible deadlocks.
595          */
596         vm_object_hold(fs.first_object);
597         fs.vp = vnode_pager_lock(fs.first_object);
598
599         fs.lookup_still_valid = TRUE;
600         fs.first_m = NULL;
601         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
602
603         /*
604          * If the entry is wired we cannot change the page protection.
605          */
606         if (fs.wired)
607                 fault_type = fs.first_prot;
608
609         /*
610          * The page we want is at (first_object, first_pindex), but if the
611          * vm_map_entry is VM_MAPTYPE_VPAGETABLE we have to traverse the
612          * page table to figure out the actual pindex.
613          *
614          * NOTE!  DEVELOPMENT IN PROGRESS, THIS IS AN INITIAL IMPLEMENTATION
615          * ONLY
616          */
617         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
618                 result = vm_fault_vpagetable(&fs, &first_pindex,
619                                              fs.entry->aux.master_pde,
620                                              fault_type);
621                 if (result == KERN_TRY_AGAIN) {
622                         vm_object_drop(fs.first_object);
623                         goto RetryFault;
624                 }
625                 if (result != KERN_SUCCESS) {
626                         *errorp = result;
627                         fs.m = NULL;
628                         goto done;
629                 }
630         }
631
632         /*
633          * Now we have the actual (object, pindex), fault in the page.  If
634          * vm_fault_object() fails it will unlock and deallocate the FS
635          * data.   If it succeeds everything remains locked and fs->object
636          * will have an additinal PIP count if it is not equal to
637          * fs->first_object
638          */
639         result = vm_fault_object(&fs, first_pindex, fault_type);
640
641         if (result == KERN_TRY_AGAIN) {
642                 vm_object_drop(fs.first_object);
643                 goto RetryFault;
644         }
645         if (result != KERN_SUCCESS) {
646                 *errorp = result;
647                 fs.m = NULL;
648                 goto done;
649         }
650
651         if ((orig_fault_type & VM_PROT_WRITE) &&
652             (fs.prot & VM_PROT_WRITE) == 0) {
653                 *errorp = KERN_PROTECTION_FAILURE;
654                 unlock_and_deallocate(&fs);
655                 fs.m = NULL;
656                 goto done;
657         }
658
659         /*
660          * On success vm_fault_object() does not unlock or deallocate, and fs.m
661          * will contain a busied page.
662          */
663         unlock_things(&fs);
664
665         /*
666          * Return a held page.  We are not doing any pmap manipulation so do
667          * not set PG_MAPPED.  However, adjust the page flags according to
668          * the fault type because the caller may not use a managed pmapping
669          * (so we don't want to lose the fact that the page will be dirtied
670          * if a write fault was specified).
671          */
672         vm_page_hold(fs.m);
673         if (fault_type & VM_PROT_WRITE)
674                 vm_page_dirty(fs.m);
675
676         /*
677          * Update the pmap.  We really only have to do this if a COW
678          * occured to replace the read-only page with the new page.  For
679          * now just do it unconditionally. XXX
680          */
681         pmap_enter(fs.map->pmap, vaddr, fs.m, fs.prot, fs.wired);
682         vm_page_flag_set(fs.m, PG_REFERENCED);
683
684         /*
685          * Unbusy the page by activating it.  It remains held and will not
686          * be reclaimed.
687          */
688         vm_page_activate(fs.m);
689
690         if (curthread->td_lwp) {
691                 if (fs.hardfault) {
692                         curthread->td_lwp->lwp_ru.ru_majflt++;
693                 } else {
694                         curthread->td_lwp->lwp_ru.ru_minflt++;
695                 }
696         }
697
698         /*
699          * Unlock everything, and return the held page.
700          */
701         vm_page_wakeup(fs.m);
702         /*vm_object_deallocate(fs.first_object);*/
703         /*fs.first_object = NULL; */
704         *errorp = 0;
705
706 done:
707         if (fs.first_object)
708                 vm_object_drop(fs.first_object);
709         lwkt_reltoken(&map->token);
710         return(fs.m);
711 }
712
713 /*
714  * Fault in the specified (object,offset), dirty the returned page as
715  * needed.  If the requested fault_type cannot be done NULL and an
716  * error is returned.
717  *
718  * A held (but not busied) page is returned.
719  *
720  * No requirements.
721  */
722 vm_page_t
723 vm_fault_object_page(vm_object_t object, vm_ooffset_t offset,
724                      vm_prot_t fault_type, int fault_flags, int *errorp)
725 {
726         int result;
727         vm_pindex_t first_pindex;
728         struct faultstate fs;
729         struct vm_map_entry entry;
730
731         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
732         bzero(&entry, sizeof(entry));
733         entry.object.vm_object = object;
734         entry.maptype = VM_MAPTYPE_NORMAL;
735         entry.protection = entry.max_protection = fault_type;
736
737         fs.didlimit = 0;
738         fs.hardfault = 0;
739         fs.fault_flags = fault_flags;
740         fs.map = NULL;
741         KKASSERT((fault_flags & VM_FAULT_WIRE_MASK) == 0);
742
743 RetryFault:
744         
745         fs.first_object = object;
746         first_pindex = OFF_TO_IDX(offset);
747         fs.entry = &entry;
748         fs.first_prot = fault_type;
749         fs.wired = 0;
750         /*fs.map_generation = 0; unused */
751
752         /*
753          * Make a reference to this object to prevent its disposal while we
754          * are messing with it.  Once we have the reference, the map is free
755          * to be diddled.  Since objects reference their shadows (and copies),
756          * they will stay around as well.
757          *
758          * The reference should also prevent an unexpected collapse of the
759          * parent that might move pages from the current object into the
760          * parent unexpectedly, resulting in corruption.
761          *
762          * Bump the paging-in-progress count to prevent size changes (e.g.
763          * truncation operations) during I/O.  This must be done after
764          * obtaining the vnode lock in order to avoid possible deadlocks.
765          */
766         fs.vp = vnode_pager_lock(fs.first_object);
767
768         fs.lookup_still_valid = TRUE;
769         fs.first_m = NULL;
770         fs.object = fs.first_object;    /* so unlock_and_deallocate works */
771
772 #if 0
773         /* XXX future - ability to operate on VM object using vpagetable */
774         if (fs.entry->maptype == VM_MAPTYPE_VPAGETABLE) {
775                 result = vm_fault_vpagetable(&fs, &first_pindex,
776                                              fs.entry->aux.master_pde,
777                                              fault_type);
778                 if (result == KERN_TRY_AGAIN)
779                         goto RetryFault;
780                 if (result != KERN_SUCCESS) {
781                         *errorp = result;
782                         return (NULL);
783                 }
784         }
785 #endif
786
787         /*
788          * Now we have the actual (object, pindex), fault in the page.  If
789          * vm_fault_object() fails it will unlock and deallocate the FS
790          * data.   If it succeeds everything remains locked and fs->object
791          * will have an additinal PIP count if it is not equal to
792          * fs->first_object
793          */
794         result = vm_fault_object(&fs, first_pindex, fault_type);
795
796         if (result == KERN_TRY_AGAIN)
797                 goto RetryFault;
798         if (result != KERN_SUCCESS) {
799                 *errorp = result;
800                 return(NULL);
801         }
802
803         if ((fault_type & VM_PROT_WRITE) && (fs.prot & VM_PROT_WRITE) == 0) {
804                 *errorp = KERN_PROTECTION_FAILURE;
805                 unlock_and_deallocate(&fs);
806                 return(NULL);
807         }
808
809         /*
810          * On success vm_fault_object() does not unlock or deallocate, and fs.m
811          * will contain a busied page.
812          */
813         unlock_things(&fs);
814
815         /*
816          * Return a held page.  We are not doing any pmap manipulation so do
817          * not set PG_MAPPED.  However, adjust the page flags according to
818          * the fault type because the caller may not use a managed pmapping
819          * (so we don't want to lose the fact that the page will be dirtied
820          * if a write fault was specified).
821          */
822         vm_page_hold(fs.m);
823         if (fault_type & VM_PROT_WRITE)
824                 vm_page_dirty(fs.m);
825
826         if (fault_flags & VM_FAULT_DIRTY)
827                 vm_page_dirty(fs.m);
828         if (fault_flags & VM_FAULT_UNSWAP)
829                 swap_pager_unswapped(fs.m);
830
831         /*
832          * Indicate that the page was accessed.
833          */
834         vm_page_flag_set(fs.m, PG_REFERENCED);
835
836         /*
837          * Unbusy the page by activating it.  It remains held and will not
838          * be reclaimed.
839          */
840         vm_page_activate(fs.m);
841
842         if (curthread->td_lwp) {
843                 if (fs.hardfault) {
844                         mycpu->gd_cnt.v_vm_faults++;
845                         curthread->td_lwp->lwp_ru.ru_majflt++;
846                 } else {
847                         curthread->td_lwp->lwp_ru.ru_minflt++;
848                 }
849         }
850
851         /*
852          * Unlock everything, and return the held page.
853          */
854         vm_page_wakeup(fs.m);
855         /*vm_object_deallocate(fs.first_object);*/
856         /*fs.first_object = NULL; */
857
858         *errorp = 0;
859         return(fs.m);
860 }
861
862 /*
863  * Translate the virtual page number (first_pindex) that is relative
864  * to the address space into a logical page number that is relative to the
865  * backing object.  Use the virtual page table pointed to by (vpte).
866  *
867  * This implements an N-level page table.  Any level can terminate the
868  * scan by setting VPTE_PS.   A linear mapping is accomplished by setting
869  * VPTE_PS in the master page directory entry set via mcontrol(MADV_SETMAP).
870  */
871 static
872 int
873 vm_fault_vpagetable(struct faultstate *fs, vm_pindex_t *pindex,
874                     vpte_t vpte, int fault_type)
875 {
876         struct lwbuf *lwb;
877         struct lwbuf lwb_cache;
878         int vshift = VPTE_FRAME_END - PAGE_SHIFT; /* index bits remaining */
879         int result = KERN_SUCCESS;
880         vpte_t *ptep;
881
882         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
883         for (;;) {
884                 /*
885                  * We cannot proceed if the vpte is not valid, not readable
886                  * for a read fault, or not writable for a write fault.
887                  */
888                 if ((vpte & VPTE_V) == 0) {
889                         unlock_and_deallocate(fs);
890                         return (KERN_FAILURE);
891                 }
892                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_R) == 0) {
893                         unlock_and_deallocate(fs);
894                         return (KERN_FAILURE);
895                 }
896                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_W) == 0) {
897                         unlock_and_deallocate(fs);
898                         return (KERN_FAILURE);
899                 }
900                 if ((vpte & VPTE_PS) || vshift == 0)
901                         break;
902                 KKASSERT(vshift >= VPTE_PAGE_BITS);
903
904                 /*
905                  * Get the page table page.  Nominally we only read the page
906                  * table, but since we are actively setting VPTE_M and VPTE_A,
907                  * tell vm_fault_object() that we are writing it. 
908                  *
909                  * There is currently no real need to optimize this.
910                  */
911                 result = vm_fault_object(fs, (vpte & VPTE_FRAME) >> PAGE_SHIFT,
912                                          VM_PROT_READ|VM_PROT_WRITE);
913                 if (result != KERN_SUCCESS)
914                         return (result);
915
916                 /*
917                  * Process the returned fs.m and look up the page table
918                  * entry in the page table page.
919                  */
920                 vshift -= VPTE_PAGE_BITS;
921                 lwb = lwbuf_alloc(fs->m, &lwb_cache);
922                 ptep = ((vpte_t *)lwbuf_kva(lwb) +
923                         ((*pindex >> vshift) & VPTE_PAGE_MASK));
924                 vpte = *ptep;
925
926                 /*
927                  * Page table write-back.  If the vpte is valid for the
928                  * requested operation, do a write-back to the page table.
929                  *
930                  * XXX VPTE_M is not set properly for page directory pages.
931                  * It doesn't get set in the page directory if the page table
932                  * is modified during a read access.
933                  */
934                 if ((fault_type & VM_PROT_WRITE) && (vpte & VPTE_V) &&
935                     (vpte & VPTE_W)) {
936                         if ((vpte & (VPTE_M|VPTE_A)) != (VPTE_M|VPTE_A)) {
937                                 atomic_set_long(ptep, VPTE_M | VPTE_A);
938                                 vm_page_dirty(fs->m);
939                         }
940                 }
941                 if ((fault_type & VM_PROT_READ) && (vpte & VPTE_V) &&
942                     (vpte & VPTE_R)) {
943                         if ((vpte & VPTE_A) == 0) {
944                                 atomic_set_long(ptep, VPTE_A);
945                                 vm_page_dirty(fs->m);
946                         }
947                 }
948                 lwbuf_free(lwb);
949                 vm_page_flag_set(fs->m, PG_REFERENCED);
950                 vm_page_activate(fs->m);
951                 vm_page_wakeup(fs->m);
952                 fs->m = NULL;
953                 cleanup_successful_fault(fs);
954         }
955         /*
956          * Combine remaining address bits with the vpte.
957          */
958         /* JG how many bits from each? */
959         *pindex = ((vpte & VPTE_FRAME) >> PAGE_SHIFT) +
960                   (*pindex & ((1L << vshift) - 1));
961         return (KERN_SUCCESS);
962 }
963
964
965 /*
966  * This is the core of the vm_fault code.
967  *
968  * Do all operations required to fault-in (fs.first_object, pindex).  Run
969  * through the shadow chain as necessary and do required COW or virtual
970  * copy operations.  The caller has already fully resolved the vm_map_entry
971  * and, if appropriate, has created a copy-on-write layer.  All we need to
972  * do is iterate the object chain.
973  *
974  * On failure (fs) is unlocked and deallocated and the caller may return or
975  * retry depending on the failure code.  On success (fs) is NOT unlocked or
976  * deallocated, fs.m will contained a resolved, busied page, and fs.object
977  * will have an additional PIP count if it is not equal to fs.first_object.
978  *
979  * fs->first_object must be held on call.
980  */
981 static
982 int
983 vm_fault_object(struct faultstate *fs,
984                 vm_pindex_t first_pindex, vm_prot_t fault_type)
985 {
986         vm_object_t next_object;
987         vm_pindex_t pindex;
988         int error;
989
990         ASSERT_LWKT_TOKEN_HELD(vm_object_token(fs->first_object));
991         fs->prot = fs->first_prot;
992         fs->object = fs->first_object;
993         pindex = first_pindex;
994
995         vm_object_chain_acquire(fs->first_object);
996         vm_object_pip_add(fs->first_object, 1);
997
998         /* 
999          * If a read fault occurs we try to make the page writable if
1000          * possible.  There are three cases where we cannot make the
1001          * page mapping writable:
1002          *
1003          * (1) The mapping is read-only or the VM object is read-only,
1004          *     fs->prot above will simply not have VM_PROT_WRITE set.
1005          *
1006          * (2) If the mapping is a virtual page table we need to be able
1007          *     to detect writes so we can set VPTE_M in the virtual page
1008          *     table.
1009          *
1010          * (3) If the VM page is read-only or copy-on-write, upgrading would
1011          *     just result in an unnecessary COW fault.
1012          *
1013          * VM_PROT_VPAGED is set if faulting via a virtual page table and
1014          * causes adjustments to the 'M'odify bit to also turn off write
1015          * access to force a re-fault.
1016          */
1017         if (fs->entry->maptype == VM_MAPTYPE_VPAGETABLE) {
1018                 if ((fault_type & VM_PROT_WRITE) == 0)
1019                         fs->prot &= ~VM_PROT_WRITE;
1020         }
1021
1022         /* vm_object_hold(fs->object); implied b/c object == first_object */
1023
1024         for (;;) {
1025                 /*
1026                  * If the object is dead, we stop here
1027                  */
1028                 if (fs->object->flags & OBJ_DEAD) {
1029                         vm_object_pip_wakeup(fs->first_object);
1030                         vm_object_chain_release_all(fs->first_object,
1031                                                     fs->object);
1032                         if (fs->object != fs->first_object)
1033                                 vm_object_drop(fs->object);
1034                         unlock_and_deallocate(fs);
1035                         return (KERN_PROTECTION_FAILURE);
1036                 }
1037
1038                 /*
1039                  * See if the page is resident.  Wait/Retry if the page is
1040                  * busy (lots of stuff may have changed so we can't continue
1041                  * in that case).
1042                  *
1043                  * We can theoretically allow the soft-busy case on a read
1044                  * fault if the page is marked valid, but since such
1045                  * pages are typically already pmap'd, putting that
1046                  * special case in might be more effort then it is
1047                  * worth.  We cannot under any circumstances mess
1048                  * around with a vm_page_t->busy page except, perhaps,
1049                  * to pmap it.
1050                  */
1051                 fs->m = vm_page_lookup_busy_try(fs->object, pindex,
1052                                                 TRUE, &error);
1053                 if (error) {
1054                         vm_object_pip_wakeup(fs->first_object);
1055                         vm_object_chain_release_all(fs->first_object,
1056                                                     fs->object);
1057                         if (fs->object != fs->first_object)
1058                                 vm_object_drop(fs->object);
1059                         unlock_things(fs);
1060                         vm_page_sleep_busy(fs->m, TRUE, "vmpfw");
1061                         mycpu->gd_cnt.v_intrans++;
1062                         /*vm_object_deallocate(fs->first_object);*/
1063                         /*fs->first_object = NULL;*/
1064                         fs->m = NULL;
1065                         return (KERN_TRY_AGAIN);
1066                 }
1067                 if (fs->m) {
1068                         /*
1069                          * The page is busied for us.
1070                          *
1071                          * If reactivating a page from PQ_CACHE we may have
1072                          * to rate-limit.
1073                          */
1074                         int queue = fs->m->queue;
1075                         vm_page_unqueue_nowakeup(fs->m);
1076
1077                         if ((queue - fs->m->pc) == PQ_CACHE && 
1078                             vm_page_count_severe()) {
1079                                 vm_page_activate(fs->m);
1080                                 vm_page_wakeup(fs->m);
1081                                 fs->m = NULL;
1082                                 vm_object_pip_wakeup(fs->first_object);
1083                                 vm_object_chain_release_all(fs->first_object,
1084                                                             fs->object);
1085                                 if (fs->object != fs->first_object)
1086                                         vm_object_drop(fs->object);
1087                                 unlock_and_deallocate(fs);
1088                                 vm_waitpfault();
1089                                 return (KERN_TRY_AGAIN);
1090                         }
1091
1092                         /*
1093                          * If it still isn't completely valid (readable),
1094                          * or if a read-ahead-mark is set on the VM page,
1095                          * jump to readrest, else we found the page and
1096                          * can return.
1097                          *
1098                          * We can release the spl once we have marked the
1099                          * page busy.
1100                          */
1101                         if (fs->m->object != &kernel_object) {
1102                                 if ((fs->m->valid & VM_PAGE_BITS_ALL) !=
1103                                     VM_PAGE_BITS_ALL) {
1104                                         goto readrest;
1105                                 }
1106                                 if (fs->m->flags & PG_RAM) {
1107                                         if (debug_cluster)
1108                                                 kprintf("R");
1109                                         vm_page_flag_clear(fs->m, PG_RAM);
1110                                         goto readrest;
1111                                 }
1112                         }
1113                         break; /* break to PAGE HAS BEEN FOUND */
1114                 }
1115
1116                 /*
1117                  * Page is not resident, If this is the search termination
1118                  * or the pager might contain the page, allocate a new page.
1119                  */
1120                 if (TRYPAGER(fs) || fs->object == fs->first_object) {
1121                         /*
1122                          * If the page is beyond the object size we fail
1123                          */
1124                         if (pindex >= fs->object->size) {
1125                                 vm_object_pip_wakeup(fs->first_object);
1126                                 vm_object_chain_release_all(fs->first_object,
1127                                                             fs->object);
1128                                 if (fs->object != fs->first_object)
1129                                         vm_object_drop(fs->object);
1130                                 unlock_and_deallocate(fs);
1131                                 return (KERN_PROTECTION_FAILURE);
1132                         }
1133
1134                         /*
1135                          * Ratelimit.
1136                          */
1137                         if (fs->didlimit == 0 && curproc != NULL) {
1138                                 int limticks;
1139
1140                                 limticks = vm_fault_ratelimit(curproc->p_vmspace);
1141                                 if (limticks) {
1142                                         vm_object_pip_wakeup(fs->first_object);
1143                                         vm_object_chain_release_all(
1144                                                 fs->first_object, fs->object);
1145                                         if (fs->object != fs->first_object)
1146                                                 vm_object_drop(fs->object);
1147                                         unlock_and_deallocate(fs);
1148                                         tsleep(curproc, 0, "vmrate", limticks);
1149                                         fs->didlimit = 1;
1150                                         return (KERN_TRY_AGAIN);
1151                                 }
1152                         }
1153
1154                         /*
1155                          * Allocate a new page for this object/offset pair.
1156                          */
1157                         fs->m = NULL;
1158                         if (!vm_page_count_severe()) {
1159                                 fs->m = vm_page_alloc(fs->object, pindex,
1160                                     ((fs->vp || fs->object->backing_object) ?
1161                                         VM_ALLOC_NORMAL :
1162                                         VM_ALLOC_NORMAL | VM_ALLOC_ZERO));
1163                         }
1164                         if (fs->m == NULL) {
1165                                 vm_object_pip_wakeup(fs->first_object);
1166                                 vm_object_chain_release_all(fs->first_object,
1167                                                             fs->object);
1168                                 if (fs->object != fs->first_object)
1169                                         vm_object_drop(fs->object);
1170                                 unlock_and_deallocate(fs);
1171                                 vm_waitpfault();
1172                                 return (KERN_TRY_AGAIN);
1173                         }
1174
1175                         /*
1176                          * Fall through to readrest.  We have a new page which
1177                          * will have to be paged (since m->valid will be 0).
1178                          */
1179                 }
1180
1181 readrest:
1182                 /*
1183                  * We have found an invalid or partially valid page, a
1184                  * page with a read-ahead mark which might be partially or
1185                  * fully valid (and maybe dirty too), or we have allocated
1186                  * a new page.
1187                  *
1188                  * Attempt to fault-in the page if there is a chance that the
1189                  * pager has it, and potentially fault in additional pages
1190                  * at the same time.
1191                  *
1192                  * We are NOT in splvm here and if TRYPAGER is true then
1193                  * fs.m will be non-NULL and will be PG_BUSY for us.
1194                  */
1195                 if (TRYPAGER(fs)) {
1196                         int rv;
1197                         int seqaccess;
1198                         u_char behavior = vm_map_entry_behavior(fs->entry);
1199
1200                         if (behavior == MAP_ENTRY_BEHAV_RANDOM)
1201                                 seqaccess = 0;
1202                         else
1203                                 seqaccess = -1;
1204
1205                         /*
1206                          * If sequential access is detected then attempt
1207                          * to deactivate/cache pages behind the scan to
1208                          * prevent resource hogging.
1209                          *
1210                          * Use of PG_RAM to detect sequential access
1211                          * also simulates multi-zone sequential access
1212                          * detection for free.
1213                          *
1214                          * NOTE: Partially valid dirty pages cannot be
1215                          *       deactivated without causing NFS picemeal
1216                          *       writes to barf.
1217                          */
1218                         if ((fs->first_object->type != OBJT_DEVICE) &&
1219                             (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
1220                                 (behavior != MAP_ENTRY_BEHAV_RANDOM &&
1221                                  (fs->m->flags & PG_RAM)))
1222                         ) {
1223                                 vm_pindex_t scan_pindex;
1224                                 int scan_count = 16;
1225
1226                                 if (first_pindex < 16) {
1227                                         scan_pindex = 0;
1228                                         scan_count = 0;
1229                                 } else {
1230                                         scan_pindex = first_pindex - 16;
1231                                         if (scan_pindex < 16)
1232                                                 scan_count = scan_pindex;
1233                                         else
1234                                                 scan_count = 16;
1235                                 }
1236
1237                                 while (scan_count) {
1238                                         vm_page_t mt;
1239
1240                                         mt = vm_page_lookup(fs->first_object,
1241                                                             scan_pindex);
1242                                         if (mt == NULL)
1243                                                 break;
1244                                         if (vm_page_busy_try(mt, TRUE))
1245                                                 goto skip;
1246
1247                                         if (mt->valid != VM_PAGE_BITS_ALL) {
1248                                                 vm_page_wakeup(mt);
1249                                                 break;
1250                                         }
1251                                         if ((mt->flags &
1252                                              (PG_FICTITIOUS | PG_UNMANAGED)) ||
1253                                             mt->hold_count ||
1254                                             mt->wire_count)  {
1255                                                 vm_page_wakeup(mt);
1256                                                 goto skip;
1257                                         }
1258                                         if (mt->dirty == 0)
1259                                                 vm_page_test_dirty(mt);
1260                                         if (mt->dirty) {
1261                                                 vm_page_protect(mt,
1262                                                                 VM_PROT_NONE);
1263                                                 vm_page_deactivate(mt);
1264                                                 vm_page_wakeup(mt);
1265                                         } else {
1266                                                 vm_page_cache(mt);
1267                                         }
1268 skip:
1269                                         --scan_count;
1270                                         --scan_pindex;
1271                                 }
1272
1273                                 seqaccess = 1;
1274                         }
1275
1276                         /*
1277                          * Avoid deadlocking against the map when doing I/O.
1278                          * fs.object and the page is PG_BUSY'd.
1279                          *
1280                          * NOTE: Once unlocked, fs->entry can become stale
1281                          *       so this will NULL it out.
1282                          *
1283                          * NOTE: fs->entry is invalid until we relock the
1284                          *       map and verify that the timestamp has not
1285                          *       changed.
1286                          */
1287                         unlock_map(fs);
1288
1289                         /*
1290                          * Acquire the page data.  We still hold a ref on
1291                          * fs.object and the page has been PG_BUSY's.
1292                          *
1293                          * The pager may replace the page (for example, in
1294                          * order to enter a fictitious page into the
1295                          * object).  If it does so it is responsible for
1296                          * cleaning up the passed page and properly setting
1297                          * the new page PG_BUSY.
1298                          *
1299                          * If we got here through a PG_RAM read-ahead
1300                          * mark the page may be partially dirty and thus
1301                          * not freeable.  Don't bother checking to see
1302                          * if the pager has the page because we can't free
1303                          * it anyway.  We have to depend on the get_page
1304                          * operation filling in any gaps whether there is
1305                          * backing store or not.
1306                          */
1307                         rv = vm_pager_get_page(fs->object, &fs->m, seqaccess);
1308
1309                         if (rv == VM_PAGER_OK) {
1310                                 /*
1311                                  * Relookup in case pager changed page. Pager
1312                                  * is responsible for disposition of old page
1313                                  * if moved.
1314                                  *
1315                                  * XXX other code segments do relookups too.
1316                                  * It's a bad abstraction that needs to be
1317                                  * fixed/removed.
1318                                  */
1319                                 fs->m = vm_page_lookup(fs->object, pindex);
1320                                 if (fs->m == NULL) {
1321                                         vm_object_pip_wakeup(fs->first_object);
1322                                         vm_object_chain_release_all(
1323                                                 fs->first_object, fs->object);
1324                                         if (fs->object != fs->first_object)
1325                                                 vm_object_drop(fs->object);
1326                                         unlock_and_deallocate(fs);
1327                                         return (KERN_TRY_AGAIN);
1328                                 }
1329
1330                                 ++fs->hardfault;
1331                                 break; /* break to PAGE HAS BEEN FOUND */
1332                         }
1333
1334                         /*
1335                          * Remove the bogus page (which does not exist at this
1336                          * object/offset); before doing so, we must get back
1337                          * our object lock to preserve our invariant.
1338                          *
1339                          * Also wake up any other process that may want to bring
1340                          * in this page.
1341                          *
1342                          * If this is the top-level object, we must leave the
1343                          * busy page to prevent another process from rushing
1344                          * past us, and inserting the page in that object at
1345                          * the same time that we are.
1346                          */
1347                         if (rv == VM_PAGER_ERROR) {
1348                                 if (curproc) {
1349                                         kprintf("vm_fault: pager read error, "
1350                                                 "pid %d (%s)\n",
1351                                                 curproc->p_pid,
1352                                                 curproc->p_comm);
1353                                 } else {
1354                                         kprintf("vm_fault: pager read error, "
1355                                                 "thread %p (%s)\n",
1356                                                 curthread,
1357                                                 curproc->p_comm);
1358                                 }
1359                         }
1360
1361                         /*
1362                          * Data outside the range of the pager or an I/O error
1363                          *
1364                          * The page may have been wired during the pagein,
1365                          * e.g. by the buffer cache, and cannot simply be
1366                          * freed.  Call vnode_pager_freepage() to deal with it.
1367                          */
1368                         /*
1369                          * XXX - the check for kernel_map is a kludge to work
1370                          * around having the machine panic on a kernel space
1371                          * fault w/ I/O error.
1372                          */
1373                         if (((fs->map != &kernel_map) &&
1374                             (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) {
1375                                 vnode_pager_freepage(fs->m);
1376                                 fs->m = NULL;
1377                                 vm_object_pip_wakeup(fs->first_object);
1378                                 vm_object_chain_release_all(fs->first_object,
1379                                                             fs->object);
1380                                 if (fs->object != fs->first_object)
1381                                         vm_object_drop(fs->object);
1382                                 unlock_and_deallocate(fs);
1383                                 if (rv == VM_PAGER_ERROR)
1384                                         return (KERN_FAILURE);
1385                                 else
1386                                         return (KERN_PROTECTION_FAILURE);
1387                                 /* NOT REACHED */
1388                         }
1389                         if (fs->object != fs->first_object) {
1390                                 vnode_pager_freepage(fs->m);
1391                                 fs->m = NULL;
1392                                 /*
1393                                  * XXX - we cannot just fall out at this
1394                                  * point, m has been freed and is invalid!
1395                                  */
1396                         }
1397                 }
1398
1399                 /*
1400                  * We get here if the object has a default pager (or unwiring) 
1401                  * or the pager doesn't have the page.
1402                  */
1403                 if (fs->object == fs->first_object)
1404                         fs->first_m = fs->m;
1405
1406                 /*
1407                  * Move on to the next object.  The chain lock should prevent
1408                  * the backing_object from getting ripped out from under us.
1409                  */
1410                 if ((next_object = fs->object->backing_object) != NULL) {
1411                         vm_object_hold(next_object);
1412                         vm_object_chain_acquire(next_object);
1413                         KKASSERT(next_object == fs->object->backing_object);
1414                         pindex += OFF_TO_IDX(fs->object->backing_object_offset);
1415                 }
1416
1417                 if (next_object == NULL) {
1418                         /*
1419                          * If there's no object left, fill the page in the top
1420                          * object with zeros.
1421                          */
1422                         if (fs->object != fs->first_object) {
1423                                 if (fs->first_object->backing_object !=
1424                                     fs->object) {
1425                                         vm_object_hold(fs->first_object->backing_object);
1426                                 }
1427                                 vm_object_chain_release_all(
1428                                         fs->first_object->backing_object,
1429                                         fs->object);
1430                                 if (fs->first_object->backing_object !=
1431                                     fs->object) {
1432                                         vm_object_drop(fs->first_object->backing_object);
1433                                 }
1434                                 vm_object_pip_wakeup(fs->object);
1435                                 vm_object_drop(fs->object);
1436                                 fs->object = fs->first_object;
1437                                 pindex = first_pindex;
1438                                 fs->m = fs->first_m;
1439                         }
1440                         fs->first_m = NULL;
1441
1442                         /*
1443                          * Zero the page if necessary and mark it valid.
1444                          */
1445                         if ((fs->m->flags & PG_ZERO) == 0) {
1446                                 vm_page_zero_fill(fs->m);
1447                         } else {
1448 #ifdef PMAP_DEBUG
1449                                 pmap_page_assertzero(VM_PAGE_TO_PHYS(fs->m));
1450 #endif
1451                                 vm_page_flag_clear(fs->m, PG_ZERO);
1452                                 mycpu->gd_cnt.v_ozfod++;
1453                         }
1454                         mycpu->gd_cnt.v_zfod++;
1455                         fs->m->valid = VM_PAGE_BITS_ALL;
1456                         break;  /* break to PAGE HAS BEEN FOUND */
1457                 }
1458                 if (fs->object != fs->first_object) {
1459                         vm_object_pip_wakeup(fs->object);
1460                         vm_object_lock_swap();
1461                         vm_object_drop(fs->object);
1462                 }
1463                 KASSERT(fs->object != next_object,
1464                         ("object loop %p", next_object));
1465                 fs->object = next_object;
1466                 vm_object_pip_add(fs->object, 1);
1467         }
1468
1469         /*
1470          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
1471          * is held.]
1472          *
1473          * object still held.
1474          *
1475          * If the page is being written, but isn't already owned by the
1476          * top-level object, we have to copy it into a new page owned by the
1477          * top-level object.
1478          */
1479         KASSERT((fs->m->flags & PG_BUSY) != 0,
1480                 ("vm_fault: not busy after main loop"));
1481
1482         if (fs->object != fs->first_object) {
1483                 /*
1484                  * We only really need to copy if we want to write it.
1485                  */
1486                 if (fault_type & VM_PROT_WRITE) {
1487                         /*
1488                          * This allows pages to be virtually copied from a 
1489                          * backing_object into the first_object, where the 
1490                          * backing object has no other refs to it, and cannot
1491                          * gain any more refs.  Instead of a bcopy, we just 
1492                          * move the page from the backing object to the 
1493                          * first object.  Note that we must mark the page 
1494                          * dirty in the first object so that it will go out 
1495                          * to swap when needed.
1496                          */
1497                         if (
1498                                 /*
1499                                  * Map, if present, has not changed
1500                                  */
1501                                 (fs->map == NULL ||
1502                                 fs->map_generation == fs->map->timestamp) &&
1503                                 /*
1504                                  * Only one shadow object
1505                                  */
1506                                 (fs->object->shadow_count == 1) &&
1507                                 /*
1508                                  * No COW refs, except us
1509                                  */
1510                                 (fs->object->ref_count == 1) &&
1511                                 /*
1512                                  * No one else can look this object up
1513                                  */
1514                                 (fs->object->handle == NULL) &&
1515                                 /*
1516                                  * No other ways to look the object up
1517                                  */
1518                                 ((fs->object->type == OBJT_DEFAULT) ||
1519                                  (fs->object->type == OBJT_SWAP)) &&
1520                                 /*
1521                                  * We don't chase down the shadow chain
1522                                  */
1523                                 (fs->object == fs->first_object->backing_object) &&
1524
1525                                 /*
1526                                  * grab the lock if we need to
1527                                  */
1528                                 (fs->lookup_still_valid ||
1529                                  fs->map == NULL ||
1530                                  lockmgr(&fs->map->lock, LK_EXCLUSIVE|LK_NOWAIT) == 0)
1531                             ) {
1532                                 /*
1533                                  * (first_m) and (m) are both busied.  We have
1534                                  * move (m) into (first_m)'s object/pindex
1535                                  * in an atomic fashion, then free (first_m).
1536                                  *
1537                                  * first_object is held so second remove
1538                                  * followed by the rename should wind
1539                                  * up being atomic.  vm_page_free() might
1540                                  * block so we don't do it until after the
1541                                  * rename.
1542                                  */
1543                                 fs->lookup_still_valid = 1;
1544                                 vm_page_protect(fs->first_m, VM_PROT_NONE);
1545                                 vm_page_remove(fs->first_m);
1546                                 vm_page_rename(fs->m, fs->first_object,
1547                                                first_pindex);
1548                                 vm_page_free(fs->first_m);
1549                                 fs->first_m = fs->m;
1550                                 fs->m = NULL;
1551                                 mycpu->gd_cnt.v_cow_optim++;
1552                         } else {
1553                                 /*
1554                                  * Oh, well, lets copy it.
1555                                  */
1556                                 vm_page_copy(fs->m, fs->first_m);
1557                                 vm_page_event(fs->m, VMEVENT_COW);
1558                         }
1559
1560                         if (fs->m) {
1561                                 /*
1562                                  * We no longer need the old page or object.
1563                                  */
1564                                 release_page(fs);
1565                         }
1566
1567                         /*
1568                          * We intend to revert to first_object, undo the
1569                          * chain lock through to that.
1570                          */
1571                         if (fs->first_object->backing_object != fs->object)
1572                                 vm_object_hold(fs->first_object->backing_object);
1573                         vm_object_chain_release_all(
1574                                         fs->first_object->backing_object,
1575                                         fs->object);
1576                         if (fs->first_object->backing_object != fs->object)
1577                                 vm_object_drop(fs->first_object->backing_object);
1578
1579                         /*
1580                          * fs->object != fs->first_object due to above 
1581                          * conditional
1582                          */
1583                         vm_object_pip_wakeup(fs->object);
1584                         vm_object_drop(fs->object);
1585
1586                         /*
1587                          * Only use the new page below...
1588                          */
1589
1590                         mycpu->gd_cnt.v_cow_faults++;
1591                         fs->m = fs->first_m;
1592                         fs->object = fs->first_object;
1593                         pindex = first_pindex;
1594                 } else {
1595                         /*
1596                          * If it wasn't a write fault avoid having to copy
1597                          * the page by mapping it read-only.
1598                          */
1599                         fs->prot &= ~VM_PROT_WRITE;
1600                 }
1601         }
1602
1603         /*
1604          * Relock the map if necessary, then check the generation count.
1605          * relock_map() will update fs->timestamp to account for the
1606          * relocking if necessary.
1607          *
1608          * If the count has changed after relocking then all sorts of
1609          * crap may have happened and we have to retry.
1610          *
1611          * NOTE: The relock_map() can fail due to a deadlock against
1612          *       the vm_page we are holding BUSY.
1613          */
1614         if (fs->lookup_still_valid == FALSE && fs->map) {
1615                 if (relock_map(fs) ||
1616                     fs->map->timestamp != fs->map_generation) {
1617                         release_page(fs);
1618                         vm_object_pip_wakeup(fs->first_object);
1619                         vm_object_chain_release_all(fs->first_object,
1620                                                     fs->object);
1621                         if (fs->object != fs->first_object)
1622                                 vm_object_drop(fs->object);
1623                         unlock_and_deallocate(fs);
1624                         return (KERN_TRY_AGAIN);
1625                 }
1626         }
1627
1628         /*
1629          * If the fault is a write, we know that this page is being
1630          * written NOW so dirty it explicitly to save on pmap_is_modified()
1631          * calls later.
1632          *
1633          * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
1634          * if the page is already dirty to prevent data written with
1635          * the expectation of being synced from not being synced.
1636          * Likewise if this entry does not request NOSYNC then make
1637          * sure the page isn't marked NOSYNC.  Applications sharing
1638          * data should use the same flags to avoid ping ponging.
1639          *
1640          * Also tell the backing pager, if any, that it should remove
1641          * any swap backing since the page is now dirty.
1642          */
1643         if (fs->prot & VM_PROT_WRITE) {
1644                 vm_object_set_writeable_dirty(fs->m->object);
1645                 vm_set_nosync(fs->m, fs->entry);
1646                 if (fs->fault_flags & VM_FAULT_DIRTY) {
1647                         vm_page_dirty(fs->m);
1648                         swap_pager_unswapped(fs->m);
1649                 }
1650         }
1651
1652         vm_object_pip_wakeup(fs->first_object);
1653         vm_object_chain_release_all(fs->first_object, fs->object);
1654         if (fs->object != fs->first_object)
1655                 vm_object_drop(fs->object);
1656
1657         /*
1658          * Page had better still be busy.  We are still locked up and 
1659          * fs->object will have another PIP reference if it is not equal
1660          * to fs->first_object.
1661          */
1662         KASSERT(fs->m->flags & PG_BUSY,
1663                 ("vm_fault: page %p not busy!", fs->m));
1664
1665         /*
1666          * Sanity check: page must be completely valid or it is not fit to
1667          * map into user space.  vm_pager_get_pages() ensures this.
1668          */
1669         if (fs->m->valid != VM_PAGE_BITS_ALL) {
1670                 vm_page_zero_invalid(fs->m, TRUE);
1671                 kprintf("Warning: page %p partially invalid on fault\n", fs->m);
1672         }
1673         vm_page_flag_clear(fs->m, PG_ZERO);
1674
1675         return (KERN_SUCCESS);
1676 }
1677
1678 /*
1679  * Wire down a range of virtual addresses in a map.  The entry in question
1680  * should be marked in-transition and the map must be locked.  We must
1681  * release the map temporarily while faulting-in the page to avoid a
1682  * deadlock.  Note that the entry may be clipped while we are blocked but
1683  * will never be freed.
1684  *
1685  * No requirements.
1686  */
1687 int
1688 vm_fault_wire(vm_map_t map, vm_map_entry_t entry, boolean_t user_wire)
1689 {
1690         boolean_t fictitious;
1691         vm_offset_t start;
1692         vm_offset_t end;
1693         vm_offset_t va;
1694         vm_paddr_t pa;
1695         vm_page_t m;
1696         pmap_t pmap;
1697         int rv;
1698
1699         lwkt_gettoken(&map->token);
1700
1701         pmap = vm_map_pmap(map);
1702         start = entry->start;
1703         end = entry->end;
1704         fictitious = entry->object.vm_object &&
1705                         (entry->object.vm_object->type == OBJT_DEVICE);
1706         if (entry->eflags & MAP_ENTRY_KSTACK)
1707                 start += PAGE_SIZE;
1708         map->timestamp++;
1709         vm_map_unlock(map);
1710
1711         /*
1712          * We simulate a fault to get the page and enter it in the physical
1713          * map.
1714          */
1715         for (va = start; va < end; va += PAGE_SIZE) {
1716                 if (user_wire) {
1717                         rv = vm_fault(map, va, VM_PROT_READ, 
1718                                         VM_FAULT_USER_WIRE);
1719                 } else {
1720                         rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
1721                                         VM_FAULT_CHANGE_WIRING);
1722                 }
1723                 if (rv) {
1724                         while (va > start) {
1725                                 va -= PAGE_SIZE;
1726                                 if ((pa = pmap_extract(pmap, va)) == 0)
1727                                         continue;
1728                                 pmap_change_wiring(pmap, va, FALSE);
1729                                 if (!fictitious) {
1730                                         m = PHYS_TO_VM_PAGE(pa);
1731                                         vm_page_busy_wait(m, FALSE, "vmwrpg");
1732                                         vm_page_unwire(m, 1);
1733                                         vm_page_wakeup(m);
1734                                 }
1735                         }
1736                         goto done;
1737                 }
1738         }
1739         rv = KERN_SUCCESS;
1740 done:
1741         vm_map_lock(map);
1742         lwkt_reltoken(&map->token);
1743         return (rv);
1744 }
1745
1746 /*
1747  * Unwire a range of virtual addresses in a map.  The map should be
1748  * locked.
1749  */
1750 void
1751 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
1752 {
1753         boolean_t fictitious;
1754         vm_offset_t start;
1755         vm_offset_t end;
1756         vm_offset_t va;
1757         vm_paddr_t pa;
1758         vm_page_t m;
1759         pmap_t pmap;
1760
1761         lwkt_gettoken(&map->token);
1762
1763         pmap = vm_map_pmap(map);
1764         start = entry->start;
1765         end = entry->end;
1766         fictitious = entry->object.vm_object &&
1767                         (entry->object.vm_object->type == OBJT_DEVICE);
1768         if (entry->eflags & MAP_ENTRY_KSTACK)
1769                 start += PAGE_SIZE;
1770
1771         /*
1772          * Since the pages are wired down, we must be able to get their
1773          * mappings from the physical map system.
1774          */
1775         for (va = start; va < end; va += PAGE_SIZE) {
1776                 pa = pmap_extract(pmap, va);
1777                 if (pa != 0) {
1778                         pmap_change_wiring(pmap, va, FALSE);
1779                         if (!fictitious) {
1780                                 m = PHYS_TO_VM_PAGE(pa);
1781                                 vm_page_busy_wait(m, FALSE, "vmwupg");
1782                                 vm_page_unwire(m, 1);
1783                                 vm_page_wakeup(m);
1784                         }
1785                 }
1786         }
1787         lwkt_reltoken(&map->token);
1788 }
1789
1790 /*
1791  * Reduce the rate at which memory is allocated to a process based
1792  * on the perceived load on the VM system. As the load increases
1793  * the allocation burst rate goes down and the delay increases. 
1794  *
1795  * Rate limiting does not apply when faulting active or inactive
1796  * pages.  When faulting 'cache' pages, rate limiting only applies
1797  * if the system currently has a severe page deficit.
1798  *
1799  * XXX vm_pagesupply should be increased when a page is freed.
1800  *
1801  * We sleep up to 1/10 of a second.
1802  */
1803 static int
1804 vm_fault_ratelimit(struct vmspace *vmspace)
1805 {
1806         if (vm_load_enable == 0)
1807                 return(0);
1808         if (vmspace->vm_pagesupply > 0) {
1809                 --vmspace->vm_pagesupply;       /* SMP race ok */
1810                 return(0);
1811         }
1812 #ifdef INVARIANTS
1813         if (vm_load_debug) {
1814                 kprintf("load %-4d give %d pgs, wait %d, pid %-5d (%s)\n",
1815                         vm_load, 
1816                         (1000 - vm_load ) / 10, vm_load * hz / 10000,
1817                         curproc->p_pid, curproc->p_comm);
1818         }
1819 #endif
1820         vmspace->vm_pagesupply = (1000 - vm_load) / 10;
1821         return(vm_load * hz / 10000);
1822 }
1823
1824 /*
1825  * Copy all of the pages from a wired-down map entry to another.
1826  *
1827  * The source and destination maps must be locked for write.
1828  * The source and destination maps token must be held
1829  * The source map entry must be wired down (or be a sharing map
1830  * entry corresponding to a main map entry that is wired down).
1831  *
1832  * No other requirements.
1833  */
1834 void
1835 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1836                     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
1837 {
1838         vm_object_t dst_object;
1839         vm_object_t src_object;
1840         vm_ooffset_t dst_offset;
1841         vm_ooffset_t src_offset;
1842         vm_prot_t prot;
1843         vm_offset_t vaddr;
1844         vm_page_t dst_m;
1845         vm_page_t src_m;
1846
1847         src_object = src_entry->object.vm_object;
1848         src_offset = src_entry->offset;
1849
1850         /*
1851          * Create the top-level object for the destination entry. (Doesn't
1852          * actually shadow anything - we copy the pages directly.)
1853          */
1854         vm_map_entry_allocate_object(dst_entry);
1855         dst_object = dst_entry->object.vm_object;
1856
1857         prot = dst_entry->max_protection;
1858
1859         /*
1860          * Loop through all of the pages in the entry's range, copying each
1861          * one from the source object (it should be there) to the destination
1862          * object.
1863          */
1864         for (vaddr = dst_entry->start, dst_offset = 0;
1865             vaddr < dst_entry->end;
1866             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
1867
1868                 /*
1869                  * Allocate a page in the destination object
1870                  */
1871                 do {
1872                         dst_m = vm_page_alloc(dst_object,
1873                                 OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
1874                         if (dst_m == NULL) {
1875                                 vm_wait(0);
1876                         }
1877                 } while (dst_m == NULL);
1878
1879                 /*
1880                  * Find the page in the source object, and copy it in.
1881                  * (Because the source is wired down, the page will be in
1882                  * memory.)
1883                  */
1884                 src_m = vm_page_lookup(src_object,
1885                                        OFF_TO_IDX(dst_offset + src_offset));
1886                 if (src_m == NULL)
1887                         panic("vm_fault_copy_wired: page missing");
1888
1889                 vm_page_copy(src_m, dst_m);
1890                 vm_page_event(src_m, VMEVENT_COW);
1891
1892                 /*
1893                  * Enter it in the pmap...
1894                  */
1895
1896                 vm_page_flag_clear(dst_m, PG_ZERO);
1897                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
1898
1899                 /*
1900                  * Mark it no longer busy, and put it on the active list.
1901                  */
1902                 vm_page_activate(dst_m);
1903                 vm_page_wakeup(dst_m);
1904         }
1905 }
1906
1907 #if 0
1908
1909 /*
1910  * This routine checks around the requested page for other pages that
1911  * might be able to be faulted in.  This routine brackets the viable
1912  * pages for the pages to be paged in.
1913  *
1914  * Inputs:
1915  *      m, rbehind, rahead
1916  *
1917  * Outputs:
1918  *  marray (array of vm_page_t), reqpage (index of requested page)
1919  *
1920  * Return value:
1921  *  number of pages in marray
1922  */
1923 static int
1924 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
1925                           vm_page_t *marray, int *reqpage)
1926 {
1927         int i,j;
1928         vm_object_t object;
1929         vm_pindex_t pindex, startpindex, endpindex, tpindex;
1930         vm_page_t rtm;
1931         int cbehind, cahead;
1932
1933         object = m->object;
1934         pindex = m->pindex;
1935
1936         /*
1937          * we don't fault-ahead for device pager
1938          */
1939         if (object->type == OBJT_DEVICE) {
1940                 *reqpage = 0;
1941                 marray[0] = m;
1942                 return 1;
1943         }
1944
1945         /*
1946          * if the requested page is not available, then give up now
1947          */
1948         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
1949                 *reqpage = 0;   /* not used by caller, fix compiler warn */
1950                 return 0;
1951         }
1952
1953         if ((cbehind == 0) && (cahead == 0)) {
1954                 *reqpage = 0;
1955                 marray[0] = m;
1956                 return 1;
1957         }
1958
1959         if (rahead > cahead) {
1960                 rahead = cahead;
1961         }
1962
1963         if (rbehind > cbehind) {
1964                 rbehind = cbehind;
1965         }
1966
1967         /*
1968          * Do not do any readahead if we have insufficient free memory.
1969          *
1970          * XXX code was broken disabled before and has instability
1971          * with this conditonal fixed, so shortcut for now.
1972          */
1973         if (burst_fault == 0 || vm_page_count_severe()) {
1974                 marray[0] = m;
1975                 *reqpage = 0;
1976                 return 1;
1977         }
1978
1979         /*
1980          * scan backward for the read behind pages -- in memory 
1981          *
1982          * Assume that if the page is not found an interrupt will not
1983          * create it.  Theoretically interrupts can only remove (busy)
1984          * pages, not create new associations.
1985          */
1986         if (pindex > 0) {
1987                 if (rbehind > pindex) {
1988                         rbehind = pindex;
1989                         startpindex = 0;
1990                 } else {
1991                         startpindex = pindex - rbehind;
1992                 }
1993
1994                 vm_object_hold(object);
1995                 for (tpindex = pindex; tpindex > startpindex; --tpindex) {
1996                         if (vm_page_lookup(object, tpindex - 1))
1997                                 break;
1998                 }
1999
2000                 i = 0;
2001                 while (tpindex < pindex) {
2002                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM);
2003                         if (rtm == NULL) {
2004                                 for (j = 0; j < i; j++) {
2005                                         vm_page_free(marray[j]);
2006                                 }
2007                                 vm_object_drop(object);
2008                                 marray[0] = m;
2009                                 *reqpage = 0;
2010                                 return 1;
2011                         }
2012                         marray[i] = rtm;
2013                         ++i;
2014                         ++tpindex;
2015                 }
2016                 vm_object_drop(object);
2017         } else {
2018                 i = 0;
2019         }
2020
2021         /*
2022          * Assign requested page
2023          */
2024         marray[i] = m;
2025         *reqpage = i;
2026         ++i;
2027
2028         /*
2029          * Scan forwards for read-ahead pages
2030          */
2031         tpindex = pindex + 1;
2032         endpindex = tpindex + rahead;
2033         if (endpindex > object->size)
2034                 endpindex = object->size;
2035
2036         vm_object_hold(object);
2037         while (tpindex < endpindex) {
2038                 if (vm_page_lookup(object, tpindex))
2039                         break;
2040                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_SYSTEM);
2041                 if (rtm == NULL)
2042                         break;
2043                 marray[i] = rtm;
2044                 ++i;
2045                 ++tpindex;
2046         }
2047         vm_object_drop(object);
2048
2049         return (i);
2050 }
2051
2052 #endif
2053
2054 /*
2055  * vm_prefault() provides a quick way of clustering pagefaults into a
2056  * processes address space.  It is a "cousin" of pmap_object_init_pt,
2057  * except it runs at page fault time instead of mmap time.
2058  *
2059  * vm.fast_fault        Enables pre-faulting zero-fill pages
2060  *
2061  * vm.prefault_pages    Number of pages (1/2 negative, 1/2 positive) to
2062  *                      prefault.  Scan stops in either direction when
2063  *                      a page is found to already exist.
2064  *
2065  * This code used to be per-platform pmap_prefault().  It is now
2066  * machine-independent and enhanced to also pre-fault zero-fill pages
2067  * (see vm.fast_fault) as well as make them writable, which greatly
2068  * reduces the number of page faults programs incur.
2069  *
2070  * Application performance when pre-faulting zero-fill pages is heavily
2071  * dependent on the application.  Very tiny applications like /bin/echo
2072  * lose a little performance while applications of any appreciable size
2073  * gain performance.  Prefaulting multiple pages also reduces SMP
2074  * congestion and can improve SMP performance significantly.
2075  *
2076  * NOTE!  prot may allow writing but this only applies to the top level
2077  *        object.  If we wind up mapping a page extracted from a backing
2078  *        object we have to make sure it is read-only.
2079  *
2080  * NOTE!  The caller has already handled any COW operations on the
2081  *        vm_map_entry via the normal fault code.  Do NOT call this
2082  *        shortcut unless the normal fault code has run on this entry.
2083  *
2084  * No other requirements.
2085  */
2086 static int vm_prefault_pages = 8;
2087 SYSCTL_INT(_vm, OID_AUTO, prefault_pages, CTLFLAG_RW, &vm_prefault_pages, 0,
2088            "Maximum number of pages to pre-fault");
2089 static int vm_fast_fault = 1;
2090 SYSCTL_INT(_vm, OID_AUTO, fast_fault, CTLFLAG_RW, &vm_fast_fault, 0,
2091            "Burst fault zero-fill regions");
2092
2093 /*
2094  * Set PG_NOSYNC if the map entry indicates so, but only if the page
2095  * is not already dirty by other means.  This will prevent passive
2096  * filesystem syncing as well as 'sync' from writing out the page.
2097  */
2098 static void
2099 vm_set_nosync(vm_page_t m, vm_map_entry_t entry)
2100 {
2101         if (entry->eflags & MAP_ENTRY_NOSYNC) {
2102                 if (m->dirty == 0)
2103                         vm_page_flag_set(m, PG_NOSYNC);
2104         } else {
2105                 vm_page_flag_clear(m, PG_NOSYNC);
2106         }
2107 }
2108
2109 static void
2110 vm_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, int prot)
2111 {
2112         struct lwp *lp;
2113         vm_page_t m;
2114         vm_offset_t addr;
2115         vm_pindex_t index;
2116         vm_pindex_t pindex;
2117         vm_object_t object;
2118         int pprot;
2119         int i;
2120         int noneg;
2121         int nopos;
2122         int maxpages;
2123
2124         /*
2125          * Get stable max count value, disabled if set to 0
2126          */
2127         maxpages = vm_prefault_pages;
2128         cpu_ccfence();
2129         if (maxpages <= 0)
2130                 return;
2131
2132         /*
2133          * We do not currently prefault mappings that use virtual page
2134          * tables.  We do not prefault foreign pmaps.
2135          */
2136         if (entry->maptype == VM_MAPTYPE_VPAGETABLE)
2137                 return;
2138         lp = curthread->td_lwp;
2139         if (lp == NULL || (pmap != vmspace_pmap(lp->lwp_vmspace)))
2140                 return;
2141
2142         /*
2143          * Limit pre-fault count to 1024 pages.
2144          */
2145         if (maxpages > 1024)
2146                 maxpages = 1024;
2147
2148         object = entry->object.vm_object;
2149         KKASSERT(object != NULL);
2150         vm_object_hold(object);
2151         KKASSERT(object == entry->object.vm_object);
2152         vm_object_chain_acquire(object);
2153
2154         noneg = 0;
2155         nopos = 0;
2156         for (i = 0; i < maxpages; ++i) {
2157                 vm_object_t lobject;
2158                 vm_object_t nobject;
2159                 int allocated = 0;
2160                 int error;
2161
2162                 /*
2163                  * Calculate the page to pre-fault, stopping the scan in
2164                  * each direction separately if the limit is reached.
2165                  */
2166                 if (i & 1) {
2167                         if (noneg)
2168                                 continue;
2169                         addr = addra - ((i + 1) >> 1) * PAGE_SIZE;
2170                 } else {
2171                         if (nopos)
2172                                 continue;
2173                         addr = addra + ((i + 2) >> 1) * PAGE_SIZE;
2174                 }
2175                 if (addr < entry->start) {
2176                         noneg = 1;
2177                         if (noneg && nopos)
2178                                 break;
2179                         continue;
2180                 }
2181                 if (addr >= entry->end) {
2182                         nopos = 1;
2183                         if (noneg && nopos)
2184                                 break;
2185                         continue;
2186                 }
2187
2188                 /*
2189                  * Skip pages already mapped, and stop scanning in that
2190                  * direction.  When the scan terminates in both directions
2191                  * we are done.
2192                  */
2193                 if (pmap_prefault_ok(pmap, addr) == 0) {
2194                         if (i & 1)
2195                                 noneg = 1;
2196                         else
2197                                 nopos = 1;
2198                         if (noneg && nopos)
2199                                 break;
2200                         continue;
2201                 }
2202
2203                 /*
2204                  * Follow the VM object chain to obtain the page to be mapped
2205                  * into the pmap.
2206                  *
2207                  * If we reach the terminal object without finding a page
2208                  * and we determine it would be advantageous, then allocate
2209                  * a zero-fill page for the base object.  The base object
2210                  * is guaranteed to be OBJT_DEFAULT for this case.
2211                  *
2212                  * In order to not have to check the pager via *haspage*()
2213                  * we stop if any non-default object is encountered.  e.g.
2214                  * a vnode or swap object would stop the loop.
2215                  */
2216                 index = ((addr - entry->start) + entry->offset) >> PAGE_SHIFT;
2217                 lobject = object;
2218                 pindex = index;
2219                 pprot = prot;
2220
2221                 KKASSERT(lobject == entry->object.vm_object);
2222                 /*vm_object_hold(lobject); implied */
2223
2224                 while ((m = vm_page_lookup_busy_try(lobject, pindex,
2225                                                     TRUE, &error)) == NULL) {
2226                         if (lobject->type != OBJT_DEFAULT)
2227                                 break;
2228                         if (lobject->backing_object == NULL) {
2229                                 if (vm_fast_fault == 0)
2230                                         break;
2231                                 if ((prot & VM_PROT_WRITE) == 0 ||
2232                                     vm_page_count_min(0)) {
2233                                         break;
2234                                 }
2235
2236                                 /*
2237                                  * NOTE: Allocated from base object
2238                                  */
2239                                 m = vm_page_alloc(object, index,
2240                                                   VM_ALLOC_NORMAL | VM_ALLOC_ZERO);
2241
2242                                 if ((m->flags & PG_ZERO) == 0) {
2243                                         vm_page_zero_fill(m);
2244                                 } else {
2245 #ifdef PMAP_DEBUG
2246                                         pmap_page_assertzero(
2247                                                         VM_PAGE_TO_PHYS(m));
2248 #endif
2249                                         vm_page_flag_clear(m, PG_ZERO);
2250                                         mycpu->gd_cnt.v_ozfod++;
2251                                 }
2252                                 mycpu->gd_cnt.v_zfod++;
2253                                 m->valid = VM_PAGE_BITS_ALL;
2254                                 allocated = 1;
2255                                 pprot = prot;
2256                                 /* lobject = object .. not needed */
2257                                 break;
2258                         }
2259                         if (lobject->backing_object_offset & PAGE_MASK)
2260                                 break;
2261                         nobject = lobject->backing_object;
2262                         vm_object_hold(nobject);
2263                         KKASSERT(nobject == lobject->backing_object);
2264                         pindex += lobject->backing_object_offset >> PAGE_SHIFT;
2265                         if (lobject != object) {
2266                                 vm_object_lock_swap();
2267                                 vm_object_drop(lobject);
2268                         }
2269                         lobject = nobject;
2270                         pprot &= ~VM_PROT_WRITE;
2271                         vm_object_chain_acquire(lobject);
2272                 }
2273
2274                 /*
2275                  * NOTE: A non-NULL (m) will be associated with lobject if
2276                  *       it was found there, otherwise it is probably a
2277                  *       zero-fill page associated with the base object.
2278                  *
2279                  * Give-up if no page is available.
2280                  */
2281                 if (m == NULL) {
2282                         if (lobject != object) {
2283                                 if (object->backing_object != lobject)
2284                                         vm_object_hold(object->backing_object);
2285                                 vm_object_chain_release_all(
2286                                         object->backing_object, lobject);
2287                                 if (object->backing_object != lobject)
2288                                         vm_object_drop(object->backing_object);
2289                                 vm_object_drop(lobject);
2290                         }
2291                         break;
2292                 }
2293
2294                 /*
2295                  * Do not conditionalize on PG_RAM.  If pages are present in
2296                  * the VM system we assume optimal caching.  If caching is
2297                  * not optimal the I/O gravy train will be restarted when we
2298                  * hit an unavailable page.  We do not want to try to restart
2299                  * the gravy train now because we really don't know how much
2300                  * of the object has been cached.  The cost for restarting
2301                  * the gravy train should be low (since accesses will likely
2302                  * be I/O bound anyway).
2303                  *
2304                  * The object must be marked dirty if we are mapping a
2305                  * writable page.  m->object is either lobject or object,
2306                  * both of which are still held.
2307                  */
2308                 if (pprot & VM_PROT_WRITE)
2309                         vm_object_set_writeable_dirty(m->object);
2310
2311                 if (lobject != object) {
2312                         if (object->backing_object != lobject)
2313                                 vm_object_hold(object->backing_object);
2314                         vm_object_chain_release_all(object->backing_object,
2315                                                     lobject);
2316                         if (object->backing_object != lobject)
2317                                 vm_object_drop(object->backing_object);
2318                         vm_object_drop(lobject);
2319                 }
2320
2321                 /*
2322                  * Enter the page into the pmap if appropriate.  If we had
2323                  * allocated the page we have to place it on a queue.  If not
2324                  * we just have to make sure it isn't on the cache queue
2325                  * (pages on the cache queue are not allowed to be mapped).
2326                  */
2327                 if (allocated) {
2328                         if (pprot & VM_PROT_WRITE)
2329                                 vm_set_nosync(m, entry);
2330                         pmap_enter(pmap, addr, m, pprot, 0);
2331                         vm_page_deactivate(m);
2332                         vm_page_wakeup(m);
2333                 } else if (error) {
2334                         /* couldn't busy page, no wakeup */
2335                 } else if (
2336                     ((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
2337                     (m->flags & PG_FICTITIOUS) == 0) {
2338                         /*
2339                          * A fully valid page not undergoing soft I/O can
2340                          * be immediately entered into the pmap.
2341                          */
2342                         if ((m->queue - m->pc) == PQ_CACHE)
2343                                 vm_page_deactivate(m);
2344                         if (pprot & VM_PROT_WRITE)
2345                                 vm_set_nosync(m, entry);
2346                         pmap_enter(pmap, addr, m, pprot, 0);
2347                         vm_page_wakeup(m);
2348                 } else {
2349                         vm_page_wakeup(m);
2350                 }
2351         }
2352         vm_object_chain_release(object);
2353         vm_object_drop(object);
2354 }