Fix a null pointer indirection, the VM fault rate limiting code only
[dragonfly.git] / sys / vm / vm_fault.c
1 /*
2  * Copyright (c) 1991, 1993
3  *      The Regents of the University of California.  All rights reserved.
4  * Copyright (c) 1994 John S. Dyson
5  * All rights reserved.
6  * Copyright (c) 1994 David Greenman
7  * All rights reserved.
8  *
9  *
10  * This code is derived from software contributed to Berkeley by
11  * The Mach Operating System project at Carnegie-Mellon University.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. All advertising materials mentioning features or use of this software
22  *    must display the following acknowledgement:
23  *      This product includes software developed by the University of
24  *      California, Berkeley and its contributors.
25  * 4. Neither the name of the University nor the names of its contributors
26  *    may be used to endorse or promote products derived from this software
27  *    without specific prior written permission.
28  *
29  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39  * SUCH DAMAGE.
40  *
41  *      from: @(#)vm_fault.c    8.4 (Berkeley) 1/12/94
42  *
43  *
44  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
45  * All rights reserved.
46  *
47  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
48  *
49  * Permission to use, copy, modify and distribute this software and
50  * its documentation is hereby granted, provided that both the copyright
51  * notice and this permission notice appear in all copies of the
52  * software, derivative works or modified versions, and any portions
53  * thereof, and that both notices appear in supporting documentation.
54  *
55  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
56  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
57  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
58  *
59  * Carnegie Mellon requests users of this software to return to
60  *
61  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
62  *  School of Computer Science
63  *  Carnegie Mellon University
64  *  Pittsburgh PA 15213-3890
65  *
66  * any improvements or extensions that they make and grant Carnegie the
67  * rights to redistribute these changes.
68  *
69  * $FreeBSD: src/sys/vm/vm_fault.c,v 1.108.2.8 2002/02/26 05:49:27 silby Exp $
70  * $DragonFly: src/sys/vm/vm_fault.c,v 1.24 2006/05/06 23:53:34 dillon Exp $
71  */
72
73 /*
74  *      Page fault handling module.
75  */
76
77 #include <sys/param.h>
78 #include <sys/systm.h>
79 #include <sys/kernel.h>
80 #include <sys/proc.h>
81 #include <sys/vnode.h>
82 #include <sys/resourcevar.h>
83 #include <sys/vmmeter.h>
84
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <sys/lock.h>
88 #include <vm/pmap.h>
89 #include <vm/vm_map.h>
90 #include <vm/vm_object.h>
91 #include <vm/vm_page.h>
92 #include <vm/vm_pageout.h>
93 #include <vm/vm_kern.h>
94 #include <vm/vm_pager.h>
95 #include <vm/vnode_pager.h>
96 #include <vm/vm_extern.h>
97
98 #include <sys/thread2.h>
99 #include <vm/vm_page2.h>
100
101 static int vm_fault_additional_pages (vm_page_t, int,
102                                           int, vm_page_t *, int *);
103 static int vm_fault_ratelimit(struct vmspace *vmspace);
104
105 #define VM_FAULT_READ_AHEAD 8
106 #define VM_FAULT_READ_BEHIND 7
107 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1)
108
109 struct faultstate {
110         vm_page_t m;
111         vm_object_t object;
112         vm_pindex_t pindex;
113         vm_page_t first_m;
114         vm_object_t     first_object;
115         vm_pindex_t first_pindex;
116         vm_map_t map;
117         vm_map_entry_t entry;
118         int lookup_still_valid;
119         struct vnode *vp;
120 };
121
122 static __inline void
123 release_page(struct faultstate *fs)
124 {
125         vm_page_wakeup(fs->m);
126         vm_page_deactivate(fs->m);
127         fs->m = NULL;
128 }
129
130 static __inline void
131 unlock_map(struct faultstate *fs)
132 {
133         if (fs->lookup_still_valid) {
134                 vm_map_lookup_done(fs->map, fs->entry, 0);
135                 fs->lookup_still_valid = FALSE;
136         }
137 }
138
139 static void
140 _unlock_things(struct faultstate *fs, int dealloc)
141 {
142         vm_object_pip_wakeup(fs->object);
143         if (fs->object != fs->first_object) {
144                 vm_page_free(fs->first_m);
145                 vm_object_pip_wakeup(fs->first_object);
146                 fs->first_m = NULL;
147         }
148         if (dealloc) {
149                 vm_object_deallocate(fs->first_object);
150         }
151         unlock_map(fs); 
152         if (fs->vp != NULL) { 
153                 vput(fs->vp);
154                 fs->vp = NULL;
155         }
156 }
157
158 #define unlock_things(fs) _unlock_things(fs, 0)
159 #define unlock_and_deallocate(fs) _unlock_things(fs, 1)
160
161 /*
162  * TRYPAGER - used by vm_fault to calculate whether the pager for the
163  *            current object *might* contain the page.
164  *
165  *            default objects are zero-fill, there is no real pager.
166  */
167
168 #define TRYPAGER        (fs.object->type != OBJT_DEFAULT && \
169                         (((fault_flags & VM_FAULT_WIRE_MASK) == 0) || wired))
170
171 /*
172  *      vm_fault:
173  *
174  *      Handle a page fault occurring at the given address,
175  *      requiring the given permissions, in the map specified.
176  *      If successful, the page is inserted into the
177  *      associated physical map.
178  *
179  *      NOTE: the given address should be truncated to the
180  *      proper page address.
181  *
182  *      KERN_SUCCESS is returned if the page fault is handled; otherwise,
183  *      a standard error specifying why the fault is fatal is returned.
184  *
185  *
186  *      The map in question must be referenced, and remains so.
187  *      Caller may hold no locks.
188  */
189 int
190 vm_fault(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags)
191 {
192         vm_prot_t prot;
193         int result;
194         boolean_t wired;
195         int map_generation;
196         vm_object_t next_object;
197         vm_page_t marray[VM_FAULT_READ];
198         int hardfault;
199         int faultcount;
200         int limticks;
201         int didlimit = 0;
202         struct faultstate fs;
203
204         mycpu->gd_cnt.v_vm_faults++;
205         hardfault = 0;
206
207 RetryFault:
208         /*
209          * Find the backing store object and offset into it to begin the
210          * search.
211          */
212         fs.map = map;
213         if ((result = vm_map_lookup(&fs.map, vaddr,
214                 fault_type, &fs.entry, &fs.first_object,
215                 &fs.first_pindex, &prot, &wired)) != KERN_SUCCESS) {
216                 if ((result != KERN_PROTECTION_FAILURE) ||
217                         ((fault_flags & VM_FAULT_WIRE_MASK) != VM_FAULT_USER_WIRE)) {
218                         return result;
219                 }
220
221                 /*
222                  * If we are user-wiring a r/w segment, and it is COW, then
223                  * we need to do the COW operation.  Note that we don't COW
224                  * currently RO sections now, because it is NOT desirable
225                  * to COW .text.  We simply keep .text from ever being COW'ed
226                  * and take the heat that one cannot debug wired .text sections.
227                  */
228                 result = vm_map_lookup(&fs.map, vaddr,
229                         VM_PROT_READ|VM_PROT_WRITE|VM_PROT_OVERRIDE_WRITE,
230                         &fs.entry, &fs.first_object, &fs.first_pindex, &prot, &wired);
231                 if (result != KERN_SUCCESS) {
232                         return result;
233                 }
234
235                 /*
236                  * If we don't COW now, on a user wire, the user will never
237                  * be able to write to the mapping.  If we don't make this
238                  * restriction, the bookkeeping would be nearly impossible.
239                  */
240                 if ((fs.entry->protection & VM_PROT_WRITE) == 0)
241                         fs.entry->max_protection &= ~VM_PROT_WRITE;
242         }
243
244         map_generation = fs.map->timestamp;
245
246         if (fs.entry->eflags & MAP_ENTRY_NOFAULT) {
247                 panic("vm_fault: fault on nofault entry, addr: %lx",
248                     (u_long)vaddr);
249         }
250
251         /*
252          * A system map entry may return a NULL object.  No object means
253          * no pager means an unrecoverable kernel fault.
254          */
255         if (fs.first_object == NULL) {
256                 panic("vm_fault: unrecoverable fault at %p in entry %p",
257                         (void *)vaddr, fs.entry);
258         }
259
260         /*
261          * Make a reference to this object to prevent its disposal while we
262          * are messing with it.  Once we have the reference, the map is free
263          * to be diddled.  Since objects reference their shadows (and copies),
264          * they will stay around as well.
265          *
266          * Bump the paging-in-progress count to prevent size changes (e.g.
267          * truncation operations) during I/O.  This must be done after
268          * obtaining the vnode lock in order to avoid possible deadlocks.
269          */
270         vm_object_reference(fs.first_object);
271         fs.vp = vnode_pager_lock(fs.first_object);
272         vm_object_pip_add(fs.first_object, 1);
273
274         if ((fault_type & VM_PROT_WRITE) &&
275                 (fs.first_object->type == OBJT_VNODE)) {
276                 vm_freeze_copyopts(fs.first_object,
277                         fs.first_pindex, fs.first_pindex + 1);
278         }
279
280         fs.lookup_still_valid = TRUE;
281
282         if (wired)
283                 fault_type = prot;
284
285         fs.first_m = NULL;
286
287         /*
288          * Search for the page at object/offset.
289          */
290
291         fs.object = fs.first_object;
292         fs.pindex = fs.first_pindex;
293
294         while (TRUE) {
295                 /*
296                  * If the object is dead, we stop here
297                  */
298
299                 if (fs.object->flags & OBJ_DEAD) {
300                         unlock_and_deallocate(&fs);
301                         return (KERN_PROTECTION_FAILURE);
302                 }
303
304                 /*
305                  * See if page is resident.  spl protection is required
306                  * to avoid an interrupt unbusy/free race against our
307                  * lookup.  We must hold the protection through a page
308                  * allocation or busy.
309                  */
310                 crit_enter();
311                 fs.m = vm_page_lookup(fs.object, fs.pindex);
312                 if (fs.m != NULL) {
313                         int queue;
314                         /*
315                          * Wait/Retry if the page is busy.  We have to do this
316                          * if the page is busy via either PG_BUSY or 
317                          * vm_page_t->busy because the vm_pager may be using
318                          * vm_page_t->busy for pageouts ( and even pageins if
319                          * it is the vnode pager ), and we could end up trying
320                          * to pagein and pageout the same page simultaneously.
321                          *
322                          * We can theoretically allow the busy case on a read
323                          * fault if the page is marked valid, but since such
324                          * pages are typically already pmap'd, putting that
325                          * special case in might be more effort then it is 
326                          * worth.  We cannot under any circumstances mess
327                          * around with a vm_page_t->busy page except, perhaps,
328                          * to pmap it.
329                          */
330                         if ((fs.m->flags & PG_BUSY) || fs.m->busy) {
331                                 unlock_things(&fs);
332                                 vm_page_sleep_busy(fs.m, TRUE, "vmpfw");
333                                 mycpu->gd_cnt.v_intrans++;
334                                 vm_object_deallocate(fs.first_object);
335                                 crit_exit();
336                                 goto RetryFault;
337                         }
338
339                         queue = fs.m->queue;
340                         vm_page_unqueue_nowakeup(fs.m);
341
342                         if ((queue - fs.m->pc) == PQ_CACHE && vm_page_count_severe()) {
343                                 vm_page_activate(fs.m);
344                                 unlock_and_deallocate(&fs);
345                                 vm_waitpfault();
346                                 crit_exit();
347                                 goto RetryFault;
348                         }
349
350                         /*
351                          * Mark page busy for other processes, and the 
352                          * pagedaemon.  If it still isn't completely valid
353                          * (readable), jump to readrest, else break-out ( we
354                          * found the page ).
355                          *
356                          * We can release the spl once we have marked the
357                          * page busy.
358                          */
359
360                         vm_page_busy(fs.m);
361                         crit_exit();
362
363                         if (((fs.m->valid & VM_PAGE_BITS_ALL) != VM_PAGE_BITS_ALL) &&
364                                 fs.m->object != kernel_object && fs.m->object != kmem_object) {
365                                 goto readrest;
366                         }
367
368                         break;
369                 }
370
371                 /*
372                  * Page is not resident, If this is the search termination
373                  * or the pager might contain the page, allocate a new page.
374                  *
375                  * note: we are still in splvm().
376                  */
377
378                 if (TRYPAGER || fs.object == fs.first_object) {
379                         if (fs.pindex >= fs.object->size) {
380                                 crit_exit();
381                                 unlock_and_deallocate(&fs);
382                                 return (KERN_PROTECTION_FAILURE);
383                         }
384
385                         /*
386                          * Ratelimit.
387                          */
388                         if (didlimit == 0 && curproc != NULL) {
389                                 limticks = 
390                                         vm_fault_ratelimit(curproc->p_vmspace);
391                                 if (limticks) {
392                                         crit_exit();
393                                         unlock_and_deallocate(&fs);
394                                         tsleep(curproc, 0, "vmrate", limticks);
395                                         didlimit = 1;
396                                         goto RetryFault;
397                                 }
398                         }
399
400                         /*
401                          * Allocate a new page for this object/offset pair.
402                          */
403                         fs.m = NULL;
404                         if (!vm_page_count_severe()) {
405                                 fs.m = vm_page_alloc(fs.object, fs.pindex,
406                                     (fs.vp || fs.object->backing_object)? VM_ALLOC_NORMAL: VM_ALLOC_NORMAL | VM_ALLOC_ZERO);
407                         }
408                         if (fs.m == NULL) {
409                                 crit_exit();
410                                 unlock_and_deallocate(&fs);
411                                 vm_waitpfault();
412                                 goto RetryFault;
413                         }
414                 }
415                 crit_exit();
416
417 readrest:
418                 /*
419                  * We have found a valid page or we have allocated a new page.
420                  * The page thus may not be valid or may not be entirely 
421                  * valid.
422                  *
423                  * Attempt to fault-in the page if there is a chance that the
424                  * pager has it, and potentially fault in additional pages
425                  * at the same time.
426                  *
427                  * We are NOT in splvm here and if TRYPAGER is true then
428                  * fs.m will be non-NULL and will be PG_BUSY for us.
429                  */
430
431                 if (TRYPAGER) {
432                         int rv;
433                         int reqpage;
434                         int ahead, behind;
435                         u_char behavior = vm_map_entry_behavior(fs.entry);
436
437                         if (behavior == MAP_ENTRY_BEHAV_RANDOM) {
438                                 ahead = 0;
439                                 behind = 0;
440                         } else {
441                                 behind = (vaddr - fs.entry->start) >> PAGE_SHIFT;
442                                 if (behind > VM_FAULT_READ_BEHIND)
443                                         behind = VM_FAULT_READ_BEHIND;
444
445                                 ahead = ((fs.entry->end - vaddr) >> PAGE_SHIFT) - 1;
446                                 if (ahead > VM_FAULT_READ_AHEAD)
447                                         ahead = VM_FAULT_READ_AHEAD;
448                         }
449
450                         if ((fs.first_object->type != OBJT_DEVICE) &&
451                             (behavior == MAP_ENTRY_BEHAV_SEQUENTIAL ||
452                                 (behavior != MAP_ENTRY_BEHAV_RANDOM &&
453                                 fs.pindex >= fs.entry->lastr &&
454                                 fs.pindex < fs.entry->lastr + VM_FAULT_READ))
455                         ) {
456                                 vm_pindex_t firstpindex, tmppindex;
457
458                                 if (fs.first_pindex < 2 * VM_FAULT_READ)
459                                         firstpindex = 0;
460                                 else
461                                         firstpindex = fs.first_pindex - 2 * VM_FAULT_READ;
462
463                                 /*
464                                  * note: partially valid pages cannot be 
465                                  * included in the lookahead - NFS piecemeal
466                                  * writes will barf on it badly.
467                                  *
468                                  * spl protection is required to avoid races
469                                  * between the lookup and an interrupt
470                                  * unbusy/free sequence occuring prior to
471                                  * our busy check.
472                                  */
473                                 crit_enter();
474                                 for (tmppindex = fs.first_pindex - 1;
475                                     tmppindex >= firstpindex;
476                                     --tmppindex
477                                 ) {
478                                         vm_page_t mt;
479                                         mt = vm_page_lookup( fs.first_object, tmppindex);
480                                         if (mt == NULL || (mt->valid != VM_PAGE_BITS_ALL))
481                                                 break;
482                                         if (mt->busy ||
483                                                 (mt->flags & (PG_BUSY | PG_FICTITIOUS | PG_UNMANAGED)) ||
484                                                 mt->hold_count ||
485                                                 mt->wire_count) 
486                                                 continue;
487                                         if (mt->dirty == 0)
488                                                 vm_page_test_dirty(mt);
489                                         if (mt->dirty) {
490                                                 vm_page_protect(mt, VM_PROT_NONE);
491                                                 vm_page_deactivate(mt);
492                                         } else {
493                                                 vm_page_cache(mt);
494                                         }
495                                 }
496                                 crit_exit();
497
498                                 ahead += behind;
499                                 behind = 0;
500                         }
501
502                         /*
503                          * now we find out if any other pages should be paged
504                          * in at this time this routine checks to see if the
505                          * pages surrounding this fault reside in the same
506                          * object as the page for this fault.  If they do,
507                          * then they are faulted in also into the object.  The
508                          * array "marray" returned contains an array of
509                          * vm_page_t structs where one of them is the
510                          * vm_page_t passed to the routine.  The reqpage
511                          * return value is the index into the marray for the
512                          * vm_page_t passed to the routine.
513                          *
514                          * fs.m plus the additional pages are PG_BUSY'd.
515                          */
516                         faultcount = vm_fault_additional_pages(
517                             fs.m, behind, ahead, marray, &reqpage);
518
519                         /*
520                          * update lastr imperfectly (we do not know how much
521                          * getpages will actually read), but good enough.
522                          */
523                         fs.entry->lastr = fs.pindex + faultcount - behind;
524
525                         /*
526                          * Call the pager to retrieve the data, if any, after
527                          * releasing the lock on the map.  We hold a ref on
528                          * fs.object and the pages are PG_BUSY'd.
529                          */
530                         unlock_map(&fs);
531
532                         rv = faultcount ?
533                             vm_pager_get_pages(fs.object, marray, faultcount,
534                                 reqpage) : VM_PAGER_FAIL;
535
536                         if (rv == VM_PAGER_OK) {
537                                 /*
538                                  * Found the page. Leave it busy while we play
539                                  * with it.
540                                  */
541
542                                 /*
543                                  * Relookup in case pager changed page. Pager
544                                  * is responsible for disposition of old page
545                                  * if moved.
546                                  *
547                                  * XXX other code segments do relookups too.
548                                  * It's a bad abstraction that needs to be
549                                  * fixed/removed.
550                                  */
551                                 fs.m = vm_page_lookup(fs.object, fs.pindex);
552                                 if (fs.m == NULL) {
553                                         unlock_and_deallocate(&fs);
554                                         goto RetryFault;
555                                 }
556
557                                 hardfault++;
558                                 break; /* break to PAGE HAS BEEN FOUND */
559                         }
560                         /*
561                          * Remove the bogus page (which does not exist at this
562                          * object/offset); before doing so, we must get back
563                          * our object lock to preserve our invariant.
564                          *
565                          * Also wake up any other process that may want to bring
566                          * in this page.
567                          *
568                          * If this is the top-level object, we must leave the
569                          * busy page to prevent another process from rushing
570                          * past us, and inserting the page in that object at
571                          * the same time that we are.
572                          */
573
574                         if (rv == VM_PAGER_ERROR) {
575                                 if (curproc)
576                                         printf("vm_fault: pager read error, pid %d (%s)\n", curproc->p_pid, curproc->p_comm);
577                                 else
578                                         printf("vm_fault: pager read error, thread %p (%s)\n", curthread, curproc->p_comm);
579                         }
580                         /*
581                          * Data outside the range of the pager or an I/O error
582                          */
583                         /*
584                          * XXX - the check for kernel_map is a kludge to work
585                          * around having the machine panic on a kernel space
586                          * fault w/ I/O error.
587                          */
588                         if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) ||
589                                 (rv == VM_PAGER_BAD)) {
590                                 vm_page_free(fs.m);
591                                 fs.m = NULL;
592                                 unlock_and_deallocate(&fs);
593                                 return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE);
594                         }
595                         if (fs.object != fs.first_object) {
596                                 vm_page_free(fs.m);
597                                 fs.m = NULL;
598                                 /*
599                                  * XXX - we cannot just fall out at this
600                                  * point, m has been freed and is invalid!
601                                  */
602                         }
603                 }
604
605                 /*
606                  * We get here if the object has default pager (or unwiring) 
607                  * or the pager doesn't have the page.
608                  */
609                 if (fs.object == fs.first_object)
610                         fs.first_m = fs.m;
611
612                 /*
613                  * Move on to the next object.  Lock the next object before
614                  * unlocking the current one.
615                  */
616
617                 fs.pindex += OFF_TO_IDX(fs.object->backing_object_offset);
618                 next_object = fs.object->backing_object;
619                 if (next_object == NULL) {
620                         /*
621                          * If there's no object left, fill the page in the top
622                          * object with zeros.
623                          */
624                         if (fs.object != fs.first_object) {
625                                 vm_object_pip_wakeup(fs.object);
626
627                                 fs.object = fs.first_object;
628                                 fs.pindex = fs.first_pindex;
629                                 fs.m = fs.first_m;
630                         }
631                         fs.first_m = NULL;
632
633                         /*
634                          * Zero the page if necessary and mark it valid.
635                          */
636                         if ((fs.m->flags & PG_ZERO) == 0) {
637                                 vm_page_zero_fill(fs.m);
638                         } else {
639                                 mycpu->gd_cnt.v_ozfod++;
640                         }
641                         mycpu->gd_cnt.v_zfod++;
642                         fs.m->valid = VM_PAGE_BITS_ALL;
643                         break;  /* break to PAGE HAS BEEN FOUND */
644                 } else {
645                         if (fs.object != fs.first_object) {
646                                 vm_object_pip_wakeup(fs.object);
647                         }
648                         KASSERT(fs.object != next_object, ("object loop %p", next_object));
649                         fs.object = next_object;
650                         vm_object_pip_add(fs.object, 1);
651                 }
652         }
653
654         KASSERT((fs.m->flags & PG_BUSY) != 0,
655             ("vm_fault: not busy after main loop"));
656
657         /*
658          * PAGE HAS BEEN FOUND. [Loop invariant still holds -- the object lock
659          * is held.]
660          */
661
662         /*
663          * If the page is being written, but isn't already owned by the
664          * top-level object, we have to copy it into a new page owned by the
665          * top-level object.
666          */
667
668         if (fs.object != fs.first_object) {
669                 /*
670                  * We only really need to copy if we want to write it.
671                  */
672
673                 if (fault_type & VM_PROT_WRITE) {
674                         /*
675                          * This allows pages to be virtually copied from a 
676                          * backing_object into the first_object, where the 
677                          * backing object has no other refs to it, and cannot
678                          * gain any more refs.  Instead of a bcopy, we just 
679                          * move the page from the backing object to the 
680                          * first object.  Note that we must mark the page 
681                          * dirty in the first object so that it will go out 
682                          * to swap when needed.
683                          */
684                         if (map_generation == fs.map->timestamp &&
685                                 /*
686                                  * Only one shadow object
687                                  */
688                                 (fs.object->shadow_count == 1) &&
689                                 /*
690                                  * No COW refs, except us
691                                  */
692                                 (fs.object->ref_count == 1) &&
693                                 /*
694                                  * No one else can look this object up
695                                  */
696                                 (fs.object->handle == NULL) &&
697                                 /*
698                                  * No other ways to look the object up
699                                  */
700                                 ((fs.object->type == OBJT_DEFAULT) ||
701                                  (fs.object->type == OBJT_SWAP)) &&
702                                 /*
703                                  * We don't chase down the shadow chain
704                                  */
705                                 (fs.object == fs.first_object->backing_object) &&
706
707                                 /*
708                                  * grab the lock if we need to
709                                  */
710                                 (fs.lookup_still_valid ||
711                                  lockmgr(&fs.map->lock, LK_EXCLUSIVE|LK_NOWAIT) == 0)
712                             ) {
713                                 
714                                 fs.lookup_still_valid = 1;
715                                 /*
716                                  * get rid of the unnecessary page
717                                  */
718                                 vm_page_protect(fs.first_m, VM_PROT_NONE);
719                                 vm_page_free(fs.first_m);
720                                 fs.first_m = NULL;
721
722                                 /*
723                                  * grab the page and put it into the 
724                                  * process'es object.  The page is 
725                                  * automatically made dirty.
726                                  */
727                                 vm_page_rename(fs.m, fs.first_object, fs.first_pindex);
728                                 fs.first_m = fs.m;
729                                 vm_page_busy(fs.first_m);
730                                 fs.m = NULL;
731                                 mycpu->gd_cnt.v_cow_optim++;
732                         } else {
733                                 /*
734                                  * Oh, well, lets copy it.
735                                  */
736                                 vm_page_copy(fs.m, fs.first_m);
737                         }
738
739                         if (fs.m) {
740                                 /*
741                                  * We no longer need the old page or object.
742                                  */
743                                 release_page(&fs);
744                         }
745
746                         /*
747                          * fs.object != fs.first_object due to above 
748                          * conditional
749                          */
750
751                         vm_object_pip_wakeup(fs.object);
752
753                         /*
754                          * Only use the new page below...
755                          */
756
757                         mycpu->gd_cnt.v_cow_faults++;
758                         fs.m = fs.first_m;
759                         fs.object = fs.first_object;
760                         fs.pindex = fs.first_pindex;
761
762                 } else {
763                         prot &= ~VM_PROT_WRITE;
764                 }
765         }
766
767         /*
768          * We must verify that the maps have not changed since our last
769          * lookup.
770          */
771
772         if (!fs.lookup_still_valid &&
773                 (fs.map->timestamp != map_generation)) {
774                 vm_object_t retry_object;
775                 vm_pindex_t retry_pindex;
776                 vm_prot_t retry_prot;
777
778                 /*
779                  * Since map entries may be pageable, make sure we can take a
780                  * page fault on them.
781                  */
782
783                 /*
784                  * Unlock vnode before the lookup to avoid deadlock.   E.G.
785                  * avoid a deadlock between the inode and exec_map that can
786                  * occur due to locks being obtained in different orders.
787                  */
788
789                 if (fs.vp != NULL) {
790                         vput(fs.vp);
791                         fs.vp = NULL;
792                 }
793                 
794                 if (fs.map->infork) {
795                         release_page(&fs);
796                         unlock_and_deallocate(&fs);
797                         goto RetryFault;
798                 }
799
800                 /*
801                  * To avoid trying to write_lock the map while another process
802                  * has it read_locked (in vm_map_wire), we do not try for
803                  * write permission.  If the page is still writable, we will
804                  * get write permission.  If it is not, or has been marked
805                  * needs_copy, we enter the mapping without write permission,
806                  * and will merely take another fault.
807                  */
808                 result = vm_map_lookup(&fs.map, vaddr, fault_type & ~VM_PROT_WRITE,
809                     &fs.entry, &retry_object, &retry_pindex, &retry_prot, &wired);
810                 map_generation = fs.map->timestamp;
811
812                 /*
813                  * If we don't need the page any longer, put it on the active
814                  * list (the easiest thing to do here).  If no one needs it,
815                  * pageout will grab it eventually.
816                  */
817
818                 if (result != KERN_SUCCESS) {
819                         release_page(&fs);
820                         unlock_and_deallocate(&fs);
821                         return (result);
822                 }
823                 fs.lookup_still_valid = TRUE;
824
825                 if ((retry_object != fs.first_object) ||
826                     (retry_pindex != fs.first_pindex)) {
827                         release_page(&fs);
828                         unlock_and_deallocate(&fs);
829                         goto RetryFault;
830                 }
831                 /*
832                  * Check whether the protection has changed or the object has
833                  * been copied while we left the map unlocked. Changing from
834                  * read to write permission is OK - we leave the page
835                  * write-protected, and catch the write fault. Changing from
836                  * write to read permission means that we can't mark the page
837                  * write-enabled after all.
838                  */
839                 prot &= retry_prot;
840         }
841
842         /*
843          * Put this page into the physical map. We had to do the unlock above
844          * because pmap_enter may cause other faults.   We don't put the page
845          * back on the active queue until later so that the page-out daemon
846          * won't find us (yet).
847          */
848
849         if (prot & VM_PROT_WRITE) {
850                 vm_page_flag_set(fs.m, PG_WRITEABLE);
851                 vm_object_set_writeable_dirty(fs.m->object);
852
853                 /*
854                  * If the fault is a write, we know that this page is being
855                  * written NOW so dirty it explicitly to save on 
856                  * pmap_is_modified() calls later.
857                  *
858                  * If this is a NOSYNC mmap we do not want to set PG_NOSYNC
859                  * if the page is already dirty to prevent data written with
860                  * the expectation of being synced from not being synced.
861                  * Likewise if this entry does not request NOSYNC then make
862                  * sure the page isn't marked NOSYNC.  Applications sharing
863                  * data should use the same flags to avoid ping ponging.
864                  *
865                  * Also tell the backing pager, if any, that it should remove
866                  * any swap backing since the page is now dirty.
867                  */
868                 if (fs.entry->eflags & MAP_ENTRY_NOSYNC) {
869                         if (fs.m->dirty == 0)
870                                 vm_page_flag_set(fs.m, PG_NOSYNC);
871                 } else {
872                         vm_page_flag_clear(fs.m, PG_NOSYNC);
873                 }
874                 if (fault_flags & VM_FAULT_DIRTY) {
875                         crit_enter();
876                         vm_page_dirty(fs.m);
877                         vm_pager_page_unswapped(fs.m);
878                         crit_exit();
879                 }
880         }
881
882         /*
883          * Page had better still be busy
884          */
885
886         KASSERT(fs.m->flags & PG_BUSY,
887                 ("vm_fault: page %p not busy!", fs.m));
888
889         unlock_things(&fs);
890
891         /*
892          * Sanity check: page must be completely valid or it is not fit to
893          * map into user space.  vm_pager_get_pages() ensures this.
894          */
895
896         if (fs.m->valid != VM_PAGE_BITS_ALL) {
897                 vm_page_zero_invalid(fs.m, TRUE);
898                 printf("Warning: page %p partially invalid on fault\n", fs.m);
899         }
900
901         pmap_enter(fs.map->pmap, vaddr, fs.m, prot, wired);
902
903         if (((fault_flags & VM_FAULT_WIRE_MASK) == 0) && (wired == 0)) {
904                 pmap_prefault(fs.map->pmap, vaddr, fs.entry);
905         }
906
907         vm_page_flag_clear(fs.m, PG_ZERO);
908         vm_page_flag_set(fs.m, PG_MAPPED|PG_REFERENCED);
909         if (fault_flags & VM_FAULT_HOLD)
910                 vm_page_hold(fs.m);
911
912         /*
913          * If the page is not wired down, then put it where the pageout daemon
914          * can find it.
915          */
916
917         if (fault_flags & VM_FAULT_WIRE_MASK) {
918                 if (wired)
919                         vm_page_wire(fs.m);
920                 else
921                         vm_page_unwire(fs.m, 1);
922         } else {
923                 vm_page_activate(fs.m);
924         }
925
926         if (curproc && (curproc->p_flag & P_SWAPPEDOUT) == 0 &&
927             curproc->p_stats) {
928                 if (hardfault) {
929                         curproc->p_stats->p_ru.ru_majflt++;
930                 } else {
931                         curproc->p_stats->p_ru.ru_minflt++;
932                 }
933         }
934
935         /*
936          * Unlock everything, and return
937          */
938
939         vm_page_wakeup(fs.m);
940         vm_object_deallocate(fs.first_object);
941
942         return (KERN_SUCCESS);
943
944 }
945
946 /*
947  * quick version of vm_fault
948  */
949 int
950 vm_fault_quick(caddr_t v, int prot)
951 {
952         int r;
953
954         if (prot & VM_PROT_WRITE)
955                 r = subyte(v, fubyte(v));
956         else
957                 r = fubyte(v);
958         return(r);
959 }
960
961 /*
962  * Wire down a range of virtual addresses in a map.  The entry in question
963  * should be marked in-transition and the map must be locked.  We must
964  * release the map temporarily while faulting-in the page to avoid a
965  * deadlock.  Note that the entry may be clipped while we are blocked but
966  * will never be freed.
967  */
968 int
969 vm_fault_wire(vm_map_t map, vm_map_entry_t entry, boolean_t user_wire)
970 {
971         boolean_t fictitious;
972         vm_offset_t start;
973         vm_offset_t end;
974         vm_offset_t va;
975         vm_paddr_t pa;
976         pmap_t pmap;
977         int rv;
978
979         pmap = vm_map_pmap(map);
980         start = entry->start;
981         end = entry->end;
982         fictitious = entry->object.vm_object &&
983                         (entry->object.vm_object->type == OBJT_DEVICE);
984
985         vm_map_unlock(map);
986         map->timestamp++;
987
988         /*
989          * We simulate a fault to get the page and enter it in the physical
990          * map.
991          */
992         for (va = start; va < end; va += PAGE_SIZE) {
993                 if (user_wire) {
994                         rv = vm_fault(map, va, VM_PROT_READ, 
995                                         VM_FAULT_USER_WIRE);
996                 } else {
997                         rv = vm_fault(map, va, VM_PROT_READ|VM_PROT_WRITE,
998                                         VM_FAULT_CHANGE_WIRING);
999                 }
1000                 if (rv) {
1001                         while (va > start) {
1002                                 va -= PAGE_SIZE;
1003                                 if ((pa = pmap_extract(pmap, va)) == 0)
1004                                         continue;
1005                                 pmap_change_wiring(pmap, va, FALSE);
1006                                 if (!fictitious)
1007                                         vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
1008                         }
1009                         vm_map_lock(map);
1010                         return (rv);
1011                 }
1012         }
1013         vm_map_lock(map);
1014         return (KERN_SUCCESS);
1015 }
1016
1017 /*
1018  * Unwire a range of virtual addresses in a map.  The map should be
1019  * locked.
1020  */
1021 void
1022 vm_fault_unwire(vm_map_t map, vm_map_entry_t entry)
1023 {
1024         boolean_t fictitious;
1025         vm_offset_t start;
1026         vm_offset_t end;
1027         vm_offset_t va;
1028         vm_paddr_t pa;
1029         pmap_t pmap;
1030
1031         pmap = vm_map_pmap(map);
1032         start = entry->start;
1033         end = entry->end;
1034         fictitious = entry->object.vm_object &&
1035                         (entry->object.vm_object->type == OBJT_DEVICE);
1036
1037         /*
1038          * Since the pages are wired down, we must be able to get their
1039          * mappings from the physical map system.
1040          */
1041         for (va = start; va < end; va += PAGE_SIZE) {
1042                 pa = pmap_extract(pmap, va);
1043                 if (pa != 0) {
1044                         pmap_change_wiring(pmap, va, FALSE);
1045                         if (!fictitious)
1046                                 vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1);
1047                 }
1048         }
1049 }
1050
1051 /*
1052  * Reduce the rate at which memory is allocated to a process based
1053  * on the perceived load on the VM system. As the load increases
1054  * the allocation burst rate goes down and the delay increases. 
1055  *
1056  * Rate limiting does not apply when faulting active or inactive
1057  * pages.  When faulting 'cache' pages, rate limiting only applies
1058  * if the system currently has a severe page deficit.
1059  *
1060  * XXX vm_pagesupply should be increased when a page is freed.
1061  *
1062  * We sleep up to 1/10 of a second.
1063  */
1064 static int
1065 vm_fault_ratelimit(struct vmspace *vmspace)
1066 {
1067         if (vm_load_enable == 0)
1068                 return(0);
1069         if (vmspace->vm_pagesupply > 0) {
1070                 --vmspace->vm_pagesupply;
1071                 return(0);
1072         }
1073 #ifdef INVARIANTS
1074         if (vm_load_debug) {
1075                 printf("load %-4d give %d pgs, wait %d, pid %-5d (%s)\n",
1076                         vm_load, 
1077                         (1000 - vm_load ) / 10, vm_load * hz / 10000,
1078                         curproc->p_pid, curproc->p_comm);
1079         }
1080 #endif
1081         vmspace->vm_pagesupply = (1000 - vm_load) / 10;
1082         return(vm_load * hz / 10000);
1083 }
1084
1085 /*
1086  *      Routine:
1087  *              vm_fault_copy_entry
1088  *      Function:
1089  *              Copy all of the pages from a wired-down map entry to another.
1090  *
1091  *      In/out conditions:
1092  *              The source and destination maps must be locked for write.
1093  *              The source map entry must be wired down (or be a sharing map
1094  *              entry corresponding to a main map entry that is wired down).
1095  */
1096
1097 void
1098 vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map,
1099     vm_map_entry_t dst_entry, vm_map_entry_t src_entry)
1100 {
1101         vm_object_t dst_object;
1102         vm_object_t src_object;
1103         vm_ooffset_t dst_offset;
1104         vm_ooffset_t src_offset;
1105         vm_prot_t prot;
1106         vm_offset_t vaddr;
1107         vm_page_t dst_m;
1108         vm_page_t src_m;
1109
1110 #ifdef  lint
1111         src_map++;
1112 #endif  /* lint */
1113
1114         src_object = src_entry->object.vm_object;
1115         src_offset = src_entry->offset;
1116
1117         /*
1118          * Create the top-level object for the destination entry. (Doesn't
1119          * actually shadow anything - we copy the pages directly.)
1120          */
1121         dst_object = vm_object_allocate(OBJT_DEFAULT,
1122             (vm_size_t) OFF_TO_IDX(dst_entry->end - dst_entry->start));
1123
1124         dst_entry->object.vm_object = dst_object;
1125         dst_entry->offset = 0;
1126
1127         prot = dst_entry->max_protection;
1128
1129         /*
1130          * Loop through all of the pages in the entry's range, copying each
1131          * one from the source object (it should be there) to the destination
1132          * object.
1133          */
1134         for (vaddr = dst_entry->start, dst_offset = 0;
1135             vaddr < dst_entry->end;
1136             vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
1137
1138                 /*
1139                  * Allocate a page in the destination object
1140                  */
1141                 do {
1142                         dst_m = vm_page_alloc(dst_object,
1143                                 OFF_TO_IDX(dst_offset), VM_ALLOC_NORMAL);
1144                         if (dst_m == NULL) {
1145                                 vm_wait();
1146                         }
1147                 } while (dst_m == NULL);
1148
1149                 /*
1150                  * Find the page in the source object, and copy it in.
1151                  * (Because the source is wired down, the page will be in
1152                  * memory.)
1153                  */
1154                 src_m = vm_page_lookup(src_object,
1155                         OFF_TO_IDX(dst_offset + src_offset));
1156                 if (src_m == NULL)
1157                         panic("vm_fault_copy_wired: page missing");
1158
1159                 vm_page_copy(src_m, dst_m);
1160
1161                 /*
1162                  * Enter it in the pmap...
1163                  */
1164
1165                 vm_page_flag_clear(dst_m, PG_ZERO);
1166                 pmap_enter(dst_map->pmap, vaddr, dst_m, prot, FALSE);
1167                 vm_page_flag_set(dst_m, PG_WRITEABLE|PG_MAPPED);
1168
1169                 /*
1170                  * Mark it no longer busy, and put it on the active list.
1171                  */
1172                 vm_page_activate(dst_m);
1173                 vm_page_wakeup(dst_m);
1174         }
1175 }
1176
1177
1178 /*
1179  * This routine checks around the requested page for other pages that
1180  * might be able to be faulted in.  This routine brackets the viable
1181  * pages for the pages to be paged in.
1182  *
1183  * Inputs:
1184  *      m, rbehind, rahead
1185  *
1186  * Outputs:
1187  *  marray (array of vm_page_t), reqpage (index of requested page)
1188  *
1189  * Return value:
1190  *  number of pages in marray
1191  */
1192 static int
1193 vm_fault_additional_pages(vm_page_t m, int rbehind, int rahead,
1194     vm_page_t *marray, int *reqpage)
1195 {
1196         int i,j;
1197         vm_object_t object;
1198         vm_pindex_t pindex, startpindex, endpindex, tpindex;
1199         vm_page_t rtm;
1200         int cbehind, cahead;
1201
1202         object = m->object;
1203         pindex = m->pindex;
1204
1205         /*
1206          * we don't fault-ahead for device pager
1207          */
1208         if (object->type == OBJT_DEVICE) {
1209                 *reqpage = 0;
1210                 marray[0] = m;
1211                 return 1;
1212         }
1213
1214         /*
1215          * if the requested page is not available, then give up now
1216          */
1217
1218         if (!vm_pager_has_page(object, pindex, &cbehind, &cahead)) {
1219                 return 0;
1220         }
1221
1222         if ((cbehind == 0) && (cahead == 0)) {
1223                 *reqpage = 0;
1224                 marray[0] = m;
1225                 return 1;
1226         }
1227
1228         if (rahead > cahead) {
1229                 rahead = cahead;
1230         }
1231
1232         if (rbehind > cbehind) {
1233                 rbehind = cbehind;
1234         }
1235
1236         /*
1237          * try to do any readahead that we might have free pages for.
1238          */
1239         if ((rahead + rbehind) >
1240                 ((vmstats.v_free_count + vmstats.v_cache_count) - vmstats.v_free_reserved)) {
1241                 pagedaemon_wakeup();
1242                 marray[0] = m;
1243                 *reqpage = 0;
1244                 return 1;
1245         }
1246
1247         /*
1248          * scan backward for the read behind pages -- in memory 
1249          *
1250          * Assume that if the page is not found an interrupt will not
1251          * create it.  Theoretically interrupts can only remove (busy)
1252          * pages, not create new associations.
1253          */
1254         if (pindex > 0) {
1255                 if (rbehind > pindex) {
1256                         rbehind = pindex;
1257                         startpindex = 0;
1258                 } else {
1259                         startpindex = pindex - rbehind;
1260                 }
1261
1262                 crit_enter();
1263                 for ( tpindex = pindex - 1; tpindex >= startpindex; tpindex -= 1) {
1264                         if (vm_page_lookup( object, tpindex)) {
1265                                 startpindex = tpindex + 1;
1266                                 break;
1267                         }
1268                         if (tpindex == 0)
1269                                 break;
1270                 }
1271
1272                 for(i = 0, tpindex = startpindex; tpindex < pindex; i++, tpindex++) {
1273
1274                         rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1275                         if (rtm == NULL) {
1276                                 crit_exit();
1277                                 for (j = 0; j < i; j++) {
1278                                         vm_page_free(marray[j]);
1279                                 }
1280                                 marray[0] = m;
1281                                 *reqpage = 0;
1282                                 return 1;
1283                         }
1284
1285                         marray[i] = rtm;
1286                 }
1287                 crit_exit();
1288         } else {
1289                 startpindex = 0;
1290                 i = 0;
1291         }
1292
1293         marray[i] = m;
1294         /* page offset of the required page */
1295         *reqpage = i;
1296
1297         tpindex = pindex + 1;
1298         i++;
1299
1300         /*
1301          * scan forward for the read ahead pages
1302          */
1303         endpindex = tpindex + rahead;
1304         if (endpindex > object->size)
1305                 endpindex = object->size;
1306
1307         crit_enter();
1308         for( ; tpindex < endpindex; i++, tpindex++) {
1309
1310                 if (vm_page_lookup(object, tpindex)) {
1311                         break;
1312                 }
1313
1314                 rtm = vm_page_alloc(object, tpindex, VM_ALLOC_NORMAL);
1315                 if (rtm == NULL) {
1316                         break;
1317                 }
1318
1319                 marray[i] = rtm;
1320         }
1321         crit_exit();
1322
1323         /* return number of bytes of pages */
1324         return i;
1325 }