7f0de90831b95a689ce7e7f633d91d5bc90502d9
[dragonfly.git] / sys / vm / vm_object.c
1 /*
2  * (MPSAFE)
3  *
4  * Copyright (c) 1991, 1993
5  *      The Regents of the University of California.  All rights reserved.
6  *
7  * This code is derived from software contributed to Berkeley by
8  * The Mach Operating System project at Carnegie-Mellon University.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *      This product includes software developed by the University of
21  *      California, Berkeley and its contributors.
22  * 4. Neither the name of the University nor the names of its contributors
23  *    may be used to endorse or promote products derived from this software
24  *    without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36  * SUCH DAMAGE.
37  *
38  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
39  *
40  *
41  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
42  * All rights reserved.
43  *
44  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
45  *
46  * Permission to use, copy, modify and distribute this software and
47  * its documentation is hereby granted, provided that both the copyright
48  * notice and this permission notice appear in all copies of the
49  * software, derivative works or modified versions, and any portions
50  * thereof, and that both notices appear in supporting documentation.
51  *
52  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
53  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
54  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
55  *
56  * Carnegie Mellon requests users of this software to return to
57  *
58  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
59  *  School of Computer Science
60  *  Carnegie Mellon University
61  *  Pittsburgh PA 15213-3890
62  *
63  * any improvements or extensions that they make and grant Carnegie the
64  * rights to redistribute these changes.
65  *
66  * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $
67  */
68
69 /*
70  *      Virtual memory object module.
71  */
72
73 #include <sys/param.h>
74 #include <sys/systm.h>
75 #include <sys/proc.h>           /* for curproc, pageproc */
76 #include <sys/thread.h>
77 #include <sys/vnode.h>
78 #include <sys/vmmeter.h>
79 #include <sys/mman.h>
80 #include <sys/mount.h>
81 #include <sys/kernel.h>
82 #include <sys/sysctl.h>
83 #include <sys/refcount.h>
84
85 #include <vm/vm.h>
86 #include <vm/vm_param.h>
87 #include <vm/pmap.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_object.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_pageout.h>
92 #include <vm/vm_pager.h>
93 #include <vm/swap_pager.h>
94 #include <vm/vm_kern.h>
95 #include <vm/vm_extern.h>
96 #include <vm/vm_zone.h>
97
98 #define EASY_SCAN_FACTOR        8
99
100 static void     vm_object_qcollapse(vm_object_t object,
101                                     vm_object_t backing_object);
102 static int      vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
103                                              int pagerflags);
104 static void     vm_object_lock_init(vm_object_t);
105
106
107 /*
108  *      Virtual memory objects maintain the actual data
109  *      associated with allocated virtual memory.  A given
110  *      page of memory exists within exactly one object.
111  *
112  *      An object is only deallocated when all "references"
113  *      are given up.  Only one "reference" to a given
114  *      region of an object should be writeable.
115  *
116  *      Associated with each object is a list of all resident
117  *      memory pages belonging to that object; this list is
118  *      maintained by the "vm_page" module, and locked by the object's
119  *      lock.
120  *
121  *      Each object also records a "pager" routine which is
122  *      used to retrieve (and store) pages to the proper backing
123  *      storage.  In addition, objects may be backed by other
124  *      objects from which they were virtual-copied.
125  *
126  *      The only items within the object structure which are
127  *      modified after time of creation are:
128  *              reference count         locked by object's lock
129  *              pager routine           locked by object's lock
130  *
131  */
132
133 struct object_q vm_object_list;         /* locked by vmobj_token */
134 struct vm_object kernel_object;
135
136 static long vm_object_count;            /* locked by vmobj_token */
137 extern int vm_pageout_page_count;
138
139 static long object_collapses;
140 static long object_bypasses;
141 static int next_index;
142 static vm_zone_t obj_zone;
143 static struct vm_zone obj_zone_store;
144 #define VM_OBJECTS_INIT 256
145 static struct vm_object vm_objects_init[VM_OBJECTS_INIT];
146
147 /*
148  * Misc low level routines
149  */
150 static void
151 vm_object_lock_init(vm_object_t obj)
152 {
153 #if defined(DEBUG_LOCKS)
154         int i;
155
156         obj->debug_hold_bitmap = 0;
157         obj->debug_hold_ovfl = 0;
158         for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) {
159                 obj->debug_hold_thrs[i] = NULL;
160                 obj->debug_hold_file[i] = NULL;
161                 obj->debug_hold_line[i] = 0;
162         }
163 #endif
164 }
165
166 void
167 vm_object_lock_swap(void)
168 {
169         lwkt_token_swap();
170 }
171
172 void
173 vm_object_lock(vm_object_t obj)
174 {
175         lwkt_getpooltoken(obj);
176 }
177
178 void
179 vm_object_unlock(vm_object_t obj)
180 {
181         lwkt_relpooltoken(obj);
182 }
183
184 static __inline void
185 vm_object_assert_held(vm_object_t obj)
186 {
187         ASSERT_LWKT_TOKEN_HELD(lwkt_token_pool_lookup(obj));
188 }
189
190 void
191 #ifndef DEBUG_LOCKS
192 vm_object_hold(vm_object_t obj)
193 #else
194 debugvm_object_hold(vm_object_t obj, char *file, int line)
195 #endif
196 {
197         KKASSERT(obj != NULL);
198
199         /*
200          * Object must be held (object allocation is stable due to callers
201          * context, typically already holding the token on a parent object)
202          * prior to potentially blocking on the lock, otherwise the object
203          * can get ripped away from us.
204          */
205         refcount_acquire(&obj->hold_count);
206         vm_object_lock(obj);
207
208 #if defined(DEBUG_LOCKS)
209         int i;
210
211         i = ffs(~obj->debug_hold_bitmap) - 1;
212         if (i == -1) {
213                 kprintf("vm_object hold count > VMOBJ_DEBUG_ARRAY_SIZE");
214                 obj->debug_hold_ovfl = 1;
215         }
216
217         obj->debug_hold_bitmap |= (1 << i);
218         obj->debug_hold_thrs[i] = curthread;
219         obj->debug_hold_file[i] = file;
220         obj->debug_hold_line[i] = line;
221 #endif
222 }
223
224 /*
225  * Drop the token and hold_count on the object.
226  */
227 void
228 vm_object_drop(vm_object_t obj)
229 {
230         if (obj == NULL)
231                 return;
232
233 #if defined(DEBUG_LOCKS)
234         int found = 0;
235         int i;
236
237         for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) {
238                 if ((obj->debug_hold_bitmap & (1 << i)) &&
239                     (obj->debug_hold_thrs[i] == curthread)) {
240                         obj->debug_hold_bitmap &= ~(1 << i);
241                         obj->debug_hold_thrs[i] = NULL;
242                         obj->debug_hold_file[i] = NULL;
243                         obj->debug_hold_line[i] = 0;
244                         found = 1;
245                         break;
246                 }
247         }
248
249         if (found == 0 && obj->debug_hold_ovfl == 0)
250                 panic("vm_object: attempt to drop hold on non-self-held obj");
251 #endif
252
253         /*
254          * The lock is a pool token, no new holders should be possible once
255          * we drop hold_count 1->0 as there is no longer any way to reference
256          * the object.
257          */
258         if (refcount_release(&obj->hold_count)) {
259                 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD))
260                         zfree(obj_zone, obj);
261         }
262         vm_object_unlock(obj);  /* uses pool token, ok to call on freed obj */
263 }
264
265 /*
266  * Initialize a freshly allocated object
267  *
268  * Used only by vm_object_allocate() and zinitna().
269  *
270  * No requirements.
271  */
272 void
273 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
274 {
275         int incr;
276
277         RB_INIT(&object->rb_memq);
278         LIST_INIT(&object->shadow_head);
279
280         object->type = type;
281         object->size = size;
282         object->ref_count = 1;
283         object->hold_count = 0;
284         object->flags = 0;
285         if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
286                 vm_object_set_flag(object, OBJ_ONEMAPPING);
287         object->paging_in_progress = 0;
288         object->resident_page_count = 0;
289         object->agg_pv_list_count = 0;
290         object->shadow_count = 0;
291 #ifdef SMP
292         /* cpu localization twist */
293         object->pg_color = (int)(intptr_t)curthread;
294 #else
295         object->pg_color = next_index;
296 #endif
297         if ( size > (PQ_L2_SIZE / 3 + PQ_PRIME1))
298                 incr = PQ_L2_SIZE / 3 + PQ_PRIME1;
299         else
300                 incr = size;
301         next_index = (next_index + incr) & PQ_L2_MASK;
302         object->handle = NULL;
303         object->backing_object = NULL;
304         object->backing_object_offset = (vm_ooffset_t)0;
305
306         object->generation++;
307         object->swblock_count = 0;
308         RB_INIT(&object->swblock_root);
309         vm_object_lock_init(object);
310
311         lwkt_gettoken(&vmobj_token);
312         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
313         vm_object_count++;
314         lwkt_reltoken(&vmobj_token);
315 }
316
317 /*
318  * Initialize the VM objects module.
319  *
320  * Called from the low level boot code only.
321  */
322 void
323 vm_object_init(void)
324 {
325         TAILQ_INIT(&vm_object_list);
326         
327         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(KvaEnd),
328                             &kernel_object);
329
330         obj_zone = &obj_zone_store;
331         zbootinit(obj_zone, "VM OBJECT", sizeof (struct vm_object),
332                 vm_objects_init, VM_OBJECTS_INIT);
333 }
334
335 void
336 vm_object_init2(void)
337 {
338         zinitna(obj_zone, NULL, NULL, 0, 0, ZONE_PANICFAIL, 1);
339 }
340
341 /*
342  * Allocate and return a new object of the specified type and size.
343  *
344  * No requirements.
345  */
346 vm_object_t
347 vm_object_allocate(objtype_t type, vm_pindex_t size)
348 {
349         vm_object_t result;
350
351         result = (vm_object_t) zalloc(obj_zone);
352
353         _vm_object_allocate(type, size, result);
354
355         return (result);
356 }
357
358 /*
359  * Add an additional reference to a vm_object.  The object must already be
360  * held.  The original non-lock version is no longer supported.  The object
361  * must NOT be chain locked by anyone at the time the reference is added.
362  *
363  * Referencing a chain-locked object can blow up the fairly sensitive
364  * ref_count and shadow_count tests in the deallocator.  Most callers
365  * will call vm_object_chain_wait() prior to calling
366  * vm_object_reference_locked() to avoid the case.
367  *
368  * The object must be held.
369  */
370 void
371 vm_object_reference_locked(vm_object_t object)
372 {
373         KKASSERT(object != NULL);
374         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
375         KKASSERT((object->flags & OBJ_CHAINLOCK) == 0);
376         object->ref_count++;
377         if (object->type == OBJT_VNODE) {
378                 vref(object->handle);
379                 /* XXX what if the vnode is being destroyed? */
380         }
381 }
382
383 /*
384  * Object OBJ_CHAINLOCK lock handling.
385  *
386  * The caller can chain-lock backing objects recursively and then
387  * use vm_object_chain_release_all() to undo the whole chain.
388  *
389  * Chain locks are used to prevent collapses and are only applicable
390  * to OBJT_DEFAULT and OBJT_SWAP objects.  Chain locking operations
391  * on other object types are ignored.  This is also important because
392  * it allows e.g. the vnode underlying a memory mapping to take concurrent
393  * faults.
394  *
395  * The object must usually be held on entry, though intermediate
396  * objects need not be held on release.
397  */
398 void
399 vm_object_chain_wait(vm_object_t object)
400 {
401         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
402         while (object->flags & OBJ_CHAINLOCK) {
403                 vm_object_set_flag(object, OBJ_CHAINWANT);
404                 tsleep(object, 0, "objchain", 0);
405         }
406 }
407
408 void
409 vm_object_chain_acquire(vm_object_t object)
410 {
411         if (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) {
412                 vm_object_chain_wait(object);
413                 vm_object_set_flag(object, OBJ_CHAINLOCK);
414         }
415 }
416
417 void
418 vm_object_chain_release(vm_object_t object)
419 {
420         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
421         if (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) {
422                 KKASSERT(object->flags & OBJ_CHAINLOCK);
423                 if (object->flags & OBJ_CHAINWANT) {
424                         vm_object_clear_flag(object,
425                                              OBJ_CHAINLOCK | OBJ_CHAINWANT);
426                         wakeup(object);
427                 } else {
428                         vm_object_clear_flag(object, OBJ_CHAINLOCK);
429                 }
430         }
431 }
432
433 /*
434  * This releases the entire chain starting with object and recursing
435  * through backing_object until stopobj is encountered.  stopobj is
436  * not released.  The caller will typically release stopobj manually
437  * before making this call (as the deepest object is the most likely
438  * to collide with other threads).
439  *
440  * object and stopobj must be held by the caller.  This code looks a
441  * bit odd but has been optimized fairly heavily.
442  */
443 void
444 vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj)
445 {
446         vm_object_t backing_object;
447         vm_object_t object;
448
449         vm_object_chain_release(stopobj);
450         object = first_object;
451
452         while (object != stopobj) {
453                 KKASSERT(object);
454                 if (object != first_object)
455                         vm_object_hold(object);
456                 backing_object = object->backing_object;
457                 vm_object_chain_release(object);
458                 if (object != first_object)
459                         vm_object_drop(object);
460                 object = backing_object;
461         }
462 }
463
464 /*
465  * Dereference an object and its underlying vnode.
466  *
467  * The object must be held and will be held on return.
468  */
469 static void
470 vm_object_vndeallocate(vm_object_t object)
471 {
472         struct vnode *vp = (struct vnode *) object->handle;
473
474         KASSERT(object->type == OBJT_VNODE,
475             ("vm_object_vndeallocate: not a vnode object"));
476         KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
477         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
478 #ifdef INVARIANTS
479         if (object->ref_count == 0) {
480                 vprint("vm_object_vndeallocate", vp);
481                 panic("vm_object_vndeallocate: bad object reference count");
482         }
483 #endif
484         object->ref_count--;
485         if (object->ref_count == 0)
486                 vclrflags(vp, VTEXT);
487         vrele(vp);
488 }
489
490 /*
491  * Release a reference to the specified object, gained either through a
492  * vm_object_allocate or a vm_object_reference call.  When all references
493  * are gone, storage associated with this object may be relinquished.
494  *
495  * The caller does not have to hold the object locked but must have control
496  * over the reference in question in order to guarantee that the object
497  * does not get ripped out from under us.
498  */
499 void
500 vm_object_deallocate(vm_object_t object)
501 {
502         if (object) {
503                 vm_object_hold(object);
504                 vm_object_deallocate_locked(object);
505                 vm_object_drop(object);
506         }
507 }
508
509 void
510 vm_object_deallocate_locked(vm_object_t object)
511 {
512         vm_object_t temp;
513         int must_drop = 0;
514
515         while (object != NULL) {
516 #if 0
517                 /*
518                  * Don't rip a ref_count out from under an object undergoing
519                  * collapse, it will confuse the collapse code.
520                  */
521                 vm_object_chain_wait(object);
522 #endif
523                 if (object->type == OBJT_VNODE) {
524                         vm_object_vndeallocate(object);
525                         break;
526                 }
527
528                 if (object->ref_count == 0) {
529                         panic("vm_object_deallocate: object deallocated "
530                               "too many times: %d", object->type);
531                 }
532                 if (object->ref_count > 2) {
533                         object->ref_count--;
534                         break;
535                 }
536
537                 /*
538                  * Here on ref_count of one or two, which are special cases for
539                  * objects.
540                  *
541                  * Nominal ref_count > 1 case if the second ref is not from
542                  * a shadow.
543                  */
544                 if (object->ref_count == 2 && object->shadow_count == 0) {
545                         vm_object_set_flag(object, OBJ_ONEMAPPING);
546                         object->ref_count--;
547                         break;
548                 }
549
550                 /*
551                  * If the second ref is from a shadow we chain along it
552                  * upwards if object's handle is exhausted.
553                  *
554                  * We have to decrement object->ref_count before potentially
555                  * collapsing the first shadow object or the collapse code
556                  * will not be able to handle the degenerate case to remove
557                  * object.  However, if we do it too early the object can
558                  * get ripped out from under us.
559                  */
560                 if (object->ref_count == 2 && object->shadow_count == 1 &&
561                     object->handle == NULL && (object->type == OBJT_DEFAULT ||
562                                                object->type == OBJT_SWAP)) {
563                         temp = LIST_FIRST(&object->shadow_head);
564                         KKASSERT(temp != NULL);
565                         vm_object_hold(temp);
566
567                         /*
568                          * Wait for any paging to complete so the collapse
569                          * doesn't (or isn't likely to) qcollapse.  pip
570                          * waiting must occur before we acquire the
571                          * chainlock.
572                          */
573                         while (
574                                 temp->paging_in_progress ||
575                                 object->paging_in_progress
576                         ) {
577                                 vm_object_pip_wait(temp, "objde1");
578                                 vm_object_pip_wait(object, "objde2");
579                         }
580
581                         /*
582                          * If the parent is locked we have to give up, as
583                          * otherwise we would be acquiring locks in the
584                          * wrong order and potentially deadlock.
585                          */
586                         if (temp->flags & OBJ_CHAINLOCK) {
587                                 vm_object_drop(temp);
588                                 goto skip;
589                         }
590                         vm_object_chain_acquire(temp);
591
592                         /*
593                          * Recheck/retry after the hold and the paging
594                          * wait, both of which can block us.
595                          */
596                         if (object->ref_count != 2 ||
597                             object->shadow_count != 1 ||
598                             object->handle ||
599                             LIST_FIRST(&object->shadow_head) != temp ||
600                             (object->type != OBJT_DEFAULT &&
601                              object->type != OBJT_SWAP)) {
602                                 vm_object_chain_release(temp);
603                                 vm_object_drop(temp);
604                                 continue;
605                         }
606
607                         /*
608                          * We can safely drop object's ref_count now.
609                          */
610                         KKASSERT(object->ref_count == 2);
611                         object->ref_count--;
612
613                         /*
614                          * If our single parent is not collapseable just
615                          * decrement ref_count (2->1) and stop.
616                          */
617                         if (temp->handle || (temp->type != OBJT_DEFAULT &&
618                                              temp->type != OBJT_SWAP)) {
619                                 vm_object_chain_release(temp);
620                                 vm_object_drop(temp);
621                                 break;
622                         }
623
624                         /*
625                          * At this point we have already dropped object's
626                          * ref_count so it is possible for a race to
627                          * deallocate obj out from under us.  Any collapse
628                          * will re-check the situation.  We must not block
629                          * until we are able to collapse.
630                          *
631                          * Bump temp's ref_count to avoid an unwanted
632                          * degenerate recursion (can't call
633                          * vm_object_reference_locked() because it asserts
634                          * that CHAINLOCK is not set).
635                          */
636                         temp->ref_count++;
637                         KKASSERT(temp->ref_count > 1);
638
639                         /*
640                          * Collapse temp, then deallocate the extra ref
641                          * formally.
642                          */
643                         vm_object_collapse(temp);
644                         vm_object_chain_release(temp);
645                         if (must_drop) {
646                                 vm_object_lock_swap();
647                                 vm_object_drop(object);
648                         }
649                         object = temp;
650                         must_drop = 1;
651                         continue;
652                 }
653
654                 /*
655                  * Drop the ref and handle termination on the 1->0 transition.
656                  * We may have blocked above so we have to recheck.
657                  */
658 skip:
659                 KKASSERT(object->ref_count != 0);
660                 if (object->ref_count >= 2) {
661                         object->ref_count--;
662                         break;
663                 }
664                 KKASSERT(object->ref_count == 1);
665
666                 /*
667                  * 1->0 transition.  Chain through the backing_object.
668                  * Maintain the ref until we've located the backing object,
669                  * then re-check.
670                  */
671                 while ((temp = object->backing_object) != NULL) {
672                         vm_object_hold(temp);
673                         if (temp == object->backing_object)
674                                 break;
675                         vm_object_drop(temp);
676                 }
677
678                 /*
679                  * 1->0 transition verified, retry if ref_count is no longer
680                  * 1.  Otherwise disconnect the backing_object (temp) and
681                  * clean up.
682                  */
683                 if (object->ref_count != 1) {
684                         vm_object_drop(temp);
685                         continue;
686                 }
687
688                 /*
689                  * It shouldn't be possible for the object to be chain locked
690                  * if we're removing the last ref on it.
691                  */
692                 KKASSERT((object->flags & OBJ_CHAINLOCK) == 0);
693
694                 if (temp) {
695                         LIST_REMOVE(object, shadow_list);
696                         temp->shadow_count--;
697                         temp->generation++;
698                         object->backing_object = NULL;
699                 }
700
701                 --object->ref_count;
702                 if ((object->flags & OBJ_DEAD) == 0)
703                         vm_object_terminate(object);
704                 if (must_drop && temp)
705                         vm_object_lock_swap();
706                 if (must_drop)
707                         vm_object_drop(object);
708                 object = temp;
709                 must_drop = 1;
710         }
711         if (must_drop && object)
712                 vm_object_drop(object);
713 }
714
715 /*
716  * Destroy the specified object, freeing up related resources.
717  *
718  * The object must have zero references.
719  *
720  * The object must held.  The caller is responsible for dropping the object
721  * after terminate returns.  Terminate does NOT drop the object.
722  */
723 static int vm_object_terminate_callback(vm_page_t p, void *data);
724
725 void
726 vm_object_terminate(vm_object_t object)
727 {
728         /*
729          * Make sure no one uses us.  Once we set OBJ_DEAD we should be
730          * able to safely block.
731          */
732         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
733         KKASSERT((object->flags & OBJ_DEAD) == 0);
734         vm_object_set_flag(object, OBJ_DEAD);
735
736         /*
737          * Wait for the pageout daemon to be done with the object
738          */
739         vm_object_pip_wait(object, "objtrm1");
740
741         KASSERT(!object->paging_in_progress,
742                 ("vm_object_terminate: pageout in progress"));
743
744         /*
745          * Clean and free the pages, as appropriate. All references to the
746          * object are gone, so we don't need to lock it.
747          */
748         if (object->type == OBJT_VNODE) {
749                 struct vnode *vp;
750
751                 /*
752                  * Clean pages and flush buffers.
753                  */
754                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
755
756                 vp = (struct vnode *) object->handle;
757                 vinvalbuf(vp, V_SAVE, 0, 0);
758         }
759
760         /*
761          * Wait for any I/O to complete, after which there had better not
762          * be any references left on the object.
763          */
764         vm_object_pip_wait(object, "objtrm2");
765
766         if (object->ref_count != 0) {
767                 panic("vm_object_terminate: object with references, "
768                       "ref_count=%d", object->ref_count);
769         }
770
771         /*
772          * Now free any remaining pages. For internal objects, this also
773          * removes them from paging queues. Don't free wired pages, just
774          * remove them from the object. 
775          */
776         vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
777                                 vm_object_terminate_callback, NULL);
778
779         /*
780          * Let the pager know object is dead.
781          */
782         vm_pager_deallocate(object);
783
784         /*
785          * Wait for the object hold count to hit 1, clean out pages as
786          * we go.  vmobj_token interlocks any race conditions that might
787          * pick the object up from the vm_object_list after we have cleared
788          * rb_memq.
789          */
790         for (;;) {
791                 if (RB_ROOT(&object->rb_memq) == NULL)
792                         break;
793                 kprintf("vm_object_terminate: Warning, object %p "
794                         "still has %d pages\n",
795                         object, object->resident_page_count);
796                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
797                                         vm_object_terminate_callback, NULL);
798         }
799
800         /*
801          * There had better not be any pages left
802          */
803         KKASSERT(object->resident_page_count == 0);
804
805         /*
806          * Remove the object from the global object list.
807          */
808         lwkt_gettoken(&vmobj_token);
809         TAILQ_REMOVE(&vm_object_list, object, object_list);
810         vm_object_count--;
811         lwkt_reltoken(&vmobj_token);
812         vm_object_dead_wakeup(object);
813
814         if (object->ref_count != 0) {
815                 panic("vm_object_terminate2: object with references, "
816                       "ref_count=%d", object->ref_count);
817         }
818
819         /*
820          * NOTE: The object hold_count is at least 1, so we cannot zfree()
821          *       the object here.  See vm_object_drop().
822          */
823 }
824
825 /*
826  * The caller must hold the object.
827  */
828 static int
829 vm_object_terminate_callback(vm_page_t p, void *data __unused)
830 {
831         vm_object_t object;
832
833         object = p->object;
834         vm_page_busy_wait(p, FALSE, "vmpgtrm");
835         if (object != p->object) {
836                 kprintf("vm_object_terminate: Warning: Encountered "
837                         "busied page %p on queue %d\n", p, p->queue);
838                 vm_page_wakeup(p);
839         } else if (p->wire_count == 0) {
840                 vm_page_free(p);
841                 mycpu->gd_cnt.v_pfree++;
842         } else {
843                 if (p->queue != PQ_NONE)
844                         kprintf("vm_object_terminate: Warning: Encountered "
845                                 "wired page %p on queue %d\n", p, p->queue);
846                 vm_page_remove(p);
847                 vm_page_wakeup(p);
848         }
849         return(0);
850 }
851
852 /*
853  * The object is dead but still has an object<->pager association.  Sleep
854  * and return.  The caller typically retests the association in a loop.
855  *
856  * The caller must hold the object.
857  */
858 void
859 vm_object_dead_sleep(vm_object_t object, const char *wmesg)
860 {
861         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
862         if (object->handle) {
863                 vm_object_set_flag(object, OBJ_DEADWNT);
864                 tsleep(object, 0, wmesg, 0);
865                 /* object may be invalid after this point */
866         }
867 }
868
869 /*
870  * Wakeup anyone waiting for the object<->pager disassociation on
871  * a dead object.
872  *
873  * The caller must hold the object.
874  */
875 void
876 vm_object_dead_wakeup(vm_object_t object)
877 {
878         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
879         if (object->flags & OBJ_DEADWNT) {
880                 vm_object_clear_flag(object, OBJ_DEADWNT);
881                 wakeup(object);
882         }
883 }
884
885 /*
886  * Clean all dirty pages in the specified range of object.  Leaves page
887  * on whatever queue it is currently on.   If NOSYNC is set then do not
888  * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC),
889  * leaving the object dirty.
890  *
891  * When stuffing pages asynchronously, allow clustering.  XXX we need a
892  * synchronous clustering mode implementation.
893  *
894  * Odd semantics: if start == end, we clean everything.
895  *
896  * The object must be locked? XXX
897  */
898 static int vm_object_page_clean_pass1(struct vm_page *p, void *data);
899 static int vm_object_page_clean_pass2(struct vm_page *p, void *data);
900
901 void
902 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
903                      int flags)
904 {
905         struct rb_vm_page_scan_info info;
906         struct vnode *vp;
907         int wholescan;
908         int pagerflags;
909         int generation;
910
911         vm_object_hold(object);
912         if (object->type != OBJT_VNODE ||
913             (object->flags & OBJ_MIGHTBEDIRTY) == 0) {
914                 vm_object_drop(object);
915                 return;
916         }
917
918         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? 
919                         VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
920         pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
921
922         vp = object->handle;
923
924         /*
925          * Interlock other major object operations.  This allows us to 
926          * temporarily clear OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY.
927          */
928         vm_object_set_flag(object, OBJ_CLEANING);
929
930         /*
931          * Handle 'entire object' case
932          */
933         info.start_pindex = start;
934         if (end == 0) {
935                 info.end_pindex = object->size - 1;
936         } else {
937                 info.end_pindex = end - 1;
938         }
939         wholescan = (start == 0 && info.end_pindex == object->size - 1);
940         info.limit = flags;
941         info.pagerflags = pagerflags;
942         info.object = object;
943
944         /*
945          * If cleaning the entire object do a pass to mark the pages read-only.
946          * If everything worked out ok, clear OBJ_WRITEABLE and
947          * OBJ_MIGHTBEDIRTY.
948          */
949         if (wholescan) {
950                 info.error = 0;
951                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
952                                         vm_object_page_clean_pass1, &info);
953                 if (info.error == 0) {
954                         vm_object_clear_flag(object,
955                                              OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
956                         if (object->type == OBJT_VNODE &&
957                             (vp = (struct vnode *)object->handle) != NULL) {
958                                 if (vp->v_flag & VOBJDIRTY) 
959                                         vclrflags(vp, VOBJDIRTY);
960                         }
961                 }
962         }
963
964         /*
965          * Do a pass to clean all the dirty pages we find.
966          */
967         do {
968                 info.error = 0;
969                 generation = object->generation;
970                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
971                                         vm_object_page_clean_pass2, &info);
972         } while (info.error || generation != object->generation);
973
974         vm_object_clear_flag(object, OBJ_CLEANING);
975         vm_object_drop(object);
976 }
977
978 /*
979  * The caller must hold the object.
980  */
981 static 
982 int
983 vm_object_page_clean_pass1(struct vm_page *p, void *data)
984 {
985         struct rb_vm_page_scan_info *info = data;
986
987         vm_page_flag_set(p, PG_CLEANCHK);
988         if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
989                 info->error = 1;
990         } else if (vm_page_busy_try(p, FALSE) == 0) {
991                 vm_page_protect(p, VM_PROT_READ);       /* must not block */
992                 vm_page_wakeup(p);
993         } else {
994                 info->error = 1;
995         }
996         return(0);
997 }
998
999 /*
1000  * The caller must hold the object
1001  */
1002 static 
1003 int
1004 vm_object_page_clean_pass2(struct vm_page *p, void *data)
1005 {
1006         struct rb_vm_page_scan_info *info = data;
1007         int generation;
1008
1009         /*
1010          * Do not mess with pages that were inserted after we started
1011          * the cleaning pass.
1012          */
1013         if ((p->flags & PG_CLEANCHK) == 0)
1014                 return(0);
1015
1016         generation = info->object->generation;
1017         vm_page_busy_wait(p, TRUE, "vpcwai");
1018         if (p->object != info->object ||
1019             info->object->generation != generation) {
1020                 info->error = 1;
1021                 vm_page_wakeup(p);
1022                 return(0);
1023         }
1024
1025         /*
1026          * Before wasting time traversing the pmaps, check for trivial
1027          * cases where the page cannot be dirty.
1028          */
1029         if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) {
1030                 KKASSERT((p->dirty & p->valid) == 0);
1031                 vm_page_wakeup(p);
1032                 return(0);
1033         }
1034
1035         /*
1036          * Check whether the page is dirty or not.  The page has been set
1037          * to be read-only so the check will not race a user dirtying the
1038          * page.
1039          */
1040         vm_page_test_dirty(p);
1041         if ((p->dirty & p->valid) == 0) {
1042                 vm_page_flag_clear(p, PG_CLEANCHK);
1043                 vm_page_wakeup(p);
1044                 return(0);
1045         }
1046
1047         /*
1048          * If we have been asked to skip nosync pages and this is a
1049          * nosync page, skip it.  Note that the object flags were
1050          * not cleared in this case (because pass1 will have returned an
1051          * error), so we do not have to set them.
1052          */
1053         if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
1054                 vm_page_flag_clear(p, PG_CLEANCHK);
1055                 vm_page_wakeup(p);
1056                 return(0);
1057         }
1058
1059         /*
1060          * Flush as many pages as we can.  PG_CLEANCHK will be cleared on
1061          * the pages that get successfully flushed.  Set info->error if
1062          * we raced an object modification.
1063          */
1064         vm_object_page_collect_flush(info->object, p, info->pagerflags);
1065         return(0);
1066 }
1067
1068 /*
1069  * Collect the specified page and nearby pages and flush them out.
1070  * The number of pages flushed is returned.  The passed page is busied
1071  * by the caller and we are responsible for its disposition.
1072  *
1073  * The caller must hold the object.
1074  */
1075 static int
1076 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags)
1077 {
1078         int runlen;
1079         int error;
1080         int maxf;
1081         int chkb;
1082         int maxb;
1083         int i;
1084         vm_pindex_t pi;
1085         vm_page_t maf[vm_pageout_page_count];
1086         vm_page_t mab[vm_pageout_page_count];
1087         vm_page_t ma[vm_pageout_page_count];
1088
1089         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1090
1091         pi = p->pindex;
1092
1093         maxf = 0;
1094         for(i = 1; i < vm_pageout_page_count; i++) {
1095                 vm_page_t tp;
1096
1097                 tp = vm_page_lookup_busy_try(object, pi + i, TRUE, &error);
1098                 if (error)
1099                         break;
1100                 if (tp == NULL)
1101                         break;
1102                 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1103                     (tp->flags & PG_CLEANCHK) == 0) {
1104                         vm_page_wakeup(tp);
1105                         break;
1106                 }
1107                 if ((tp->queue - tp->pc) == PQ_CACHE) {
1108                         vm_page_flag_clear(tp, PG_CLEANCHK);
1109                         vm_page_wakeup(tp);
1110                         break;
1111                 }
1112                 vm_page_test_dirty(tp);
1113                 if ((tp->dirty & tp->valid) == 0) {
1114                         vm_page_flag_clear(tp, PG_CLEANCHK);
1115                         vm_page_wakeup(tp);
1116                         break;
1117                 }
1118                 maf[i - 1] = tp;
1119                 maxf++;
1120         }
1121
1122         maxb = 0;
1123         chkb = vm_pageout_page_count -  maxf;
1124         /*
1125          * NOTE: chkb can be 0
1126          */
1127         for(i = 1; chkb && i < chkb; i++) {
1128                 vm_page_t tp;
1129
1130                 tp = vm_page_lookup_busy_try(object, pi - i, TRUE, &error);
1131                 if (error)
1132                         break;
1133                 if (tp == NULL)
1134                         break;
1135                 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1136                     (tp->flags & PG_CLEANCHK) == 0) {
1137                         vm_page_wakeup(tp);
1138                         break;
1139                 }
1140                 if ((tp->queue - tp->pc) == PQ_CACHE) {
1141                         vm_page_flag_clear(tp, PG_CLEANCHK);
1142                         vm_page_wakeup(tp);
1143                         break;
1144                 }
1145                 vm_page_test_dirty(tp);
1146                 if ((tp->dirty & tp->valid) == 0) {
1147                         vm_page_flag_clear(tp, PG_CLEANCHK);
1148                         vm_page_wakeup(tp);
1149                         break;
1150                 }
1151                 mab[i - 1] = tp;
1152                 maxb++;
1153         }
1154
1155         /*
1156          * All pages in the maf[] and mab[] array are busied.
1157          */
1158         for (i = 0; i < maxb; i++) {
1159                 int index = (maxb - i) - 1;
1160                 ma[index] = mab[i];
1161                 vm_page_flag_clear(ma[index], PG_CLEANCHK);
1162         }
1163         vm_page_flag_clear(p, PG_CLEANCHK);
1164         ma[maxb] = p;
1165         for(i = 0; i < maxf; i++) {
1166                 int index = (maxb + i) + 1;
1167                 ma[index] = maf[i];
1168                 vm_page_flag_clear(ma[index], PG_CLEANCHK);
1169         }
1170         runlen = maxb + maxf + 1;
1171
1172         for (i = 0; i < runlen; i++)
1173                 vm_page_hold(ma[i]);
1174
1175         vm_pageout_flush(ma, runlen, pagerflags);
1176
1177         for (i = 0; i < runlen; i++) {
1178                 if (ma[i]->valid & ma[i]->dirty) {
1179                         vm_page_protect(ma[i], VM_PROT_READ);
1180                         vm_page_flag_set(ma[i], PG_CLEANCHK);
1181
1182                         /*
1183                          * maxf will end up being the actual number of pages
1184                          * we wrote out contiguously, non-inclusive of the
1185                          * first page.  We do not count look-behind pages.
1186                          */
1187                         if (i >= maxb + 1 && (maxf > i - maxb - 1))
1188                                 maxf = i - maxb - 1;
1189                 }
1190                 vm_page_unhold(ma[i]);
1191         }
1192         return(maxf + 1);
1193 }
1194
1195 /*
1196  * Same as vm_object_pmap_copy, except range checking really
1197  * works, and is meant for small sections of an object.
1198  *
1199  * This code protects resident pages by making them read-only
1200  * and is typically called on a fork or split when a page
1201  * is converted to copy-on-write.  
1202  *
1203  * NOTE: If the page is already at VM_PROT_NONE, calling
1204  * vm_page_protect will have no effect.
1205  */
1206 void
1207 vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1208 {
1209         vm_pindex_t idx;
1210         vm_page_t p;
1211
1212         if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
1213                 return;
1214
1215         vm_object_hold(object);
1216         for (idx = start; idx < end; idx++) {
1217                 p = vm_page_lookup(object, idx);
1218                 if (p == NULL)
1219                         continue;
1220                 vm_page_protect(p, VM_PROT_READ);
1221         }
1222         vm_object_drop(object);
1223 }
1224
1225 /*
1226  * Removes all physical pages in the specified object range from all
1227  * physical maps.
1228  *
1229  * The object must *not* be locked.
1230  */
1231
1232 static int vm_object_pmap_remove_callback(vm_page_t p, void *data);
1233
1234 void
1235 vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1236 {
1237         struct rb_vm_page_scan_info info;
1238
1239         if (object == NULL)
1240                 return;
1241         info.start_pindex = start;
1242         info.end_pindex = end - 1;
1243
1244         vm_object_hold(object);
1245         vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1246                                 vm_object_pmap_remove_callback, &info);
1247         if (start == 0 && end == object->size)
1248                 vm_object_clear_flag(object, OBJ_WRITEABLE);
1249         vm_object_drop(object);
1250 }
1251
1252 /*
1253  * The caller must hold the object
1254  */
1255 static int
1256 vm_object_pmap_remove_callback(vm_page_t p, void *data __unused)
1257 {
1258         vm_page_protect(p, VM_PROT_NONE);
1259         return(0);
1260 }
1261
1262 /*
1263  * Implements the madvise function at the object/page level.
1264  *
1265  * MADV_WILLNEED        (any object)
1266  *
1267  *      Activate the specified pages if they are resident.
1268  *
1269  * MADV_DONTNEED        (any object)
1270  *
1271  *      Deactivate the specified pages if they are resident.
1272  *
1273  * MADV_FREE    (OBJT_DEFAULT/OBJT_SWAP objects, OBJ_ONEMAPPING only)
1274  *
1275  *      Deactivate and clean the specified pages if they are
1276  *      resident.  This permits the process to reuse the pages
1277  *      without faulting or the kernel to reclaim the pages
1278  *      without I/O.
1279  *
1280  * No requirements.
1281  */
1282 void
1283 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise)
1284 {
1285         vm_pindex_t end, tpindex;
1286         vm_object_t tobject;
1287         vm_object_t xobj;
1288         vm_page_t m;
1289         int error;
1290
1291         if (object == NULL)
1292                 return;
1293
1294         end = pindex + count;
1295
1296         vm_object_hold(object);
1297         tobject = object;
1298
1299         /*
1300          * Locate and adjust resident pages
1301          */
1302         for (; pindex < end; pindex += 1) {
1303 relookup:
1304                 if (tobject != object)
1305                         vm_object_drop(tobject);
1306                 tobject = object;
1307                 tpindex = pindex;
1308 shadowlookup:
1309                 /*
1310                  * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
1311                  * and those pages must be OBJ_ONEMAPPING.
1312                  */
1313                 if (advise == MADV_FREE) {
1314                         if ((tobject->type != OBJT_DEFAULT &&
1315                              tobject->type != OBJT_SWAP) ||
1316                             (tobject->flags & OBJ_ONEMAPPING) == 0) {
1317                                 continue;
1318                         }
1319                 }
1320
1321                 m = vm_page_lookup_busy_try(tobject, tpindex, TRUE, &error);
1322
1323                 if (error) {
1324                         vm_page_sleep_busy(m, TRUE, "madvpo");
1325                         goto relookup;
1326                 }
1327                 if (m == NULL) {
1328                         /*
1329                          * There may be swap even if there is no backing page
1330                          */
1331                         if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1332                                 swap_pager_freespace(tobject, tpindex, 1);
1333
1334                         /*
1335                          * next object
1336                          */
1337                         while ((xobj = tobject->backing_object) != NULL) {
1338                                 KKASSERT(xobj != object);
1339                                 vm_object_hold(xobj);
1340                                 if (xobj == tobject->backing_object)
1341                                         break;
1342                                 vm_object_drop(xobj);
1343                         }
1344                         if (xobj == NULL)
1345                                 continue;
1346                         tpindex += OFF_TO_IDX(tobject->backing_object_offset);
1347                         if (tobject != object) {
1348                                 vm_object_lock_swap();
1349                                 vm_object_drop(tobject);
1350                         }
1351                         tobject = xobj;
1352                         goto shadowlookup;
1353                 }
1354
1355                 /*
1356                  * If the page is not in a normal active state, we skip it.
1357                  * If the page is not managed there are no page queues to
1358                  * mess with.  Things can break if we mess with pages in
1359                  * any of the below states.
1360                  */
1361                 if (
1362                     /*m->hold_count ||*/
1363                     m->wire_count ||
1364                     (m->flags & PG_UNMANAGED) ||
1365                     m->valid != VM_PAGE_BITS_ALL
1366                 ) {
1367                         vm_page_wakeup(m);
1368                         continue;
1369                 }
1370
1371                 /*
1372                  * Theoretically once a page is known not to be busy, an
1373                  * interrupt cannot come along and rip it out from under us.
1374                  */
1375
1376                 if (advise == MADV_WILLNEED) {
1377                         vm_page_activate(m);
1378                 } else if (advise == MADV_DONTNEED) {
1379                         vm_page_dontneed(m);
1380                 } else if (advise == MADV_FREE) {
1381                         /*
1382                          * Mark the page clean.  This will allow the page
1383                          * to be freed up by the system.  However, such pages
1384                          * are often reused quickly by malloc()/free()
1385                          * so we do not do anything that would cause
1386                          * a page fault if we can help it.
1387                          *
1388                          * Specifically, we do not try to actually free
1389                          * the page now nor do we try to put it in the
1390                          * cache (which would cause a page fault on reuse).
1391                          *
1392                          * But we do make the page is freeable as we
1393                          * can without actually taking the step of unmapping
1394                          * it.
1395                          */
1396                         pmap_clear_modify(m);
1397                         m->dirty = 0;
1398                         m->act_count = 0;
1399                         vm_page_dontneed(m);
1400                         if (tobject->type == OBJT_SWAP)
1401                                 swap_pager_freespace(tobject, tpindex, 1);
1402                 }
1403                 vm_page_wakeup(m);
1404         }       
1405         if (tobject != object)
1406                 vm_object_drop(tobject);
1407         vm_object_drop(object);
1408 }
1409
1410 /*
1411  * Create a new object which is backed by the specified existing object
1412  * range.  Replace the pointer and offset that was pointing at the existing
1413  * object with the pointer/offset for the new object.
1414  *
1415  * No other requirements.
1416  */
1417 void
1418 vm_object_shadow(vm_object_t *objectp, vm_ooffset_t *offset, vm_size_t length,
1419                  int addref)
1420 {
1421         vm_object_t source;
1422         vm_object_t result;
1423
1424         source = *objectp;
1425
1426         /*
1427          * Don't create the new object if the old object isn't shared.
1428          * We have to chain wait before adding the reference to avoid
1429          * racing a collapse or deallocation.
1430          *
1431          * Add the additional ref to source here to avoid racing a later
1432          * collapse or deallocation. Clear the ONEMAPPING flag whether
1433          * addref is TRUE or not in this case because the original object
1434          * will be shadowed.
1435          */
1436         if (source) {
1437                 vm_object_hold(source);
1438                 vm_object_chain_wait(source);
1439                 if (source->ref_count == 1 &&
1440                     source->handle == NULL &&
1441                     (source->type == OBJT_DEFAULT ||
1442                      source->type == OBJT_SWAP)) {
1443                         vm_object_drop(source);
1444                         if (addref) {
1445                                 vm_object_reference_locked(source);
1446                                 vm_object_clear_flag(source, OBJ_ONEMAPPING);
1447                         }
1448                         return;
1449                 }
1450                 vm_object_reference_locked(source);
1451                 vm_object_clear_flag(source, OBJ_ONEMAPPING);
1452         }
1453
1454         /*
1455          * Allocate a new object with the given length.  The new object
1456          * is returned referenced but we may have to add another one.
1457          * If we are adding a second reference we must clear OBJ_ONEMAPPING.
1458          * (typically because the caller is about to clone a vm_map_entry).
1459          *
1460          * The source object currently has an extra reference to prevent
1461          * collapses into it while we mess with its shadow list, which
1462          * we will remove later in this routine.
1463          */
1464         if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL)
1465                 panic("vm_object_shadow: no object for shadowing");
1466         vm_object_hold(result);
1467         if (addref) {
1468                 vm_object_reference_locked(result);
1469                 vm_object_clear_flag(result, OBJ_ONEMAPPING);
1470         }
1471
1472         /*
1473          * The new object shadows the source object.  Chain wait before
1474          * adjusting shadow_count or the shadow list to avoid races.
1475          *
1476          * Try to optimize the result object's page color when shadowing
1477          * in order to maintain page coloring consistency in the combined 
1478          * shadowed object.
1479          */
1480         KKASSERT(result->backing_object == NULL);
1481         result->backing_object = source;
1482         if (source) {
1483                 vm_object_chain_wait(source);
1484                 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
1485                 source->shadow_count++;
1486                 source->generation++;
1487 #ifdef SMP
1488                 /* cpu localization twist */
1489                 result->pg_color = (int)(intptr_t)curthread;
1490 #else
1491                 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) &
1492                                    PQ_L2_MASK;
1493 #endif
1494         }
1495
1496         /*
1497          * Adjust the return storage.  Drop the ref on source before
1498          * returning.
1499          */
1500         result->backing_object_offset = *offset;
1501         vm_object_drop(result);
1502         *offset = 0;
1503         if (source) {
1504                 vm_object_deallocate_locked(source);
1505                 vm_object_drop(source);
1506         }
1507
1508         /*
1509          * Return the new things
1510          */
1511         *objectp = result;
1512 }
1513
1514 #define OBSC_TEST_ALL_SHADOWED  0x0001
1515 #define OBSC_COLLAPSE_NOWAIT    0x0002
1516 #define OBSC_COLLAPSE_WAIT      0x0004
1517
1518 static int vm_object_backing_scan_callback(vm_page_t p, void *data);
1519
1520 /*
1521  * The caller must hold the object.
1522  */
1523 static __inline int
1524 vm_object_backing_scan(vm_object_t object, vm_object_t backing_object, int op)
1525 {
1526         struct rb_vm_page_scan_info info;
1527
1528         vm_object_assert_held(object);
1529         vm_object_assert_held(backing_object);
1530
1531         KKASSERT(backing_object == object->backing_object);
1532         info.backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1533
1534         /*
1535          * Initial conditions
1536          */
1537         if (op & OBSC_TEST_ALL_SHADOWED) {
1538                 /*
1539                  * We do not want to have to test for the existence of
1540                  * swap pages in the backing object.  XXX but with the
1541                  * new swapper this would be pretty easy to do.
1542                  *
1543                  * XXX what about anonymous MAP_SHARED memory that hasn't
1544                  * been ZFOD faulted yet?  If we do not test for this, the
1545                  * shadow test may succeed! XXX
1546                  */
1547                 if (backing_object->type != OBJT_DEFAULT)
1548                         return(0);
1549         }
1550         if (op & OBSC_COLLAPSE_WAIT) {
1551                 KKASSERT((backing_object->flags & OBJ_DEAD) == 0);
1552                 vm_object_set_flag(backing_object, OBJ_DEAD);
1553                 lwkt_gettoken(&vmobj_token);
1554                 TAILQ_REMOVE(&vm_object_list, backing_object, object_list);
1555                 vm_object_count--;
1556                 lwkt_reltoken(&vmobj_token);
1557                 vm_object_dead_wakeup(backing_object);
1558         }
1559
1560         /*
1561          * Our scan.   We have to retry if a negative error code is returned,
1562          * otherwise 0 or 1 will be returned in info.error.  0 Indicates that
1563          * the scan had to be stopped because the parent does not completely
1564          * shadow the child.
1565          */
1566         info.object = object;
1567         info.backing_object = backing_object;
1568         info.limit = op;
1569         do {
1570                 info.error = 1;
1571                 vm_page_rb_tree_RB_SCAN(&backing_object->rb_memq, NULL,
1572                                         vm_object_backing_scan_callback,
1573                                         &info);
1574         } while (info.error < 0);
1575
1576         return(info.error);
1577 }
1578
1579 /*
1580  * The caller must hold the object.
1581  */
1582 static int
1583 vm_object_backing_scan_callback(vm_page_t p, void *data)
1584 {
1585         struct rb_vm_page_scan_info *info = data;
1586         vm_object_t backing_object;
1587         vm_object_t object;
1588         vm_pindex_t new_pindex;
1589         vm_pindex_t backing_offset_index;
1590         int op;
1591
1592         new_pindex = p->pindex - info->backing_offset_index;
1593         op = info->limit;
1594         object = info->object;
1595         backing_object = info->backing_object;
1596         backing_offset_index = info->backing_offset_index;
1597
1598         if (op & OBSC_TEST_ALL_SHADOWED) {
1599                 vm_page_t pp;
1600
1601                 /*
1602                  * Ignore pages outside the parent object's range
1603                  * and outside the parent object's mapping of the 
1604                  * backing object.
1605                  *
1606                  * note that we do not busy the backing object's
1607                  * page.
1608                  */
1609                 if (
1610                     p->pindex < backing_offset_index ||
1611                     new_pindex >= object->size
1612                 ) {
1613                         return(0);
1614                 }
1615
1616                 /*
1617                  * See if the parent has the page or if the parent's
1618                  * object pager has the page.  If the parent has the
1619                  * page but the page is not valid, the parent's
1620                  * object pager must have the page.
1621                  *
1622                  * If this fails, the parent does not completely shadow
1623                  * the object and we might as well give up now.
1624                  */
1625
1626                 pp = vm_page_lookup(object, new_pindex);
1627                 if ((pp == NULL || pp->valid == 0) &&
1628                     !vm_pager_has_page(object, new_pindex)
1629                 ) {
1630                         info->error = 0;        /* problemo */
1631                         return(-1);             /* stop the scan */
1632                 }
1633         }
1634
1635         /*
1636          * Check for busy page
1637          */
1638         if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
1639                 vm_page_t pp;
1640
1641                 if (vm_page_busy_try(p, TRUE)) {
1642                         if (op & OBSC_COLLAPSE_NOWAIT) {
1643                                 return(0);
1644                         } else {
1645                                 /*
1646                                  * If we slept, anything could have
1647                                  * happened.   Ask that the scan be restarted.
1648                                  *
1649                                  * Since the object is marked dead, the
1650                                  * backing offset should not have changed.  
1651                                  */
1652                                 vm_page_sleep_busy(p, TRUE, "vmocol");
1653                                 info->error = -1;
1654                                 return(-1);
1655                         }
1656                 }
1657                 if (op & OBSC_COLLAPSE_NOWAIT) {
1658                         if (p->valid == 0 /*|| p->hold_count*/ ||
1659                             p->wire_count) {
1660                                 vm_page_wakeup(p);
1661                                 return(0);
1662                         }
1663                 } else {
1664                         /* XXX what if p->valid == 0 , hold_count, etc? */
1665                 }
1666
1667                 KASSERT(
1668                     p->object == backing_object,
1669                     ("vm_object_qcollapse(): object mismatch")
1670                 );
1671
1672                 /*
1673                  * Destroy any associated swap
1674                  */
1675                 if (backing_object->type == OBJT_SWAP)
1676                         swap_pager_freespace(backing_object, p->pindex, 1);
1677
1678                 if (
1679                     p->pindex < backing_offset_index ||
1680                     new_pindex >= object->size
1681                 ) {
1682                         /*
1683                          * Page is out of the parent object's range, we 
1684                          * can simply destroy it. 
1685                          */
1686                         vm_page_protect(p, VM_PROT_NONE);
1687                         vm_page_free(p);
1688                         return(0);
1689                 }
1690
1691                 pp = vm_page_lookup(object, new_pindex);
1692                 if (pp != NULL || vm_pager_has_page(object, new_pindex)) {
1693                         /*
1694                          * page already exists in parent OR swap exists
1695                          * for this location in the parent.  Destroy 
1696                          * the original page from the backing object.
1697                          *
1698                          * Leave the parent's page alone
1699                          */
1700                         vm_page_protect(p, VM_PROT_NONE);
1701                         vm_page_free(p);
1702                         return(0);
1703                 }
1704
1705                 /*
1706                  * Page does not exist in parent, rename the
1707                  * page from the backing object to the main object. 
1708                  *
1709                  * If the page was mapped to a process, it can remain 
1710                  * mapped through the rename.
1711                  */
1712                 if ((p->queue - p->pc) == PQ_CACHE)
1713                         vm_page_deactivate(p);
1714
1715                 vm_page_rename(p, object, new_pindex);
1716                 vm_page_wakeup(p);
1717                 /* page automatically made dirty by rename */
1718         }
1719         return(0);
1720 }
1721
1722 /*
1723  * This version of collapse allows the operation to occur earlier and
1724  * when paging_in_progress is true for an object...  This is not a complete
1725  * operation, but should plug 99.9% of the rest of the leaks.
1726  *
1727  * The caller must hold the object and backing_object and both must be
1728  * chainlocked.
1729  *
1730  * (only called from vm_object_collapse)
1731  */
1732 static void
1733 vm_object_qcollapse(vm_object_t object, vm_object_t backing_object)
1734 {
1735         if (backing_object->ref_count == 1) {
1736                 backing_object->ref_count += 2;
1737                 vm_object_backing_scan(object, backing_object,
1738                                        OBSC_COLLAPSE_NOWAIT);
1739                 backing_object->ref_count -= 2;
1740         }
1741 }
1742
1743 /*
1744  * Collapse an object with the object backing it.  Pages in the backing
1745  * object are moved into the parent, and the backing object is deallocated.
1746  *
1747  * object must be held and chain-locked on call.
1748  *
1749  * The caller must have an extra ref on object to prevent a race from
1750  * destroying it during the collapse.
1751  */
1752 void
1753 vm_object_collapse(vm_object_t object)
1754 {
1755         vm_object_t backing_object;
1756
1757         /*
1758          * Only one thread is attempting a collapse at any given moment.
1759          * There are few restrictions for (object) that callers of this
1760          * function check so reentrancy is likely.
1761          */
1762         KKASSERT(object != NULL);
1763         vm_object_assert_held(object);
1764         KKASSERT(object->flags & OBJ_CHAINLOCK);
1765
1766         for (;;) {
1767                 vm_object_t bbobj;
1768                 int dodealloc;
1769
1770                 /*
1771                  * We have to hold the backing object, check races.
1772                  */
1773                 while ((backing_object = object->backing_object) != NULL) {
1774                         vm_object_hold(backing_object);
1775                         if (backing_object == object->backing_object)
1776                                 break;
1777                         vm_object_drop(backing_object);
1778                 }
1779
1780                 /*
1781                  * No backing object?  Nothing to collapse then.
1782                  */
1783                 if (backing_object == NULL)
1784                         break;
1785
1786                 /*
1787                  * You can't collapse with a non-default/non-swap object.
1788                  */
1789                 if (backing_object->type != OBJT_DEFAULT &&
1790                     backing_object->type != OBJT_SWAP) {
1791                         vm_object_drop(backing_object);
1792                         backing_object = NULL;
1793                         break;
1794                 }
1795
1796                 /*
1797                  * Chain-lock the backing object too because if we
1798                  * successfully merge its pages into the top object we
1799                  * will collapse backing_object->backing_object as the
1800                  * new backing_object.  Re-check that it is still our
1801                  * backing object.
1802                  */
1803                 vm_object_chain_acquire(backing_object);
1804                 if (backing_object != object->backing_object) {
1805                         vm_object_chain_release(backing_object);
1806                         vm_object_drop(backing_object);
1807                         continue;
1808                 }
1809
1810                 /*
1811                  * we check the backing object first, because it is most likely
1812                  * not collapsable.
1813                  */
1814                 if (backing_object->handle != NULL ||
1815                     (backing_object->type != OBJT_DEFAULT &&
1816                      backing_object->type != OBJT_SWAP) ||
1817                     (backing_object->flags & OBJ_DEAD) ||
1818                     object->handle != NULL ||
1819                     (object->type != OBJT_DEFAULT &&
1820                      object->type != OBJT_SWAP) ||
1821                     (object->flags & OBJ_DEAD)) {
1822                         break;
1823                 }
1824
1825                 /*
1826                  * If paging is in progress we can't do a normal collapse.
1827                  */
1828                 if (
1829                     object->paging_in_progress != 0 ||
1830                     backing_object->paging_in_progress != 0
1831                 ) {
1832                         vm_object_qcollapse(object, backing_object);
1833                         break;
1834                 }
1835
1836                 /*
1837                  * We know that we can either collapse the backing object (if
1838                  * the parent is the only reference to it) or (perhaps) have
1839                  * the parent bypass the object if the parent happens to shadow
1840                  * all the resident pages in the entire backing object.
1841                  *
1842                  * This is ignoring pager-backed pages such as swap pages.
1843                  * vm_object_backing_scan fails the shadowing test in this
1844                  * case.
1845                  */
1846                 if (backing_object->ref_count == 1) {
1847                         /*
1848                          * If there is exactly one reference to the backing
1849                          * object, we can collapse it into the parent.  
1850                          */
1851                         KKASSERT(object->backing_object == backing_object);
1852                         vm_object_backing_scan(object, backing_object,
1853                                                OBSC_COLLAPSE_WAIT);
1854
1855                         /*
1856                          * Move the pager from backing_object to object.
1857                          */
1858                         if (backing_object->type == OBJT_SWAP) {
1859                                 vm_object_pip_add(backing_object, 1);
1860
1861                                 /*
1862                                  * scrap the paging_offset junk and do a 
1863                                  * discrete copy.  This also removes major 
1864                                  * assumptions about how the swap-pager 
1865                                  * works from where it doesn't belong.  The
1866                                  * new swapper is able to optimize the
1867                                  * destroy-source case.
1868                                  */
1869                                 vm_object_pip_add(object, 1);
1870                                 swap_pager_copy(backing_object, object,
1871                                     OFF_TO_IDX(object->backing_object_offset),
1872                                     TRUE);
1873                                 vm_object_pip_wakeup(object);
1874                                 vm_object_pip_wakeup(backing_object);
1875                         }
1876
1877                         /*
1878                          * Object now shadows whatever backing_object did.
1879                          * Remove object from backing_object's shadow_list.
1880                          *
1881                          * NOTE: backing_object->backing_object moves from
1882                          *       within backing_object to within object.
1883                          */
1884                         LIST_REMOVE(object, shadow_list);
1885                         KKASSERT(object->backing_object == backing_object);
1886                         backing_object->shadow_count--;
1887                         backing_object->generation++;
1888
1889                         while ((bbobj = backing_object->backing_object) != NULL) {
1890                                 vm_object_hold(bbobj);
1891                                 if (bbobj == backing_object->backing_object)
1892                                         break;
1893                                 vm_object_drop(bbobj);
1894                         }
1895                         if (bbobj) {
1896                                 LIST_REMOVE(backing_object, shadow_list);
1897                                 bbobj->shadow_count--;
1898                                 bbobj->generation++;
1899                         }
1900                         object->backing_object = bbobj;
1901                         if (bbobj) {
1902                                 LIST_INSERT_HEAD(&bbobj->shadow_head,
1903                                                  object, shadow_list);
1904                                 bbobj->shadow_count++;
1905                                 bbobj->generation++;
1906                         }
1907
1908                         object->backing_object_offset +=
1909                                 backing_object->backing_object_offset;
1910
1911                         vm_object_drop(bbobj);
1912
1913                         /*
1914                          * Discard the old backing_object.  Nothing should be
1915                          * able to ref it, other than a vm_map_split(),
1916                          * and vm_map_split() will stall on our chain lock.
1917                          * And we control the parent so it shouldn't be
1918                          * possible for it to go away either.
1919                          *
1920                          * Since the backing object has no pages, no pager
1921                          * left, and no object references within it, all
1922                          * that is necessary is to dispose of it.
1923                          */
1924                         KASSERT(backing_object->ref_count == 1,
1925                                 ("backing_object %p was somehow "
1926                                  "re-referenced during collapse!",
1927                                  backing_object));
1928                         KASSERT(RB_EMPTY(&backing_object->rb_memq),
1929                                 ("backing_object %p somehow has left "
1930                                  "over pages during collapse!",
1931                                  backing_object));
1932
1933                         /*
1934                          * The object can be destroyed.
1935                          *
1936                          * XXX just fall through and dodealloc instead
1937                          *     of forcing destruction?
1938                          */
1939                         --backing_object->ref_count;
1940                         if ((backing_object->flags & OBJ_DEAD) == 0)
1941                                 vm_object_terminate(backing_object);
1942                         object_collapses++;
1943                         dodealloc = 1;
1944                         dodealloc = 0;
1945                 } else {
1946                         /*
1947                          * If we do not entirely shadow the backing object,
1948                          * there is nothing we can do so we give up.
1949                          */
1950                         if (vm_object_backing_scan(object, backing_object,
1951                                                 OBSC_TEST_ALL_SHADOWED) == 0) {
1952                                 break;
1953                         }
1954
1955                         while ((bbobj = backing_object->backing_object) != NULL) {
1956                                 vm_object_hold(bbobj);
1957                                 if (bbobj == backing_object->backing_object)
1958                                         break;
1959                                 vm_object_drop(bbobj);
1960                         }
1961                         /*
1962                          * Make the parent shadow the next object in the
1963                          * chain.  Remove object from backing_object's
1964                          * shadow list.
1965                          *
1966                          * Deallocating backing_object will not remove
1967                          * it, since its reference count is at least 2.
1968                          */
1969                         KKASSERT(object->backing_object == backing_object);
1970                         LIST_REMOVE(object, shadow_list);
1971                         backing_object->shadow_count--;
1972                         backing_object->generation++;
1973
1974                         /*
1975                          * Add a ref to bbobj
1976                          */
1977                         if (bbobj) {
1978                                 vm_object_chain_wait(bbobj);
1979                                 vm_object_reference_locked(bbobj);
1980                                 LIST_INSERT_HEAD(&bbobj->shadow_head,
1981                                                  object, shadow_list);
1982                                 bbobj->shadow_count++;
1983                                 bbobj->generation++;
1984                                 object->backing_object_offset +=
1985                                         backing_object->backing_object_offset;
1986                                 object->backing_object = bbobj;
1987                                 vm_object_drop(bbobj);
1988                         } else {
1989                                 object->backing_object = NULL;
1990                         }
1991
1992                         /*
1993                          * Drop the reference count on backing_object.  To
1994                          * handle ref_count races properly we can't assume
1995                          * that the ref_count is still at least 2 so we
1996                          * have to actually call vm_object_deallocate()
1997                          * (after clearing the chainlock).
1998                          */
1999                         object_bypasses++;
2000                         dodealloc = 1;
2001                 }
2002
2003                 /*
2004                  * Clean up the original backing_object and try again with
2005                  * this object's new backing object (loop).
2006                  */
2007                 vm_object_chain_release(backing_object);
2008
2009                 /*
2010                  * The backing_object was
2011                  */
2012                 if (dodealloc)
2013                         vm_object_deallocate_locked(backing_object);
2014                 vm_object_drop(backing_object);
2015                 /* loop */
2016         }
2017
2018         /*
2019          * Clean up any left over backing_object
2020          */
2021         if (backing_object) {
2022 #if 1
2023                 vm_object_chain_release(backing_object);
2024 #endif
2025                 vm_object_drop(backing_object);
2026         }
2027 }
2028
2029 /*
2030  * Removes all physical pages in the specified object range from the
2031  * object's list of pages.
2032  *
2033  * No requirements.
2034  */
2035 static int vm_object_page_remove_callback(vm_page_t p, void *data);
2036
2037 void
2038 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
2039                       boolean_t clean_only)
2040 {
2041         struct rb_vm_page_scan_info info;
2042         int all;
2043
2044         /*
2045          * Degenerate cases and assertions
2046          */
2047         vm_object_hold(object);
2048         if (object == NULL ||
2049             (object->resident_page_count == 0 && object->swblock_count == 0)) {
2050                 vm_object_drop(object);
2051                 return;
2052         }
2053         KASSERT(object->type != OBJT_PHYS, 
2054                 ("attempt to remove pages from a physical object"));
2055
2056         /*
2057          * Indicate that paging is occuring on the object
2058          */
2059         vm_object_pip_add(object, 1);
2060
2061         /*
2062          * Figure out the actual removal range and whether we are removing
2063          * the entire contents of the object or not.  If removing the entire
2064          * contents, be sure to get all pages, even those that might be 
2065          * beyond the end of the object.
2066          */
2067         info.start_pindex = start;
2068         if (end == 0)
2069                 info.end_pindex = (vm_pindex_t)-1;
2070         else
2071                 info.end_pindex = end - 1;
2072         info.limit = clean_only;
2073         all = (start == 0 && info.end_pindex >= object->size - 1);
2074
2075         /*
2076          * Loop until we are sure we have gotten them all.
2077          */
2078         do {
2079                 info.error = 0;
2080                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
2081                                         vm_object_page_remove_callback, &info);
2082         } while (info.error);
2083
2084         /*
2085          * Remove any related swap if throwing away pages, or for
2086          * non-swap objects (the swap is a clean copy in that case).
2087          */
2088         if (object->type != OBJT_SWAP || clean_only == FALSE) {
2089                 if (all)
2090                         swap_pager_freespace_all(object);
2091                 else
2092                         swap_pager_freespace(object, info.start_pindex,
2093                              info.end_pindex - info.start_pindex + 1);
2094         }
2095
2096         /*
2097          * Cleanup
2098          */
2099         vm_object_pip_wakeup(object);
2100         vm_object_drop(object);
2101 }
2102
2103 /*
2104  * The caller must hold the object
2105  */
2106 static int
2107 vm_object_page_remove_callback(vm_page_t p, void *data)
2108 {
2109         struct rb_vm_page_scan_info *info = data;
2110
2111         if (vm_page_busy_try(p, TRUE)) {
2112                 vm_page_sleep_busy(p, TRUE, "vmopar");
2113                 info->error = 1;
2114                 return(0);
2115         }
2116
2117         /*
2118          * Wired pages cannot be destroyed, but they can be invalidated
2119          * and we do so if clean_only (limit) is not set.
2120          *
2121          * WARNING!  The page may be wired due to being part of a buffer
2122          *           cache buffer, and the buffer might be marked B_CACHE.
2123          *           This is fine as part of a truncation but VFSs must be
2124          *           sure to fix the buffer up when re-extending the file.
2125          */
2126         if (p->wire_count != 0) {
2127                 vm_page_protect(p, VM_PROT_NONE);
2128                 if (info->limit == 0)
2129                         p->valid = 0;
2130                 vm_page_wakeup(p);
2131                 return(0);
2132         }
2133
2134         /*
2135          * limit is our clean_only flag.  If set and the page is dirty, do
2136          * not free it.  If set and the page is being held by someone, do
2137          * not free it.
2138          */
2139         if (info->limit && p->valid) {
2140                 vm_page_test_dirty(p);
2141                 if (p->valid & p->dirty) {
2142                         vm_page_wakeup(p);
2143                         return(0);
2144                 }
2145 #if 0
2146                 if (p->hold_count) {
2147                         vm_page_wakeup(p);
2148                         return(0);
2149                 }
2150 #endif
2151         }
2152
2153         /*
2154          * Destroy the page
2155          */
2156         vm_page_protect(p, VM_PROT_NONE);
2157         vm_page_free(p);
2158         return(0);
2159 }
2160
2161 /*
2162  * Coalesces two objects backing up adjoining regions of memory into a
2163  * single object.
2164  *
2165  * returns TRUE if objects were combined.
2166  *
2167  * NOTE: Only works at the moment if the second object is NULL -
2168  *       if it's not, which object do we lock first?
2169  *
2170  * Parameters:
2171  *      prev_object     First object to coalesce
2172  *      prev_offset     Offset into prev_object
2173  *      next_object     Second object into coalesce
2174  *      next_offset     Offset into next_object
2175  *
2176  *      prev_size       Size of reference to prev_object
2177  *      next_size       Size of reference to next_object
2178  *
2179  * The caller does not need to hold (prev_object) but must have a stable
2180  * pointer to it (typically by holding the vm_map locked).
2181  */
2182 boolean_t
2183 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
2184                    vm_size_t prev_size, vm_size_t next_size)
2185 {
2186         vm_pindex_t next_pindex;
2187
2188         if (prev_object == NULL)
2189                 return (TRUE);
2190
2191         vm_object_hold(prev_object);
2192
2193         if (prev_object->type != OBJT_DEFAULT &&
2194             prev_object->type != OBJT_SWAP) {
2195                 vm_object_drop(prev_object);
2196                 return (FALSE);
2197         }
2198
2199         /*
2200          * Try to collapse the object first
2201          */
2202         vm_object_chain_acquire(prev_object);
2203         vm_object_collapse(prev_object);
2204
2205         /*
2206          * Can't coalesce if: . more than one reference . paged out . shadows
2207          * another object . has a copy elsewhere (any of which mean that the
2208          * pages not mapped to prev_entry may be in use anyway)
2209          */
2210
2211         if (prev_object->backing_object != NULL) {
2212                 vm_object_chain_release(prev_object);
2213                 vm_object_drop(prev_object);
2214                 return (FALSE);
2215         }
2216
2217         prev_size >>= PAGE_SHIFT;
2218         next_size >>= PAGE_SHIFT;
2219         next_pindex = prev_pindex + prev_size;
2220
2221         if ((prev_object->ref_count > 1) &&
2222             (prev_object->size != next_pindex)) {
2223                 vm_object_chain_release(prev_object);
2224                 vm_object_drop(prev_object);
2225                 return (FALSE);
2226         }
2227
2228         /*
2229          * Remove any pages that may still be in the object from a previous
2230          * deallocation.
2231          */
2232         if (next_pindex < prev_object->size) {
2233                 vm_object_page_remove(prev_object,
2234                                       next_pindex,
2235                                       next_pindex + next_size, FALSE);
2236                 if (prev_object->type == OBJT_SWAP)
2237                         swap_pager_freespace(prev_object,
2238                                              next_pindex, next_size);
2239         }
2240
2241         /*
2242          * Extend the object if necessary.
2243          */
2244         if (next_pindex + next_size > prev_object->size)
2245                 prev_object->size = next_pindex + next_size;
2246
2247         vm_object_chain_release(prev_object);
2248         vm_object_drop(prev_object);
2249         return (TRUE);
2250 }
2251
2252 /*
2253  * Make the object writable and flag is being possibly dirty.
2254  *
2255  * The caller must hold the object. XXX called from vm_page_dirty(),
2256  * There is currently no requirement to hold the object.
2257  */
2258 void
2259 vm_object_set_writeable_dirty(vm_object_t object)
2260 {
2261         struct vnode *vp;
2262
2263         /*vm_object_assert_held(object);*/
2264         vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
2265         if (object->type == OBJT_VNODE &&
2266             (vp = (struct vnode *)object->handle) != NULL) {
2267                 if ((vp->v_flag & VOBJDIRTY) == 0) {
2268                         vsetflags(vp, VOBJDIRTY);
2269                 }
2270         }
2271 }
2272
2273 #include "opt_ddb.h"
2274 #ifdef DDB
2275 #include <sys/kernel.h>
2276
2277 #include <sys/cons.h>
2278
2279 #include <ddb/ddb.h>
2280
2281 static int      _vm_object_in_map (vm_map_t map, vm_object_t object,
2282                                        vm_map_entry_t entry);
2283 static int      vm_object_in_map (vm_object_t object);
2284
2285 /*
2286  * The caller must hold the object.
2287  */
2288 static int
2289 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2290 {
2291         vm_map_t tmpm;
2292         vm_map_entry_t tmpe;
2293         vm_object_t obj, nobj;
2294         int entcount;
2295
2296         if (map == 0)
2297                 return 0;
2298         if (entry == 0) {
2299                 tmpe = map->header.next;
2300                 entcount = map->nentries;
2301                 while (entcount-- && (tmpe != &map->header)) {
2302                         if( _vm_object_in_map(map, object, tmpe)) {
2303                                 return 1;
2304                         }
2305                         tmpe = tmpe->next;
2306                 }
2307                 return (0);
2308         }
2309         switch(entry->maptype) {
2310         case VM_MAPTYPE_SUBMAP:
2311                 tmpm = entry->object.sub_map;
2312                 tmpe = tmpm->header.next;
2313                 entcount = tmpm->nentries;
2314                 while (entcount-- && tmpe != &tmpm->header) {
2315                         if( _vm_object_in_map(tmpm, object, tmpe)) {
2316                                 return 1;
2317                         }
2318                         tmpe = tmpe->next;
2319                 }
2320                 break;
2321         case VM_MAPTYPE_NORMAL:
2322         case VM_MAPTYPE_VPAGETABLE:
2323                 obj = entry->object.vm_object;
2324                 while (obj) {
2325                         if (obj == object) {
2326                                 if (obj != entry->object.vm_object)
2327                                         vm_object_drop(obj);
2328                                 return 1;
2329                         }
2330                         while ((nobj = obj->backing_object) != NULL) {
2331                                 vm_object_hold(nobj);
2332                                 if (nobj == obj->backing_object)
2333                                         break;
2334                                 vm_object_drop(nobj);
2335                         }
2336                         if (obj != entry->object.vm_object) {
2337                                 if (nobj)
2338                                         vm_object_lock_swap();
2339                                 vm_object_drop(obj);
2340                         }
2341                         obj = nobj;
2342                 }
2343                 break;
2344         default:
2345                 break;
2346         }
2347         return 0;
2348 }
2349
2350 static int vm_object_in_map_callback(struct proc *p, void *data);
2351
2352 struct vm_object_in_map_info {
2353         vm_object_t object;
2354         int rv;
2355 };
2356
2357 /*
2358  * Debugging only
2359  */
2360 static int
2361 vm_object_in_map(vm_object_t object)
2362 {
2363         struct vm_object_in_map_info info;
2364
2365         info.rv = 0;
2366         info.object = object;
2367
2368         allproc_scan(vm_object_in_map_callback, &info);
2369         if (info.rv)
2370                 return 1;
2371         if( _vm_object_in_map(&kernel_map, object, 0))
2372                 return 1;
2373         if( _vm_object_in_map(&pager_map, object, 0))
2374                 return 1;
2375         if( _vm_object_in_map(&buffer_map, object, 0))
2376                 return 1;
2377         return 0;
2378 }
2379
2380 /*
2381  * Debugging only
2382  */
2383 static int
2384 vm_object_in_map_callback(struct proc *p, void *data)
2385 {
2386         struct vm_object_in_map_info *info = data;
2387
2388         if (p->p_vmspace) {
2389                 if (_vm_object_in_map(&p->p_vmspace->vm_map, info->object, 0)) {
2390                         info->rv = 1;
2391                         return -1;
2392                 }
2393         }
2394         return (0);
2395 }
2396
2397 DB_SHOW_COMMAND(vmochk, vm_object_check)
2398 {
2399         vm_object_t object;
2400
2401         /*
2402          * make sure that internal objs are in a map somewhere
2403          * and none have zero ref counts.
2404          */
2405         for (object = TAILQ_FIRST(&vm_object_list);
2406                         object != NULL;
2407                         object = TAILQ_NEXT(object, object_list)) {
2408                 if (object->type == OBJT_MARKER)
2409                         continue;
2410                 if (object->handle == NULL &&
2411                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
2412                         if (object->ref_count == 0) {
2413                                 db_printf("vmochk: internal obj has zero ref count: %ld\n",
2414                                         (long)object->size);
2415                         }
2416                         if (!vm_object_in_map(object)) {
2417                                 db_printf(
2418                         "vmochk: internal obj is not in a map: "
2419                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2420                                     object->ref_count, (u_long)object->size, 
2421                                     (u_long)object->size,
2422                                     (void *)object->backing_object);
2423                         }
2424                 }
2425         }
2426 }
2427
2428 /*
2429  * Debugging only
2430  */
2431 DB_SHOW_COMMAND(object, vm_object_print_static)
2432 {
2433         /* XXX convert args. */
2434         vm_object_t object = (vm_object_t)addr;
2435         boolean_t full = have_addr;
2436
2437         vm_page_t p;
2438
2439         /* XXX count is an (unused) arg.  Avoid shadowing it. */
2440 #define count   was_count
2441
2442         int count;
2443
2444         if (object == NULL)
2445                 return;
2446
2447         db_iprintf(
2448             "Object %p: type=%d, size=0x%lx, res=%d, ref=%d, flags=0x%x\n",
2449             object, (int)object->type, (u_long)object->size,
2450             object->resident_page_count, object->ref_count, object->flags);
2451         /*
2452          * XXX no %qd in kernel.  Truncate object->backing_object_offset.
2453          */
2454         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n",
2455             object->shadow_count, 
2456             object->backing_object ? object->backing_object->ref_count : 0,
2457             object->backing_object, (long)object->backing_object_offset);
2458
2459         if (!full)
2460                 return;
2461
2462         db_indent += 2;
2463         count = 0;
2464         RB_FOREACH(p, vm_page_rb_tree, &object->rb_memq) {
2465                 if (count == 0)
2466                         db_iprintf("memory:=");
2467                 else if (count == 6) {
2468                         db_printf("\n");
2469                         db_iprintf(" ...");
2470                         count = 0;
2471                 } else
2472                         db_printf(",");
2473                 count++;
2474
2475                 db_printf("(off=0x%lx,page=0x%lx)",
2476                     (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
2477         }
2478         if (count != 0)
2479                 db_printf("\n");
2480         db_indent -= 2;
2481 }
2482
2483 /* XXX. */
2484 #undef count
2485
2486 /*
2487  * XXX need this non-static entry for calling from vm_map_print.
2488  *
2489  * Debugging only
2490  */
2491 void
2492 vm_object_print(/* db_expr_t */ long addr,
2493                 boolean_t have_addr,
2494                 /* db_expr_t */ long count,
2495                 char *modif)
2496 {
2497         vm_object_print_static(addr, have_addr, count, modif);
2498 }
2499
2500 /*
2501  * Debugging only
2502  */
2503 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2504 {
2505         vm_object_t object;
2506         int nl = 0;
2507         int c;
2508         for (object = TAILQ_FIRST(&vm_object_list);
2509                         object != NULL;
2510                         object = TAILQ_NEXT(object, object_list)) {
2511                 vm_pindex_t idx, fidx;
2512                 vm_pindex_t osize;
2513                 vm_paddr_t pa = -1, padiff;
2514                 int rcount;
2515                 vm_page_t m;
2516
2517                 if (object->type == OBJT_MARKER)
2518                         continue;
2519                 db_printf("new object: %p\n", (void *)object);
2520                 if ( nl > 18) {
2521                         c = cngetc();
2522                         if (c != ' ')
2523                                 return;
2524                         nl = 0;
2525                 }
2526                 nl++;
2527                 rcount = 0;
2528                 fidx = 0;
2529                 osize = object->size;
2530                 if (osize > 128)
2531                         osize = 128;
2532                 for (idx = 0; idx < osize; idx++) {
2533                         m = vm_page_lookup(object, idx);
2534                         if (m == NULL) {
2535                                 if (rcount) {
2536                                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2537                                                 (long)fidx, rcount, (long)pa);
2538                                         if ( nl > 18) {
2539                                                 c = cngetc();
2540                                                 if (c != ' ')
2541                                                         return;
2542                                                 nl = 0;
2543                                         }
2544                                         nl++;
2545                                         rcount = 0;
2546                                 }
2547                                 continue;
2548                         }
2549
2550                                 
2551                         if (rcount &&
2552                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2553                                 ++rcount;
2554                                 continue;
2555                         }
2556                         if (rcount) {
2557                                 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
2558                                 padiff >>= PAGE_SHIFT;
2559                                 padiff &= PQ_L2_MASK;
2560                                 if (padiff == 0) {
2561                                         pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
2562                                         ++rcount;
2563                                         continue;
2564                                 }
2565                                 db_printf(" index(%ld)run(%d)pa(0x%lx)",
2566                                         (long)fidx, rcount, (long)pa);
2567                                 db_printf("pd(%ld)\n", (long)padiff);
2568                                 if ( nl > 18) {
2569                                         c = cngetc();
2570                                         if (c != ' ')
2571                                                 return;
2572                                         nl = 0;
2573                                 }
2574                                 nl++;
2575                         }
2576                         fidx = idx;
2577                         pa = VM_PAGE_TO_PHYS(m);
2578                         rcount = 1;
2579                 }
2580                 if (rcount) {
2581                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2582                                 (long)fidx, rcount, (long)pa);
2583                         if ( nl > 18) {
2584                                 c = cngetc();
2585                                 if (c != ' ')
2586                                         return;
2587                                 nl = 0;
2588                         }
2589                         nl++;
2590                 }
2591         }
2592 }
2593 #endif /* DDB */