263c977252fd8074a9afb7aff2459bf6a703c99f
[dragonfly.git] / sys / vm / vm_object.c
1 /*
2  * Copyright (c) 1991, 1993, 2013
3  *      The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. Neither the name of the University nor the names of its contributors
17  *    may be used to endorse or promote products derived from this software
18  *    without specific prior written permission.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  *
32  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
33  *
34  *
35  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36  * All rights reserved.
37  *
38  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39  *
40  * Permission to use, copy, modify and distribute this software and
41  * its documentation is hereby granted, provided that both the copyright
42  * notice and this permission notice appear in all copies of the
43  * software, derivative works or modified versions, and any portions
44  * thereof, and that both notices appear in supporting documentation.
45  *
46  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49  *
50  * Carnegie Mellon requests users of this software to return to
51  *
52  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
53  *  School of Computer Science
54  *  Carnegie Mellon University
55  *  Pittsburgh PA 15213-3890
56  *
57  * any improvements or extensions that they make and grant Carnegie the
58  * rights to redistribute these changes.
59  *
60  * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $
61  */
62
63 /*
64  *      Virtual memory object module.
65  */
66
67 #include <sys/param.h>
68 #include <sys/systm.h>
69 #include <sys/proc.h>           /* for curproc, pageproc */
70 #include <sys/thread.h>
71 #include <sys/vnode.h>
72 #include <sys/vmmeter.h>
73 #include <sys/mman.h>
74 #include <sys/mount.h>
75 #include <sys/kernel.h>
76 #include <sys/sysctl.h>
77 #include <sys/refcount.h>
78
79 #include <vm/vm.h>
80 #include <vm/vm_param.h>
81 #include <vm/pmap.h>
82 #include <vm/vm_map.h>
83 #include <vm/vm_object.h>
84 #include <vm/vm_page.h>
85 #include <vm/vm_pageout.h>
86 #include <vm/vm_pager.h>
87 #include <vm/swap_pager.h>
88 #include <vm/vm_kern.h>
89 #include <vm/vm_extern.h>
90 #include <vm/vm_zone.h>
91
92 #include <vm/vm_page2.h>
93
94 #include <machine/specialreg.h>
95
96 #define EASY_SCAN_FACTOR        8
97
98 static void     vm_object_qcollapse(vm_object_t object,
99                                     vm_object_t backing_object);
100 static void     vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
101                                              int pagerflags);
102 static void     vm_object_lock_init(vm_object_t);
103
104
105 /*
106  *      Virtual memory objects maintain the actual data
107  *      associated with allocated virtual memory.  A given
108  *      page of memory exists within exactly one object.
109  *
110  *      An object is only deallocated when all "references"
111  *      are given up.  Only one "reference" to a given
112  *      region of an object should be writeable.
113  *
114  *      Associated with each object is a list of all resident
115  *      memory pages belonging to that object; this list is
116  *      maintained by the "vm_page" module, and locked by the object's
117  *      lock.
118  *
119  *      Each object also records a "pager" routine which is
120  *      used to retrieve (and store) pages to the proper backing
121  *      storage.  In addition, objects may be backed by other
122  *      objects from which they were virtual-copied.
123  *
124  *      The only items within the object structure which are
125  *      modified after time of creation are:
126  *              reference count         locked by object's lock
127  *              pager routine           locked by object's lock
128  *
129  */
130
131 struct vm_object kernel_object;
132
133 static long object_collapses;
134 static long object_bypasses;
135
136 struct vm_object_hash vm_object_hash[VMOBJ_HSIZE];
137
138 MALLOC_DEFINE(M_VM_OBJECT, "vm_object", "vm_object structures");
139
140 #if defined(DEBUG_LOCKS)
141
142 #define vm_object_vndeallocate(obj, vpp)        \
143                 debugvm_object_vndeallocate(obj, vpp, __FILE__, __LINE__)
144
145 /*
146  * Debug helper to track hold/drop/ref/deallocate calls.
147  */
148 static void
149 debugvm_object_add(vm_object_t obj, char *file, int line, int addrem)
150 {
151         int i;
152
153         i = atomic_fetchadd_int(&obj->debug_index, 1);
154         i = i & (VMOBJ_DEBUG_ARRAY_SIZE - 1);
155         ksnprintf(obj->debug_hold_thrs[i],
156                   sizeof(obj->debug_hold_thrs[i]),
157                   "%c%d:(%d):%s",
158                   (addrem == -1 ? '-' : (addrem == 1 ? '+' : '=')),
159                   (curthread->td_proc ? curthread->td_proc->p_pid : -1),
160                   obj->ref_count,
161                   curthread->td_comm);
162         obj->debug_hold_file[i] = file;
163         obj->debug_hold_line[i] = line;
164 #if 0
165         /* Uncomment for debugging obj refs/derefs in reproducable cases */
166         if (strcmp(curthread->td_comm, "sshd") == 0) {
167                 kprintf("%d %p refs=%d ar=%d file: %s/%d\n",
168                         (curthread->td_proc ? curthread->td_proc->p_pid : -1),
169                         obj, obj->ref_count, addrem, file, line);
170         }
171 #endif
172 }
173
174 #endif
175
176 /*
177  * Misc low level routines
178  */
179 static void
180 vm_object_lock_init(vm_object_t obj)
181 {
182 #if defined(DEBUG_LOCKS)
183         int i;
184
185         obj->debug_index = 0;
186         for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) {
187                 obj->debug_hold_thrs[i][0] = 0;
188                 obj->debug_hold_file[i] = NULL;
189                 obj->debug_hold_line[i] = 0;
190         }
191 #endif
192 }
193
194 void
195 vm_object_lock_swap(void)
196 {
197         lwkt_token_swap();
198 }
199
200 void
201 vm_object_lock(vm_object_t obj)
202 {
203         lwkt_gettoken(&obj->token);
204 }
205
206 /*
207  * Returns TRUE on sucesss
208  */
209 static int
210 vm_object_lock_try(vm_object_t obj)
211 {
212         return(lwkt_trytoken(&obj->token));
213 }
214
215 void
216 vm_object_lock_shared(vm_object_t obj)
217 {
218         lwkt_gettoken_shared(&obj->token);
219 }
220
221 void
222 vm_object_unlock(vm_object_t obj)
223 {
224         lwkt_reltoken(&obj->token);
225 }
226
227 void
228 vm_object_upgrade(vm_object_t obj)
229 {
230         lwkt_reltoken(&obj->token);
231         lwkt_gettoken(&obj->token);
232 }
233
234 void
235 vm_object_downgrade(vm_object_t obj)
236 {
237         lwkt_reltoken(&obj->token);
238         lwkt_gettoken_shared(&obj->token);
239 }
240
241 static __inline void
242 vm_object_assert_held(vm_object_t obj)
243 {
244         ASSERT_LWKT_TOKEN_HELD(&obj->token);
245 }
246
247 static __inline int
248 vm_quickcolor(void)
249 {
250         globaldata_t gd = mycpu;
251         int pg_color;
252
253         pg_color = (int)(intptr_t)gd->gd_curthread >> 10;
254         pg_color += gd->gd_quick_color;
255         gd->gd_quick_color += PQ_PRIME2;
256
257         return pg_color;
258 }
259
260 void
261 VMOBJDEBUG(vm_object_hold)(vm_object_t obj VMOBJDBARGS)
262 {
263         KKASSERT(obj != NULL);
264
265         /*
266          * Object must be held (object allocation is stable due to callers
267          * context, typically already holding the token on a parent object)
268          * prior to potentially blocking on the lock, otherwise the object
269          * can get ripped away from us.
270          */
271         refcount_acquire(&obj->hold_count);
272         vm_object_lock(obj);
273
274 #if defined(DEBUG_LOCKS)
275         debugvm_object_add(obj, file, line, 1);
276 #endif
277 }
278
279 int
280 VMOBJDEBUG(vm_object_hold_try)(vm_object_t obj VMOBJDBARGS)
281 {
282         KKASSERT(obj != NULL);
283
284         /*
285          * Object must be held (object allocation is stable due to callers
286          * context, typically already holding the token on a parent object)
287          * prior to potentially blocking on the lock, otherwise the object
288          * can get ripped away from us.
289          */
290         refcount_acquire(&obj->hold_count);
291         if (vm_object_lock_try(obj) == 0) {
292                 if (refcount_release(&obj->hold_count)) {
293                         if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD))
294                                 kfree(obj, M_VM_OBJECT);
295                 }
296                 return(0);
297         }
298
299 #if defined(DEBUG_LOCKS)
300         debugvm_object_add(obj, file, line, 1);
301 #endif
302         return(1);
303 }
304
305 void
306 VMOBJDEBUG(vm_object_hold_shared)(vm_object_t obj VMOBJDBARGS)
307 {
308         KKASSERT(obj != NULL);
309
310         /*
311          * Object must be held (object allocation is stable due to callers
312          * context, typically already holding the token on a parent object)
313          * prior to potentially blocking on the lock, otherwise the object
314          * can get ripped away from us.
315          */
316         refcount_acquire(&obj->hold_count);
317         vm_object_lock_shared(obj);
318
319 #if defined(DEBUG_LOCKS)
320         debugvm_object_add(obj, file, line, 1);
321 #endif
322 }
323
324 /*
325  * Drop the token and hold_count on the object.
326  *
327  * WARNING! Token might be shared.
328  */
329 void
330 VMOBJDEBUG(vm_object_drop)(vm_object_t obj VMOBJDBARGS)
331 {
332         if (obj == NULL)
333                 return;
334
335         /*
336          * No new holders should be possible once we drop hold_count 1->0 as
337          * there is no longer any way to reference the object.
338          */
339         KKASSERT(obj->hold_count > 0);
340         if (refcount_release(&obj->hold_count)) {
341 #if defined(DEBUG_LOCKS)
342                 debugvm_object_add(obj, file, line, -1);
343 #endif
344
345                 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) {
346                         vm_object_unlock(obj);
347                         kfree(obj, M_VM_OBJECT);
348                 } else {
349                         vm_object_unlock(obj);
350                 }
351         } else {
352 #if defined(DEBUG_LOCKS)
353                 debugvm_object_add(obj, file, line, -1);
354 #endif
355                 vm_object_unlock(obj);
356         }
357 }
358
359 /*
360  * Initialize a freshly allocated object, returning a held object.
361  *
362  * Used only by vm_object_allocate(), zinitna() and vm_object_init().
363  *
364  * No requirements.
365  */
366 void
367 _vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
368 {
369         struct vm_object_hash *hash;
370
371         RB_INIT(&object->rb_memq);
372         LIST_INIT(&object->shadow_head);
373         lwkt_token_init(&object->token, "vmobj");
374
375         object->type = type;
376         object->size = size;
377         object->ref_count = 1;
378         object->memattr = VM_MEMATTR_DEFAULT;
379         object->hold_count = 0;
380         object->flags = 0;
381         if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
382                 vm_object_set_flag(object, OBJ_ONEMAPPING);
383         object->paging_in_progress = 0;
384         object->resident_page_count = 0;
385         object->agg_pv_list_count = 0;
386         object->shadow_count = 0;
387         /* cpu localization twist */
388         object->pg_color = vm_quickcolor();
389         object->handle = NULL;
390         object->backing_object = NULL;
391         object->backing_object_offset = (vm_ooffset_t)0;
392
393         object->generation++;
394         object->swblock_count = 0;
395         RB_INIT(&object->swblock_root);
396         vm_object_lock_init(object);
397         pmap_object_init(object);
398
399         vm_object_hold(object);
400
401         hash = VMOBJ_HASH(object);
402         lwkt_gettoken(&hash->token);
403         TAILQ_INSERT_TAIL(&hash->list, object, object_list);
404         lwkt_reltoken(&hash->token);
405 }
406
407 /*
408  * Initialize a VM object.
409  */
410 void
411 vm_object_init(vm_object_t object, vm_pindex_t size)
412 {
413         _vm_object_allocate(OBJT_DEFAULT, size, object);
414         vm_object_drop(object);
415 }
416
417 /*
418  * Initialize the VM objects module.
419  *
420  * Called from the low level boot code only.  Note that this occurs before
421  * kmalloc is initialized so we cannot allocate any VM objects.
422  */
423 void
424 vm_object_init1(void)
425 {
426         int i;
427
428         for (i = 0; i < VMOBJ_HSIZE; ++i) {
429                 TAILQ_INIT(&vm_object_hash[i].list);
430                 lwkt_token_init(&vm_object_hash[i].token, "vmobjlst");
431         }
432
433         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(KvaEnd),
434                             &kernel_object);
435         vm_object_drop(&kernel_object);
436 }
437
438 void
439 vm_object_init2(void)
440 {
441         kmalloc_set_unlimited(M_VM_OBJECT);
442 }
443
444 /*
445  * Allocate and return a new object of the specified type and size.
446  *
447  * No requirements.
448  */
449 vm_object_t
450 vm_object_allocate(objtype_t type, vm_pindex_t size)
451 {
452         vm_object_t obj;
453
454         obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO);
455         _vm_object_allocate(type, size, obj);
456         vm_object_drop(obj);
457
458         return (obj);
459 }
460
461 /*
462  * This version returns a held object, allowing further atomic initialization
463  * of the object.
464  */
465 vm_object_t
466 vm_object_allocate_hold(objtype_t type, vm_pindex_t size)
467 {
468         vm_object_t obj;
469
470         obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO);
471         _vm_object_allocate(type, size, obj);
472
473         return (obj);
474 }
475
476 /*
477  * Add an additional reference to a vm_object.  The object must already be
478  * held.  The original non-lock version is no longer supported.  The object
479  * must NOT be chain locked by anyone at the time the reference is added.
480  *
481  * Referencing a chain-locked object can blow up the fairly sensitive
482  * ref_count and shadow_count tests in the deallocator.  Most callers
483  * will call vm_object_chain_wait() prior to calling
484  * vm_object_reference_locked() to avoid the case.
485  *
486  * The object must be held, but may be held shared if desired (hence why
487  * we use an atomic op).
488  */
489 void
490 VMOBJDEBUG(vm_object_reference_locked)(vm_object_t object VMOBJDBARGS)
491 {
492         KKASSERT(object != NULL);
493         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
494         KKASSERT((object->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) == 0);
495         atomic_add_int(&object->ref_count, 1);
496         if (object->type == OBJT_VNODE) {
497                 vref(object->handle);
498                 /* XXX what if the vnode is being destroyed? */
499         }
500 #if defined(DEBUG_LOCKS)
501         debugvm_object_add(object, file, line, 1);
502 #endif
503 }
504
505 /*
506  * This version is only allowed for vnode objects.
507  */
508 void
509 VMOBJDEBUG(vm_object_reference_quick)(vm_object_t object VMOBJDBARGS)
510 {
511         KKASSERT(object->type == OBJT_VNODE);
512         atomic_add_int(&object->ref_count, 1);
513         vref(object->handle);
514 #if defined(DEBUG_LOCKS)
515         debugvm_object_add(object, file, line, 1);
516 #endif
517 }
518
519 /*
520  * Object OBJ_CHAINLOCK lock handling.
521  *
522  * The caller can chain-lock backing objects recursively and then
523  * use vm_object_chain_release_all() to undo the whole chain.
524  *
525  * Chain locks are used to prevent collapses and are only applicable
526  * to OBJT_DEFAULT and OBJT_SWAP objects.  Chain locking operations
527  * on other object types are ignored.  This is also important because
528  * it allows e.g. the vnode underlying a memory mapping to take concurrent
529  * faults.
530  *
531  * The object must usually be held on entry, though intermediate
532  * objects need not be held on release.  The object must be held exclusively,
533  * NOT shared.  Note that the prefault path checks the shared state and
534  * avoids using the chain functions.
535  */
536 void
537 vm_object_chain_wait(vm_object_t object, int shared)
538 {
539         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
540         for (;;) {
541                 uint32_t chainlk = object->chainlk;
542
543                 cpu_ccfence();
544                 if (shared) {
545                         if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) {
546                                 tsleep_interlock(object, 0);
547                                 if (atomic_cmpset_int(&object->chainlk,
548                                                       chainlk,
549                                                       chainlk | CHAINLK_WAIT)) {
550                                         tsleep(object, PINTERLOCKED,
551                                                "objchns", 0);
552                                 }
553                                 /* retry */
554                         } else {
555                                 break;
556                         }
557                         /* retry */
558                 } else {
559                         if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) {
560                                 tsleep_interlock(object, 0);
561                                 if (atomic_cmpset_int(&object->chainlk,
562                                                       chainlk,
563                                                       chainlk | CHAINLK_WAIT))
564                                 {
565                                         tsleep(object, PINTERLOCKED,
566                                                "objchnx", 0);
567                                 }
568                                 /* retry */
569                         } else {
570                                 if (atomic_cmpset_int(&object->chainlk,
571                                                       chainlk,
572                                                       chainlk & ~CHAINLK_WAIT))
573                                 {
574                                         if (chainlk & CHAINLK_WAIT)
575                                                 wakeup(object);
576                                         break;
577                                 }
578                                 /* retry */
579                         }
580                 }
581                 /* retry */
582         }
583 }
584
585 void
586 vm_object_chain_acquire(vm_object_t object, int shared)
587 {
588         if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP)
589                 return;
590         if (vm_shared_fault == 0)
591                 shared = 0;
592
593         for (;;) {
594                 uint32_t chainlk = object->chainlk;
595
596                 cpu_ccfence();
597                 if (shared) {
598                         if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) {
599                                 tsleep_interlock(object, 0);
600                                 if (atomic_cmpset_int(&object->chainlk,
601                                                       chainlk,
602                                                       chainlk | CHAINLK_WAIT)) {
603                                         tsleep(object, PINTERLOCKED,
604                                                "objchns", 0);
605                                 }
606                                 /* retry */
607                         } else if (atomic_cmpset_int(&object->chainlk,
608                                               chainlk, chainlk + 1)) {
609                                 break;
610                         }
611                         /* retry */
612                 } else {
613                         if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) {
614                                 tsleep_interlock(object, 0);
615                                 if (atomic_cmpset_int(&object->chainlk,
616                                                       chainlk,
617                                                       chainlk |
618                                                        CHAINLK_WAIT |
619                                                        CHAINLK_EXCLREQ)) {
620                                         tsleep(object, PINTERLOCKED,
621                                                "objchnx", 0);
622                                 }
623                                 /* retry */
624                         } else {
625                                 if (atomic_cmpset_int(&object->chainlk,
626                                                       chainlk,
627                                                       (chainlk | CHAINLK_EXCL) &
628                                                       ~(CHAINLK_EXCLREQ |
629                                                         CHAINLK_WAIT))) {
630                                         if (chainlk & CHAINLK_WAIT)
631                                                 wakeup(object);
632                                         break;
633                                 }
634                                 /* retry */
635                         }
636                 }
637                 /* retry */
638         }
639 }
640
641 void
642 vm_object_chain_release(vm_object_t object)
643 {
644         /*ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));*/
645         if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP)
646                 return;
647         KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL));
648         for (;;) {
649                 uint32_t chainlk = object->chainlk;
650
651                 cpu_ccfence();
652                 if (chainlk & CHAINLK_MASK) {
653                         if ((chainlk & CHAINLK_MASK) == 1 &&
654                             atomic_cmpset_int(&object->chainlk,
655                                               chainlk,
656                                               (chainlk - 1) & ~CHAINLK_WAIT)) {
657                                 if (chainlk & CHAINLK_WAIT)
658                                         wakeup(object);
659                                 break;
660                         }
661                         if ((chainlk & CHAINLK_MASK) > 1 &&
662                             atomic_cmpset_int(&object->chainlk,
663                                               chainlk, chainlk - 1)) {
664                                 break;
665                         }
666                         /* retry */
667                 } else {
668                         KKASSERT(chainlk & CHAINLK_EXCL);
669                         if (atomic_cmpset_int(&object->chainlk,
670                                               chainlk,
671                                               chainlk & ~(CHAINLK_EXCL |
672                                                           CHAINLK_WAIT))) {
673                                 if (chainlk & CHAINLK_WAIT)
674                                         wakeup(object);
675                                 break;
676                         }
677                 }
678         }
679 }
680
681 /*
682  * Release the chain from first_object through and including stopobj.
683  * The caller is typically holding the first and last object locked
684  * (shared or exclusive) to prevent destruction races.
685  *
686  * We release stopobj first as an optimization as this object is most
687  * likely to be shared across multiple processes.
688  */
689 void
690 vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj)
691 {
692         vm_object_t backing_object;
693         vm_object_t object;
694
695         vm_object_chain_release(stopobj);
696         object = first_object;
697
698         while (object != stopobj) {
699                 KKASSERT(object);
700                 backing_object = object->backing_object;
701                 vm_object_chain_release(object);
702                 object = backing_object;
703         }
704 }
705
706 /*
707  * Dereference an object and its underlying vnode.  The object may be
708  * held shared.  On return the object will remain held.
709  *
710  * This function may return a vnode in *vpp which the caller must release
711  * after the caller drops its own lock.  If vpp is NULL, we assume that
712  * the caller was holding an exclusive lock on the object and we vrele()
713  * the vp ourselves.
714  */
715 static void
716 VMOBJDEBUG(vm_object_vndeallocate)(vm_object_t object, struct vnode **vpp
717                                    VMOBJDBARGS)
718 {
719         struct vnode *vp = (struct vnode *) object->handle;
720
721         KASSERT(object->type == OBJT_VNODE,
722             ("vm_object_vndeallocate: not a vnode object"));
723         KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
724         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
725 #ifdef INVARIANTS
726         if (object->ref_count == 0) {
727                 vprint("vm_object_vndeallocate", vp);
728                 panic("vm_object_vndeallocate: bad object reference count");
729         }
730 #endif
731         for (;;) {
732                 int count = object->ref_count;
733                 cpu_ccfence();
734                 if (count == 1) {
735                         vm_object_upgrade(object);
736                         if (atomic_cmpset_int(&object->ref_count, count, 0)) {
737                                 vclrflags(vp, VTEXT);
738                                 break;
739                         }
740                 } else {
741                         if (atomic_cmpset_int(&object->ref_count,
742                                               count, count - 1)) {
743                                 break;
744                         }
745                 }
746                 /* retry */
747         }
748 #if defined(DEBUG_LOCKS)
749         debugvm_object_add(object, file, line, -1);
750 #endif
751
752         /*
753          * vrele or return the vp to vrele.  We can only safely vrele(vp)
754          * if the object was locked exclusively.  But there are two races
755          * here.
756          *
757          * We had to upgrade the object above to safely clear VTEXT
758          * but the alternative path where the shared lock is retained
759          * can STILL race to 0 in other paths and cause our own vrele()
760          * to terminate the vnode.  We can't allow that if the VM object
761          * is still locked shared.
762          */
763         if (vpp)
764                 *vpp = vp;
765         else
766                 vrele(vp);
767 }
768
769 /*
770  * Release a reference to the specified object, gained either through a
771  * vm_object_allocate or a vm_object_reference call.  When all references
772  * are gone, storage associated with this object may be relinquished.
773  *
774  * The caller does not have to hold the object locked but must have control
775  * over the reference in question in order to guarantee that the object
776  * does not get ripped out from under us.
777  *
778  * XXX Currently all deallocations require an exclusive lock.
779  */
780 void
781 VMOBJDEBUG(vm_object_deallocate)(vm_object_t object VMOBJDBARGS)
782 {
783         struct vnode *vp;
784         int count;
785
786         if (object == NULL)
787                 return;
788
789         for (;;) {
790                 count = object->ref_count;
791                 cpu_ccfence();
792
793                 /*
794                  * If decrementing the count enters into special handling
795                  * territory (0, 1, or 2) we have to do it the hard way.
796                  * Fortunate though, objects with only a few refs like this
797                  * are not likely to be heavily contended anyway.
798                  *
799                  * For vnode objects we only care about 1->0 transitions.
800                  */
801                 if (count <= 3 || (object->type == OBJT_VNODE && count <= 1)) {
802 #if defined(DEBUG_LOCKS)
803                         debugvm_object_add(object, file, line, 0);
804 #endif
805                         vm_object_hold(object);
806                         vm_object_deallocate_locked(object);
807                         vm_object_drop(object);
808                         break;
809                 }
810
811                 /*
812                  * Try to decrement ref_count without acquiring a hold on
813                  * the object.  This is particularly important for the exec*()
814                  * and exit*() code paths because the program binary may
815                  * have a great deal of sharing and an exclusive lock will
816                  * crowbar performance in those circumstances.
817                  */
818                 if (object->type == OBJT_VNODE) {
819                         vp = (struct vnode *)object->handle;
820                         if (atomic_cmpset_int(&object->ref_count,
821                                               count, count - 1)) {
822 #if defined(DEBUG_LOCKS)
823                                 debugvm_object_add(object, file, line, -1);
824 #endif
825
826                                 vrele(vp);
827                                 break;
828                         }
829                         /* retry */
830                 } else {
831                         if (atomic_cmpset_int(&object->ref_count,
832                                               count, count - 1)) {
833 #if defined(DEBUG_LOCKS)
834                                 debugvm_object_add(object, file, line, -1);
835 #endif
836                                 break;
837                         }
838                         /* retry */
839                 }
840                 /* retry */
841         }
842 }
843
844 void
845 VMOBJDEBUG(vm_object_deallocate_locked)(vm_object_t object VMOBJDBARGS)
846 {
847         struct vm_object_dealloc_list *dlist = NULL;
848         struct vm_object_dealloc_list *dtmp;
849         vm_object_t temp;
850         int must_drop = 0;
851
852         /*
853          * We may chain deallocate object, but additional objects may
854          * collect on the dlist which also have to be deallocated.  We
855          * must avoid a recursion, vm_object chains can get deep.
856          */
857
858 again:
859         while (object != NULL) {
860                 /*
861                  * vnode case, caller either locked the object exclusively
862                  * or this is a recursion with must_drop != 0 and the vnode
863                  * object will be locked shared.
864                  *
865                  * If locked shared we have to drop the object before we can
866                  * call vrele() or risk a shared/exclusive livelock.
867                  */
868                 if (object->type == OBJT_VNODE) {
869                         ASSERT_LWKT_TOKEN_HELD(&object->token);
870                         if (must_drop) {
871                                 struct vnode *tmp_vp;
872
873                                 vm_object_vndeallocate(object, &tmp_vp);
874                                 vm_object_drop(object);
875                                 must_drop = 0;
876                                 object = NULL;
877                                 vrele(tmp_vp);
878                         } else {
879                                 vm_object_vndeallocate(object, NULL);
880                         }
881                         break;
882                 }
883                 ASSERT_LWKT_TOKEN_HELD_EXCL(&object->token);
884
885                 /*
886                  * Normal case (object is locked exclusively)
887                  */
888                 if (object->ref_count == 0) {
889                         panic("vm_object_deallocate: object deallocated "
890                               "too many times: %d", object->type);
891                 }
892                 if (object->ref_count > 2) {
893                         atomic_add_int(&object->ref_count, -1);
894 #if defined(DEBUG_LOCKS)
895                         debugvm_object_add(object, file, line, -1);
896 #endif
897                         break;
898                 }
899
900                 /*
901                  * Here on ref_count of one or two, which are special cases for
902                  * objects.
903                  *
904                  * Nominal ref_count > 1 case if the second ref is not from
905                  * a shadow.
906                  *
907                  * (ONEMAPPING only applies to DEFAULT AND SWAP objects)
908                  */
909                 if (object->ref_count == 2 && object->shadow_count == 0) {
910                         if (object->type == OBJT_DEFAULT ||
911                             object->type == OBJT_SWAP) {
912                                 vm_object_set_flag(object, OBJ_ONEMAPPING);
913                         }
914                         atomic_add_int(&object->ref_count, -1);
915 #if defined(DEBUG_LOCKS)
916                         debugvm_object_add(object, file, line, -1);
917 #endif
918                         break;
919                 }
920
921                 /*
922                  * If the second ref is from a shadow we chain along it
923                  * upwards if object's handle is exhausted.
924                  *
925                  * We have to decrement object->ref_count before potentially
926                  * collapsing the first shadow object or the collapse code
927                  * will not be able to handle the degenerate case to remove
928                  * object.  However, if we do it too early the object can
929                  * get ripped out from under us.
930                  */
931                 if (object->ref_count == 2 && object->shadow_count == 1 &&
932                     object->handle == NULL && (object->type == OBJT_DEFAULT ||
933                                                object->type == OBJT_SWAP)) {
934                         temp = LIST_FIRST(&object->shadow_head);
935                         KKASSERT(temp != NULL);
936                         vm_object_hold(temp);
937
938                         /*
939                          * Wait for any paging to complete so the collapse
940                          * doesn't (or isn't likely to) qcollapse.  pip
941                          * waiting must occur before we acquire the
942                          * chainlock.
943                          */
944                         while (
945                                 temp->paging_in_progress ||
946                                 object->paging_in_progress
947                         ) {
948                                 vm_object_pip_wait(temp, "objde1");
949                                 vm_object_pip_wait(object, "objde2");
950                         }
951
952                         /*
953                          * If the parent is locked we have to give up, as
954                          * otherwise we would be acquiring locks in the
955                          * wrong order and potentially deadlock.
956                          */
957                         if (temp->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) {
958                                 vm_object_drop(temp);
959                                 goto skip;
960                         }
961                         vm_object_chain_acquire(temp, 0);
962
963                         /*
964                          * Recheck/retry after the hold and the paging
965                          * wait, both of which can block us.
966                          */
967                         if (object->ref_count != 2 ||
968                             object->shadow_count != 1 ||
969                             object->handle ||
970                             LIST_FIRST(&object->shadow_head) != temp ||
971                             (object->type != OBJT_DEFAULT &&
972                              object->type != OBJT_SWAP)) {
973                                 vm_object_chain_release(temp);
974                                 vm_object_drop(temp);
975                                 continue;
976                         }
977
978                         /*
979                          * We can safely drop object's ref_count now.
980                          */
981                         KKASSERT(object->ref_count == 2);
982                         atomic_add_int(&object->ref_count, -1);
983 #if defined(DEBUG_LOCKS)
984                         debugvm_object_add(object, file, line, -1);
985 #endif
986
987                         /*
988                          * If our single parent is not collapseable just
989                          * decrement ref_count (2->1) and stop.
990                          */
991                         if (temp->handle || (temp->type != OBJT_DEFAULT &&
992                                              temp->type != OBJT_SWAP)) {
993                                 vm_object_chain_release(temp);
994                                 vm_object_drop(temp);
995                                 break;
996                         }
997
998                         /*
999                          * At this point we have already dropped object's
1000                          * ref_count so it is possible for a race to
1001                          * deallocate obj out from under us.  Any collapse
1002                          * will re-check the situation.  We must not block
1003                          * until we are able to collapse.
1004                          *
1005                          * Bump temp's ref_count to avoid an unwanted
1006                          * degenerate recursion (can't call
1007                          * vm_object_reference_locked() because it asserts
1008                          * that CHAINLOCK is not set).
1009                          */
1010                         atomic_add_int(&temp->ref_count, 1);
1011                         KKASSERT(temp->ref_count > 1);
1012
1013                         /*
1014                          * Collapse temp, then deallocate the extra ref
1015                          * formally.
1016                          */
1017                         vm_object_collapse(temp, &dlist);
1018                         vm_object_chain_release(temp);
1019                         if (must_drop) {
1020                                 vm_object_lock_swap();
1021                                 vm_object_drop(object);
1022                         }
1023                         object = temp;
1024                         must_drop = 1;
1025                         continue;
1026                 }
1027
1028                 /*
1029                  * Drop the ref and handle termination on the 1->0 transition.
1030                  * We may have blocked above so we have to recheck.
1031                  */
1032 skip:
1033                 KKASSERT(object->ref_count != 0);
1034                 if (object->ref_count >= 2) {
1035                         atomic_add_int(&object->ref_count, -1);
1036 #if defined(DEBUG_LOCKS)
1037                         debugvm_object_add(object, file, line, -1);
1038 #endif
1039                         break;
1040                 }
1041                 KKASSERT(object->ref_count == 1);
1042
1043                 /*
1044                  * 1->0 transition.  Chain through the backing_object.
1045                  * Maintain the ref until we've located the backing object,
1046                  * then re-check.
1047                  */
1048                 while ((temp = object->backing_object) != NULL) {
1049                         if (temp->type == OBJT_VNODE)
1050                                 vm_object_hold_shared(temp);
1051                         else
1052                                 vm_object_hold(temp);
1053                         if (temp == object->backing_object)
1054                                 break;
1055                         vm_object_drop(temp);
1056                 }
1057
1058                 /*
1059                  * 1->0 transition verified, retry if ref_count is no longer
1060                  * 1.  Otherwise disconnect the backing_object (temp) and
1061                  * clean up.
1062                  */
1063                 if (object->ref_count != 1) {
1064                         vm_object_drop(temp);
1065                         continue;
1066                 }
1067
1068                 /*
1069                  * It shouldn't be possible for the object to be chain locked
1070                  * if we're removing the last ref on it.
1071                  *
1072                  * Removing object from temp's shadow list requires dropping
1073                  * temp, which we will do on loop.
1074                  *
1075                  * NOTE! vnodes do not use the shadow list, but still have
1076                  *       the backing_object reference.
1077                  */
1078                 KKASSERT((object->chainlk & (CHAINLK_EXCL|CHAINLK_MASK)) == 0);
1079
1080                 if (temp) {
1081                         if (object->flags & OBJ_ONSHADOW) {
1082                                 LIST_REMOVE(object, shadow_list);
1083                                 temp->shadow_count--;
1084                                 temp->generation++;
1085                                 vm_object_clear_flag(object, OBJ_ONSHADOW);
1086                         }
1087                         object->backing_object = NULL;
1088                 }
1089
1090                 atomic_add_int(&object->ref_count, -1);
1091                 if ((object->flags & OBJ_DEAD) == 0)
1092                         vm_object_terminate(object);
1093                 if (must_drop && temp)
1094                         vm_object_lock_swap();
1095                 if (must_drop)
1096                         vm_object_drop(object);
1097                 object = temp;
1098                 must_drop = 1;
1099         }
1100
1101         if (must_drop && object)
1102                 vm_object_drop(object);
1103
1104         /*
1105          * Additional tail recursion on dlist.  Avoid a recursion.  Objects
1106          * on the dlist have a hold count but are not locked.
1107          */
1108         if ((dtmp = dlist) != NULL) {
1109                 dlist = dtmp->next;
1110                 object = dtmp->object;
1111                 kfree(dtmp, M_TEMP);
1112
1113                 vm_object_lock(object); /* already held, add lock */
1114                 must_drop = 1;          /* and we're responsible for it */
1115                 goto again;
1116         }
1117 }
1118
1119 /*
1120  * Destroy the specified object, freeing up related resources.
1121  *
1122  * The object must have zero references.
1123  *
1124  * The object must held.  The caller is responsible for dropping the object
1125  * after terminate returns.  Terminate does NOT drop the object.
1126  */
1127 static int vm_object_terminate_callback(vm_page_t p, void *data);
1128
1129 void
1130 vm_object_terminate(vm_object_t object)
1131 {
1132         struct rb_vm_page_scan_info info;
1133         struct vm_object_hash *hash;
1134
1135         /*
1136          * Make sure no one uses us.  Once we set OBJ_DEAD we should be
1137          * able to safely block.
1138          */
1139         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1140         KKASSERT((object->flags & OBJ_DEAD) == 0);
1141         vm_object_set_flag(object, OBJ_DEAD);
1142
1143         /*
1144          * Wait for the pageout daemon to be done with the object
1145          */
1146         vm_object_pip_wait(object, "objtrm1");
1147
1148         KASSERT(!object->paging_in_progress,
1149                 ("vm_object_terminate: pageout in progress"));
1150
1151         /*
1152          * Clean and free the pages, as appropriate. All references to the
1153          * object are gone, so we don't need to lock it.
1154          */
1155         if (object->type == OBJT_VNODE) {
1156                 struct vnode *vp;
1157
1158                 /*
1159                  * Clean pages and flush buffers.
1160                  *
1161                  * NOTE!  TMPFS buffer flushes do not typically flush the
1162                  *        actual page to swap as this would be highly
1163                  *        inefficient, and normal filesystems usually wrap
1164                  *        page flushes with buffer cache buffers.
1165                  *
1166                  *        To deal with this we have to call vinvalbuf() both
1167                  *        before and after the vm_object_page_clean().
1168                  */
1169                 vp = (struct vnode *) object->handle;
1170                 vinvalbuf(vp, V_SAVE, 0, 0);
1171                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
1172                 vinvalbuf(vp, V_SAVE, 0, 0);
1173         }
1174
1175         /*
1176          * Wait for any I/O to complete, after which there had better not
1177          * be any references left on the object.
1178          */
1179         vm_object_pip_wait(object, "objtrm2");
1180
1181         if (object->ref_count != 0) {
1182                 panic("vm_object_terminate: object with references, "
1183                       "ref_count=%d", object->ref_count);
1184         }
1185
1186         /*
1187          * Cleanup any shared pmaps associated with this object.
1188          */
1189         pmap_object_free(object);
1190
1191         /*
1192          * Now free any remaining pages. For internal objects, this also
1193          * removes them from paging queues. Don't free wired pages, just
1194          * remove them from the object. 
1195          */
1196         info.count = 0;
1197         info.object = object;
1198         vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
1199                                 vm_object_terminate_callback, &info);
1200
1201         /*
1202          * Let the pager know object is dead.
1203          */
1204         vm_pager_deallocate(object);
1205
1206         /*
1207          * Wait for the object hold count to hit 1, clean out pages as
1208          * we go.  vmobj_token interlocks any race conditions that might
1209          * pick the object up from the vm_object_list after we have cleared
1210          * rb_memq.
1211          */
1212         for (;;) {
1213                 if (RB_ROOT(&object->rb_memq) == NULL)
1214                         break;
1215                 kprintf("vm_object_terminate: Warning, object %p "
1216                         "still has %ld pages\n",
1217                         object, object->resident_page_count);
1218                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
1219                                         vm_object_terminate_callback, &info);
1220         }
1221
1222         /*
1223          * There had better not be any pages left
1224          */
1225         KKASSERT(object->resident_page_count == 0);
1226
1227         /*
1228          * Remove the object from the global object list.
1229          */
1230         hash = VMOBJ_HASH(object);
1231         lwkt_gettoken(&hash->token);
1232         TAILQ_REMOVE(&hash->list, object, object_list);
1233         lwkt_reltoken(&hash->token);
1234
1235         if (object->ref_count != 0) {
1236                 panic("vm_object_terminate2: object with references, "
1237                       "ref_count=%d", object->ref_count);
1238         }
1239
1240         /*
1241          * NOTE: The object hold_count is at least 1, so we cannot kfree()
1242          *       the object here.  See vm_object_drop().
1243          */
1244 }
1245
1246 /*
1247  * The caller must hold the object.
1248  */
1249 static int
1250 vm_object_terminate_callback(vm_page_t p, void *data)
1251 {
1252         struct rb_vm_page_scan_info *info = data;
1253         vm_object_t object;
1254
1255         if ((++info->count & 63) == 0)
1256                 lwkt_user_yield();
1257         object = p->object;
1258         if (object != info->object) {
1259                 kprintf("vm_object_terminate_callback: obj/pg race %p/%p\n",
1260                         info->object, p);
1261                 return(0);
1262         }
1263         vm_page_busy_wait(p, TRUE, "vmpgtrm");
1264         if (object != p->object) {
1265                 kprintf("vm_object_terminate: Warning: Encountered "
1266                         "busied page %p on queue %d\n", p, p->queue);
1267                 vm_page_wakeup(p);
1268         } else if (p->wire_count == 0) {
1269                 /*
1270                  * NOTE: p->dirty and PG_NEED_COMMIT are ignored.
1271                  */
1272                 vm_page_free(p);
1273                 mycpu->gd_cnt.v_pfree++;
1274         } else {
1275                 if (p->queue != PQ_NONE)
1276                         kprintf("vm_object_terminate: Warning: Encountered "
1277                                 "wired page %p on queue %d\n", p, p->queue);
1278                 vm_page_remove(p);
1279                 vm_page_wakeup(p);
1280         }
1281         return(0);
1282 }
1283
1284 /*
1285  * Clean all dirty pages in the specified range of object.  Leaves page
1286  * on whatever queue it is currently on.   If NOSYNC is set then do not
1287  * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC),
1288  * leaving the object dirty.
1289  *
1290  * When stuffing pages asynchronously, allow clustering.  XXX we need a
1291  * synchronous clustering mode implementation.
1292  *
1293  * Odd semantics: if start == end, we clean everything.
1294  *
1295  * The object must be locked? XXX
1296  */
1297 static int vm_object_page_clean_pass1(struct vm_page *p, void *data);
1298 static int vm_object_page_clean_pass2(struct vm_page *p, void *data);
1299
1300 void
1301 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1302                      int flags)
1303 {
1304         struct rb_vm_page_scan_info info;
1305         struct vnode *vp;
1306         int wholescan;
1307         int pagerflags;
1308         int generation;
1309
1310         vm_object_hold(object);
1311         if (object->type != OBJT_VNODE ||
1312             (object->flags & OBJ_MIGHTBEDIRTY) == 0) {
1313                 vm_object_drop(object);
1314                 return;
1315         }
1316
1317         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? 
1318                         VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1319         pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
1320
1321         vp = object->handle;
1322
1323         /*
1324          * Interlock other major object operations.  This allows us to 
1325          * temporarily clear OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY.
1326          */
1327         vm_object_set_flag(object, OBJ_CLEANING);
1328
1329         /*
1330          * Handle 'entire object' case
1331          */
1332         info.start_pindex = start;
1333         if (end == 0) {
1334                 info.end_pindex = object->size - 1;
1335         } else {
1336                 info.end_pindex = end - 1;
1337         }
1338         wholescan = (start == 0 && info.end_pindex == object->size - 1);
1339         info.limit = flags;
1340         info.pagerflags = pagerflags;
1341         info.object = object;
1342
1343         /*
1344          * If cleaning the entire object do a pass to mark the pages read-only.
1345          * If everything worked out ok, clear OBJ_WRITEABLE and
1346          * OBJ_MIGHTBEDIRTY.
1347          */
1348         if (wholescan) {
1349                 info.error = 0;
1350                 info.count = 0;
1351                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1352                                         vm_object_page_clean_pass1, &info);
1353                 if (info.error == 0) {
1354                         vm_object_clear_flag(object,
1355                                              OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
1356                         if (object->type == OBJT_VNODE &&
1357                             (vp = (struct vnode *)object->handle) != NULL) {
1358                                 /*
1359                                  * Use new-style interface to clear VISDIRTY
1360                                  * because the vnode is not necessarily removed
1361                                  * from the syncer list(s) as often as it was
1362                                  * under the old interface, which can leave
1363                                  * the vnode on the syncer list after reclaim.
1364                                  */
1365                                 vclrobjdirty(vp);
1366                         }
1367                 }
1368         }
1369
1370         /*
1371          * Do a pass to clean all the dirty pages we find.
1372          */
1373         do {
1374                 info.error = 0;
1375                 info.count = 0;
1376                 generation = object->generation;
1377                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1378                                         vm_object_page_clean_pass2, &info);
1379         } while (info.error || generation != object->generation);
1380
1381         vm_object_clear_flag(object, OBJ_CLEANING);
1382         vm_object_drop(object);
1383 }
1384
1385 /*
1386  * The caller must hold the object.
1387  */
1388 static 
1389 int
1390 vm_object_page_clean_pass1(struct vm_page *p, void *data)
1391 {
1392         struct rb_vm_page_scan_info *info = data;
1393
1394         if ((++info->count & 63) == 0)
1395                 lwkt_user_yield();
1396         if (p->object != info->object ||
1397             p->pindex < info->start_pindex ||
1398             p->pindex > info->end_pindex) {
1399                 kprintf("vm_object_page_clean_pass1: obj/pg race %p/%p\n",
1400                         info->object, p);
1401                 return(0);
1402         }
1403         vm_page_flag_set(p, PG_CLEANCHK);
1404         if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
1405                 info->error = 1;
1406         } else if (vm_page_busy_try(p, FALSE) == 0) {
1407                 if (p->object == info->object)
1408                         vm_page_protect(p, VM_PROT_READ);
1409                 vm_page_wakeup(p);
1410         } else {
1411                 info->error = 1;
1412         }
1413         return(0);
1414 }
1415
1416 /*
1417  * The caller must hold the object
1418  */
1419 static 
1420 int
1421 vm_object_page_clean_pass2(struct vm_page *p, void *data)
1422 {
1423         struct rb_vm_page_scan_info *info = data;
1424         int generation;
1425
1426         if (p->object != info->object ||
1427             p->pindex < info->start_pindex ||
1428             p->pindex > info->end_pindex) {
1429                 kprintf("vm_object_page_clean_pass2: obj/pg race %p/%p\n",
1430                         info->object, p);
1431                 return(0);
1432         }
1433
1434         /*
1435          * Do not mess with pages that were inserted after we started
1436          * the cleaning pass.
1437          */
1438         if ((p->flags & PG_CLEANCHK) == 0)
1439                 goto done;
1440
1441         generation = info->object->generation;
1442         vm_page_busy_wait(p, TRUE, "vpcwai");
1443
1444         if (p->object != info->object ||
1445             p->pindex < info->start_pindex ||
1446             p->pindex > info->end_pindex ||
1447             info->object->generation != generation) {
1448                 info->error = 1;
1449                 vm_page_wakeup(p);
1450                 goto done;
1451         }
1452
1453         /*
1454          * Before wasting time traversing the pmaps, check for trivial
1455          * cases where the page cannot be dirty.
1456          */
1457         if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) {
1458                 KKASSERT((p->dirty & p->valid) == 0 &&
1459                          (p->flags & PG_NEED_COMMIT) == 0);
1460                 vm_page_wakeup(p);
1461                 goto done;
1462         }
1463
1464         /*
1465          * Check whether the page is dirty or not.  The page has been set
1466          * to be read-only so the check will not race a user dirtying the
1467          * page.
1468          */
1469         vm_page_test_dirty(p);
1470         if ((p->dirty & p->valid) == 0 && (p->flags & PG_NEED_COMMIT) == 0) {
1471                 vm_page_flag_clear(p, PG_CLEANCHK);
1472                 vm_page_wakeup(p);
1473                 goto done;
1474         }
1475
1476         /*
1477          * If we have been asked to skip nosync pages and this is a
1478          * nosync page, skip it.  Note that the object flags were
1479          * not cleared in this case (because pass1 will have returned an
1480          * error), so we do not have to set them.
1481          */
1482         if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
1483                 vm_page_flag_clear(p, PG_CLEANCHK);
1484                 vm_page_wakeup(p);
1485                 goto done;
1486         }
1487
1488         /*
1489          * Flush as many pages as we can.  PG_CLEANCHK will be cleared on
1490          * the pages that get successfully flushed.  Set info->error if
1491          * we raced an object modification.
1492          */
1493         vm_object_page_collect_flush(info->object, p, info->pagerflags);
1494         /* vm_wait_nominal(); this can deadlock the system in syncer/pageout */
1495 done:
1496         if ((++info->count & 63) == 0)
1497                 lwkt_user_yield();
1498
1499         return(0);
1500 }
1501
1502 /*
1503  * Collect the specified page and nearby pages and flush them out.
1504  * The number of pages flushed is returned.  The passed page is busied
1505  * by the caller and we are responsible for its disposition.
1506  *
1507  * The caller must hold the object.
1508  */
1509 static void
1510 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags)
1511 {
1512         int error;
1513         int is;
1514         int ib;
1515         int i;
1516         int page_base;
1517         vm_pindex_t pi;
1518         vm_page_t ma[BLIST_MAX_ALLOC];
1519
1520         ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1521
1522         pi = p->pindex;
1523         page_base = pi % BLIST_MAX_ALLOC;
1524         ma[page_base] = p;
1525         ib = page_base - 1;
1526         is = page_base + 1;
1527
1528         while (ib >= 0) {
1529                 vm_page_t tp;
1530
1531                 tp = vm_page_lookup_busy_try(object, pi - page_base + ib,
1532                                              TRUE, &error);
1533                 if (error)
1534                         break;
1535                 if (tp == NULL)
1536                         break;
1537                 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1538                     (tp->flags & PG_CLEANCHK) == 0) {
1539                         vm_page_wakeup(tp);
1540                         break;
1541                 }
1542                 if ((tp->queue - tp->pc) == PQ_CACHE) {
1543                         vm_page_flag_clear(tp, PG_CLEANCHK);
1544                         vm_page_wakeup(tp);
1545                         break;
1546                 }
1547                 vm_page_test_dirty(tp);
1548                 if ((tp->dirty & tp->valid) == 0 &&
1549                     (tp->flags & PG_NEED_COMMIT) == 0) {
1550                         vm_page_flag_clear(tp, PG_CLEANCHK);
1551                         vm_page_wakeup(tp);
1552                         break;
1553                 }
1554                 ma[ib] = tp;
1555                 --ib;
1556         }
1557         ++ib;   /* fixup */
1558
1559         while (is < BLIST_MAX_ALLOC &&
1560                pi - page_base + is < object->size) {
1561                 vm_page_t tp;
1562
1563                 tp = vm_page_lookup_busy_try(object, pi - page_base + is,
1564                                              TRUE, &error);
1565                 if (error)
1566                         break;
1567                 if (tp == NULL)
1568                         break;
1569                 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1570                     (tp->flags & PG_CLEANCHK) == 0) {
1571                         vm_page_wakeup(tp);
1572                         break;
1573                 }
1574                 if ((tp->queue - tp->pc) == PQ_CACHE) {
1575                         vm_page_flag_clear(tp, PG_CLEANCHK);
1576                         vm_page_wakeup(tp);
1577                         break;
1578                 }
1579                 vm_page_test_dirty(tp);
1580                 if ((tp->dirty & tp->valid) == 0 &&
1581                     (tp->flags & PG_NEED_COMMIT) == 0) {
1582                         vm_page_flag_clear(tp, PG_CLEANCHK);
1583                         vm_page_wakeup(tp);
1584                         break;
1585                 }
1586                 ma[is] = tp;
1587                 ++is;
1588         }
1589
1590         /*
1591          * All pages in the ma[] array are busied now
1592          */
1593         for (i = ib; i < is; ++i) {
1594                 vm_page_flag_clear(ma[i], PG_CLEANCHK);
1595                 vm_page_hold(ma[i]);    /* XXX need this any more? */
1596         }
1597         vm_pageout_flush(&ma[ib], is - ib, pagerflags);
1598         for (i = ib; i < is; ++i)       /* XXX need this any more? */
1599                 vm_page_unhold(ma[i]);
1600 }
1601
1602 /*
1603  * Same as vm_object_pmap_copy, except range checking really
1604  * works, and is meant for small sections of an object.
1605  *
1606  * This code protects resident pages by making them read-only
1607  * and is typically called on a fork or split when a page
1608  * is converted to copy-on-write.  
1609  *
1610  * NOTE: If the page is already at VM_PROT_NONE, calling
1611  * vm_page_protect will have no effect.
1612  */
1613 void
1614 vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1615 {
1616         vm_pindex_t idx;
1617         vm_page_t p;
1618
1619         if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
1620                 return;
1621
1622         vm_object_hold(object);
1623         for (idx = start; idx < end; idx++) {
1624                 p = vm_page_lookup(object, idx);
1625                 if (p == NULL)
1626                         continue;
1627                 vm_page_protect(p, VM_PROT_READ);
1628         }
1629         vm_object_drop(object);
1630 }
1631
1632 /*
1633  * Removes all physical pages in the specified object range from all
1634  * physical maps.
1635  *
1636  * The object must *not* be locked.
1637  */
1638
1639 static int vm_object_pmap_remove_callback(vm_page_t p, void *data);
1640
1641 void
1642 vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1643 {
1644         struct rb_vm_page_scan_info info;
1645
1646         if (object == NULL)
1647                 return;
1648         info.start_pindex = start;
1649         info.end_pindex = end - 1;
1650         info.count = 0;
1651         info.object = object;
1652
1653         vm_object_hold(object);
1654         vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1655                                 vm_object_pmap_remove_callback, &info);
1656         if (start == 0 && end == object->size)
1657                 vm_object_clear_flag(object, OBJ_WRITEABLE);
1658         vm_object_drop(object);
1659 }
1660
1661 /*
1662  * The caller must hold the object
1663  */
1664 static int
1665 vm_object_pmap_remove_callback(vm_page_t p, void *data)
1666 {
1667         struct rb_vm_page_scan_info *info = data;
1668
1669         if ((++info->count & 63) == 0)
1670                 lwkt_user_yield();
1671
1672         if (info->object != p->object ||
1673             p->pindex < info->start_pindex ||
1674             p->pindex > info->end_pindex) {
1675                 kprintf("vm_object_pmap_remove_callback: obj/pg race %p/%p\n",
1676                         info->object, p);
1677                 return(0);
1678         }
1679
1680         vm_page_protect(p, VM_PROT_NONE);
1681
1682         return(0);
1683 }
1684
1685 /*
1686  * Implements the madvise function at the object/page level.
1687  *
1688  * MADV_WILLNEED        (any object)
1689  *
1690  *      Activate the specified pages if they are resident.
1691  *
1692  * MADV_DONTNEED        (any object)
1693  *
1694  *      Deactivate the specified pages if they are resident.
1695  *
1696  * MADV_FREE    (OBJT_DEFAULT/OBJT_SWAP objects, OBJ_ONEMAPPING only)
1697  *
1698  *      Deactivate and clean the specified pages if they are
1699  *      resident.  This permits the process to reuse the pages
1700  *      without faulting or the kernel to reclaim the pages
1701  *      without I/O.
1702  *
1703  * No requirements.
1704  */
1705 void
1706 vm_object_madvise(vm_object_t object, vm_pindex_t pindex,
1707                   vm_pindex_t count, int advise)
1708 {
1709         vm_pindex_t end, tpindex;
1710         vm_object_t tobject;
1711         vm_object_t xobj;
1712         vm_page_t m;
1713         int error;
1714
1715         if (object == NULL)
1716                 return;
1717
1718         end = pindex + count;
1719
1720         vm_object_hold(object);
1721         tobject = object;
1722
1723         /*
1724          * Locate and adjust resident pages
1725          */
1726         for (; pindex < end; pindex += 1) {
1727 relookup:
1728                 if (tobject != object)
1729                         vm_object_drop(tobject);
1730                 tobject = object;
1731                 tpindex = pindex;
1732 shadowlookup:
1733                 /*
1734                  * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
1735                  * and those pages must be OBJ_ONEMAPPING.
1736                  */
1737                 if (advise == MADV_FREE) {
1738                         if ((tobject->type != OBJT_DEFAULT &&
1739                              tobject->type != OBJT_SWAP) ||
1740                             (tobject->flags & OBJ_ONEMAPPING) == 0) {
1741                                 continue;
1742                         }
1743                 }
1744
1745                 m = vm_page_lookup_busy_try(tobject, tpindex, TRUE, &error);
1746
1747                 if (error) {
1748                         vm_page_sleep_busy(m, TRUE, "madvpo");
1749                         goto relookup;
1750                 }
1751                 if (m == NULL) {
1752                         /*
1753                          * There may be swap even if there is no backing page
1754                          */
1755                         if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1756                                 swap_pager_freespace(tobject, tpindex, 1);
1757
1758                         /*
1759                          * next object
1760                          */
1761                         while ((xobj = tobject->backing_object) != NULL) {
1762                                 KKASSERT(xobj != object);
1763                                 vm_object_hold(xobj);
1764                                 if (xobj == tobject->backing_object)
1765                                         break;
1766                                 vm_object_drop(xobj);
1767                         }
1768                         if (xobj == NULL)
1769                                 continue;
1770                         tpindex += OFF_TO_IDX(tobject->backing_object_offset);
1771                         if (tobject != object) {
1772                                 vm_object_lock_swap();
1773                                 vm_object_drop(tobject);
1774                         }
1775                         tobject = xobj;
1776                         goto shadowlookup;
1777                 }
1778
1779                 /*
1780                  * If the page is not in a normal active state, we skip it.
1781                  * If the page is not managed there are no page queues to
1782                  * mess with.  Things can break if we mess with pages in
1783                  * any of the below states.
1784                  */
1785                 if (m->wire_count ||
1786                     (m->flags & (PG_UNMANAGED | PG_NEED_COMMIT)) ||
1787                     m->valid != VM_PAGE_BITS_ALL
1788                 ) {
1789                         vm_page_wakeup(m);
1790                         continue;
1791                 }
1792
1793                 /*
1794                  * Theoretically once a page is known not to be busy, an
1795                  * interrupt cannot come along and rip it out from under us.
1796                  */
1797
1798                 if (advise == MADV_WILLNEED) {
1799                         vm_page_activate(m);
1800                 } else if (advise == MADV_DONTNEED) {
1801                         vm_page_dontneed(m);
1802                 } else if (advise == MADV_FREE) {
1803                         /*
1804                          * Mark the page clean.  This will allow the page
1805                          * to be freed up by the system.  However, such pages
1806                          * are often reused quickly by malloc()/free()
1807                          * so we do not do anything that would cause
1808                          * a page fault if we can help it.
1809                          *
1810                          * Specifically, we do not try to actually free
1811                          * the page now nor do we try to put it in the
1812                          * cache (which would cause a page fault on reuse).
1813                          *
1814                          * But we do make the page is freeable as we
1815                          * can without actually taking the step of unmapping
1816                          * it.
1817                          */
1818                         pmap_clear_modify(m);
1819                         m->dirty = 0;
1820                         m->act_count = 0;
1821                         vm_page_dontneed(m);
1822                         if (tobject->type == OBJT_SWAP)
1823                                 swap_pager_freespace(tobject, tpindex, 1);
1824                 }
1825                 vm_page_wakeup(m);
1826         }       
1827         if (tobject != object)
1828                 vm_object_drop(tobject);
1829         vm_object_drop(object);
1830 }
1831
1832 /*
1833  * Create a new object which is backed by the specified existing object
1834  * range.  Replace the pointer and offset that was pointing at the existing
1835  * object with the pointer/offset for the new object.
1836  *
1837  * If addref is non-zero the returned object is given an additional reference.
1838  * This mechanic exists to avoid the situation where refs might be 1 and
1839  * race against a collapse when the caller intends to bump it.  So the
1840  * caller cannot add the ref after the fact.  Used when the caller is
1841  * duplicating a vm_map_entry.
1842  *
1843  * No other requirements.
1844  */
1845 void
1846 vm_object_shadow(vm_object_t *objectp, vm_ooffset_t *offset, vm_size_t length,
1847                  int addref)
1848 {
1849         vm_object_t source;
1850         vm_object_t result;
1851         int useshadowlist;
1852
1853         source = *objectp;
1854
1855         /*
1856          * Don't create the new object if the old object isn't shared.
1857          * We have to chain wait before adding the reference to avoid
1858          * racing a collapse or deallocation.
1859          *
1860          * Clear OBJ_ONEMAPPING flag when shadowing.
1861          *
1862          * The caller owns a ref on source via *objectp which we are going
1863          * to replace.  This ref is inherited by the backing_object assignment.
1864          * from nobject and does not need to be incremented here.
1865          *
1866          * However, we add a temporary extra reference to the original source
1867          * prior to holding nobject in case we block, to avoid races where
1868          * someone else might believe that the source can be collapsed.
1869          */
1870         useshadowlist = 0;
1871         if (source) {
1872                 if (source->type != OBJT_VNODE) {
1873                         useshadowlist = 1;
1874                         vm_object_hold(source);
1875                         vm_object_chain_wait(source, 0);
1876                         if (source->ref_count == 1 &&
1877                             source->handle == NULL &&
1878                             (source->type == OBJT_DEFAULT ||
1879                              source->type == OBJT_SWAP)) {
1880                                 if (addref) {
1881                                         vm_object_reference_locked(source);
1882                                         vm_object_clear_flag(source,
1883                                                              OBJ_ONEMAPPING);
1884                                 }
1885                                 vm_object_drop(source);
1886                                 return;
1887                         }
1888                         vm_object_reference_locked(source);
1889                         vm_object_clear_flag(source, OBJ_ONEMAPPING);
1890                 } else {
1891                         vm_object_reference_quick(source);
1892                         vm_object_clear_flag(source, OBJ_ONEMAPPING);
1893                 }
1894         }
1895
1896         /*
1897          * Allocate a new object with the given length.  The new object
1898          * is returned referenced but we may have to add another one.
1899          * If we are adding a second reference we must clear OBJ_ONEMAPPING.
1900          * (typically because the caller is about to clone a vm_map_entry).
1901          *
1902          * The source object currently has an extra reference to prevent
1903          * collapses into it while we mess with its shadow list, which
1904          * we will remove later in this routine.
1905          *
1906          * The target object may require a second reference if asked for one
1907          * by the caller.
1908          */
1909         result = vm_object_allocate(OBJT_DEFAULT, length);
1910         if (result == NULL)
1911                 panic("vm_object_shadow: no object for shadowing");
1912         vm_object_hold(result);
1913         if (addref) {
1914                 vm_object_reference_locked(result);
1915                 vm_object_clear_flag(result, OBJ_ONEMAPPING);
1916         }
1917
1918         /*
1919          * The new object shadows the source object.  Chain wait before
1920          * adjusting shadow_count or the shadow list to avoid races.
1921          *
1922          * Try to optimize the result object's page color when shadowing
1923          * in order to maintain page coloring consistency in the combined 
1924          * shadowed object.
1925          *
1926          * The backing_object reference to source requires adding a ref to
1927          * source.  We simply inherit the ref from the original *objectp
1928          * (which we are replacing) so no additional refs need to be added.
1929          * (we must still clean up the extra ref we had to prevent collapse
1930          * races).
1931          *
1932          * SHADOWING IS NOT APPLICABLE TO OBJT_VNODE OBJECTS
1933          */
1934         KKASSERT(result->backing_object == NULL);
1935         result->backing_object = source;
1936         if (source) {
1937                 if (useshadowlist) {
1938                         vm_object_chain_wait(source, 0);
1939                         LIST_INSERT_HEAD(&source->shadow_head,
1940                                          result, shadow_list);
1941                         source->shadow_count++;
1942                         source->generation++;
1943                         vm_object_set_flag(result, OBJ_ONSHADOW);
1944                 }
1945                 /* cpu localization twist */
1946                 result->pg_color = vm_quickcolor();
1947         }
1948
1949         /*
1950          * Adjust the return storage.  Drop the ref on source before
1951          * returning.
1952          */
1953         result->backing_object_offset = *offset;
1954         vm_object_drop(result);
1955         *offset = 0;
1956         if (source) {
1957                 if (useshadowlist) {
1958                         vm_object_deallocate_locked(source);
1959                         vm_object_drop(source);
1960                 } else {
1961                         vm_object_deallocate(source);
1962                 }
1963         }
1964
1965         /*
1966          * Return the new things
1967          */
1968         *objectp = result;
1969 }
1970
1971 #define OBSC_TEST_ALL_SHADOWED  0x0001
1972 #define OBSC_COLLAPSE_NOWAIT    0x0002
1973 #define OBSC_COLLAPSE_WAIT      0x0004
1974
1975 static int vm_object_backing_scan_callback(vm_page_t p, void *data);
1976
1977 /*
1978  * The caller must hold the object.
1979  */
1980 static __inline int
1981 vm_object_backing_scan(vm_object_t object, vm_object_t backing_object, int op)
1982 {
1983         struct rb_vm_page_scan_info info;
1984         struct vm_object_hash *hash;
1985
1986         vm_object_assert_held(object);
1987         vm_object_assert_held(backing_object);
1988
1989         KKASSERT(backing_object == object->backing_object);
1990         info.backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1991
1992         /*
1993          * Initial conditions
1994          */
1995         if (op & OBSC_TEST_ALL_SHADOWED) {
1996                 /*
1997                  * We do not want to have to test for the existence of
1998                  * swap pages in the backing object.  XXX but with the
1999                  * new swapper this would be pretty easy to do.
2000                  *
2001                  * XXX what about anonymous MAP_SHARED memory that hasn't
2002                  * been ZFOD faulted yet?  If we do not test for this, the
2003                  * shadow test may succeed! XXX
2004                  */
2005                 if (backing_object->type != OBJT_DEFAULT)
2006                         return(0);
2007         }
2008         if (op & OBSC_COLLAPSE_WAIT) {
2009                 KKASSERT((backing_object->flags & OBJ_DEAD) == 0);
2010                 vm_object_set_flag(backing_object, OBJ_DEAD);
2011
2012                 hash = VMOBJ_HASH(backing_object);
2013                 lwkt_gettoken(&hash->token);
2014                 TAILQ_REMOVE(&hash->list, backing_object, object_list);
2015                 lwkt_reltoken(&hash->token);
2016         }
2017
2018         /*
2019          * Our scan.   We have to retry if a negative error code is returned,
2020          * otherwise 0 or 1 will be returned in info.error.  0 Indicates that
2021          * the scan had to be stopped because the parent does not completely
2022          * shadow the child.
2023          */
2024         info.object = object;
2025         info.backing_object = backing_object;
2026         info.limit = op;
2027         info.count = 0;
2028         do {
2029                 info.error = 1;
2030                 vm_page_rb_tree_RB_SCAN(&backing_object->rb_memq, NULL,
2031                                         vm_object_backing_scan_callback,
2032                                         &info);
2033         } while (info.error < 0);
2034
2035         return(info.error);
2036 }
2037
2038 /*
2039  * The caller must hold the object.
2040  */
2041 static int
2042 vm_object_backing_scan_callback(vm_page_t p, void *data)
2043 {
2044         struct rb_vm_page_scan_info *info = data;
2045         vm_object_t backing_object;
2046         vm_object_t object;
2047         vm_pindex_t pindex;
2048         vm_pindex_t new_pindex;
2049         vm_pindex_t backing_offset_index;
2050         int op;
2051
2052         pindex = p->pindex;
2053         new_pindex = pindex - info->backing_offset_index;
2054         op = info->limit;
2055         object = info->object;
2056         backing_object = info->backing_object;
2057         backing_offset_index = info->backing_offset_index;
2058
2059         if (op & OBSC_TEST_ALL_SHADOWED) {
2060                 vm_page_t pp;
2061
2062                 /*
2063                  * Ignore pages outside the parent object's range
2064                  * and outside the parent object's mapping of the 
2065                  * backing object.
2066                  *
2067                  * note that we do not busy the backing object's
2068                  * page.
2069                  */
2070                 if (pindex < backing_offset_index ||
2071                     new_pindex >= object->size
2072                 ) {
2073                         return(0);
2074                 }
2075
2076                 /*
2077                  * See if the parent has the page or if the parent's
2078                  * object pager has the page.  If the parent has the
2079                  * page but the page is not valid, the parent's
2080                  * object pager must have the page.
2081                  *
2082                  * If this fails, the parent does not completely shadow
2083                  * the object and we might as well give up now.
2084                  */
2085                 pp = vm_page_lookup(object, new_pindex);
2086                 if ((pp == NULL || pp->valid == 0) &&
2087                     !vm_pager_has_page(object, new_pindex)
2088                 ) {
2089                         info->error = 0;        /* problemo */
2090                         return(-1);             /* stop the scan */
2091                 }
2092         }
2093
2094         /*
2095          * Check for busy page.  Note that we may have lost (p) when we
2096          * possibly blocked above.
2097          */
2098         if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
2099                 vm_page_t pp;
2100
2101                 if (vm_page_busy_try(p, TRUE)) {
2102                         if (op & OBSC_COLLAPSE_NOWAIT) {
2103                                 return(0);
2104                         } else {
2105                                 /*
2106                                  * If we slept, anything could have
2107                                  * happened.   Ask that the scan be restarted.
2108                                  *
2109                                  * Since the object is marked dead, the
2110                                  * backing offset should not have changed.  
2111                                  */
2112                                 vm_page_sleep_busy(p, TRUE, "vmocol");
2113                                 info->error = -1;
2114                                 return(-1);
2115                         }
2116                 }
2117
2118                 /*
2119                  * If (p) is no longer valid restart the scan.
2120                  */
2121                 if (p->object != backing_object || p->pindex != pindex) {
2122                         kprintf("vm_object_backing_scan: Warning: page "
2123                                 "%p ripped out from under us\n", p);
2124                         vm_page_wakeup(p);
2125                         info->error = -1;
2126                         return(-1);
2127                 }
2128
2129                 if (op & OBSC_COLLAPSE_NOWAIT) {
2130                         if (p->valid == 0 ||
2131                             p->wire_count ||
2132                             (p->flags & PG_NEED_COMMIT)) {
2133                                 vm_page_wakeup(p);
2134                                 return(0);
2135                         }
2136                 } else {
2137                         /* XXX what if p->valid == 0 , hold_count, etc? */
2138                 }
2139
2140                 KASSERT(
2141                     p->object == backing_object,
2142                     ("vm_object_qcollapse(): object mismatch")
2143                 );
2144
2145                 /*
2146                  * Destroy any associated swap
2147                  */
2148                 if (backing_object->type == OBJT_SWAP)
2149                         swap_pager_freespace(backing_object, p->pindex, 1);
2150
2151                 if (
2152                     p->pindex < backing_offset_index ||
2153                     new_pindex >= object->size
2154                 ) {
2155                         /*
2156                          * Page is out of the parent object's range, we 
2157                          * can simply destroy it. 
2158                          */
2159                         vm_page_protect(p, VM_PROT_NONE);
2160                         vm_page_free(p);
2161                         return(0);
2162                 }
2163
2164                 pp = vm_page_lookup(object, new_pindex);
2165                 if (pp != NULL || vm_pager_has_page(object, new_pindex)) {
2166                         /*
2167                          * page already exists in parent OR swap exists
2168                          * for this location in the parent.  Destroy 
2169                          * the original page from the backing object.
2170                          *
2171                          * Leave the parent's page alone
2172                          */
2173                         vm_page_protect(p, VM_PROT_NONE);
2174                         vm_page_free(p);
2175                         return(0);
2176                 }
2177
2178                 /*
2179                  * Page does not exist in parent, rename the
2180                  * page from the backing object to the main object. 
2181                  *
2182                  * If the page was mapped to a process, it can remain 
2183                  * mapped through the rename.
2184                  */
2185                 if ((p->queue - p->pc) == PQ_CACHE)
2186                         vm_page_deactivate(p);
2187
2188                 vm_page_rename(p, object, new_pindex);
2189                 vm_page_wakeup(p);
2190                 /* page automatically made dirty by rename */
2191         }
2192         return(0);
2193 }
2194
2195 /*
2196  * This version of collapse allows the operation to occur earlier and
2197  * when paging_in_progress is true for an object...  This is not a complete
2198  * operation, but should plug 99.9% of the rest of the leaks.
2199  *
2200  * The caller must hold the object and backing_object and both must be
2201  * chainlocked.
2202  *
2203  * (only called from vm_object_collapse)
2204  */
2205 static void
2206 vm_object_qcollapse(vm_object_t object, vm_object_t backing_object)
2207 {
2208         if (backing_object->ref_count == 1) {
2209                 atomic_add_int(&backing_object->ref_count, 2);
2210 #if defined(DEBUG_LOCKS)
2211                 debugvm_object_add(backing_object, "qcollapse", 1, 2);
2212 #endif
2213                 vm_object_backing_scan(object, backing_object,
2214                                        OBSC_COLLAPSE_NOWAIT);
2215                 atomic_add_int(&backing_object->ref_count, -2);
2216 #if defined(DEBUG_LOCKS)
2217                 debugvm_object_add(backing_object, "qcollapse", 2, -2);
2218 #endif
2219         }
2220 }
2221
2222 /*
2223  * Collapse an object with the object backing it.  Pages in the backing
2224  * object are moved into the parent, and the backing object is deallocated.
2225  * Any conflict is resolved in favor of the parent's existing pages.
2226  *
2227  * object must be held and chain-locked on call.
2228  *
2229  * The caller must have an extra ref on object to prevent a race from
2230  * destroying it during the collapse.
2231  */
2232 void
2233 vm_object_collapse(vm_object_t object, struct vm_object_dealloc_list **dlistp)
2234 {
2235         struct vm_object_dealloc_list *dlist = NULL;
2236         vm_object_t backing_object;
2237
2238         /*
2239          * Only one thread is attempting a collapse at any given moment.
2240          * There are few restrictions for (object) that callers of this
2241          * function check so reentrancy is likely.
2242          */
2243         KKASSERT(object != NULL);
2244         vm_object_assert_held(object);
2245         KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL));
2246
2247         for (;;) {
2248                 vm_object_t bbobj;
2249                 int dodealloc;
2250
2251                 /*
2252                  * We can only collapse a DEFAULT/SWAP object with a
2253                  * DEFAULT/SWAP object.
2254                  */
2255                 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) {
2256                         backing_object = NULL;
2257                         break;
2258                 }
2259
2260                 backing_object = object->backing_object;
2261                 if (backing_object == NULL)
2262                         break;
2263                 if (backing_object->type != OBJT_DEFAULT &&
2264                     backing_object->type != OBJT_SWAP) {
2265                         backing_object = NULL;
2266                         break;
2267                 }
2268
2269                 /*
2270                  * Hold the backing_object and check for races
2271                  */
2272                 vm_object_hold(backing_object);
2273                 if (backing_object != object->backing_object ||
2274                     (backing_object->type != OBJT_DEFAULT &&
2275                      backing_object->type != OBJT_SWAP)) {
2276                         vm_object_drop(backing_object);
2277                         continue;
2278                 }
2279
2280                 /*
2281                  * Chain-lock the backing object too because if we
2282                  * successfully merge its pages into the top object we
2283                  * will collapse backing_object->backing_object as the
2284                  * new backing_object.  Re-check that it is still our
2285                  * backing object.
2286                  */
2287                 vm_object_chain_acquire(backing_object, 0);
2288                 if (backing_object != object->backing_object) {
2289                         vm_object_chain_release(backing_object);
2290                         vm_object_drop(backing_object);
2291                         continue;
2292                 }
2293
2294                 /*
2295                  * we check the backing object first, because it is most likely
2296                  * not collapsable.
2297                  */
2298                 if (backing_object->handle != NULL ||
2299                     (backing_object->type != OBJT_DEFAULT &&
2300                      backing_object->type != OBJT_SWAP) ||
2301                     (backing_object->flags & OBJ_DEAD) ||
2302                     object->handle != NULL ||
2303                     (object->type != OBJT_DEFAULT &&
2304                      object->type != OBJT_SWAP) ||
2305                     (object->flags & OBJ_DEAD)) {
2306                         break;
2307                 }
2308
2309                 /*
2310                  * If paging is in progress we can't do a normal collapse.
2311                  */
2312                 if (
2313                     object->paging_in_progress != 0 ||
2314                     backing_object->paging_in_progress != 0
2315                 ) {
2316                         vm_object_qcollapse(object, backing_object);
2317                         break;
2318                 }
2319
2320                 /*
2321                  * We know that we can either collapse the backing object (if
2322                  * the parent is the only reference to it) or (perhaps) have
2323                  * the parent bypass the object if the parent happens to shadow
2324                  * all the resident pages in the entire backing object.
2325                  *
2326                  * This is ignoring pager-backed pages such as swap pages.
2327                  * vm_object_backing_scan fails the shadowing test in this
2328                  * case.
2329                  */
2330                 if (backing_object->ref_count == 1) {
2331                         /*
2332                          * If there is exactly one reference to the backing
2333                          * object, we can collapse it into the parent.  
2334                          */
2335                         KKASSERT(object->backing_object == backing_object);
2336                         vm_object_backing_scan(object, backing_object,
2337                                                OBSC_COLLAPSE_WAIT);
2338
2339                         /*
2340                          * Move the pager from backing_object to object.
2341                          */
2342                         if (backing_object->type == OBJT_SWAP) {
2343                                 vm_object_pip_add(backing_object, 1);
2344
2345                                 /*
2346                                  * scrap the paging_offset junk and do a 
2347                                  * discrete copy.  This also removes major 
2348                                  * assumptions about how the swap-pager 
2349                                  * works from where it doesn't belong.  The
2350                                  * new swapper is able to optimize the
2351                                  * destroy-source case.
2352                                  */
2353                                 vm_object_pip_add(object, 1);
2354                                 swap_pager_copy(backing_object, object,
2355                                     OFF_TO_IDX(object->backing_object_offset),
2356                                     TRUE);
2357                                 vm_object_pip_wakeup(object);
2358                                 vm_object_pip_wakeup(backing_object);
2359                         }
2360
2361                         /*
2362                          * Object now shadows whatever backing_object did.
2363                          * Remove object from backing_object's shadow_list.
2364                          *
2365                          * Removing object from backing_objects shadow list
2366                          * requires releasing object, which we will do below.
2367                          */
2368                         KKASSERT(object->backing_object == backing_object);
2369                         if (object->flags & OBJ_ONSHADOW) {
2370                                 LIST_REMOVE(object, shadow_list);
2371                                 backing_object->shadow_count--;
2372                                 backing_object->generation++;
2373                                 vm_object_clear_flag(object, OBJ_ONSHADOW);
2374                         }
2375
2376                         /*
2377                          * backing_object->backing_object moves from within
2378                          * backing_object to within object.
2379                          *
2380                          * OBJT_VNODE bbobj's should have empty shadow lists.
2381                          */
2382                         while ((bbobj = backing_object->backing_object) != NULL) {
2383                                 if (bbobj->type == OBJT_VNODE)
2384                                         vm_object_hold_shared(bbobj);
2385                                 else
2386                                         vm_object_hold(bbobj);
2387                                 if (bbobj == backing_object->backing_object)
2388                                         break;
2389                                 vm_object_drop(bbobj);
2390                         }
2391
2392                         /*
2393                          * We are removing backing_object from bbobj's
2394                          * shadow list and adding object to bbobj's shadow
2395                          * list, so the ref_count on bbobj is unchanged.
2396                          */
2397                         if (bbobj) {
2398                                 if (backing_object->flags & OBJ_ONSHADOW) {
2399                                         /* not locked exclusively if vnode */
2400                                         KKASSERT(bbobj->type != OBJT_VNODE);
2401                                         LIST_REMOVE(backing_object,
2402                                                     shadow_list);
2403                                         bbobj->shadow_count--;
2404                                         bbobj->generation++;
2405                                         vm_object_clear_flag(backing_object,
2406                                                              OBJ_ONSHADOW);
2407                                 }
2408                                 backing_object->backing_object = NULL;
2409                         }
2410                         object->backing_object = bbobj;
2411                         if (bbobj) {
2412                                 if (bbobj->type != OBJT_VNODE) {
2413                                         LIST_INSERT_HEAD(&bbobj->shadow_head,
2414                                                          object, shadow_list);
2415                                         bbobj->shadow_count++;
2416                                         bbobj->generation++;
2417                                         vm_object_set_flag(object,
2418                                                            OBJ_ONSHADOW);
2419                                 }
2420                         }
2421
2422                         object->backing_object_offset +=
2423                                 backing_object->backing_object_offset;
2424
2425                         vm_object_drop(bbobj);
2426
2427                         /*
2428                          * Discard the old backing_object.  Nothing should be
2429                          * able to ref it, other than a vm_map_split(),
2430                          * and vm_map_split() will stall on our chain lock.
2431                          * And we control the parent so it shouldn't be
2432                          * possible for it to go away either.
2433                          *
2434                          * Since the backing object has no pages, no pager
2435                          * left, and no object references within it, all
2436                          * that is necessary is to dispose of it.
2437                          */
2438                         KASSERT(backing_object->ref_count == 1,
2439                                 ("backing_object %p was somehow "
2440                                  "re-referenced during collapse!",
2441                                  backing_object));
2442                         KASSERT(RB_EMPTY(&backing_object->rb_memq),
2443                                 ("backing_object %p somehow has left "
2444                                  "over pages during collapse!",
2445                                  backing_object));
2446
2447                         /*
2448                          * The object can be destroyed.
2449                          *
2450                          * XXX just fall through and dodealloc instead
2451                          *     of forcing destruction?
2452                          */
2453                         atomic_add_int(&backing_object->ref_count, -1);
2454 #if defined(DEBUG_LOCKS)
2455                         debugvm_object_add(backing_object, "collapse", 1, -1);
2456 #endif
2457                         if ((backing_object->flags & OBJ_DEAD) == 0)
2458                                 vm_object_terminate(backing_object);
2459                         object_collapses++;
2460                         dodealloc = 0;
2461                 } else {
2462                         /*
2463                          * If we do not entirely shadow the backing object,
2464                          * there is nothing we can do so we give up.
2465                          */
2466                         if (vm_object_backing_scan(object, backing_object,
2467                                                 OBSC_TEST_ALL_SHADOWED) == 0) {
2468                                 break;
2469                         }
2470
2471                         /*
2472                          * bbobj is backing_object->backing_object.  Since
2473                          * object completely shadows backing_object we can
2474                          * bypass it and become backed by bbobj instead.
2475                          *
2476                          * The shadow list for vnode backing objects is not
2477                          * used and a shared hold is allowed.
2478                          */
2479                         while ((bbobj = backing_object->backing_object) != NULL) {
2480                                 if (bbobj->type == OBJT_VNODE)
2481                                         vm_object_hold_shared(bbobj);
2482                                 else
2483                                         vm_object_hold(bbobj);
2484                                 if (bbobj == backing_object->backing_object)
2485                                         break;
2486                                 vm_object_drop(bbobj);
2487                         }
2488
2489                         /*
2490                          * Make object shadow bbobj instead of backing_object.
2491                          * Remove object from backing_object's shadow list.
2492                          *
2493                          * Deallocating backing_object will not remove
2494                          * it, since its reference count is at least 2.
2495                          *
2496                          * Removing object from backing_object's shadow
2497                          * list requires releasing a ref, which we do
2498                          * below by setting dodealloc to 1.
2499                          */
2500                         KKASSERT(object->backing_object == backing_object);
2501                         if (object->flags & OBJ_ONSHADOW) {
2502                                 LIST_REMOVE(object, shadow_list);
2503                                 backing_object->shadow_count--;
2504                                 backing_object->generation++;
2505                                 vm_object_clear_flag(object, OBJ_ONSHADOW);
2506                         }
2507
2508                         /*
2509                          * Add a ref to bbobj, bbobj now shadows object.
2510                          *
2511                          * NOTE: backing_object->backing_object still points
2512                          *       to bbobj.  That relationship remains intact
2513                          *       because backing_object has > 1 ref, so
2514                          *       someone else is pointing to it (hence why
2515                          *       we can't collapse it into object and can
2516                          *       only handle the all-shadowed bypass case).
2517                          */
2518                         if (bbobj) {
2519                                 if (bbobj->type != OBJT_VNODE) {
2520                                         vm_object_chain_wait(bbobj, 0);
2521                                         vm_object_reference_locked(bbobj);
2522                                         LIST_INSERT_HEAD(&bbobj->shadow_head,
2523                                                          object, shadow_list);
2524                                         bbobj->shadow_count++;
2525                                         bbobj->generation++;
2526                                         vm_object_set_flag(object,
2527                                                            OBJ_ONSHADOW);
2528                                 } else {
2529                                         vm_object_reference_quick(bbobj);
2530                                 }
2531                                 object->backing_object_offset +=
2532                                         backing_object->backing_object_offset;
2533                                 object->backing_object = bbobj;
2534                                 vm_object_drop(bbobj);
2535                         } else {
2536                                 object->backing_object = NULL;
2537                         }
2538
2539                         /*
2540                          * Drop the reference count on backing_object.  To
2541                          * handle ref_count races properly we can't assume
2542                          * that the ref_count is still at least 2 so we
2543                          * have to actually call vm_object_deallocate()
2544                          * (after clearing the chainlock).
2545                          */
2546                         object_bypasses++;
2547                         dodealloc = 1;
2548                 }
2549
2550                 /*
2551                  * Ok, we want to loop on the new object->bbobj association,
2552                  * possibly collapsing it further.  However if dodealloc is
2553                  * non-zero we have to deallocate the backing_object which
2554                  * itself can potentially undergo a collapse, creating a
2555                  * recursion depth issue with the LWKT token subsystem.
2556                  *
2557                  * In the case where we must deallocate the backing_object
2558                  * it is possible now that the backing_object has a single
2559                  * shadow count on some other object (not represented here
2560                  * as yet), since it no longer shadows us.  Thus when we
2561                  * call vm_object_deallocate() it may attempt to collapse
2562                  * itself into its remaining parent.
2563                  */
2564                 if (dodealloc) {
2565                         struct vm_object_dealloc_list *dtmp;
2566
2567                         vm_object_chain_release(backing_object);
2568                         vm_object_unlock(backing_object);
2569                         /* backing_object remains held */
2570
2571                         /*
2572                          * Auto-deallocation list for caller convenience.
2573                          */
2574                         if (dlistp == NULL)
2575                                 dlistp = &dlist;
2576
2577                         dtmp = kmalloc(sizeof(*dtmp), M_TEMP, M_WAITOK);
2578                         dtmp->object = backing_object;
2579                         dtmp->next = *dlistp;
2580                         *dlistp = dtmp;
2581                 } else {
2582                         vm_object_chain_release(backing_object);
2583                         vm_object_drop(backing_object);
2584                 }
2585                 /* backing_object = NULL; not needed */
2586                 /* loop */
2587         }
2588
2589         /*
2590          * Clean up any left over backing_object
2591          */
2592         if (backing_object) {
2593                 vm_object_chain_release(backing_object);
2594                 vm_object_drop(backing_object);
2595         }
2596
2597         /*
2598          * Clean up any auto-deallocation list.  This is a convenience
2599          * for top-level callers so they don't have to pass &dlist.
2600          * Do not clean up any caller-passed dlistp, the caller will
2601          * do that.
2602          */
2603         if (dlist)
2604                 vm_object_deallocate_list(&dlist);
2605
2606 }
2607
2608 /*
2609  * vm_object_collapse() may collect additional objects in need of
2610  * deallocation.  This routine deallocates these objects.  The
2611  * deallocation itself can trigger additional collapses (which the
2612  * deallocate function takes care of).  This procedure is used to
2613  * reduce procedural recursion since these vm_object shadow chains
2614  * can become quite long.
2615  */
2616 void
2617 vm_object_deallocate_list(struct vm_object_dealloc_list **dlistp)
2618 {
2619         struct vm_object_dealloc_list *dlist;
2620
2621         while ((dlist = *dlistp) != NULL) {
2622                 *dlistp = dlist->next;
2623                 vm_object_lock(dlist->object);
2624                 vm_object_deallocate_locked(dlist->object);
2625                 vm_object_drop(dlist->object);
2626                 kfree(dlist, M_TEMP);
2627         }
2628 }
2629
2630 /*
2631  * Removes all physical pages in the specified object range from the
2632  * object's list of pages.
2633  *
2634  * No requirements.
2635  */
2636 static int vm_object_page_remove_callback(vm_page_t p, void *data);
2637
2638 void
2639 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
2640                       boolean_t clean_only)
2641 {
2642         struct rb_vm_page_scan_info info;
2643         int all;
2644
2645         /*
2646          * Degenerate cases and assertions
2647          */
2648         vm_object_hold(object);
2649         if (object == NULL ||
2650             (object->resident_page_count == 0 && object->swblock_count == 0)) {
2651                 vm_object_drop(object);
2652                 return;
2653         }
2654         KASSERT(object->type != OBJT_PHYS, 
2655                 ("attempt to remove pages from a physical object"));
2656
2657         /*
2658          * Indicate that paging is occuring on the object
2659          */
2660         vm_object_pip_add(object, 1);
2661
2662         /*
2663          * Figure out the actual removal range and whether we are removing
2664          * the entire contents of the object or not.  If removing the entire
2665          * contents, be sure to get all pages, even those that might be 
2666          * beyond the end of the object.
2667          */
2668         info.object = object;
2669         info.start_pindex = start;
2670         if (end == 0)
2671                 info.end_pindex = (vm_pindex_t)-1;
2672         else
2673                 info.end_pindex = end - 1;
2674         info.limit = clean_only;
2675         info.count = 0;
2676         all = (start == 0 && info.end_pindex >= object->size - 1);
2677
2678         /*
2679          * Loop until we are sure we have gotten them all.
2680          */
2681         do {
2682                 info.error = 0;
2683                 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
2684                                         vm_object_page_remove_callback, &info);
2685         } while (info.error);
2686
2687         /*
2688          * Remove any related swap if throwing away pages, or for
2689          * non-swap objects (the swap is a clean copy in that case).
2690          */
2691         if (object->type != OBJT_SWAP || clean_only == FALSE) {
2692                 if (all)
2693                         swap_pager_freespace_all(object);
2694                 else
2695                         swap_pager_freespace(object, info.start_pindex,
2696                              info.end_pindex - info.start_pindex + 1);
2697         }
2698
2699         /*
2700          * Cleanup
2701          */
2702         vm_object_pip_wakeup(object);
2703         vm_object_drop(object);
2704 }
2705
2706 /*
2707  * The caller must hold the object.
2708  *
2709  * NOTE: User yields are allowed when removing more than one page, but not
2710  *       allowed if only removing one page (the path for single page removals
2711  *       might hold a spinlock).
2712  */
2713 static int
2714 vm_object_page_remove_callback(vm_page_t p, void *data)
2715 {
2716         struct rb_vm_page_scan_info *info = data;
2717
2718         if ((++info->count & 63) == 0)
2719                 lwkt_user_yield();
2720
2721         if (info->object != p->object ||
2722             p->pindex < info->start_pindex ||
2723             p->pindex > info->end_pindex) {
2724                 kprintf("vm_object_page_remove_callbackA: obj/pg race %p/%p\n",
2725                         info->object, p);
2726                 return(0);
2727         }
2728         if (vm_page_busy_try(p, TRUE)) {
2729                 vm_page_sleep_busy(p, TRUE, "vmopar");
2730                 info->error = 1;
2731                 return(0);
2732         }
2733         if (info->object != p->object) {
2734                 /* this should never happen */
2735                 kprintf("vm_object_page_remove_callbackB: obj/pg race %p/%p\n",
2736                         info->object, p);
2737                 vm_page_wakeup(p);
2738                 return(0);
2739         }
2740
2741         /*
2742          * Wired pages cannot be destroyed, but they can be invalidated
2743          * and we do so if clean_only (limit) is not set.
2744          *
2745          * WARNING!  The page may be wired due to being part of a buffer
2746          *           cache buffer, and the buffer might be marked B_CACHE.
2747          *           This is fine as part of a truncation but VFSs must be
2748          *           sure to fix the buffer up when re-extending the file.
2749          *
2750          * NOTE!     PG_NEED_COMMIT is ignored.
2751          */
2752         if (p->wire_count != 0) {
2753                 vm_page_protect(p, VM_PROT_NONE);
2754                 if (info->limit == 0)
2755                         p->valid = 0;
2756                 vm_page_wakeup(p);
2757                 return(0);
2758         }
2759
2760         /*
2761          * limit is our clean_only flag.  If set and the page is dirty or
2762          * requires a commit, do not free it.  If set and the page is being
2763          * held by someone, do not free it.
2764          */
2765         if (info->limit && p->valid) {
2766                 vm_page_test_dirty(p);
2767                 if ((p->valid & p->dirty) || (p->flags & PG_NEED_COMMIT)) {
2768                         vm_page_wakeup(p);
2769                         return(0);
2770                 }
2771         }
2772
2773         /*
2774          * Destroy the page
2775          */
2776         vm_page_protect(p, VM_PROT_NONE);
2777         vm_page_free(p);
2778
2779         return(0);
2780 }
2781
2782 /*
2783  * Coalesces two objects backing up adjoining regions of memory into a
2784  * single object.
2785  *
2786  * returns TRUE if objects were combined.
2787  *
2788  * NOTE: Only works at the moment if the second object is NULL -
2789  *       if it's not, which object do we lock first?
2790  *
2791  * Parameters:
2792  *      prev_object     First object to coalesce
2793  *      prev_offset     Offset into prev_object
2794  *      next_object     Second object into coalesce
2795  *      next_offset     Offset into next_object
2796  *
2797  *      prev_size       Size of reference to prev_object
2798  *      next_size       Size of reference to next_object
2799  *
2800  * The caller does not need to hold (prev_object) but must have a stable
2801  * pointer to it (typically by holding the vm_map locked).
2802  */
2803 boolean_t
2804 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
2805                    vm_size_t prev_size, vm_size_t next_size)
2806 {
2807         vm_pindex_t next_pindex;
2808
2809         if (prev_object == NULL)
2810                 return (TRUE);
2811
2812         vm_object_hold(prev_object);
2813
2814         if (prev_object->type != OBJT_DEFAULT &&
2815             prev_object->type != OBJT_SWAP) {
2816                 vm_object_drop(prev_object);
2817                 return (FALSE);
2818         }
2819
2820         /*
2821          * Try to collapse the object first
2822          */
2823         vm_object_chain_acquire(prev_object, 0);
2824         vm_object_collapse(prev_object, NULL);
2825
2826         /*
2827          * Can't coalesce if: . more than one reference . paged out . shadows
2828          * another object . has a copy elsewhere (any of which mean that the
2829          * pages not mapped to prev_entry may be in use anyway)
2830          */
2831
2832         if (prev_object->backing_object != NULL) {
2833                 vm_object_chain_release(prev_object);
2834                 vm_object_drop(prev_object);
2835                 return (FALSE);
2836         }
2837
2838         prev_size >>= PAGE_SHIFT;
2839         next_size >>= PAGE_SHIFT;
2840         next_pindex = prev_pindex + prev_size;
2841
2842         if ((prev_object->ref_count > 1) &&
2843             (prev_object->size != next_pindex)) {
2844                 vm_object_chain_release(prev_object);
2845                 vm_object_drop(prev_object);
2846                 return (FALSE);
2847         }
2848
2849         /*
2850          * Remove any pages that may still be in the object from a previous
2851          * deallocation.
2852          */
2853         if (next_pindex < prev_object->size) {
2854                 vm_object_page_remove(prev_object,
2855                                       next_pindex,
2856                                       next_pindex + next_size, FALSE);
2857                 if (prev_object->type == OBJT_SWAP)
2858                         swap_pager_freespace(prev_object,
2859                                              next_pindex, next_size);
2860         }
2861
2862         /*
2863          * Extend the object if necessary.
2864          */
2865         if (next_pindex + next_size > prev_object->size)
2866                 prev_object->size = next_pindex + next_size;
2867
2868         vm_object_chain_release(prev_object);
2869         vm_object_drop(prev_object);
2870         return (TRUE);
2871 }
2872
2873 /*
2874  * Make the object writable and flag is being possibly dirty.
2875  *
2876  * The object might not be held (or might be held but held shared),
2877  * the related vnode is probably not held either.  Object and vnode are
2878  * stable by virtue of the vm_page busied by the caller preventing
2879  * destruction.
2880  *
2881  * If the related mount is flagged MNTK_THR_SYNC we need to call
2882  * vsetobjdirty().  Filesystems using this option usually shortcut
2883  * synchronization by only scanning the syncer list.
2884  */
2885 void
2886 vm_object_set_writeable_dirty(vm_object_t object)
2887 {
2888         struct vnode *vp;
2889
2890         /*vm_object_assert_held(object);*/
2891         /*
2892          * Avoid contention in vm fault path by checking the state before
2893          * issuing an atomic op on it.
2894          */
2895         if ((object->flags & (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) !=
2896             (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) {
2897                 vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
2898         }
2899         if (object->type == OBJT_VNODE &&
2900             (vp = (struct vnode *)object->handle) != NULL) {
2901                 if ((vp->v_flag & VOBJDIRTY) == 0) {
2902                         if (vp->v_mount &&
2903                             (vp->v_mount->mnt_kern_flag & MNTK_THR_SYNC)) {
2904                                 /*
2905                                  * New style THR_SYNC places vnodes on the
2906                                  * syncer list more deterministically.
2907                                  */
2908                                 vsetobjdirty(vp);
2909                         } else {
2910                                 /*
2911                                  * Old style scan would not necessarily place
2912                                  * a vnode on the syncer list when possibly
2913                                  * modified via mmap.
2914                                  */
2915                                 vsetflags(vp, VOBJDIRTY);
2916                         }
2917                 }
2918         }
2919 }
2920
2921 #include "opt_ddb.h"
2922 #ifdef DDB
2923 #include <sys/kernel.h>
2924
2925 #include <sys/cons.h>
2926
2927 #include <ddb/ddb.h>
2928
2929 static int      _vm_object_in_map (vm_map_t map, vm_object_t object,
2930                                        vm_map_entry_t entry);
2931 static int      vm_object_in_map (vm_object_t object);
2932
2933 /*
2934  * The caller must hold the object.
2935  */
2936 static int
2937 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2938 {
2939         vm_map_t tmpm;
2940         vm_map_entry_t tmpe;
2941         vm_object_t obj, nobj;
2942         int entcount;
2943
2944         if (map == 0)
2945                 return 0;
2946         if (entry == 0) {
2947                 tmpe = map->header.next;
2948                 entcount = map->nentries;
2949                 while (entcount-- && (tmpe != &map->header)) {
2950                         if( _vm_object_in_map(map, object, tmpe)) {
2951                                 return 1;
2952                         }
2953                         tmpe = tmpe->next;
2954                 }
2955                 return (0);
2956         }
2957         switch(entry->maptype) {
2958         case VM_MAPTYPE_SUBMAP:
2959                 tmpm = entry->object.sub_map;
2960                 tmpe = tmpm->header.next;
2961                 entcount = tmpm->nentries;
2962                 while (entcount-- && tmpe != &tmpm->header) {
2963                         if( _vm_object_in_map(tmpm, object, tmpe)) {
2964                                 return 1;
2965                         }
2966                         tmpe = tmpe->next;
2967                 }
2968                 break;
2969         case VM_MAPTYPE_NORMAL:
2970         case VM_MAPTYPE_VPAGETABLE:
2971                 obj = entry->object.vm_object;
2972                 while (obj) {
2973                         if (obj == object) {
2974                                 if (obj != entry->object.vm_object)
2975                                         vm_object_drop(obj);
2976                                 return 1;
2977                         }
2978                         while ((nobj = obj->backing_object) != NULL) {
2979                                 vm_object_hold(nobj);
2980                                 if (nobj == obj->backing_object)
2981                                         break;
2982                                 vm_object_drop(nobj);
2983                         }
2984                         if (obj != entry->object.vm_object) {
2985                                 if (nobj)
2986                                         vm_object_lock_swap();
2987                                 vm_object_drop(obj);
2988                         }
2989                         obj = nobj;
2990                 }
2991                 break;
2992         default:
2993                 break;
2994         }
2995         return 0;
2996 }
2997
2998 static int vm_object_in_map_callback(struct proc *p, void *data);
2999
3000 struct vm_object_in_map_info {
3001         vm_object_t object;
3002         int rv;
3003 };
3004
3005 /*
3006  * Debugging only
3007  */
3008 static int
3009 vm_object_in_map(vm_object_t object)
3010 {
3011         struct vm_object_in_map_info info;
3012
3013         info.rv = 0;
3014         info.object = object;
3015
3016         allproc_scan(vm_object_in_map_callback, &info);
3017         if (info.rv)
3018                 return 1;
3019         if( _vm_object_in_map(&kernel_map, object, 0))
3020                 return 1;
3021         if( _vm_object_in_map(&pager_map, object, 0))
3022                 return 1;
3023         if( _vm_object_in_map(&buffer_map, object, 0))
3024                 return 1;
3025         return 0;
3026 }
3027
3028 /*
3029  * Debugging only
3030  */
3031 static int
3032 vm_object_in_map_callback(struct proc *p, void *data)
3033 {
3034         struct vm_object_in_map_info *info = data;
3035
3036         if (p->p_vmspace) {
3037                 if (_vm_object_in_map(&p->p_vmspace->vm_map, info->object, 0)) {
3038                         info->rv = 1;
3039                         return -1;
3040                 }
3041         }
3042         return (0);
3043 }
3044
3045 DB_SHOW_COMMAND(vmochk, vm_object_check)
3046 {
3047         struct vm_object_hash *hash;
3048         vm_object_t object;
3049         int n;
3050
3051         /*
3052          * make sure that internal objs are in a map somewhere
3053          * and none have zero ref counts.
3054          */
3055         for (n = 0; n < VMOBJ_HSIZE; ++n) {
3056                 hash = &vm_object_hash[n];
3057                 for (object = TAILQ_FIRST(&hash->list);
3058                                 object != NULL;
3059                                 object = TAILQ_NEXT(object, object_list)) {
3060                         if (object->type == OBJT_MARKER)
3061                                 continue;
3062                         if (object->handle != NULL ||
3063                             (object->type != OBJT_DEFAULT &&
3064                              object->type != OBJT_SWAP)) {
3065                                 continue;
3066                         }
3067                         if (object->ref_count == 0) {
3068                                 db_printf("vmochk: internal obj has "
3069                                           "zero ref count: %ld\n",
3070                                           (long)object->size);
3071                         }
3072                         if (vm_object_in_map(object))
3073                                 continue;
3074                         db_printf("vmochk: internal obj is not in a map: "
3075                                   "ref: %d, size: %lu: 0x%lx, "
3076                                   "backing_object: %p\n",
3077                                   object->ref_count, (u_long)object->size,
3078                                   (u_long)object->size,
3079                                   (void *)object->backing_object);
3080                 }
3081         }
3082 }
3083
3084 /*
3085  * Debugging only
3086  */
3087 DB_SHOW_COMMAND(object, vm_object_print_static)
3088 {
3089         /* XXX convert args. */
3090         vm_object_t object = (vm_object_t)addr;
3091         boolean_t full = have_addr;
3092
3093         vm_page_t p;
3094
3095         /* XXX count is an (unused) arg.  Avoid shadowing it. */
3096 #define count   was_count
3097
3098         int count;
3099
3100         if (object == NULL)
3101                 return;
3102
3103         db_iprintf(
3104             "Object %p: type=%d, size=0x%lx, res=%ld, ref=%d, flags=0x%x\n",
3105             object, (int)object->type, (u_long)object->size,
3106             object->resident_page_count, object->ref_count, object->flags);
3107         /*
3108          * XXX no %qd in kernel.  Truncate object->backing_object_offset.
3109          */
3110         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n",
3111             object->shadow_count, 
3112             object->backing_object ? object->backing_object->ref_count : 0,
3113             object->backing_object, (long)object->backing_object_offset);
3114
3115         if (!full)
3116                 return;
3117
3118         db_indent += 2;
3119         count = 0;
3120         RB_FOREACH(p, vm_page_rb_tree, &object->rb_memq) {
3121                 if (count == 0)
3122                         db_iprintf("memory:=");
3123                 else if (count == 6) {
3124                         db_printf("\n");
3125                         db_iprintf(" ...");
3126                         count = 0;
3127                 } else
3128                         db_printf(",");
3129                 count++;
3130
3131                 db_printf("(off=0x%lx,page=0x%lx)",
3132                     (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
3133         }
3134         if (count != 0)
3135                 db_printf("\n");
3136         db_indent -= 2;
3137 }
3138
3139 /* XXX. */
3140 #undef count
3141
3142 /*
3143  * XXX need this non-static entry for calling from vm_map_print.
3144  *
3145  * Debugging only
3146  */
3147 void
3148 vm_object_print(/* db_expr_t */ long addr,
3149                 boolean_t have_addr,
3150                 /* db_expr_t */ long count,
3151                 char *modif)
3152 {
3153         vm_object_print_static(addr, have_addr, count, modif);
3154 }
3155
3156 /*
3157  * Debugging only
3158  */
3159 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
3160 {
3161         struct vm_object_hash *hash;
3162         vm_object_t object;
3163         int nl = 0;
3164         int c;
3165         int n;
3166
3167         for (n = 0; n < VMOBJ_HSIZE; ++n) {
3168                 hash = &vm_object_hash[n];
3169                 for (object = TAILQ_FIRST(&hash->list);
3170                                 object != NULL;
3171                                 object = TAILQ_NEXT(object, object_list)) {
3172                         vm_pindex_t idx, fidx;
3173                         vm_pindex_t osize;
3174                         vm_paddr_t pa = -1, padiff;
3175                         int rcount;
3176                         vm_page_t m;
3177
3178                         if (object->type == OBJT_MARKER)
3179                                 continue;
3180                         db_printf("new object: %p\n", (void *)object);
3181                         if ( nl > 18) {
3182                                 c = cngetc();
3183                                 if (c != ' ')
3184                                         return;
3185                                 nl = 0;
3186                         }
3187                         nl++;
3188                         rcount = 0;
3189                         fidx = 0;
3190                         osize = object->size;
3191                         if (osize > 128)
3192                                 osize = 128;
3193                         for (idx = 0; idx < osize; idx++) {
3194                                 m = vm_page_lookup(object, idx);
3195                                 if (m == NULL) {
3196                                         if (rcount) {
3197                                                 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
3198                                                         (long)fidx, rcount, (long)pa);
3199                                                 if ( nl > 18) {
3200                                                         c = cngetc();
3201                                                         if (c != ' ')
3202                                                                 return;
3203                                                         nl = 0;
3204                                                 }
3205                                                 nl++;
3206                                                 rcount = 0;
3207                                         }
3208                                         continue;
3209                                 }
3210
3211                                 if (rcount &&
3212                                         (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
3213                                         ++rcount;
3214                                         continue;
3215                                 }
3216                                 if (rcount) {
3217                                         padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
3218                                         padiff >>= PAGE_SHIFT;
3219                                         padiff &= PQ_L2_MASK;
3220                                         if (padiff == 0) {
3221                                                 pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
3222                                                 ++rcount;
3223                                                 continue;
3224                                         }
3225                                         db_printf(" index(%ld)run(%d)pa(0x%lx)",
3226                                                 (long)fidx, rcount, (long)pa);
3227                                         db_printf("pd(%ld)\n", (long)padiff);
3228                                         if ( nl > 18) {
3229                                                 c = cngetc();
3230                                                 if (c != ' ')
3231                                                         return;
3232                                                 nl = 0;
3233                                         }
3234                                         nl++;
3235                                 }
3236                                 fidx = idx;
3237                                 pa = VM_PAGE_TO_PHYS(m);
3238                                 rcount = 1;
3239                         }
3240                         if (rcount) {
3241                                 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
3242                                         (long)fidx, rcount, (long)pa);
3243                                 if ( nl > 18) {
3244                                         c = cngetc();
3245                                         if (c != ' ')
3246                                                 return;
3247                                         nl = 0;
3248                                 }
3249                                 nl++;
3250                         }
3251                 }
3252         }
3253 }
3254 #endif /* DDB */