kernel - Fix more wiring / fictitious bugs w/recent VM work
[dragonfly.git] / sys / vm / vm_object.c
CommitLineData
984263bc 1/*
501747bf 2 * Copyright (c) 1991, 1993, 2013
984263bc
MD
3 * The Regents of the University of California. All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
dc71b7ab 16 * 3. Neither the name of the University nor the names of its contributors
984263bc
MD
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94
33 *
34 *
35 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
36 * All rights reserved.
37 *
38 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
39 *
40 * Permission to use, copy, modify and distribute this software and
41 * its documentation is hereby granted, provided that both the copyright
42 * notice and this permission notice appear in all copies of the
43 * software, derivative works or modified versions, and any portions
44 * thereof, and that both notices appear in supporting documentation.
45 *
46 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
47 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
48 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
49 *
50 * Carnegie Mellon requests users of this software to return to
51 *
52 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
53 * School of Computer Science
54 * Carnegie Mellon University
55 * Pittsburgh PA 15213-3890
56 *
57 * any improvements or extensions that they make and grant Carnegie the
58 * rights to redistribute these changes.
59 *
60 * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $
61 */
62
63/*
64 * Virtual memory object module.
65 */
66
67#include <sys/param.h>
68#include <sys/systm.h>
69#include <sys/proc.h> /* for curproc, pageproc */
e32ad78d 70#include <sys/thread.h>
984263bc
MD
71#include <sys/vnode.h>
72#include <sys/vmmeter.h>
73#include <sys/mman.h>
74#include <sys/mount.h>
75#include <sys/kernel.h>
76#include <sys/sysctl.h>
e1c14c82 77#include <sys/refcount.h>
984263bc
MD
78
79#include <vm/vm.h>
80#include <vm/vm_param.h>
81#include <vm/pmap.h>
82#include <vm/vm_map.h>
83#include <vm/vm_object.h>
84#include <vm/vm_page.h>
85#include <vm/vm_pageout.h>
86#include <vm/vm_pager.h>
87#include <vm/swap_pager.h>
88#include <vm/vm_kern.h>
89#include <vm/vm_extern.h>
90#include <vm/vm_zone.h>
91
a86ce0cd
MD
92#include <vm/vm_page2.h>
93
b524ca76
MD
94#include <machine/specialreg.h>
95
984263bc
MD
96#define EASY_SCAN_FACTOR 8
97
b12defdc
MD
98static void vm_object_qcollapse(vm_object_t object,
99 vm_object_t backing_object);
ab3e1edd 100static void vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
1f804340 101 int pagerflags);
e1c14c82 102static void vm_object_lock_init(vm_object_t);
e1c14c82 103
984263bc
MD
104/*
105 * Virtual memory objects maintain the actual data
106 * associated with allocated virtual memory. A given
107 * page of memory exists within exactly one object.
108 *
109 * An object is only deallocated when all "references"
110 * are given up. Only one "reference" to a given
111 * region of an object should be writeable.
112 *
113 * Associated with each object is a list of all resident
114 * memory pages belonging to that object; this list is
115 * maintained by the "vm_page" module, and locked by the object's
116 * lock.
117 *
118 * Each object also records a "pager" routine which is
119 * used to retrieve (and store) pages to the proper backing
120 * storage. In addition, objects may be backed by other
121 * objects from which they were virtual-copied.
122 *
123 * The only items within the object structure which are
124 * modified after time of creation are:
125 * reference count locked by object's lock
126 * pager routine locked by object's lock
127 *
128 */
129
c439ad8f
MD
130struct vm_object kernel_object;
131
984263bc
MD
132static long object_collapses;
133static long object_bypasses;
984263bc 134
fde6be6a
MD
135struct vm_object_hash vm_object_hash[VMOBJ_HSIZE];
136
137MALLOC_DEFINE(M_VM_OBJECT, "vm_object", "vm_object structures");
7b00fbb4 138
562ffbba
MD
139#define VMOBJ_HASH_PRIME1 66555444443333333ULL
140#define VMOBJ_HASH_PRIME2 989042931893ULL
141
0ca81fbe
MD
142int vm_object_debug;
143SYSCTL_INT(_vm, OID_AUTO, object_debug, CTLFLAG_RW, &vm_object_debug, 0, "");
144
562ffbba
MD
145static __inline
146struct vm_object_hash *
147vmobj_hash(vm_object_t obj)
148{
149 uintptr_t hash1;
150 uintptr_t hash2;
151
152 hash1 = (uintptr_t)obj + ((uintptr_t)obj >> 18);
153 hash1 %= VMOBJ_HASH_PRIME1;
154 hash2 = ((uintptr_t)obj >> 8) + ((uintptr_t)obj >> 24);
155 hash2 %= VMOBJ_HASH_PRIME2;
156 return (&vm_object_hash[(hash1 ^ hash2) & VMOBJ_HMASK]);
157}
158
15553805
MD
159#if defined(DEBUG_LOCKS)
160
161#define vm_object_vndeallocate(obj, vpp) \
162 debugvm_object_vndeallocate(obj, vpp, __FILE__, __LINE__)
163
164/*
165 * Debug helper to track hold/drop/ref/deallocate calls.
166 */
167static void
168debugvm_object_add(vm_object_t obj, char *file, int line, int addrem)
169{
170 int i;
171
172 i = atomic_fetchadd_int(&obj->debug_index, 1);
173 i = i & (VMOBJ_DEBUG_ARRAY_SIZE - 1);
174 ksnprintf(obj->debug_hold_thrs[i],
175 sizeof(obj->debug_hold_thrs[i]),
176 "%c%d:(%d):%s",
177 (addrem == -1 ? '-' : (addrem == 1 ? '+' : '=')),
178 (curthread->td_proc ? curthread->td_proc->p_pid : -1),
179 obj->ref_count,
180 curthread->td_comm);
181 obj->debug_hold_file[i] = file;
182 obj->debug_hold_line[i] = line;
183#if 0
184 /* Uncomment for debugging obj refs/derefs in reproducable cases */
185 if (strcmp(curthread->td_comm, "sshd") == 0) {
186 kprintf("%d %p refs=%d ar=%d file: %s/%d\n",
187 (curthread->td_proc ? curthread->td_proc->p_pid : -1),
188 obj, obj->ref_count, addrem, file, line);
189 }
190#endif
191}
192
193#endif
194
212f39f5
MD
195/*
196 * Misc low level routines
197 */
198static void
199vm_object_lock_init(vm_object_t obj)
200{
201#if defined(DEBUG_LOCKS)
202 int i;
203
15553805 204 obj->debug_index = 0;
212f39f5 205 for (i = 0; i < VMOBJ_DEBUG_ARRAY_SIZE; i++) {
15553805 206 obj->debug_hold_thrs[i][0] = 0;
18a4c8dc
VS
207 obj->debug_hold_file[i] = NULL;
208 obj->debug_hold_line[i] = 0;
212f39f5
MD
209 }
210#endif
211}
212
a31129d8 213void
212f39f5
MD
214vm_object_lock_swap(void)
215{
216 lwkt_token_swap();
217}
218
219void
220vm_object_lock(vm_object_t obj)
221{
c9958a5a 222 lwkt_gettoken(&obj->token);
212f39f5
MD
223}
224
609c9aae
MD
225/*
226 * Returns TRUE on sucesss
227 */
228static int
229vm_object_lock_try(vm_object_t obj)
230{
231 return(lwkt_trytoken(&obj->token));
232}
233
54341a3b
MD
234void
235vm_object_lock_shared(vm_object_t obj)
236{
c9958a5a 237 lwkt_gettoken_shared(&obj->token);
54341a3b
MD
238}
239
212f39f5
MD
240void
241vm_object_unlock(vm_object_t obj)
242{
c9958a5a 243 lwkt_reltoken(&obj->token);
212f39f5
MD
244}
245
501747bf
MD
246void
247vm_object_upgrade(vm_object_t obj)
248{
249 lwkt_reltoken(&obj->token);
250 lwkt_gettoken(&obj->token);
251}
252
253void
254vm_object_downgrade(vm_object_t obj)
255{
256 lwkt_reltoken(&obj->token);
257 lwkt_gettoken_shared(&obj->token);
258}
259
212f39f5
MD
260static __inline void
261vm_object_assert_held(vm_object_t obj)
262{
c9958a5a 263 ASSERT_LWKT_TOKEN_HELD(&obj->token);
212f39f5
MD
264}
265
33ee48c4
MD
266static __inline int
267vm_quickcolor(void)
268{
269 globaldata_t gd = mycpu;
270 int pg_color;
271
272 pg_color = (int)(intptr_t)gd->gd_curthread >> 10;
9002b0d5
MD
273 pg_color += gd->gd_quick_color;
274 gd->gd_quick_color += PQ_PRIME2;
33ee48c4
MD
275
276 return pg_color;
277}
278
212f39f5 279void
15553805 280VMOBJDEBUG(vm_object_hold)(vm_object_t obj VMOBJDBARGS)
212f39f5 281{
b12defdc 282 KKASSERT(obj != NULL);
212f39f5
MD
283
284 /*
285 * Object must be held (object allocation is stable due to callers
286 * context, typically already holding the token on a parent object)
287 * prior to potentially blocking on the lock, otherwise the object
288 * can get ripped away from us.
289 */
290 refcount_acquire(&obj->hold_count);
291 vm_object_lock(obj);
292
293#if defined(DEBUG_LOCKS)
15553805 294 debugvm_object_add(obj, file, line, 1);
212f39f5
MD
295#endif
296}
297
609c9aae 298int
15553805 299VMOBJDEBUG(vm_object_hold_try)(vm_object_t obj VMOBJDBARGS)
609c9aae
MD
300{
301 KKASSERT(obj != NULL);
302
303 /*
304 * Object must be held (object allocation is stable due to callers
305 * context, typically already holding the token on a parent object)
306 * prior to potentially blocking on the lock, otherwise the object
307 * can get ripped away from us.
308 */
309 refcount_acquire(&obj->hold_count);
310 if (vm_object_lock_try(obj) == 0) {
311 if (refcount_release(&obj->hold_count)) {
312 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD))
fde6be6a 313 kfree(obj, M_VM_OBJECT);
609c9aae
MD
314 }
315 return(0);
316 }
317
318#if defined(DEBUG_LOCKS)
15553805 319 debugvm_object_add(obj, file, line, 1);
609c9aae
MD
320#endif
321 return(1);
322}
323
54341a3b 324void
15553805 325VMOBJDEBUG(vm_object_hold_shared)(vm_object_t obj VMOBJDBARGS)
54341a3b
MD
326{
327 KKASSERT(obj != NULL);
328
329 /*
330 * Object must be held (object allocation is stable due to callers
331 * context, typically already holding the token on a parent object)
332 * prior to potentially blocking on the lock, otherwise the object
333 * can get ripped away from us.
334 */
335 refcount_acquire(&obj->hold_count);
336 vm_object_lock_shared(obj);
337
338#if defined(DEBUG_LOCKS)
15553805 339 debugvm_object_add(obj, file, line, 1);
54341a3b
MD
340#endif
341}
342
b12defdc
MD
343/*
344 * Drop the token and hold_count on the object.
501747bf
MD
345 *
346 * WARNING! Token might be shared.
b12defdc 347 */
212f39f5 348void
15553805 349VMOBJDEBUG(vm_object_drop)(vm_object_t obj VMOBJDBARGS)
212f39f5
MD
350{
351 if (obj == NULL)
352 return;
353
212f39f5 354 /*
c9958a5a
MD
355 * No new holders should be possible once we drop hold_count 1->0 as
356 * there is no longer any way to reference the object.
212f39f5 357 */
c17a6852 358 KKASSERT(obj->hold_count > 0);
b12defdc 359 if (refcount_release(&obj->hold_count)) {
15553805
MD
360#if defined(DEBUG_LOCKS)
361 debugvm_object_add(obj, file, line, -1);
362#endif
363
c9958a5a
MD
364 if (obj->ref_count == 0 && (obj->flags & OBJ_DEAD)) {
365 vm_object_unlock(obj);
fde6be6a 366 kfree(obj, M_VM_OBJECT);
c9958a5a
MD
367 } else {
368 vm_object_unlock(obj);
369 }
370 } else {
15553805
MD
371#if defined(DEBUG_LOCKS)
372 debugvm_object_add(obj, file, line, -1);
373#endif
c9958a5a 374 vm_object_unlock(obj);
212f39f5 375 }
212f39f5
MD
376}
377
6846fd23 378/*
a2ee730d 379 * Initialize a freshly allocated object, returning a held object.
6846fd23 380 *
a17c6c05 381 * Used only by vm_object_allocate(), zinitna() and vm_object_init().
6846fd23
MD
382 *
383 * No requirements.
384 */
984263bc 385void
aecf2182 386_vm_object_allocate(objtype_t type, vm_pindex_t size, vm_object_t object)
984263bc 387{
fde6be6a 388 struct vm_object_hash *hash;
6846fd23 389
1f804340 390 RB_INIT(&object->rb_memq);
984263bc 391 LIST_INIT(&object->shadow_head);
c9958a5a 392 lwkt_token_init(&object->token, "vmobj");
984263bc
MD
393
394 object->type = type;
395 object->size = size;
396 object->ref_count = 1;
6ed30774 397 object->memattr = VM_MEMATTR_DEFAULT;
e1c14c82 398 object->hold_count = 0;
984263bc
MD
399 object->flags = 0;
400 if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
401 vm_object_set_flag(object, OBJ_ONEMAPPING);
402 object->paging_in_progress = 0;
403 object->resident_page_count = 0;
404 object->shadow_count = 0;
85946b6c 405 /* cpu localization twist */
33ee48c4 406 object->pg_color = vm_quickcolor();
984263bc
MD
407 object->handle = NULL;
408 object->backing_object = NULL;
b12defdc 409 object->backing_object_offset = (vm_ooffset_t)0;
984263bc 410
95270b7e 411 atomic_add_int(&object->generation, 1);
96adc753
MD
412 object->swblock_count = 0;
413 RB_INIT(&object->swblock_root);
e1c14c82 414 vm_object_lock_init(object);
921c891e 415 pmap_object_init(object);
984263bc 416
a2ee730d 417 vm_object_hold(object);
7b00fbb4 418
562ffbba 419 hash = vmobj_hash(object);
fde6be6a
MD
420 lwkt_gettoken(&hash->token);
421 TAILQ_INSERT_TAIL(&hash->list, object, object_list);
422 lwkt_reltoken(&hash->token);
984263bc
MD
423}
424
a17c6c05
AHJ
425/*
426 * Initialize a VM object.
427 */
428void
429vm_object_init(vm_object_t object, vm_pindex_t size)
430{
431 _vm_object_allocate(OBJT_DEFAULT, size, object);
432 vm_object_drop(object);
433}
434
984263bc 435/*
6846fd23 436 * Initialize the VM objects module.
984263bc 437 *
fde6be6a
MD
438 * Called from the low level boot code only. Note that this occurs before
439 * kmalloc is initialized so we cannot allocate any VM objects.
984263bc
MD
440 */
441void
4f077c8a 442vm_object_init1(void)
984263bc 443{
7b00fbb4
MD
444 int i;
445
446 for (i = 0; i < VMOBJ_HSIZE; ++i) {
fde6be6a
MD
447 TAILQ_INIT(&vm_object_hash[i].list);
448 lwkt_token_init(&vm_object_hash[i].token, "vmobjlst");
7b00fbb4 449 }
4f077c8a 450
e4846942 451 _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(KvaEnd),
c439ad8f 452 &kernel_object);
a2ee730d 453 vm_object_drop(&kernel_object);
984263bc
MD
454}
455
456void
57e43348 457vm_object_init2(void)
c97ef899 458{
fde6be6a 459 kmalloc_set_unlimited(M_VM_OBJECT);
984263bc
MD
460}
461
462/*
6846fd23 463 * Allocate and return a new object of the specified type and size.
984263bc 464 *
6846fd23 465 * No requirements.
984263bc 466 */
984263bc 467vm_object_t
aecf2182 468vm_object_allocate(objtype_t type, vm_pindex_t size)
984263bc 469{
fde6be6a 470 vm_object_t obj;
984263bc 471
fde6be6a
MD
472 obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO);
473 _vm_object_allocate(type, size, obj);
474 vm_object_drop(obj);
984263bc 475
fde6be6a 476 return (obj);
a2ee730d
MD
477}
478
479/*
480 * This version returns a held object, allowing further atomic initialization
481 * of the object.
482 */
483vm_object_t
484vm_object_allocate_hold(objtype_t type, vm_pindex_t size)
485{
fde6be6a 486 vm_object_t obj;
a2ee730d 487
fde6be6a
MD
488 obj = kmalloc(sizeof(*obj), M_VM_OBJECT, M_INTWAIT|M_ZERO);
489 _vm_object_allocate(type, size, obj);
984263bc 490
fde6be6a 491 return (obj);
984263bc
MD
492}
493
984263bc 494/*
b12defdc
MD
495 * Add an additional reference to a vm_object. The object must already be
496 * held. The original non-lock version is no longer supported. The object
497 * must NOT be chain locked by anyone at the time the reference is added.
498 *
499 * Referencing a chain-locked object can blow up the fairly sensitive
500 * ref_count and shadow_count tests in the deallocator. Most callers
501 * will call vm_object_chain_wait() prior to calling
641f3b0a
MD
502 * vm_object_reference_locked() to avoid the case. The held token
503 * allows the caller to pair the wait and ref.
984263bc 504 *
ce94514e
MD
505 * The object must be held, but may be held shared if desired (hence why
506 * we use an atomic op).
984263bc
MD
507 */
508void
15553805 509VMOBJDEBUG(vm_object_reference_locked)(vm_object_t object VMOBJDBARGS)
984263bc 510{
b12defdc
MD
511 KKASSERT(object != NULL);
512 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
501747bf 513 KKASSERT((object->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) == 0);
ce94514e 514 atomic_add_int(&object->ref_count, 1);
b12defdc
MD
515 if (object->type == OBJT_VNODE) {
516 vref(object->handle);
517 /* XXX what if the vnode is being destroyed? */
518 }
15553805
MD
519#if defined(DEBUG_LOCKS)
520 debugvm_object_add(object, file, line, 1);
521#endif
2de4f77e 522}
984263bc 523
641f3b0a
MD
524/*
525 * This version explicitly allows the chain to be held (i.e. by the
526 * caller). The token must also be held.
527 */
528void
529VMOBJDEBUG(vm_object_reference_locked_chain_held)(vm_object_t object
530 VMOBJDBARGS)
531{
532 KKASSERT(object != NULL);
533 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
534 atomic_add_int(&object->ref_count, 1);
535 if (object->type == OBJT_VNODE) {
536 vref(object->handle);
537 /* XXX what if the vnode is being destroyed? */
538 }
539#if defined(DEBUG_LOCKS)
540 debugvm_object_add(object, file, line, 1);
541#endif
542}
543
501747bf
MD
544/*
545 * This version is only allowed for vnode objects.
546 */
547void
15553805 548VMOBJDEBUG(vm_object_reference_quick)(vm_object_t object VMOBJDBARGS)
501747bf
MD
549{
550 KKASSERT(object->type == OBJT_VNODE);
551 atomic_add_int(&object->ref_count, 1);
552 vref(object->handle);
15553805
MD
553#if defined(DEBUG_LOCKS)
554 debugvm_object_add(object, file, line, 1);
555#endif
501747bf
MD
556}
557
b12defdc
MD
558/*
559 * Object OBJ_CHAINLOCK lock handling.
560 *
561 * The caller can chain-lock backing objects recursively and then
562 * use vm_object_chain_release_all() to undo the whole chain.
563 *
564 * Chain locks are used to prevent collapses and are only applicable
565 * to OBJT_DEFAULT and OBJT_SWAP objects. Chain locking operations
566 * on other object types are ignored. This is also important because
567 * it allows e.g. the vnode underlying a memory mapping to take concurrent
568 * faults.
569 *
570 * The object must usually be held on entry, though intermediate
501747bf
MD
571 * objects need not be held on release. The object must be held exclusively,
572 * NOT shared. Note that the prefault path checks the shared state and
573 * avoids using the chain functions.
b12defdc 574 */
2de4f77e 575void
501747bf 576vm_object_chain_wait(vm_object_t object, int shared)
2de4f77e 577{
b12defdc 578 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
501747bf
MD
579 for (;;) {
580 uint32_t chainlk = object->chainlk;
581
582 cpu_ccfence();
583 if (shared) {
584 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) {
585 tsleep_interlock(object, 0);
586 if (atomic_cmpset_int(&object->chainlk,
587 chainlk,
588 chainlk | CHAINLK_WAIT)) {
589 tsleep(object, PINTERLOCKED,
590 "objchns", 0);
591 }
592 /* retry */
593 } else {
594 break;
595 }
596 /* retry */
597 } else {
598 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) {
599 tsleep_interlock(object, 0);
600 if (atomic_cmpset_int(&object->chainlk,
601 chainlk,
602 chainlk | CHAINLK_WAIT))
603 {
604 tsleep(object, PINTERLOCKED,
605 "objchnx", 0);
606 }
607 /* retry */
608 } else {
609 if (atomic_cmpset_int(&object->chainlk,
610 chainlk,
611 chainlk & ~CHAINLK_WAIT))
612 {
613 if (chainlk & CHAINLK_WAIT)
614 wakeup(object);
615 break;
616 }
617 /* retry */
618 }
619 }
620 /* retry */
b12defdc
MD
621 }
622}
623
624void
501747bf 625vm_object_chain_acquire(vm_object_t object, int shared)
b12defdc 626{
501747bf
MD
627 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP)
628 return;
629 if (vm_shared_fault == 0)
630 shared = 0;
631
632 for (;;) {
633 uint32_t chainlk = object->chainlk;
634
635 cpu_ccfence();
636 if (shared) {
637 if (chainlk & (CHAINLK_EXCL | CHAINLK_EXCLREQ)) {
638 tsleep_interlock(object, 0);
639 if (atomic_cmpset_int(&object->chainlk,
640 chainlk,
641 chainlk | CHAINLK_WAIT)) {
642 tsleep(object, PINTERLOCKED,
643 "objchns", 0);
644 }
645 /* retry */
646 } else if (atomic_cmpset_int(&object->chainlk,
647 chainlk, chainlk + 1)) {
648 break;
649 }
650 /* retry */
651 } else {
652 if (chainlk & (CHAINLK_MASK | CHAINLK_EXCL)) {
653 tsleep_interlock(object, 0);
654 if (atomic_cmpset_int(&object->chainlk,
655 chainlk,
656 chainlk |
657 CHAINLK_WAIT |
658 CHAINLK_EXCLREQ)) {
659 tsleep(object, PINTERLOCKED,
660 "objchnx", 0);
661 }
662 /* retry */
663 } else {
664 if (atomic_cmpset_int(&object->chainlk,
665 chainlk,
666 (chainlk | CHAINLK_EXCL) &
667 ~(CHAINLK_EXCLREQ |
668 CHAINLK_WAIT))) {
669 if (chainlk & CHAINLK_WAIT)
670 wakeup(object);
671 break;
672 }
673 /* retry */
674 }
675 }
676 /* retry */
b12defdc
MD
677 }
678}
679
680void
681vm_object_chain_release(vm_object_t object)
682{
501747bf
MD
683 /*ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));*/
684 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP)
685 return;
686 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL));
687 for (;;) {
688 uint32_t chainlk = object->chainlk;
689
690 cpu_ccfence();
691 if (chainlk & CHAINLK_MASK) {
692 if ((chainlk & CHAINLK_MASK) == 1 &&
693 atomic_cmpset_int(&object->chainlk,
694 chainlk,
695 (chainlk - 1) & ~CHAINLK_WAIT)) {
696 if (chainlk & CHAINLK_WAIT)
697 wakeup(object);
698 break;
699 }
700 if ((chainlk & CHAINLK_MASK) > 1 &&
701 atomic_cmpset_int(&object->chainlk,
702 chainlk, chainlk - 1)) {
703 break;
704 }
705 /* retry */
b12defdc 706 } else {
501747bf
MD
707 KKASSERT(chainlk & CHAINLK_EXCL);
708 if (atomic_cmpset_int(&object->chainlk,
709 chainlk,
710 chainlk & ~(CHAINLK_EXCL |
711 CHAINLK_WAIT))) {
712 if (chainlk & CHAINLK_WAIT)
713 wakeup(object);
714 break;
715 }
2de4f77e 716 }
984263bc
MD
717 }
718}
719
b12defdc 720/*
501747bf
MD
721 * Release the chain from first_object through and including stopobj.
722 * The caller is typically holding the first and last object locked
723 * (shared or exclusive) to prevent destruction races.
d2d8515b
MD
724 *
725 * We release stopobj first as an optimization as this object is most
726 * likely to be shared across multiple processes.
b12defdc
MD
727 */
728void
729vm_object_chain_release_all(vm_object_t first_object, vm_object_t stopobj)
730{
731 vm_object_t backing_object;
732 vm_object_t object;
733
734 vm_object_chain_release(stopobj);
735 object = first_object;
736
737 while (object != stopobj) {
738 KKASSERT(object);
b12defdc
MD
739 backing_object = object->backing_object;
740 vm_object_chain_release(object);
b12defdc
MD
741 object = backing_object;
742 }
743}
744
6846fd23 745/*
519883f3
MD
746 * Dereference an object and its underlying vnode. The object may be
747 * held shared. On return the object will remain held.
6846fd23 748 *
519883f3
MD
749 * This function may return a vnode in *vpp which the caller must release
750 * after the caller drops its own lock. If vpp is NULL, we assume that
751 * the caller was holding an exclusive lock on the object and we vrele()
752 * the vp ourselves.
6846fd23 753 */
b33a2ac2 754static void
15553805
MD
755VMOBJDEBUG(vm_object_vndeallocate)(vm_object_t object, struct vnode **vpp
756 VMOBJDBARGS)
984263bc
MD
757{
758 struct vnode *vp = (struct vnode *) object->handle;
759
760 KASSERT(object->type == OBJT_VNODE,
761 ("vm_object_vndeallocate: not a vnode object"));
762 KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
b12defdc 763 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
984263bc
MD
764#ifdef INVARIANTS
765 if (object->ref_count == 0) {
766 vprint("vm_object_vndeallocate", vp);
767 panic("vm_object_vndeallocate: bad object reference count");
768 }
769#endif
519883f3
MD
770 for (;;) {
771 int count = object->ref_count;
772 cpu_ccfence();
773 if (count == 1) {
774 vm_object_upgrade(object);
775 if (atomic_cmpset_int(&object->ref_count, count, 0)) {
776 vclrflags(vp, VTEXT);
777 break;
778 }
779 } else {
780 if (atomic_cmpset_int(&object->ref_count,
781 count, count - 1)) {
782 break;
783 }
784 }
785 /* retry */
786 }
15553805
MD
787#if defined(DEBUG_LOCKS)
788 debugvm_object_add(object, file, line, -1);
789#endif
519883f3
MD
790
791 /*
792 * vrele or return the vp to vrele. We can only safely vrele(vp)
793 * if the object was locked exclusively. But there are two races
794 * here.
795 *
796 * We had to upgrade the object above to safely clear VTEXT
797 * but the alternative path where the shared lock is retained
798 * can STILL race to 0 in other paths and cause our own vrele()
799 * to terminate the vnode. We can't allow that if the VM object
800 * is still locked shared.
801 */
802 if (vpp)
803 *vpp = vp;
804 else
805 vrele(vp);
984263bc
MD
806}
807
808/*
6846fd23
MD
809 * Release a reference to the specified object, gained either through a
810 * vm_object_allocate or a vm_object_reference call. When all references
811 * are gone, storage associated with this object may be relinquished.
212f39f5
MD
812 *
813 * The caller does not have to hold the object locked but must have control
814 * over the reference in question in order to guarantee that the object
815 * does not get ripped out from under us.
ce94514e
MD
816 *
817 * XXX Currently all deallocations require an exclusive lock.
984263bc
MD
818 */
819void
15553805 820VMOBJDEBUG(vm_object_deallocate)(vm_object_t object VMOBJDBARGS)
2de4f77e 821{
501747bf
MD
822 struct vnode *vp;
823 int count;
824
825 if (object == NULL)
826 return;
15553805 827
501747bf
MD
828 for (;;) {
829 count = object->ref_count;
830 cpu_ccfence();
831
832 /*
833 * If decrementing the count enters into special handling
834 * territory (0, 1, or 2) we have to do it the hard way.
835 * Fortunate though, objects with only a few refs like this
836 * are not likely to be heavily contended anyway.
2734d278
MD
837 *
838 * For vnode objects we only care about 1->0 transitions.
501747bf 839 */
2734d278 840 if (count <= 3 || (object->type == OBJT_VNODE && count <= 1)) {
15553805
MD
841#if defined(DEBUG_LOCKS)
842 debugvm_object_add(object, file, line, 0);
843#endif
501747bf
MD
844 vm_object_hold(object);
845 vm_object_deallocate_locked(object);
846 vm_object_drop(object);
847 break;
848 }
849
850 /*
851 * Try to decrement ref_count without acquiring a hold on
852 * the object. This is particularly important for the exec*()
853 * and exit*() code paths because the program binary may
854 * have a great deal of sharing and an exclusive lock will
855 * crowbar performance in those circumstances.
856 */
857 if (object->type == OBJT_VNODE) {
858 vp = (struct vnode *)object->handle;
859 if (atomic_cmpset_int(&object->ref_count,
860 count, count - 1)) {
15553805
MD
861#if defined(DEBUG_LOCKS)
862 debugvm_object_add(object, file, line, -1);
863#endif
864
501747bf
MD
865 vrele(vp);
866 break;
867 }
868 /* retry */
869 } else {
870 if (atomic_cmpset_int(&object->ref_count,
871 count, count - 1)) {
15553805
MD
872#if defined(DEBUG_LOCKS)
873 debugvm_object_add(object, file, line, -1);
874#endif
501747bf
MD
875 break;
876 }
877 /* retry */
878 }
879 /* retry */
b12defdc 880 }
2de4f77e
MD
881}
882
883void
15553805 884VMOBJDEBUG(vm_object_deallocate_locked)(vm_object_t object VMOBJDBARGS)
984263bc 885{
e806bedd
MD
886 struct vm_object_dealloc_list *dlist = NULL;
887 struct vm_object_dealloc_list *dtmp;
984263bc 888 vm_object_t temp;
b12defdc 889 int must_drop = 0;
212f39f5 890
e806bedd
MD
891 /*
892 * We may chain deallocate object, but additional objects may
893 * collect on the dlist which also have to be deallocated. We
894 * must avoid a recursion, vm_object chains can get deep.
895 */
519883f3 896
e806bedd 897again:
984263bc 898 while (object != NULL) {
b12defdc 899 /*
519883f3
MD
900 * vnode case, caller either locked the object exclusively
901 * or this is a recursion with must_drop != 0 and the vnode
902 * object will be locked shared.
903 *
904 * If locked shared we have to drop the object before we can
905 * call vrele() or risk a shared/exclusive livelock.
b12defdc 906 */
984263bc 907 if (object->type == OBJT_VNODE) {
519883f3
MD
908 ASSERT_LWKT_TOKEN_HELD(&object->token);
909 if (must_drop) {
910 struct vnode *tmp_vp;
911
912 vm_object_vndeallocate(object, &tmp_vp);
913 vm_object_drop(object);
914 must_drop = 0;
915 object = NULL;
916 vrele(tmp_vp);
917 } else {
918 vm_object_vndeallocate(object, NULL);
919 }
6846fd23 920 break;
984263bc 921 }
519883f3 922 ASSERT_LWKT_TOKEN_HELD_EXCL(&object->token);
984263bc 923
519883f3
MD
924 /*
925 * Normal case (object is locked exclusively)
926 */
984263bc 927 if (object->ref_count == 0) {
6846fd23
MD
928 panic("vm_object_deallocate: object deallocated "
929 "too many times: %d", object->type);
930 }
931 if (object->ref_count > 2) {
501747bf 932 atomic_add_int(&object->ref_count, -1);
15553805
MD
933#if defined(DEBUG_LOCKS)
934 debugvm_object_add(object, file, line, -1);
935#endif
2de4f77e
MD
936 break;
937 }
938
984263bc
MD
939 /*
940 * Here on ref_count of one or two, which are special cases for
941 * objects.
212f39f5
MD
942 *
943 * Nominal ref_count > 1 case if the second ref is not from
944 * a shadow.
ce94514e
MD
945 *
946 * (ONEMAPPING only applies to DEFAULT AND SWAP objects)
984263bc 947 */
212f39f5 948 if (object->ref_count == 2 && object->shadow_count == 0) {
ce94514e
MD
949 if (object->type == OBJT_DEFAULT ||
950 object->type == OBJT_SWAP) {
951 vm_object_set_flag(object, OBJ_ONEMAPPING);
952 }
501747bf 953 atomic_add_int(&object->ref_count, -1);
15553805
MD
954#if defined(DEBUG_LOCKS)
955 debugvm_object_add(object, file, line, -1);
956#endif
6846fd23
MD
957 break;
958 }
212f39f5
MD
959
960 /*
961 * If the second ref is from a shadow we chain along it
b12defdc 962 * upwards if object's handle is exhausted.
1f8fc82a
MD
963 *
964 * We have to decrement object->ref_count before potentially
965 * collapsing the first shadow object or the collapse code
b12defdc
MD
966 * will not be able to handle the degenerate case to remove
967 * object. However, if we do it too early the object can
968 * get ripped out from under us.
212f39f5 969 */
b12defdc
MD
970 if (object->ref_count == 2 && object->shadow_count == 1 &&
971 object->handle == NULL && (object->type == OBJT_DEFAULT ||
972 object->type == OBJT_SWAP)) {
973 temp = LIST_FIRST(&object->shadow_head);
974 KKASSERT(temp != NULL);
975 vm_object_hold(temp);
976
977 /*
978 * Wait for any paging to complete so the collapse
979 * doesn't (or isn't likely to) qcollapse. pip
980 * waiting must occur before we acquire the
981 * chainlock.
982 */
983 while (
984 temp->paging_in_progress ||
985 object->paging_in_progress
986 ) {
987 vm_object_pip_wait(temp, "objde1");
988 vm_object_pip_wait(object, "objde2");
989 }
990
991 /*
992 * If the parent is locked we have to give up, as
993 * otherwise we would be acquiring locks in the
994 * wrong order and potentially deadlock.
995 */
501747bf 996 if (temp->chainlk & (CHAINLK_EXCL | CHAINLK_MASK)) {
b12defdc
MD
997 vm_object_drop(temp);
998 goto skip;
999 }
501747bf 1000 vm_object_chain_acquire(temp, 0);
b12defdc
MD
1001
1002 /*
1003 * Recheck/retry after the hold and the paging
1004 * wait, both of which can block us.
1005 */
1006 if (object->ref_count != 2 ||
1007 object->shadow_count != 1 ||
1008 object->handle ||
1009 LIST_FIRST(&object->shadow_head) != temp ||
1010 (object->type != OBJT_DEFAULT &&
1011 object->type != OBJT_SWAP)) {
1012 vm_object_chain_release(temp);
1013 vm_object_drop(temp);
1014 continue;
1015 }
1016
1017 /*
1018 * We can safely drop object's ref_count now.
1019 */
1020 KKASSERT(object->ref_count == 2);
501747bf 1021 atomic_add_int(&object->ref_count, -1);
15553805
MD
1022#if defined(DEBUG_LOCKS)
1023 debugvm_object_add(object, file, line, -1);
1024#endif
984263bc 1025
b12defdc
MD
1026 /*
1027 * If our single parent is not collapseable just
1028 * decrement ref_count (2->1) and stop.
1029 */
1030 if (temp->handle || (temp->type != OBJT_DEFAULT &&
1031 temp->type != OBJT_SWAP)) {
1032 vm_object_chain_release(temp);
1033 vm_object_drop(temp);
1034 break;
984263bc 1035 }
b12defdc
MD
1036
1037 /*
1038 * At this point we have already dropped object's
1039 * ref_count so it is possible for a race to
1040 * deallocate obj out from under us. Any collapse
1041 * will re-check the situation. We must not block
1042 * until we are able to collapse.
1043 *
1044 * Bump temp's ref_count to avoid an unwanted
1045 * degenerate recursion (can't call
1046 * vm_object_reference_locked() because it asserts
1047 * that CHAINLOCK is not set).
1048 */
501747bf 1049 atomic_add_int(&temp->ref_count, 1);
b12defdc
MD
1050 KKASSERT(temp->ref_count > 1);
1051
1052 /*
1053 * Collapse temp, then deallocate the extra ref
1054 * formally.
1055 */
e806bedd 1056 vm_object_collapse(temp, &dlist);
b12defdc
MD
1057 vm_object_chain_release(temp);
1058 if (must_drop) {
1059 vm_object_lock_swap();
1060 vm_object_drop(object);
1061 }
1062 object = temp;
1063 must_drop = 1;
1064 continue;
984263bc
MD
1065 }
1066
6846fd23 1067 /*
b12defdc
MD
1068 * Drop the ref and handle termination on the 1->0 transition.
1069 * We may have blocked above so we have to recheck.
6846fd23 1070 */
b12defdc
MD
1071skip:
1072 KKASSERT(object->ref_count != 0);
1073 if (object->ref_count >= 2) {
501747bf 1074 atomic_add_int(&object->ref_count, -1);
15553805
MD
1075#if defined(DEBUG_LOCKS)
1076 debugvm_object_add(object, file, line, -1);
1077#endif
6846fd23 1078 break;
2de4f77e 1079 }
b12defdc 1080 KKASSERT(object->ref_count == 1);
6846fd23
MD
1081
1082 /*
b12defdc
MD
1083 * 1->0 transition. Chain through the backing_object.
1084 * Maintain the ref until we've located the backing object,
1085 * then re-check.
6846fd23 1086 */
212f39f5 1087 while ((temp = object->backing_object) != NULL) {
519883f3
MD
1088 if (temp->type == OBJT_VNODE)
1089 vm_object_hold_shared(temp);
1090 else
1091 vm_object_hold(temp);
212f39f5
MD
1092 if (temp == object->backing_object)
1093 break;
b12defdc 1094 vm_object_drop(temp);
212f39f5 1095 }
b12defdc
MD
1096
1097 /*
1098 * 1->0 transition verified, retry if ref_count is no longer
1099 * 1. Otherwise disconnect the backing_object (temp) and
1100 * clean up.
1101 */
1102 if (object->ref_count != 1) {
1103 vm_object_drop(temp);
1104 continue;
1105 }
1106
1107 /*
1108 * It shouldn't be possible for the object to be chain locked
1109 * if we're removing the last ref on it.
15553805
MD
1110 *
1111 * Removing object from temp's shadow list requires dropping
1112 * temp, which we will do on loop.
1113 *
1114 * NOTE! vnodes do not use the shadow list, but still have
1115 * the backing_object reference.
b12defdc 1116 */
501747bf 1117 KKASSERT((object->chainlk & (CHAINLK_EXCL|CHAINLK_MASK)) == 0);
b12defdc 1118
984263bc 1119 if (temp) {
501747bf
MD
1120 if (object->flags & OBJ_ONSHADOW) {
1121 LIST_REMOVE(object, shadow_list);
1122 temp->shadow_count--;
95270b7e 1123 atomic_add_int(&temp->generation, 1);
501747bf
MD
1124 vm_object_clear_flag(object, OBJ_ONSHADOW);
1125 }
984263bc
MD
1126 object->backing_object = NULL;
1127 }
1128
501747bf 1129 atomic_add_int(&object->ref_count, -1);
b12defdc 1130 if ((object->flags & OBJ_DEAD) == 0)
984263bc 1131 vm_object_terminate(object);
b12defdc
MD
1132 if (must_drop && temp)
1133 vm_object_lock_swap();
1134 if (must_drop)
1135 vm_object_drop(object);
984263bc 1136 object = temp;
b12defdc 1137 must_drop = 1;
984263bc 1138 }
519883f3 1139
b12defdc
MD
1140 if (must_drop && object)
1141 vm_object_drop(object);
e806bedd
MD
1142
1143 /*
1144 * Additional tail recursion on dlist. Avoid a recursion. Objects
1145 * on the dlist have a hold count but are not locked.
1146 */
1147 if ((dtmp = dlist) != NULL) {
1148 dlist = dtmp->next;
1149 object = dtmp->object;
1150 kfree(dtmp, M_TEMP);
1151
1152 vm_object_lock(object); /* already held, add lock */
1153 must_drop = 1; /* and we're responsible for it */
1154 goto again;
1155 }
984263bc
MD
1156}
1157
1158/*
6846fd23
MD
1159 * Destroy the specified object, freeing up related resources.
1160 *
1161 * The object must have zero references.
984263bc 1162 *
b12defdc
MD
1163 * The object must held. The caller is responsible for dropping the object
1164 * after terminate returns. Terminate does NOT drop the object.
984263bc 1165 */
1f804340
MD
1166static int vm_object_terminate_callback(vm_page_t p, void *data);
1167
984263bc 1168void
57e43348 1169vm_object_terminate(vm_object_t object)
984263bc 1170{
e674353b 1171 struct rb_vm_page_scan_info info;
fde6be6a 1172 struct vm_object_hash *hash;
7b00fbb4 1173
984263bc 1174 /*
2de4f77e
MD
1175 * Make sure no one uses us. Once we set OBJ_DEAD we should be
1176 * able to safely block.
984263bc 1177 */
b12defdc 1178 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
2de4f77e 1179 KKASSERT((object->flags & OBJ_DEAD) == 0);
984263bc
MD
1180 vm_object_set_flag(object, OBJ_DEAD);
1181
1182 /*
2de4f77e 1183 * Wait for the pageout daemon to be done with the object
984263bc 1184 */
e1c14c82 1185 vm_object_pip_wait(object, "objtrm1");
984263bc
MD
1186
1187 KASSERT(!object->paging_in_progress,
1188 ("vm_object_terminate: pageout in progress"));
1189
1190 /*
1191 * Clean and free the pages, as appropriate. All references to the
1192 * object are gone, so we don't need to lock it.
1193 */
1194 if (object->type == OBJT_VNODE) {
1195 struct vnode *vp;
1196
984263bc
MD
1197 /*
1198 * Clean pages and flush buffers.
d86d27a8
MD
1199 *
1200 * NOTE! TMPFS buffer flushes do not typically flush the
1201 * actual page to swap as this would be highly
1202 * inefficient, and normal filesystems usually wrap
1203 * page flushes with buffer cache buffers.
1204 *
1205 * To deal with this we have to call vinvalbuf() both
1206 * before and after the vm_object_page_clean().
984263bc 1207 */
984263bc 1208 vp = (struct vnode *) object->handle;
87de5057 1209 vinvalbuf(vp, V_SAVE, 0, 0);
d86d27a8
MD
1210 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
1211 vinvalbuf(vp, V_SAVE, 0, 0);
984263bc
MD
1212 }
1213
1214 /*
1215 * Wait for any I/O to complete, after which there had better not
1216 * be any references left on the object.
1217 */
e1c14c82 1218 vm_object_pip_wait(object, "objtrm2");
984263bc 1219
2de4f77e
MD
1220 if (object->ref_count != 0) {
1221 panic("vm_object_terminate: object with references, "
1222 "ref_count=%d", object->ref_count);
1223 }
984263bc 1224
921c891e
MD
1225 /*
1226 * Cleanup any shared pmaps associated with this object.
1227 */
1228 pmap_object_free(object);
1229
984263bc
MD
1230 /*
1231 * Now free any remaining pages. For internal objects, this also
1232 * removes them from paging queues. Don't free wired pages, just
1233 * remove them from the object.
1234 */
e674353b 1235 info.count = 0;
120ac5dd 1236 info.object = object;
07540d37
MD
1237 do {
1238 info.error = 0;
1239 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
1240 vm_object_terminate_callback, &info);
1241 } while (info.error);
984263bc 1242
e1c14c82 1243 /*
398c240d 1244 * Let the pager know object is dead.
e1c14c82 1245 */
398c240d 1246 vm_pager_deallocate(object);
e1c14c82 1247
984263bc 1248 /*
b12defdc
MD
1249 * Wait for the object hold count to hit 1, clean out pages as
1250 * we go. vmobj_token interlocks any race conditions that might
1251 * pick the object up from the vm_object_list after we have cleared
1252 * rb_memq.
984263bc 1253 */
34542daf 1254 for (;;) {
34542daf
MD
1255 if (RB_ROOT(&object->rb_memq) == NULL)
1256 break;
1257 kprintf("vm_object_terminate: Warning, object %p "
534ee349 1258 "still has %ld pages\n",
34542daf
MD
1259 object, object->resident_page_count);
1260 vm_page_rb_tree_RB_SCAN(&object->rb_memq, NULL,
e674353b 1261 vm_object_terminate_callback, &info);
34542daf 1262 }
34542daf
MD
1263
1264 /*
1265 * There had better not be any pages left
1266 */
1267 KKASSERT(object->resident_page_count == 0);
984263bc
MD
1268
1269 /*
1270 * Remove the object from the global object list.
1271 */
562ffbba 1272 hash = vmobj_hash(object);
fde6be6a
MD
1273 lwkt_gettoken(&hash->token);
1274 TAILQ_REMOVE(&hash->list, object, object_list);
1275 lwkt_reltoken(&hash->token);
2de4f77e
MD
1276
1277 if (object->ref_count != 0) {
1278 panic("vm_object_terminate2: object with references, "
1279 "ref_count=%d", object->ref_count);
1280 }
984263bc
MD
1281
1282 /*
fde6be6a 1283 * NOTE: The object hold_count is at least 1, so we cannot kfree()
b12defdc 1284 * the object here. See vm_object_drop().
984263bc 1285 */
984263bc
MD
1286}
1287
6846fd23 1288/*
b12defdc 1289 * The caller must hold the object.
6846fd23 1290 */
1f804340 1291static int
e674353b 1292vm_object_terminate_callback(vm_page_t p, void *data)
1f804340 1293{
e674353b 1294 struct rb_vm_page_scan_info *info = data;
b12defdc
MD
1295 vm_object_t object;
1296
1297 object = p->object;
07540d37
MD
1298 KKASSERT(object == info->object);
1299 if (vm_page_busy_try(p, TRUE)) {
1300 vm_page_sleep_busy(p, TRUE, "vmotrm");
1301 info->error = 1;
1302 return 0;
120ac5dd 1303 }
b12defdc 1304 if (object != p->object) {
07540d37 1305 /* XXX remove once we determine it can't happen */
b12defdc
MD
1306 kprintf("vm_object_terminate: Warning: Encountered "
1307 "busied page %p on queue %d\n", p, p->queue);
1308 vm_page_wakeup(p);
07540d37 1309 info->error = 1;
b12defdc 1310 } else if (p->wire_count == 0) {
9bf025db 1311 /*
d86d27a8 1312 * NOTE: p->dirty and PG_NEED_COMMIT are ignored.
9bf025db 1313 */
1f804340
MD
1314 vm_page_free(p);
1315 mycpu->gd_cnt.v_pfree++;
1316 } else {
0ca81fbe 1317 if (p->queue != PQ_NONE) {
b12defdc
MD
1318 kprintf("vm_object_terminate: Warning: Encountered "
1319 "wired page %p on queue %d\n", p, p->queue);
0ca81fbe
MD
1320 if (vm_object_debug > 0) {
1321 --vm_object_debug;
1322 print_backtrace(10);
1323 }
1324 }
1f804340
MD
1325 vm_page_remove(p);
1326 vm_page_wakeup(p);
1327 }
07540d37
MD
1328
1329 /*
1330 * Must be at end to avoid SMP races, caller holds object token
1331 */
1332 if ((++info->count & 63) == 0)
1333 lwkt_user_yield();
1f804340
MD
1334 return(0);
1335}
1336
984263bc 1337/*
6846fd23
MD
1338 * Clean all dirty pages in the specified range of object. Leaves page
1339 * on whatever queue it is currently on. If NOSYNC is set then do not
1340 * write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC),
1341 * leaving the object dirty.
984263bc 1342 *
6846fd23
MD
1343 * When stuffing pages asynchronously, allow clustering. XXX we need a
1344 * synchronous clustering mode implementation.
984263bc 1345 *
6846fd23 1346 * Odd semantics: if start == end, we clean everything.
984263bc 1347 *
6846fd23 1348 * The object must be locked? XXX
984263bc 1349 */
1f804340
MD
1350static int vm_object_page_clean_pass1(struct vm_page *p, void *data);
1351static int vm_object_page_clean_pass2(struct vm_page *p, void *data);
984263bc
MD
1352
1353void
57e43348 1354vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1f804340 1355 int flags)
984263bc 1356{
1f804340 1357 struct rb_vm_page_scan_info info;
984263bc 1358 struct vnode *vp;
1f804340 1359 int wholescan;
984263bc 1360 int pagerflags;
b12defdc 1361 int generation;
984263bc 1362
edd6da8e 1363 vm_object_hold(object);
984263bc 1364 if (object->type != OBJT_VNODE ||
6846fd23 1365 (object->flags & OBJ_MIGHTBEDIRTY) == 0) {
edd6da8e 1366 vm_object_drop(object);
984263bc 1367 return;
6846fd23 1368 }
984263bc 1369
1f804340
MD
1370 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ?
1371 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
984263bc
MD
1372 pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
1373
1374 vp = object->handle;
1375
1f804340
MD
1376 /*
1377 * Interlock other major object operations. This allows us to
1378 * temporarily clear OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY.
1379 */
984263bc
MD
1380 vm_object_set_flag(object, OBJ_CLEANING);
1381
1382 /*
1383 * Handle 'entire object' case
1384 */
1f804340 1385 info.start_pindex = start;
984263bc 1386 if (end == 0) {
1f804340 1387 info.end_pindex = object->size - 1;
984263bc 1388 } else {
1f804340 1389 info.end_pindex = end - 1;
984263bc 1390 }
1f804340
MD
1391 wholescan = (start == 0 && info.end_pindex == object->size - 1);
1392 info.limit = flags;
1393 info.pagerflags = pagerflags;
1394 info.object = object;
984263bc
MD
1395
1396 /*
1f804340
MD
1397 * If cleaning the entire object do a pass to mark the pages read-only.
1398 * If everything worked out ok, clear OBJ_WRITEABLE and
1399 * OBJ_MIGHTBEDIRTY.
984263bc 1400 */
1f804340
MD
1401 if (wholescan) {
1402 info.error = 0;
51424739 1403 info.count = 0;
1f804340
MD
1404 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1405 vm_object_page_clean_pass1, &info);
1406 if (info.error == 0) {
1407 vm_object_clear_flag(object,
1408 OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
1409 if (object->type == OBJT_VNODE &&
1410 (vp = (struct vnode *)object->handle) != NULL) {
518cbb49
MD
1411 /*
1412 * Use new-style interface to clear VISDIRTY
1413 * because the vnode is not necessarily removed
1414 * from the syncer list(s) as often as it was
1415 * under the old interface, which can leave
1416 * the vnode on the syncer list after reclaim.
1417 */
1418 vclrobjdirty(vp);
984263bc 1419 }
984263bc 1420 }
984263bc
MD
1421 }
1422
1423 /*
1f804340 1424 * Do a pass to clean all the dirty pages we find.
984263bc 1425 */
1f804340
MD
1426 do {
1427 info.error = 0;
51424739 1428 info.count = 0;
b12defdc 1429 generation = object->generation;
1f804340
MD
1430 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1431 vm_object_page_clean_pass2, &info);
b12defdc 1432 } while (info.error || generation != object->generation);
984263bc 1433
1f804340 1434 vm_object_clear_flag(object, OBJ_CLEANING);
edd6da8e 1435 vm_object_drop(object);
1f804340 1436}
984263bc 1437
6846fd23 1438/*
b12defdc 1439 * The caller must hold the object.
6846fd23 1440 */
1f804340
MD
1441static
1442int
1443vm_object_page_clean_pass1(struct vm_page *p, void *data)
1444{
1445 struct rb_vm_page_scan_info *info = data;
984263bc 1446
07540d37
MD
1447 KKASSERT(p->object == info->object);
1448
1f804340 1449 vm_page_flag_set(p, PG_CLEANCHK);
b12defdc 1450 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
1f804340 1451 info->error = 1;
07540d37 1452 } else if (vm_page_busy_try(p, FALSE)) {
b12defdc 1453 info->error = 1;
07540d37
MD
1454 } else {
1455 KKASSERT(p->object == info->object);
1456 vm_page_protect(p, VM_PROT_READ);
1457 vm_page_wakeup(p);
b12defdc 1458 }
07540d37
MD
1459
1460 /*
1461 * Must be at end to avoid SMP races, caller holds object token
1462 */
1463 if ((++info->count & 63) == 0)
1464 lwkt_user_yield();
1f804340
MD
1465 return(0);
1466}
1467
6846fd23 1468/*
b12defdc 1469 * The caller must hold the object
6846fd23 1470 */
1f804340
MD
1471static
1472int
1473vm_object_page_clean_pass2(struct vm_page *p, void *data)
1474{
1475 struct rb_vm_page_scan_info *info = data;
b12defdc 1476 int generation;
984263bc 1477
07540d37 1478 KKASSERT(p->object == info->object);
120ac5dd 1479
06ecca5a 1480 /*
1f804340
MD
1481 * Do not mess with pages that were inserted after we started
1482 * the cleaning pass.
06ecca5a 1483 */
1f804340 1484 if ((p->flags & PG_CLEANCHK) == 0)
fc9ed34d 1485 goto done;
984263bc 1486
b12defdc 1487 generation = info->object->generation;
120ac5dd 1488
07540d37
MD
1489 if (vm_page_busy_try(p, TRUE)) {
1490 vm_page_sleep_busy(p, TRUE, "vpcwai");
b12defdc 1491 info->error = 1;
fc9ed34d 1492 goto done;
b12defdc
MD
1493 }
1494
07540d37
MD
1495 KKASSERT(p->object == info->object &&
1496 info->object->generation == generation);
1497
1f804340
MD
1498 /*
1499 * Before wasting time traversing the pmaps, check for trivial
1500 * cases where the page cannot be dirty.
1501 */
1502 if (p->valid == 0 || (p->queue - p->pc) == PQ_CACHE) {
d86d27a8
MD
1503 KKASSERT((p->dirty & p->valid) == 0 &&
1504 (p->flags & PG_NEED_COMMIT) == 0);
b12defdc 1505 vm_page_wakeup(p);
fc9ed34d 1506 goto done;
1f804340 1507 }
984263bc 1508
1f804340
MD
1509 /*
1510 * Check whether the page is dirty or not. The page has been set
1511 * to be read-only so the check will not race a user dirtying the
1512 * page.
1513 */
1514 vm_page_test_dirty(p);
d86d27a8 1515 if ((p->dirty & p->valid) == 0 && (p->flags & PG_NEED_COMMIT) == 0) {
1f804340 1516 vm_page_flag_clear(p, PG_CLEANCHK);
b12defdc 1517 vm_page_wakeup(p);
fc9ed34d 1518 goto done;
1f804340 1519 }
984263bc 1520
1f804340
MD
1521 /*
1522 * If we have been asked to skip nosync pages and this is a
1523 * nosync page, skip it. Note that the object flags were
1524 * not cleared in this case (because pass1 will have returned an
1525 * error), so we do not have to set them.
1526 */
1527 if ((info->limit & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
1528 vm_page_flag_clear(p, PG_CLEANCHK);
b12defdc 1529 vm_page_wakeup(p);
fc9ed34d 1530 goto done;
984263bc
MD
1531 }
1532
1f804340
MD
1533 /*
1534 * Flush as many pages as we can. PG_CLEANCHK will be cleared on
1535 * the pages that get successfully flushed. Set info->error if
1536 * we raced an object modification.
1537 */
b12defdc 1538 vm_object_page_collect_flush(info->object, p, info->pagerflags);
d64622f6 1539 /* vm_wait_nominal(); this can deadlock the system in syncer/pageout */
07540d37
MD
1540
1541 /*
1542 * Must be at end to avoid SMP races, caller holds object token
1543 */
fc9ed34d 1544done:
cfffe7b1 1545 if ((++info->count & 63) == 0)
e674353b 1546 lwkt_user_yield();
1f804340 1547 return(0);
984263bc
MD
1548}
1549
06ecca5a 1550/*
6846fd23 1551 * Collect the specified page and nearby pages and flush them out.
b12defdc
MD
1552 * The number of pages flushed is returned. The passed page is busied
1553 * by the caller and we are responsible for its disposition.
06ecca5a 1554 *
b12defdc 1555 * The caller must hold the object.
06ecca5a 1556 */
ab3e1edd 1557static void
1f804340 1558vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags)
984263bc 1559{
b12defdc 1560 int error;
79ce07d0
MD
1561 int is;
1562 int ib;
984263bc 1563 int i;
79ce07d0 1564 int page_base;
984263bc 1565 vm_pindex_t pi;
79ce07d0 1566 vm_page_t ma[BLIST_MAX_ALLOC];
984263bc 1567
b12defdc 1568 ASSERT_LWKT_TOKEN_HELD(vm_object_token(object));
1f804340 1569
984263bc 1570 pi = p->pindex;
79ce07d0 1571 page_base = pi % BLIST_MAX_ALLOC;
8cfbc019 1572 ma[page_base] = p;
79ce07d0
MD
1573 ib = page_base - 1;
1574 is = page_base + 1;
984263bc 1575
79ce07d0 1576 while (ib >= 0) {
984263bc
MD
1577 vm_page_t tp;
1578
79ce07d0
MD
1579 tp = vm_page_lookup_busy_try(object, pi - page_base + ib,
1580 TRUE, &error);
b12defdc
MD
1581 if (error)
1582 break;
1583 if (tp == NULL)
1584 break;
1585 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1586 (tp->flags & PG_CLEANCHK) == 0) {
1587 vm_page_wakeup(tp);
1588 break;
984263bc 1589 }
b12defdc
MD
1590 if ((tp->queue - tp->pc) == PQ_CACHE) {
1591 vm_page_flag_clear(tp, PG_CLEANCHK);
1592 vm_page_wakeup(tp);
1593 break;
1594 }
1595 vm_page_test_dirty(tp);
d86d27a8
MD
1596 if ((tp->dirty & tp->valid) == 0 &&
1597 (tp->flags & PG_NEED_COMMIT) == 0) {
b12defdc
MD
1598 vm_page_flag_clear(tp, PG_CLEANCHK);
1599 vm_page_wakeup(tp);
1600 break;
1601 }
79ce07d0
MD
1602 ma[ib] = tp;
1603 --ib;
984263bc 1604 }
79ce07d0 1605 ++ib; /* fixup */
984263bc 1606
79ce07d0
MD
1607 while (is < BLIST_MAX_ALLOC &&
1608 pi - page_base + is < object->size) {
b12defdc
MD
1609 vm_page_t tp;
1610
79ce07d0
MD
1611 tp = vm_page_lookup_busy_try(object, pi - page_base + is,
1612 TRUE, &error);
b12defdc
MD
1613 if (error)
1614 break;
1615 if (tp == NULL)
1616 break;
1617 if ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
1618 (tp->flags & PG_CLEANCHK) == 0) {
1619 vm_page_wakeup(tp);
1620 break;
1621 }
1622 if ((tp->queue - tp->pc) == PQ_CACHE) {
1623 vm_page_flag_clear(tp, PG_CLEANCHK);
1624 vm_page_wakeup(tp);
984263bc
MD
1625 break;
1626 }
b12defdc 1627 vm_page_test_dirty(tp);
d86d27a8
MD
1628 if ((tp->dirty & tp->valid) == 0 &&
1629 (tp->flags & PG_NEED_COMMIT) == 0) {
b12defdc
MD
1630 vm_page_flag_clear(tp, PG_CLEANCHK);
1631 vm_page_wakeup(tp);
1632 break;
1633 }
79ce07d0
MD
1634 ma[is] = tp;
1635 ++is;
984263bc
MD
1636 }
1637
b12defdc 1638 /*
79ce07d0 1639 * All pages in the ma[] array are busied now
b12defdc 1640 */
79ce07d0
MD
1641 for (i = ib; i < is; ++i) {
1642 vm_page_flag_clear(ma[i], PG_CLEANCHK);
1643 vm_page_hold(ma[i]); /* XXX need this any more? */
984263bc 1644 }
79ce07d0
MD
1645 vm_pageout_flush(&ma[ib], is - ib, pagerflags);
1646 for (i = ib; i < is; ++i) /* XXX need this any more? */
b12defdc 1647 vm_page_unhold(ma[i]);
984263bc
MD
1648}
1649
984263bc
MD
1650/*
1651 * Same as vm_object_pmap_copy, except range checking really
1652 * works, and is meant for small sections of an object.
1653 *
1654 * This code protects resident pages by making them read-only
1655 * and is typically called on a fork or split when a page
1656 * is converted to copy-on-write.
1657 *
1658 * NOTE: If the page is already at VM_PROT_NONE, calling
1659 * vm_page_protect will have no effect.
1660 */
984263bc 1661void
57e43348 1662vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
984263bc
MD
1663{
1664 vm_pindex_t idx;
1665 vm_page_t p;
1666
1667 if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
1668 return;
1669
b12defdc 1670 vm_object_hold(object);
984263bc
MD
1671 for (idx = start; idx < end; idx++) {
1672 p = vm_page_lookup(object, idx);
1673 if (p == NULL)
1674 continue;
1675 vm_page_protect(p, VM_PROT_READ);
1676 }
b12defdc 1677 vm_object_drop(object);
984263bc
MD
1678}
1679
1680/*
6846fd23
MD
1681 * Removes all physical pages in the specified object range from all
1682 * physical maps.
984263bc 1683 *
6846fd23 1684 * The object must *not* be locked.
984263bc 1685 */
1f804340
MD
1686
1687static int vm_object_pmap_remove_callback(vm_page_t p, void *data);
1688
984263bc 1689void
57e43348 1690vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
984263bc 1691{
1f804340 1692 struct rb_vm_page_scan_info info;
984263bc
MD
1693
1694 if (object == NULL)
1695 return;
07540d37
MD
1696 if (start == end)
1697 return;
1f804340
MD
1698 info.start_pindex = start;
1699 info.end_pindex = end - 1;
e674353b 1700 info.count = 0;
120ac5dd 1701 info.object = object;
6846fd23 1702
b12defdc 1703 vm_object_hold(object);
07540d37
MD
1704 do {
1705 info.error = 0;
1706 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
1707 vm_object_pmap_remove_callback, &info);
1708 } while (info.error);
1f804340 1709 if (start == 0 && end == object->size)
984263bc 1710 vm_object_clear_flag(object, OBJ_WRITEABLE);
b12defdc 1711 vm_object_drop(object);
1f804340
MD
1712}
1713
6846fd23 1714/*
b12defdc 1715 * The caller must hold the object
6846fd23 1716 */
1f804340 1717static int
e674353b 1718vm_object_pmap_remove_callback(vm_page_t p, void *data)
1f804340 1719{
e674353b
MD
1720 struct rb_vm_page_scan_info *info = data;
1721
120ac5dd
MD
1722 if (info->object != p->object ||
1723 p->pindex < info->start_pindex ||
1724 p->pindex > info->end_pindex) {
1725 kprintf("vm_object_pmap_remove_callback: obj/pg race %p/%p\n",
1726 info->object, p);
07540d37 1727 info->error = 1;
120ac5dd
MD
1728 return(0);
1729 }
1730
1f804340 1731 vm_page_protect(p, VM_PROT_NONE);
120ac5dd 1732
07540d37
MD
1733 /*
1734 * Must be at end to avoid SMP races, caller holds object token
1735 */
1736 if ((++info->count & 63) == 0)
1737 lwkt_user_yield();
1f804340 1738 return(0);
984263bc
MD
1739}
1740
1741/*
6846fd23 1742 * Implements the madvise function at the object/page level.
984263bc 1743 *
6846fd23 1744 * MADV_WILLNEED (any object)
984263bc 1745 *
6846fd23 1746 * Activate the specified pages if they are resident.
984263bc 1747 *
6846fd23 1748 * MADV_DONTNEED (any object)
984263bc 1749 *
6846fd23 1750 * Deactivate the specified pages if they are resident.
984263bc 1751 *
6846fd23 1752 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects, OBJ_ONEMAPPING only)
984263bc 1753 *
6846fd23
MD
1754 * Deactivate and clean the specified pages if they are
1755 * resident. This permits the process to reuse the pages
1756 * without faulting or the kernel to reclaim the pages
1757 * without I/O.
984263bc 1758 *
6846fd23 1759 * No requirements.
984263bc
MD
1760 */
1761void
76f1911e
MD
1762vm_object_madvise(vm_object_t object, vm_pindex_t pindex,
1763 vm_pindex_t count, int advise)
984263bc
MD
1764{
1765 vm_pindex_t end, tpindex;
1766 vm_object_t tobject;
b12defdc 1767 vm_object_t xobj;
984263bc 1768 vm_page_t m;
b12defdc 1769 int error;
984263bc
MD
1770
1771 if (object == NULL)
1772 return;
1773
1774 end = pindex + count;
1775
b12defdc
MD
1776 vm_object_hold(object);
1777 tobject = object;
6846fd23 1778
984263bc
MD
1779 /*
1780 * Locate and adjust resident pages
1781 */
984263bc
MD
1782 for (; pindex < end; pindex += 1) {
1783relookup:
b12defdc
MD
1784 if (tobject != object)
1785 vm_object_drop(tobject);
984263bc
MD
1786 tobject = object;
1787 tpindex = pindex;
1788shadowlookup:
1789 /*
1790 * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
1791 * and those pages must be OBJ_ONEMAPPING.
1792 */
1793 if (advise == MADV_FREE) {
1794 if ((tobject->type != OBJT_DEFAULT &&
1795 tobject->type != OBJT_SWAP) ||
1796 (tobject->flags & OBJ_ONEMAPPING) == 0) {
1797 continue;
1798 }
1799 }
1800
b12defdc 1801 m = vm_page_lookup_busy_try(tobject, tpindex, TRUE, &error);
984263bc 1802
b12defdc
MD
1803 if (error) {
1804 vm_page_sleep_busy(m, TRUE, "madvpo");
1805 goto relookup;
1806 }
984263bc
MD
1807 if (m == NULL) {
1808 /*
1809 * There may be swap even if there is no backing page
1810 */
1811 if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1812 swap_pager_freespace(tobject, tpindex, 1);
1813
1814 /*
1815 * next object
1816 */
b12defdc
MD
1817 while ((xobj = tobject->backing_object) != NULL) {
1818 KKASSERT(xobj != object);
1819 vm_object_hold(xobj);
1820 if (xobj == tobject->backing_object)
1821 break;
1822 vm_object_drop(xobj);
1823 }
1824 if (xobj == NULL)
984263bc
MD
1825 continue;
1826 tpindex += OFF_TO_IDX(tobject->backing_object_offset);
b12defdc
MD
1827 if (tobject != object) {
1828 vm_object_lock_swap();
1829 vm_object_drop(tobject);
1830 }
1831 tobject = xobj;
984263bc
MD
1832 goto shadowlookup;
1833 }
1834
1835 /*
b12defdc
MD
1836 * If the page is not in a normal active state, we skip it.
1837 * If the page is not managed there are no page queues to
1838 * mess with. Things can break if we mess with pages in
1839 * any of the below states.
984263bc 1840 */
9bf025db
MD
1841 if (m->wire_count ||
1842 (m->flags & (PG_UNMANAGED | PG_NEED_COMMIT)) ||
984263bc
MD
1843 m->valid != VM_PAGE_BITS_ALL
1844 ) {
b12defdc 1845 vm_page_wakeup(m);
984263bc
MD
1846 continue;
1847 }
1848
06ecca5a
MD
1849 /*
1850 * Theoretically once a page is known not to be busy, an
1851 * interrupt cannot come along and rip it out from under us.
1852 */
984263bc
MD
1853
1854 if (advise == MADV_WILLNEED) {
1855 vm_page_activate(m);
1856 } else if (advise == MADV_DONTNEED) {
1857 vm_page_dontneed(m);
1858 } else if (advise == MADV_FREE) {
1859 /*
1860 * Mark the page clean. This will allow the page
1861 * to be freed up by the system. However, such pages
1862 * are often reused quickly by malloc()/free()
1863 * so we do not do anything that would cause
1864 * a page fault if we can help it.
1865 *
1866 * Specifically, we do not try to actually free
1867 * the page now nor do we try to put it in the
1868 * cache (which would cause a page fault on reuse).
1869 *
1870 * But we do make the page is freeable as we
1871 * can without actually taking the step of unmapping
1872 * it.
1873 */
1874 pmap_clear_modify(m);
1875 m->dirty = 0;
1876 m->act_count = 0;
1877 vm_page_dontneed(m);
1878 if (tobject->type == OBJT_SWAP)
1879 swap_pager_freespace(tobject, tpindex, 1);
1880 }
a491077e 1881 vm_page_wakeup(m);
984263bc 1882 }
b12defdc
MD
1883 if (tobject != object)
1884 vm_object_drop(tobject);
1885 vm_object_drop(object);
984263bc
MD
1886}
1887
1888/*
6846fd23 1889 * Create a new object which is backed by the specified existing object
b12defdc
MD
1890 * range. Replace the pointer and offset that was pointing at the existing
1891 * object with the pointer/offset for the new object.
984263bc 1892 *
15553805
MD
1893 * If addref is non-zero the returned object is given an additional reference.
1894 * This mechanic exists to avoid the situation where refs might be 1 and
1895 * race against a collapse when the caller intends to bump it. So the
1896 * caller cannot add the ref after the fact. Used when the caller is
1897 * duplicating a vm_map_entry.
1898 *
6846fd23 1899 * No other requirements.
984263bc 1900 */
984263bc 1901void
b12defdc
MD
1902vm_object_shadow(vm_object_t *objectp, vm_ooffset_t *offset, vm_size_t length,
1903 int addref)
984263bc
MD
1904{
1905 vm_object_t source;
1906 vm_object_t result;
501747bf 1907 int useshadowlist;
984263bc 1908
b12defdc 1909 source = *objectp;
984263bc
MD
1910
1911 /*
1912 * Don't create the new object if the old object isn't shared.
b12defdc
MD
1913 * We have to chain wait before adding the reference to avoid
1914 * racing a collapse or deallocation.
1915 *
15553805
MD
1916 * Clear OBJ_ONEMAPPING flag when shadowing.
1917 *
1918 * The caller owns a ref on source via *objectp which we are going
1919 * to replace. This ref is inherited by the backing_object assignment.
1920 * from nobject and does not need to be incremented here.
1921 *
1922 * However, we add a temporary extra reference to the original source
1923 * prior to holding nobject in case we block, to avoid races where
1924 * someone else might believe that the source can be collapsed.
984263bc 1925 */
501747bf 1926 useshadowlist = 0;
b12defdc 1927 if (source) {
501747bf
MD
1928 if (source->type != OBJT_VNODE) {
1929 useshadowlist = 1;
1930 vm_object_hold(source);
1931 vm_object_chain_wait(source, 0);
1932 if (source->ref_count == 1 &&
1933 source->handle == NULL &&
1934 (source->type == OBJT_DEFAULT ||
1935 source->type == OBJT_SWAP)) {
1936 if (addref) {
1937 vm_object_reference_locked(source);
15553805
MD
1938 vm_object_clear_flag(source,
1939 OBJ_ONEMAPPING);
501747bf
MD
1940 }
1941 vm_object_drop(source);
1942 return;
b12defdc 1943 }
501747bf
MD
1944 vm_object_reference_locked(source);
1945 vm_object_clear_flag(source, OBJ_ONEMAPPING);
1946 } else {
1947 vm_object_reference_quick(source);
1948 vm_object_clear_flag(source, OBJ_ONEMAPPING);
b12defdc 1949 }
6846fd23 1950 }
984263bc
MD
1951
1952 /*
b12defdc
MD
1953 * Allocate a new object with the given length. The new object
1954 * is returned referenced but we may have to add another one.
1955 * If we are adding a second reference we must clear OBJ_ONEMAPPING.
1956 * (typically because the caller is about to clone a vm_map_entry).
1957 *
1958 * The source object currently has an extra reference to prevent
1959 * collapses into it while we mess with its shadow list, which
1960 * we will remove later in this routine.
15553805
MD
1961 *
1962 * The target object may require a second reference if asked for one
1963 * by the caller.
984263bc 1964 */
15553805
MD
1965 result = vm_object_allocate(OBJT_DEFAULT, length);
1966 if (result == NULL)
984263bc 1967 panic("vm_object_shadow: no object for shadowing");
b12defdc
MD
1968 vm_object_hold(result);
1969 if (addref) {
1970 vm_object_reference_locked(result);
1971 vm_object_clear_flag(result, OBJ_ONEMAPPING);
1972 }
984263bc
MD
1973
1974 /*
b12defdc
MD
1975 * The new object shadows the source object. Chain wait before
1976 * adjusting shadow_count or the shadow list to avoid races.
984263bc
MD
1977 *
1978 * Try to optimize the result object's page color when shadowing
1979 * in order to maintain page coloring consistency in the combined
1980 * shadowed object.
501747bf 1981 *
15553805
MD
1982 * The backing_object reference to source requires adding a ref to
1983 * source. We simply inherit the ref from the original *objectp
1984 * (which we are replacing) so no additional refs need to be added.
1985 * (we must still clean up the extra ref we had to prevent collapse
1986 * races).
1987 *
501747bf 1988 * SHADOWING IS NOT APPLICABLE TO OBJT_VNODE OBJECTS
984263bc 1989 */
b12defdc 1990 KKASSERT(result->backing_object == NULL);
984263bc
MD
1991 result->backing_object = source;
1992 if (source) {
501747bf
MD
1993 if (useshadowlist) {
1994 vm_object_chain_wait(source, 0);
1995 LIST_INSERT_HEAD(&source->shadow_head,
1996 result, shadow_list);
1997 source->shadow_count++;
95270b7e 1998 atomic_add_int(&source->generation, 1);
501747bf
MD
1999 vm_object_set_flag(result, OBJ_ONSHADOW);
2000 }
85946b6c 2001 /* cpu localization twist */
33ee48c4 2002 result->pg_color = vm_quickcolor();
984263bc
MD
2003 }
2004
2005 /*
b12defdc
MD
2006 * Adjust the return storage. Drop the ref on source before
2007 * returning.
984263bc 2008 */
984263bc 2009 result->backing_object_offset = *offset;
b12defdc
MD
2010 vm_object_drop(result);
2011 *offset = 0;
2012 if (source) {
501747bf
MD
2013 if (useshadowlist) {
2014 vm_object_deallocate_locked(source);
2015 vm_object_drop(source);
2016 } else {
2017 vm_object_deallocate(source);
2018 }
b12defdc 2019 }
984263bc
MD
2020
2021 /*
2022 * Return the new things
2023 */
b12defdc 2024 *objectp = result;
984263bc
MD
2025}
2026
2027#define OBSC_TEST_ALL_SHADOWED 0x0001
2028#define OBSC_COLLAPSE_NOWAIT 0x0002
2029#define OBSC_COLLAPSE_WAIT 0x0004
2030
1f804340
MD
2031static int vm_object_backing_scan_callback(vm_page_t p, void *data);
2032
6846fd23 2033/*
b12defdc 2034 * The caller must hold the object.
6846fd23 2035 */
984263bc 2036static __inline int
b12defdc 2037vm_object_backing_scan(vm_object_t object, vm_object_t backing_object, int op)
984263bc 2038{
1f804340 2039 struct rb_vm_page_scan_info info;
fde6be6a 2040 struct vm_object_hash *hash;
984263bc 2041
b12defdc
MD
2042 vm_object_assert_held(object);
2043 vm_object_assert_held(backing_object);
2044
2045 KKASSERT(backing_object == object->backing_object);
1f804340 2046 info.backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
984263bc
MD
2047
2048 /*
2049 * Initial conditions
2050 */
984263bc
MD
2051 if (op & OBSC_TEST_ALL_SHADOWED) {
2052 /*
2053 * We do not want to have to test for the existence of
2054 * swap pages in the backing object. XXX but with the
2055 * new swapper this would be pretty easy to do.
2056 *
2057 * XXX what about anonymous MAP_SHARED memory that hasn't
2058 * been ZFOD faulted yet? If we do not test for this, the
2059 * shadow test may succeed! XXX
2060 */
b12defdc 2061 if (backing_object->type != OBJT_DEFAULT)
984263bc 2062 return(0);
984263bc
MD
2063 }
2064 if (op & OBSC_COLLAPSE_WAIT) {
fad57d0e 2065 KKASSERT((backing_object->flags & OBJ_DEAD) == 0);
984263bc 2066 vm_object_set_flag(backing_object, OBJ_DEAD);
7b00fbb4 2067
562ffbba 2068 hash = vmobj_hash(backing_object);
fde6be6a
MD
2069 lwkt_gettoken(&hash->token);
2070 TAILQ_REMOVE(&hash->list, backing_object, object_list);
2071 lwkt_reltoken(&hash->token);
984263bc
MD
2072 }
2073
2074 /*
1f804340
MD
2075 * Our scan. We have to retry if a negative error code is returned,
2076 * otherwise 0 or 1 will be returned in info.error. 0 Indicates that
2077 * the scan had to be stopped because the parent does not completely
2078 * shadow the child.
984263bc 2079 */
1f804340
MD
2080 info.object = object;
2081 info.backing_object = backing_object;
2082 info.limit = op;
51424739 2083 info.count = 0;
1f804340
MD
2084 do {
2085 info.error = 1;
2086 vm_page_rb_tree_RB_SCAN(&backing_object->rb_memq, NULL,
2087 vm_object_backing_scan_callback,
2088 &info);
2089 } while (info.error < 0);
a5fc46c9 2090
1f804340
MD
2091 return(info.error);
2092}
984263bc 2093
6846fd23 2094/*
b12defdc 2095 * The caller must hold the object.
6846fd23 2096 */
1f804340
MD
2097static int
2098vm_object_backing_scan_callback(vm_page_t p, void *data)
2099{
2100 struct rb_vm_page_scan_info *info = data;
2101 vm_object_t backing_object;
2102 vm_object_t object;
54341a3b 2103 vm_pindex_t pindex;
1f804340
MD
2104 vm_pindex_t new_pindex;
2105 vm_pindex_t backing_offset_index;
2106 int op;
984263bc 2107
54341a3b
MD
2108 pindex = p->pindex;
2109 new_pindex = pindex - info->backing_offset_index;
1f804340
MD
2110 op = info->limit;
2111 object = info->object;
2112 backing_object = info->backing_object;
2113 backing_offset_index = info->backing_offset_index;
984263bc 2114
1f804340
MD
2115 if (op & OBSC_TEST_ALL_SHADOWED) {
2116 vm_page_t pp;
984263bc 2117
1f804340
MD
2118 /*
2119 * Ignore pages outside the parent object's range
2120 * and outside the parent object's mapping of the
2121 * backing object.
2122 *
2123 * note that we do not busy the backing object's
2124 * page.
2125 */
54341a3b 2126 if (pindex < backing_offset_index ||
1f804340
MD
2127 new_pindex >= object->size
2128 ) {
2129 return(0);
984263bc
MD
2130 }
2131
2132 /*
1f804340
MD
2133 * See if the parent has the page or if the parent's
2134 * object pager has the page. If the parent has the
2135 * page but the page is not valid, the parent's
2136 * object pager must have the page.
2137 *
2138 * If this fails, the parent does not completely shadow
2139 * the object and we might as well give up now.
984263bc 2140 */
1f804340 2141 pp = vm_page_lookup(object, new_pindex);
1b9d3514
MD
2142 if ((pp == NULL || pp->valid == 0) &&
2143 !vm_pager_has_page(object, new_pindex)
1f804340
MD
2144 ) {
2145 info->error = 0; /* problemo */
2146 return(-1); /* stop the scan */
2147 }
2148 }
984263bc 2149
1f804340 2150 /*
54341a3b
MD
2151 * Check for busy page. Note that we may have lost (p) when we
2152 * possibly blocked above.
1f804340 2153 */
1f804340
MD
2154 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
2155 vm_page_t pp;
984263bc 2156
b12defdc
MD
2157 if (vm_page_busy_try(p, TRUE)) {
2158 if (op & OBSC_COLLAPSE_NOWAIT) {
1f804340 2159 return(0);
b12defdc 2160 } else {
984263bc 2161 /*
1f804340
MD
2162 * If we slept, anything could have
2163 * happened. Ask that the scan be restarted.
984263bc 2164 *
1f804340
MD
2165 * Since the object is marked dead, the
2166 * backing offset should not have changed.
984263bc 2167 */
b12defdc 2168 vm_page_sleep_busy(p, TRUE, "vmocol");
1f804340
MD
2169 info->error = -1;
2170 return(-1);
984263bc 2171 }
1f804340 2172 }
54341a3b
MD
2173
2174 /*
2175 * If (p) is no longer valid restart the scan.
2176 */
2177 if (p->object != backing_object || p->pindex != pindex) {
2178 kprintf("vm_object_backing_scan: Warning: page "
2179 "%p ripped out from under us\n", p);
2180 vm_page_wakeup(p);
2181 info->error = -1;
2182 return(-1);
2183 }
2184
b12defdc 2185 if (op & OBSC_COLLAPSE_NOWAIT) {
9bf025db
MD
2186 if (p->valid == 0 ||
2187 p->wire_count ||
2188 (p->flags & PG_NEED_COMMIT)) {
b12defdc
MD
2189 vm_page_wakeup(p);
2190 return(0);
2191 }
2192 } else {
2193 /* XXX what if p->valid == 0 , hold_count, etc? */
2194 }
984263bc 2195
1f804340
MD
2196 KASSERT(
2197 p->object == backing_object,
2198 ("vm_object_qcollapse(): object mismatch")
2199 );
2200
2201 /*
2202 * Destroy any associated swap
2203 */
8d292090
MD
2204 if (backing_object->type == OBJT_SWAP)
2205 swap_pager_freespace(backing_object, p->pindex, 1);
1f804340
MD
2206
2207 if (
2208 p->pindex < backing_offset_index ||
2209 new_pindex >= object->size
2210 ) {
984263bc 2211 /*
1f804340
MD
2212 * Page is out of the parent object's range, we
2213 * can simply destroy it.
984263bc 2214 */
1f804340
MD
2215 vm_page_protect(p, VM_PROT_NONE);
2216 vm_page_free(p);
2217 return(0);
2218 }
984263bc 2219
1f804340 2220 pp = vm_page_lookup(object, new_pindex);
1b9d3514 2221 if (pp != NULL || vm_pager_has_page(object, new_pindex)) {
1f804340
MD
2222 /*
2223 * page already exists in parent OR swap exists
2224 * for this location in the parent. Destroy
2225 * the original page from the backing object.
2226 *
2227 * Leave the parent's page alone
2228 */
2229 vm_page_protect(p, VM_PROT_NONE);
2230 vm_page_free(p);
2231 return(0);
984263bc 2232 }
1f804340
MD
2233
2234 /*
2235 * Page does not exist in parent, rename the
2236 * page from the backing object to the main object.
2237 *
2238 * If the page was mapped to a process, it can remain
2239 * mapped through the rename.
2240 */
2241 if ((p->queue - p->pc) == PQ_CACHE)
2242 vm_page_deactivate(p);
2243
2244 vm_page_rename(p, object, new_pindex);
b12defdc 2245 vm_page_wakeup(p);
1f804340 2246 /* page automatically made dirty by rename */
984263bc 2247 }
1f804340 2248 return(0);
984263bc
MD
2249}
2250
984263bc 2251/*
6846fd23 2252 * This version of collapse allows the operation to occur earlier and
984263bc
MD
2253 * when paging_in_progress is true for an object... This is not a complete
2254 * operation, but should plug 99.9% of the rest of the leaks.
6846fd23 2255 *
b12defdc
MD
2256 * The caller must hold the object and backing_object and both must be
2257 * chainlocked.
2258 *
2de4f77e 2259 * (only called from vm_object_collapse)
984263bc
MD
2260 */
2261static void
b12defdc 2262vm_object_qcollapse(vm_object_t object, vm_object_t backing_object)
984263bc 2263{
b12defdc 2264 if (backing_object->ref_count == 1) {
501747bf 2265 atomic_add_int(&backing_object->ref_count, 2);
15553805
MD
2266#if defined(DEBUG_LOCKS)
2267 debugvm_object_add(backing_object, "qcollapse", 1, 2);
2268#endif
b12defdc
MD
2269 vm_object_backing_scan(object, backing_object,
2270 OBSC_COLLAPSE_NOWAIT);
501747bf 2271 atomic_add_int(&backing_object->ref_count, -2);
15553805
MD
2272#if defined(DEBUG_LOCKS)
2273 debugvm_object_add(backing_object, "qcollapse", 2, -2);
2274#endif
b12defdc 2275 }
984263bc
MD
2276}
2277
2278/*
6846fd23
MD
2279 * Collapse an object with the object backing it. Pages in the backing
2280 * object are moved into the parent, and the backing object is deallocated.
e806bedd 2281 * Any conflict is resolved in favor of the parent's existing pages.
212f39f5 2282 *
b12defdc
MD
2283 * object must be held and chain-locked on call.
2284 *
2285 * The caller must have an extra ref on object to prevent a race from
2286 * destroying it during the collapse.
984263bc
MD
2287 */
2288void
e806bedd 2289vm_object_collapse(vm_object_t object, struct vm_object_dealloc_list **dlistp)
984263bc 2290{
e806bedd 2291 struct vm_object_dealloc_list *dlist = NULL;
b12defdc
MD
2292 vm_object_t backing_object;
2293
2294 /*
2295 * Only one thread is attempting a collapse at any given moment.
2296 * There are few restrictions for (object) that callers of this
2297 * function check so reentrancy is likely.
2298 */
2299 KKASSERT(object != NULL);
212f39f5 2300 vm_object_assert_held(object);
501747bf 2301 KKASSERT(object->chainlk & (CHAINLK_MASK | CHAINLK_EXCL));
b12defdc
MD
2302
2303 for (;;) {
2304 vm_object_t bbobj;
2305 int dodealloc;
6846fd23 2306
b12defdc 2307 /*
501747bf
MD
2308 * We can only collapse a DEFAULT/SWAP object with a
2309 * DEFAULT/SWAP object.
b12defdc 2310 */
501747bf
MD
2311 if (object->type != OBJT_DEFAULT && object->type != OBJT_SWAP) {
2312 backing_object = NULL;
2313 break;
b12defdc 2314 }
984263bc 2315
501747bf 2316 backing_object = object->backing_object;
b12defdc 2317 if (backing_object == NULL)
984263bc 2318 break;
b12defdc
MD
2319 if (backing_object->type != OBJT_DEFAULT &&
2320 backing_object->type != OBJT_SWAP) {
b12defdc 2321 backing_object = NULL;
984263bc 2322 break;
b12defdc 2323 }
984263bc 2324
501747bf 2325 /*
641f3b0a 2326 * Hold (token lock) the backing_object and retest conditions.
501747bf
MD
2327 */
2328 vm_object_hold(backing_object);
2329 if (backing_object != object->backing_object ||
2330 (backing_object->type != OBJT_DEFAULT &&
2331 backing_object->type != OBJT_SWAP)) {
2332 vm_object_drop(backing_object);
2333 continue;
2334 }
2335
b12defdc
MD
2336 /*
2337 * Chain-lock the backing object too because if we
2338 * successfully merge its pages into the top object we
2339 * will collapse backing_object->backing_object as the
2340 * new backing_object. Re-check that it is still our
2341 * backing object.
2342 */
501747bf 2343 vm_object_chain_acquire(backing_object, 0);
212f39f5 2344 if (backing_object != object->backing_object) {
b12defdc 2345 vm_object_chain_release(backing_object);
212f39f5
MD
2346 vm_object_drop(backing_object);
2347 continue;
2348 }
2349
984263bc 2350 /*
641f3b0a
MD
2351 * We check the backing object first, because it is most
2352 * likely not collapsable.
984263bc
MD
2353 */
2354 if (backing_object->handle != NULL ||
2355 (backing_object->type != OBJT_DEFAULT &&
2356 backing_object->type != OBJT_SWAP) ||
2357 (backing_object->flags & OBJ_DEAD) ||
2358 object->handle != NULL ||
2359 (object->type != OBJT_DEFAULT &&
2360 object->type != OBJT_SWAP) ||
2361 (object->flags & OBJ_DEAD)) {
2362 break;
2363 }
2364
b12defdc
MD
2365 /*
2366 * If paging is in progress we can't do a normal collapse.
2367 */
641f3b0a 2368 if (object->paging_in_progress != 0 ||
984263bc
MD
2369 backing_object->paging_in_progress != 0
2370 ) {
b12defdc 2371 vm_object_qcollapse(object, backing_object);
984263bc
MD
2372 break;
2373 }
2374
2375 /*
2376 * We know that we can either collapse the backing object (if
2377 * the parent is the only reference to it) or (perhaps) have
2378 * the parent bypass the object if the parent happens to shadow
2379 * all the resident pages in the entire backing object.
2380 *
2381 * This is ignoring pager-backed pages such as swap pages.
2382 * vm_object_backing_scan fails the shadowing test in this
2383 * case.
2384 */
984263bc
MD
2385 if (backing_object->ref_count == 1) {
2386 /*
2387 * If there is exactly one reference to the backing
2388 * object, we can collapse it into the parent.
2389 */
b12defdc
MD
2390 KKASSERT(object->backing_object == backing_object);
2391 vm_object_backing_scan(object, backing_object,
2392 OBSC_COLLAPSE_WAIT);
984263bc
MD
2393
2394 /*
2395 * Move the pager from backing_object to object.
2396 */
984263bc
MD
2397 if (backing_object->type == OBJT_SWAP) {
2398 vm_object_pip_add(backing_object, 1);
2399
2400 /*
2401 * scrap the paging_offset junk and do a
2402 * discrete copy. This also removes major
2403 * assumptions about how the swap-pager
2404 * works from where it doesn't belong. The
2405 * new swapper is able to optimize the
2406 * destroy-source case.
2407 */
984263bc 2408 vm_object_pip_add(object, 1);
b12defdc
MD
2409 swap_pager_copy(backing_object, object,
2410 OFF_TO_IDX(object->backing_object_offset),
2411 TRUE);
984263bc 2412 vm_object_pip_wakeup(object);
984263bc
MD
2413 vm_object_pip_wakeup(backing_object);
2414 }
b12defdc 2415
984263bc
MD
2416 /*
2417 * Object now shadows whatever backing_object did.
b12defdc 2418 * Remove object from backing_object's shadow_list.
15553805
MD
2419 *
2420 * Removing object from backing_objects shadow list
2421 * requires releasing object, which we will do below.
984263bc 2422 */
b12defdc 2423 KKASSERT(object->backing_object == backing_object);
501747bf
MD
2424 if (object->flags & OBJ_ONSHADOW) {
2425 LIST_REMOVE(object, shadow_list);
2426 backing_object->shadow_count--;
95270b7e 2427 atomic_add_int(&backing_object->generation, 1);
501747bf
MD
2428 vm_object_clear_flag(object, OBJ_ONSHADOW);
2429 }
b12defdc 2430
e806bedd
MD
2431 /*
2432 * backing_object->backing_object moves from within
2433 * backing_object to within object.
501747bf
MD
2434 *
2435 * OBJT_VNODE bbobj's should have empty shadow lists.
e806bedd 2436 */
b12defdc 2437 while ((bbobj = backing_object->backing_object) != NULL) {
501747bf
MD
2438 if (bbobj->type == OBJT_VNODE)
2439 vm_object_hold_shared(bbobj);
2440 else
2441 vm_object_hold(bbobj);
b12defdc
MD
2442 if (bbobj == backing_object->backing_object)
2443 break;
2444 vm_object_drop(bbobj);
2445 }
15553805
MD
2446
2447 /*
2448 * We are removing backing_object from bbobj's
2449 * shadow list and adding object to bbobj's shadow
2450 * list, so the ref_count on bbobj is unchanged.
2451 */
b12defdc 2452 if (bbobj) {
501747bf
MD
2453 if (backing_object->flags & OBJ_ONSHADOW) {
2454 /* not locked exclusively if vnode */
2455 KKASSERT(bbobj->type != OBJT_VNODE);
2456 LIST_REMOVE(backing_object,
2457 shadow_list);
2458 bbobj->shadow_count--;
95270b7e 2459 atomic_add_int(&bbobj->generation, 1);
501747bf
MD
2460 vm_object_clear_flag(backing_object,
2461 OBJ_ONSHADOW);
2462 }
e806bedd 2463 backing_object->backing_object = NULL;
984263bc 2464 }
b12defdc
MD
2465 object->backing_object = bbobj;
2466 if (bbobj) {
501747bf
MD
2467 if (bbobj->type != OBJT_VNODE) {
2468 LIST_INSERT_HEAD(&bbobj->shadow_head,
2469 object, shadow_list);
2470 bbobj->shadow_count++;
95270b7e 2471 atomic_add_int(&bbobj->generation, 1);
501747bf
MD
2472 vm_object_set_flag(object,
2473 OBJ_ONSHADOW);
2474 }
984263bc
MD
2475 }
2476
2477 object->backing_object_offset +=
b12defdc
MD
2478 backing_object->backing_object_offset;
2479
2480 vm_object_drop(bbobj);
984263bc
MD
2481
2482 /*
b12defdc
MD
2483 * Discard the old backing_object. Nothing should be
2484 * able to ref it, other than a vm_map_split(),
2485 * and vm_map_split() will stall on our chain lock.
2486 * And we control the parent so it shouldn't be
2487 * possible for it to go away either.
984263bc 2488 *
b12defdc
MD
2489 * Since the backing object has no pages, no pager
2490 * left, and no object references within it, all
2491 * that is necessary is to dispose of it.
984263bc 2492 */
2de4f77e
MD
2493 KASSERT(backing_object->ref_count == 1,
2494 ("backing_object %p was somehow "
2495 "re-referenced during collapse!",
2496 backing_object));
2497 KASSERT(RB_EMPTY(&backing_object->rb_memq),
2498 ("backing_object %p somehow has left "
2499 "over pages during collapse!",
2500 backing_object));
2501
e1c14c82 2502 /*
b12defdc
MD
2503 * The object can be destroyed.
2504 *
2505 * XXX just fall through and dodealloc instead
2506 * of forcing destruction?
e1c14c82 2507 */
501747bf 2508 atomic_add_int(&backing_object->ref_count, -1);
15553805
MD
2509#if defined(DEBUG_LOCKS)
2510 debugvm_object_add(backing_object, "collapse", 1, -1);
2511#endif
b12defdc
MD
2512 if ((backing_object->flags & OBJ_DEAD) == 0)
2513 vm_object_terminate(backing_object);
984263bc 2514 object_collapses++;
b12defdc 2515 dodealloc = 0;
984263bc 2516 } else {
984263bc
MD
2517 /*
2518 * If we do not entirely shadow the backing object,
2519 * there is nothing we can do so we give up.
2520 */
b12defdc
MD
2521 if (vm_object_backing_scan(object, backing_object,
2522 OBSC_TEST_ALL_SHADOWED) == 0) {
984263bc
MD
2523 break;
2524 }
2525
e806bedd
MD
2526 /*
2527 * bbobj is backing_object->backing_object. Since
2528 * object completely shadows backing_object we can
2529 * bypass it and become backed by bbobj instead.
501747bf
MD
2530 *
2531 * The shadow list for vnode backing objects is not
2532 * used and a shared hold is allowed.
e806bedd 2533 */
b12defdc 2534 while ((bbobj = backing_object->backing_object) != NULL) {
501747bf
MD
2535 if (bbobj->type == OBJT_VNODE)
2536 vm_object_hold_shared(bbobj);
2537 else
2538 vm_object_hold(bbobj);
b12defdc
MD
2539 if (bbobj == backing_object->backing_object)
2540 break;
2541 vm_object_drop(bbobj);
2542 }
e806bedd 2543
984263bc 2544 /*
e806bedd
MD
2545 * Make object shadow bbobj instead of backing_object.
2546 * Remove object from backing_object's shadow list.
b12defdc
MD
2547 *
2548 * Deallocating backing_object will not remove
984263bc 2549 * it, since its reference count is at least 2.
15553805
MD
2550 *
2551 * Removing object from backing_object's shadow
2552 * list requires releasing a ref, which we do
2553 * below by setting dodealloc to 1.
984263bc 2554 */
b12defdc 2555 KKASSERT(object->backing_object == backing_object);
501747bf
MD
2556 if (object->flags & OBJ_ONSHADOW) {
2557 LIST_REMOVE(object, shadow_list);
2558 backing_object->shadow_count--;
95270b7e 2559 atomic_add_int(&backing_object->generation, 1);
501747bf
MD
2560 vm_object_clear_flag(object, OBJ_ONSHADOW);
2561 }
984263bc 2562
b12defdc 2563 /*
e806bedd
MD
2564 * Add a ref to bbobj, bbobj now shadows object.
2565 *
2566 * NOTE: backing_object->backing_object still points
2567 * to bbobj. That relationship remains intact
2568 * because backing_object has > 1 ref, so
2569 * someone else is pointing to it (hence why
2570 * we can't collapse it into object and can
2571 * only handle the all-shadowed bypass case).
b12defdc
MD
2572 */
2573 if (bbobj) {
501747bf
MD
2574 if (bbobj->type != OBJT_VNODE) {
2575 vm_object_chain_wait(bbobj, 0);
2576 vm_object_reference_locked(bbobj);
2577 LIST_INSERT_HEAD(&bbobj->shadow_head,
2578 object, shadow_list);
2579 bbobj->shadow_count++;
95270b7e 2580 atomic_add_int(&bbobj->generation, 1);
501747bf
MD
2581 vm_object_set_flag(object,
2582 OBJ_ONSHADOW);
2583 } else {
2584 vm_object_reference_quick(bbobj);
2585 }
984263bc
MD
2586 object->backing_object_offset +=
2587 backing_object->backing_object_offset;
b12defdc
MD
2588 object->backing_object = bbobj;
2589 vm_object_drop(bbobj);
2590 } else {
2591 object->backing_object = NULL;
984263bc
MD
2592 }
2593
2594 /*
b12defdc
MD
2595 * Drop the reference count on backing_object. To
2596 * handle ref_count races properly we can't assume
2597 * that the ref_count is still at least 2 so we
2598 * have to actually call vm_object_deallocate()
2599 * (after clearing the chainlock).
984263bc 2600 */
984263bc 2601 object_bypasses++;
b12defdc 2602 dodealloc = 1;
984263bc
MD
2603 }
2604
2605 /*
e806bedd
MD
2606 * Ok, we want to loop on the new object->bbobj association,
2607 * possibly collapsing it further. However if dodealloc is
2608 * non-zero we have to deallocate the backing_object which
2609 * itself can potentially undergo a collapse, creating a
2610 * recursion depth issue with the LWKT token subsystem.
2611 *
2612 * In the case where we must deallocate the backing_object
2613 * it is possible now that the backing_object has a single
2614 * shadow count on some other object (not represented here
2615 * as yet), since it no longer shadows us. Thus when we
2616 * call vm_object_deallocate() it may attempt to collapse
2617 * itself into its remaining parent.
984263bc 2618 */
e806bedd
MD
2619 if (dodealloc) {
2620 struct vm_object_dealloc_list *dtmp;
b12defdc 2621
e806bedd
MD
2622 vm_object_chain_release(backing_object);
2623 vm_object_unlock(backing_object);
2624 /* backing_object remains held */
2625
2626 /*
2627 * Auto-deallocation list for caller convenience.
2628 */
2629 if (dlistp == NULL)
2630 dlistp = &dlist;
2631
2632 dtmp = kmalloc(sizeof(*dtmp), M_TEMP, M_WAITOK);
2633 dtmp->object = backing_object;
2634 dtmp->next = *dlistp;
2635 *dlistp = dtmp;
2636 } else {
2637 vm_object_chain_release(backing_object);
2638 vm_object_drop(backing_object);
2639 }
2640 /* backing_object = NULL; not needed */
b12defdc
MD
2641 /* loop */
2642 }
2643
b59b553b
MD
2644 /*
2645 * Clean up any left over backing_object
2646 */
2647 if (backing_object) {
2648 vm_object_chain_release(backing_object);
2649 vm_object_drop(backing_object);
2650 }
2651
e806bedd
MD
2652 /*
2653 * Clean up any auto-deallocation list. This is a convenience
2654 * for top-level callers so they don't have to pass &dlist.
2655 * Do not clean up any caller-passed dlistp, the caller will
2656 * do that.
2657 */
2658 if (dlist)
2659 vm_object_deallocate_list(&dlist);
2660
984263bc
MD
2661}
2662
e806bedd
MD
2663/*
2664 * vm_object_collapse() may collect additional objects in need of
2665 * deallocation. This routine deallocates these objects. The
2666 * deallocation itself can trigger additional collapses (which the
2667 * deallocate function takes care of). This procedure is used to
2668 * reduce procedural recursion since these vm_object shadow chains
2669 * can become quite long.
2670 */
2671void
2672vm_object_deallocate_list(struct vm_object_dealloc_list **dlistp)
2673{
2674 struct vm_object_dealloc_list *dlist;
2675
2676 while ((dlist = *dlistp) != NULL) {
2677 *dlistp = dlist->next;
2678 vm_object_lock(dlist->object);
2679 vm_object_deallocate_locked(dlist->object);
2680 vm_object_drop(dlist->object);
2681 kfree(dlist, M_TEMP);
2682 }
2683}
2684
984263bc 2685/*
6846fd23
MD
2686 * Removes all physical pages in the specified object range from the
2687 * object's list of pages.
984263bc 2688 *
6846fd23 2689 * No requirements.
984263bc 2690 */
1f804340
MD
2691static int vm_object_page_remove_callback(vm_page_t p, void *data);
2692
984263bc 2693void
57e43348 2694vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1f804340 2695 boolean_t clean_only)
984263bc 2696{
1f804340 2697 struct rb_vm_page_scan_info info;
984263bc
MD
2698 int all;
2699
1f804340
MD
2700 /*
2701 * Degenerate cases and assertions
2702 */
b12defdc 2703 vm_object_hold(object);
b8158020
MD
2704 if (object == NULL ||
2705 (object->resident_page_count == 0 && object->swblock_count == 0)) {
b12defdc 2706 vm_object_drop(object);
984263bc 2707 return;
b8158020 2708 }
1f804340
MD
2709 KASSERT(object->type != OBJT_PHYS,
2710 ("attempt to remove pages from a physical object"));
984263bc 2711
1f804340
MD
2712 /*
2713 * Indicate that paging is occuring on the object
2714 */
1f804340 2715 vm_object_pip_add(object, 1);
984263bc
MD
2716
2717 /*
1f804340
MD
2718 * Figure out the actual removal range and whether we are removing
2719 * the entire contents of the object or not. If removing the entire
2720 * contents, be sure to get all pages, even those that might be
2721 * beyond the end of the object.
984263bc 2722 */
120ac5dd 2723 info.object = object;
1f804340
MD
2724 info.start_pindex = start;
2725 if (end == 0)
2726 info.end_pindex = (vm_pindex_t)-1;
2727 else
2728 info.end_pindex = end - 1;
2729 info.limit = clean_only;
51424739 2730 info.count = 0;
1f804340 2731 all = (start == 0 && info.end_pindex >= object->size - 1);
984263bc 2732
06ecca5a 2733 /*
1f804340 2734 * Loop until we are sure we have gotten them all.
06ecca5a 2735 */
1f804340
MD
2736 do {
2737 info.error = 0;
2738 vm_page_rb_tree_RB_SCAN(&object->rb_memq, rb_vm_page_scancmp,
2739 vm_object_page_remove_callback, &info);
2740 } while (info.error);
984263bc 2741
8d292090 2742 /*
5ac04117
MD
2743 * Remove any related swap if throwing away pages, or for
2744 * non-swap objects (the swap is a clean copy in that case).
8d292090 2745 */
5ac04117 2746 if (object->type != OBJT_SWAP || clean_only == FALSE) {
8d292090
MD
2747 if (all)
2748 swap_pager_freespace_all(object);
2749 else
2750 swap_pager_freespace(object, info.start_pindex,
2751 info.end_pindex - info.start_pindex + 1);
2752 }
2753
1f804340
MD
2754 /*
2755 * Cleanup
2756 */
2757 vm_object_pip_wakeup(object);
b12defdc 2758 vm_object_drop(object);
1f804340 2759}
984263bc 2760
6846fd23 2761/*
51424739
MD
2762 * The caller must hold the object.
2763 *
2764 * NOTE: User yields are allowed when removing more than one page, but not
2765 * allowed if only removing one page (the path for single page removals
2766 * might hold a spinlock).
6846fd23 2767 */
1f804340
MD
2768static int
2769vm_object_page_remove_callback(vm_page_t p, void *data)
2770{
2771 struct rb_vm_page_scan_info *info = data;
984263bc 2772
120ac5dd
MD
2773 if (info->object != p->object ||
2774 p->pindex < info->start_pindex ||
2775 p->pindex > info->end_pindex) {
2776 kprintf("vm_object_page_remove_callbackA: obj/pg race %p/%p\n",
2777 info->object, p);
2778 return(0);
2779 }
b12defdc
MD
2780 if (vm_page_busy_try(p, TRUE)) {
2781 vm_page_sleep_busy(p, TRUE, "vmopar");
2782 info->error = 1;
2783 return(0);
2784 }
120ac5dd
MD
2785 if (info->object != p->object) {
2786 /* this should never happen */
2787 kprintf("vm_object_page_remove_callbackB: obj/pg race %p/%p\n",
2788 info->object, p);
2789 vm_page_wakeup(p);
2790 return(0);
2791 }
b12defdc 2792
1f804340
MD
2793 /*
2794 * Wired pages cannot be destroyed, but they can be invalidated
2795 * and we do so if clean_only (limit) is not set.
1c9602b3
MD
2796 *
2797 * WARNING! The page may be wired due to being part of a buffer
2798 * cache buffer, and the buffer might be marked B_CACHE.
2799 * This is fine as part of a truncation but VFSs must be
2800 * sure to fix the buffer up when re-extending the file.
9bf025db
MD
2801 *
2802 * NOTE! PG_NEED_COMMIT is ignored.
1f804340
MD
2803 */
2804 if (p->wire_count != 0) {
2805 vm_page_protect(p, VM_PROT_NONE);
2806 if (info->limit == 0)
2807 p->valid = 0;
b12defdc 2808 vm_page_wakeup(p);
07540d37 2809 goto done;
1f804340 2810 }
984263bc 2811
1f804340 2812 /*
d86d27a8
MD
2813 * limit is our clean_only flag. If set and the page is dirty or
2814 * requires a commit, do not free it. If set and the page is being
2815 * held by someone, do not free it.
1f804340
MD
2816 */
2817 if (info->limit && p->valid) {
2818 vm_page_test_dirty(p);
d86d27a8 2819 if ((p->valid & p->dirty) || (p->flags & PG_NEED_COMMIT)) {
b12defdc 2820 vm_page_wakeup(p);
07540d37 2821 goto done;
b12defdc 2822 }
984263bc 2823 }
1f804340
MD
2824
2825 /*
2826 * Destroy the page
2827 */
1f804340
MD
2828 vm_page_protect(p, VM_PROT_NONE);
2829 vm_page_free(p);
cfffe7b1 2830
07540d37
MD
2831 /*
2832 * Must be at end to avoid SMP races, caller holds object token
2833 */
2834done:
2835 if ((++info->count & 63) == 0)
2836 lwkt_user_yield();
2837
1f804340 2838 return(0);
984263bc
MD
2839}
2840
2841/*
641f3b0a
MD
2842 * Try to extend prev_object into an adjoining region of virtual
2843 * memory, return TRUE on success.
984263bc 2844 *
212f39f5
MD
2845 * The caller does not need to hold (prev_object) but must have a stable
2846 * pointer to it (typically by holding the vm_map locked).
641f3b0a
MD
2847 *
2848 * This function only works for anonymous memory objects which either
2849 * have (a) one reference or (b) we are extending the object's size.
2850 * Otherwise the related VM pages we want to use for the object might
2851 * be in use by another mapping.
984263bc
MD
2852 */
2853boolean_t
57e43348 2854vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
8d292090 2855 vm_size_t prev_size, vm_size_t next_size)
984263bc
MD
2856{
2857 vm_pindex_t next_pindex;
2858
b12defdc 2859 if (prev_object == NULL)
984263bc 2860 return (TRUE);
984263bc 2861
212f39f5
MD
2862 vm_object_hold(prev_object);
2863
984263bc
MD
2864 if (prev_object->type != OBJT_DEFAULT &&
2865 prev_object->type != OBJT_SWAP) {
212f39f5 2866 vm_object_drop(prev_object);
984263bc
MD
2867 return (FALSE);
2868 }
2869
2870 /*
2871 * Try to collapse the object first
2872 */
501747bf 2873 vm_object_chain_acquire(prev_object, 0);
e806bedd 2874 vm_object_collapse(prev_object, NULL);
984263bc
MD
2875
2876 /*
641f3b0a
MD
2877 * We can't coalesce if we shadow another object (figuring out the
2878 * relationships become too complex).
984263bc 2879 */
70de66db 2880 if (prev_object->backing_object != NULL) {
b12defdc 2881 vm_object_chain_release(prev_object);
212f39f5 2882 vm_object_drop(prev_object);
984263bc 2883 return (FALSE);
70de66db 2884 }
984263bc
MD
2885
2886 prev_size >>= PAGE_SHIFT;
2887 next_size >>= PAGE_SHIFT;
2888 next_pindex = prev_pindex + prev_size;
2889
641f3b0a
MD
2890 /*
2891 * We can't if the object has more than one ref count unless we
2892 * are extending it into newly minted space.
2893 */
2894 if (prev_object->ref_count > 1 &&
2895 prev_object->size != next_pindex) {
b12defdc 2896 vm_object_chain_release(prev_object);
212f39f5 2897 vm_object_drop(prev_object);
984263bc
MD
2898 return (FALSE);
2899 }
2900
2901 /*
2902 * Remove any pages that may still be in the object from a previous
2903 * deallocation.
2904 */
2905 if (next_pindex < prev_object->size) {
2906 vm_object_page_remove(prev_object,
2907 next_pindex,
2908 next_pindex + next_size, FALSE);
2909 if (prev_object->type == OBJT_SWAP)
2910 swap_pager_freespace(prev_object,
2911 next_pindex, next_size);
2912 }
2913
2914 /*
2915 * Extend the object if necessary.
2916 */
2917 if (next_pindex + next_size > prev_object->size)
2918 prev_object->size = next_pindex + next_size;
b12defdc 2919 vm_object_chain_release(prev_object);
212f39f5 2920 vm_object_drop(prev_object);
641f3b0a 2921
984263bc
MD
2922 return (TRUE);
2923}
2924
6846fd23
MD
2925/*
2926 * Make the object writable and flag is being possibly dirty.
2927 *
501747bf
MD
2928 * The object might not be held (or might be held but held shared),
2929 * the related vnode is probably not held either. Object and vnode are
2930 * stable by virtue of the vm_page busied by the caller preventing
2931 * destruction.
eddc656a
MD
2932 *
2933 * If the related mount is flagged MNTK_THR_SYNC we need to call
fd2da346 2934 * vsetobjdirty(). Filesystems using this option usually shortcut
eddc656a 2935 * synchronization by only scanning the syncer list.
6846fd23 2936 */
984263bc
MD
2937void
2938vm_object_set_writeable_dirty(vm_object_t object)
2939{
2940 struct vnode *vp;
2941
b12defdc 2942 /*vm_object_assert_held(object);*/
54341a3b
MD
2943 /*
2944 * Avoid contention in vm fault path by checking the state before
2945 * issuing an atomic op on it.
2946 */
2947 if ((object->flags & (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) !=
2948 (OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY)) {
2949 vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
2950 }
984263bc
MD
2951 if (object->type == OBJT_VNODE &&
2952 (vp = (struct vnode *)object->handle) != NULL) {
2953 if ((vp->v_flag & VOBJDIRTY) == 0) {
eddc656a
MD
2954 if (vp->v_mount &&
2955 (vp->v_mount->mnt_kern_flag & MNTK_THR_SYNC)) {
518cbb49
MD
2956 /*
2957 * New style THR_SYNC places vnodes on the
2958 * syncer list more deterministically.
2959 */
fd2da346
MD
2960 vsetobjdirty(vp);
2961 } else {
518cbb49
MD
2962 /*
2963 * Old style scan would not necessarily place
2964 * a vnode on the syncer list when possibly
2965 * modified via mmap.
2966 */
fd2da346 2967 vsetflags(vp, VOBJDIRTY);
eddc656a 2968 }
984263bc
MD
2969 }
2970 }
2971}
2972
984263bc
MD
2973#include "opt_ddb.h"
2974#ifdef DDB
984263bc
MD
2975#include <sys/cons.h>
2976
2977#include <ddb/ddb.h>
2978
1388df65
RG
2979static int _vm_object_in_map (vm_map_t map, vm_object_t object,
2980 vm_map_entry_t entry);
2981static int vm_object_in_map (vm_object_t object);
984263bc 2982
6846fd23 2983/*
b12defdc 2984 * The caller must hold the object.
6846fd23 2985 */
984263bc 2986static int
57e43348 2987_vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
984263bc
MD
2988{
2989 vm_map_t tmpm;
2990 vm_map_entry_t tmpe;
b12defdc 2991 vm_object_t obj, nobj;
984263bc
MD
2992 int entcount;
2993
47ec0953 2994 if (map == NULL)
984263bc 2995 return 0;
47ec0953
MD
2996 if (entry == NULL) {
2997 tmpe = RB_MIN(vm_map_rb_tree, &map->rb_root);
984263bc 2998 entcount = map->nentries;
47ec0953 2999 while (entcount-- && tmpe) {
984263bc
MD
3000 if( _vm_object_in_map(map, object, tmpe)) {
3001 return 1;
3002 }
47ec0953 3003 tmpe = vm_map_rb_tree_RB_NEXT(tmpe);
984263bc 3004 }
1b874851
MD
3005 return (0);
3006 }
3007 switch(entry->maptype) {
3008 case VM_MAPTYPE_SUBMAP:
984263bc 3009 tmpm = entry->object.sub_map;
47ec0953 3010 tmpe = RB_MIN(vm_map_rb_tree, &tmpm->rb_root);
984263bc 3011 entcount = tmpm->nentries;
47ec0953 3012 while (entcount-- && tmpe) {
984263bc
MD
3013 if( _vm_object_in_map(tmpm, object, tmpe)) {
3014 return 1;
3015 }
47ec0953 3016 tmpe = vm_map_rb_tree_RB_NEXT(tmpe);
984263bc 3017 }
1b874851
MD
3018 break;
3019 case VM_MAPTYPE_NORMAL:
3020 case VM_MAPTYPE_VPAGETABLE:
3021 obj = entry->object.vm_object;
3022 while (obj) {
b12defdc
MD
3023 if (obj == object) {
3024 if (obj != entry->object.vm_object)
3025 vm_object_drop(obj);
984263bc 3026 return 1;
b12defdc
MD
3027 }
3028 while ((nobj = obj->backing_object) != NULL) {
3029 vm_object_hold(nobj);
3030 if (nobj == obj->backing_object)
3031 break;
3032 vm_object_drop(nobj);
3033 }
3034 if (obj != entry->object.vm_object) {
3035 if (nobj)
3036 vm_object_lock_swap();
3037 vm_object_drop(obj);
3038 }
3039 obj = nobj;
1b874851
MD
3040 }
3041 break;
3042 default:
3043 break;
984263bc
MD
3044 }
3045 return 0;
3046}
3047
8fa76237
MD
3048static int vm_object_in_map_callback(struct proc *p, void *data);
3049
3050struct vm_object_in_map_info {
3051 vm_object_t object;
3052 int rv;
3053};
3054
6846fd23
MD
3055/*
3056 * Debugging only
3057 */
984263bc 3058static int
57e43348 3059vm_object_in_map(vm_object_t object)
984263bc 3060{
8fa76237
MD
3061 struct vm_object_in_map_info info;
3062
3063 info.rv = 0;
3064 info.object = object;
3065
586c4308 3066 allproc_scan(vm_object_in_map_callback, &info, 0);
8fa76237
MD
3067 if (info.rv)
3068 return 1;
e4846942 3069 if( _vm_object_in_map(&kernel_map, object, 0))
984263bc 3070 return 1;
e4846942 3071 if( _vm_object_in_map(&pager_map, object, 0))
984263bc 3072 return 1;
e4846942 3073 if( _vm_object_in_map(&buffer_map, object, 0))
984263bc 3074 return 1;
984263bc
MD
3075 return 0;
3076}
3077
6846fd23
MD
3078/*
3079 * Debugging only
3080 */
8fa76237
MD
3081static int
3082vm_object_in_map_callback(struct proc *p, void *data)
3083{
3084 struct vm_object_in_map_info *info = data;
3085
3086 if (p->p_vmspace) {
3087 if (_vm_object_in_map(&p->p_vmspace->vm_map, info->object, 0)) {
3088 info->rv = 1;
3089 return -1;
3090 }
3091 }
3092 return (0);
3093}
3094
984263bc
MD
3095DB_SHOW_COMMAND(vmochk, vm_object_check)
3096{
fde6be6a 3097 struct vm_object_hash *hash;
984263bc 3098 vm_object_t object;
7b00fbb4 3099 int n;
984263bc
MD
3100
3101 /*
3102 * make sure that internal objs are in a map somewhere
3103 * and none have zero ref counts.
3104 */
7b00fbb4 3105 for (n = 0; n < VMOBJ_HSIZE; ++n) {
fde6be6a
MD
3106 hash = &vm_object_hash[n];
3107 for (object = TAILQ_FIRST(&hash->list);
7b00fbb4
MD
3108 object != NULL;
3109 object = TAILQ_NEXT(object, object_list)) {
3110 if (object->type == OBJT_MARKER)
3111 continue;
3112 if (object->handle != NULL ||
3113 (object->type != OBJT_DEFAULT &&
3114 object->type != OBJT_SWAP)) {
3115 continue;
984263bc 3116 }
7b00fbb4
MD
3117 if (object->ref_count == 0) {
3118 db_printf("vmochk: internal obj has "
3119 "zero ref count: %ld\n",
3120 (long)object->size);
984263bc 3121 }
7b00fbb4
MD
3122 if (vm_object_in_map(object))
3123 continue;
3124 db_printf("vmochk: internal obj is not in a map: "
3125 "ref: %d, size: %lu: 0x%lx, "
3126 "backing_object: %p\n",
3127 object->ref_count, (u_long)object->size,
3128 (u_long)object->size,
3129 (void *)object->backing_object);
984263bc
MD
3130 }
3131 }
3132}
3133
3134/*
6846fd23 3135 * Debugging only
984263bc
MD
3136 */
3137DB_SHOW_COMMAND(object, vm_object_print_static)
3138{
3139 /* XXX convert args. */
3140 vm_object_t object = (vm_object_t)addr;
3141 boolean_t full = have_addr;
3142
3143 vm_page_t p;
3144
3145 /* XXX count is an (unused) arg. Avoid shadowing it. */
3146#define count was_count
3147
3148 int count;
3149
3150 if (object == NULL)
3151 return;
3152
3153 db_iprintf(
534ee349 3154 "Object %p: type=%d, size=0x%lx, res=%ld, ref=%d, flags=0x%x\n",
984263bc
MD
3155 object, (int)object->type, (u_long)object->size,
3156 object->resident_page_count, object->ref_count, object->flags);
3157 /*
3158 * XXX no %qd in kernel. Truncate object->backing_object_offset.
3159 */
3160 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n",
3161 object->shadow_count,
3162 object->backing_object ? object->backing_object->ref_count : 0,
3163 object->backing_object, (long)object->backing_object_offset);
3164
3165 if (!full)
3166 return;
3167
3168 db_indent += 2;
3169 count = 0;
1f804340 3170 RB_FOREACH(p, vm_page_rb_tree, &object->rb_memq) {
984263bc
MD
3171 if (count == 0)
3172 db_iprintf("memory:=");
3173 else if (count == 6) {
3174 db_printf("\n");
3175 db_iprintf(" ...");
3176 count = 0;
3177 } else
3178 db_printf(",");
3179 count++;
3180
3181 db_printf("(off=0x%lx,page=0x%lx)",
3182 (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
3183 }
3184 if (count != 0)
3185 db_printf("\n");
3186 db_indent -= 2;
3187}
3188
3189/* XXX. */
3190#undef count
3191
6846fd23
MD
3192/*
3193 * XXX need this non-static entry for calling from vm_map_print.
3194 *
3195 * Debugging only
3196 */
984263bc 3197void
57e43348
MD
3198vm_object_print(/* db_expr_t */ long addr,
3199 boolean_t have_addr,
3200 /* db_expr_t */ long count,
3201 char *modif)
984263bc
MD
3202{
3203 vm_object_print_static(addr, have_addr, count, modif);
3204}
3205
6846fd23
MD
3206/*
3207 * Debugging only
3208 */
984263bc
MD
3209DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
3210{
fde6be6a 3211 struct vm_object_hash *hash;
984263bc
MD
3212 vm_object_t object;
3213 int nl = 0;
3214 int c;
7b00fbb4
MD
3215 int n;
3216
3217 for (n = 0; n < VMOBJ_HSIZE; ++n) {
fde6be6a
MD
3218 hash = &vm_object_hash[n];
3219 for (object = TAILQ_FIRST(&hash->list);
7b00fbb4
MD
3220 object != NULL;
3221 object = TAILQ_NEXT(object, object_list)) {
3222 vm_pindex_t idx, fidx;
3223 vm_pindex_t osize;
3224 vm_paddr_t pa = -1, padiff;
3225 int rcount;
3226 vm_page_t m;
3227
3228 if (object->type == OBJT_MARKER)
3229 continue;
3230 db_printf("new object: %p\n", (void *)object);
3231 if ( nl > 18) {
3232 c = cngetc();
3233 if (c != ' ')
3234 return;
3235 nl = 0;
3236 }
3237 nl++;
3238 rcount = 0;
3239 fidx = 0;
3240 osize = object->size;
3241 if (osize > 128)
3242 osize = 128;
3243 for (idx = 0; idx < osize; idx++) {
3244 m = vm_page_lookup(object, idx);
3245 if (m == NULL) {
3246 if (rcount) {
3247 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
3248 (long)fidx, rcount, (long)pa);
3249 if ( nl > 18) {
3250 c = cngetc();
3251 if (c != ' ')
3252 return;
3253 nl = 0;
3254 }
3255 nl++;
3256 rcount = 0;
3257 }
3258 continue;
3259 }
3260
3261 if (rcount &&
3262 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
3263 ++rcount;
3264 continue;
3265 }
984263bc 3266 if (rcount) {
7b00fbb4
MD
3267 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
3268 padiff >>= PAGE_SHIFT;
3269 padiff &= PQ_L2_MASK;
3270 if (padiff == 0) {
3271 pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
3272 ++rcount;
3273 continue;
3274 }
3275 db_printf(" index(%ld)run(%d)pa(0x%lx)",
984263bc 3276 (long)fidx, rcount, (long)pa);
7b00fbb4 3277 db_printf("pd(%ld)\n", (long)padiff);
984263bc
MD
3278 if ( nl > 18) {
3279 c = cngetc();
3280 if (c != ' ')
3281 return;
3282 nl = 0;
3283 }
3284 nl++;
984263bc 3285 }
7b00fbb4
MD
3286 fidx = idx;
3287 pa = VM_PAGE_TO_PHYS(m);
3288 rcount = 1;
984263bc
MD
3289 }
3290 if (rcount) {
7b00fbb4 3291 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
984263bc 3292 (long)fidx, rcount, (long)pa);
984263bc
MD
3293 if ( nl > 18) {
3294 c = cngetc();
3295 if (c != ' ')
3296 return;
3297 nl = 0;
3298 }
3299 nl++;
3300 }
984263bc
MD
3301 }
3302 }
3303}
3304#endif /* DDB */