Get rid of mb_map. Retool the mbuf and mbuf cluster allocator to use
[dragonfly.git] / sys / vm / vm_object.c
1 /*
2  * Copyright (c) 1991, 1993
3  *      The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. All advertising materials mentioning features or use of this software
17  *    must display the following acknowledgement:
18  *      This product includes software developed by the University of
19  *      California, Berkeley and its contributors.
20  * 4. Neither the name of the University nor the names of its contributors
21  *    may be used to endorse or promote products derived from this software
22  *    without specific prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  *      from: @(#)vm_object.c   8.5 (Berkeley) 3/22/94
37  *
38  *
39  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40  * All rights reserved.
41  *
42  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43  *
44  * Permission to use, copy, modify and distribute this software and
45  * its documentation is hereby granted, provided that both the copyright
46  * notice and this permission notice appear in all copies of the
47  * software, derivative works or modified versions, and any portions
48  * thereof, and that both notices appear in supporting documentation.
49  *
50  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53  *
54  * Carnegie Mellon requests users of this software to return to
55  *
56  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
57  *  School of Computer Science
58  *  Carnegie Mellon University
59  *  Pittsburgh PA 15213-3890
60  *
61  * any improvements or extensions that they make and grant Carnegie the
62  * rights to redistribute these changes.
63  *
64  * $FreeBSD: src/sys/vm/vm_object.c,v 1.171.2.8 2003/05/26 19:17:56 alc Exp $
65  * $DragonFly: src/sys/vm/vm_object.c,v 1.18 2004/07/31 07:52:51 dillon Exp $
66  */
67
68 /*
69  *      Virtual memory object module.
70  */
71
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/proc.h>           /* for curproc, pageproc */
75 #include <sys/vnode.h>
76 #include <sys/vmmeter.h>
77 #include <sys/mman.h>
78 #include <sys/mount.h>
79 #include <sys/kernel.h>
80 #include <sys/sysctl.h>
81
82 #include <vm/vm.h>
83 #include <vm/vm_param.h>
84 #include <vm/pmap.h>
85 #include <vm/vm_map.h>
86 #include <vm/vm_object.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_pageout.h>
89 #include <vm/vm_pager.h>
90 #include <vm/swap_pager.h>
91 #include <vm/vm_kern.h>
92 #include <vm/vm_extern.h>
93 #include <vm/vm_zone.h>
94
95 #define EASY_SCAN_FACTOR        8
96
97 #define MSYNC_FLUSH_HARDSEQ     0x01
98 #define MSYNC_FLUSH_SOFTSEQ     0x02
99
100 static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ;
101 SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags,
102         CTLFLAG_RW, &msync_flush_flags, 0, "");
103
104 static void     vm_object_qcollapse (vm_object_t object);
105 static int      vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags);
106
107 /*
108  *      Virtual memory objects maintain the actual data
109  *      associated with allocated virtual memory.  A given
110  *      page of memory exists within exactly one object.
111  *
112  *      An object is only deallocated when all "references"
113  *      are given up.  Only one "reference" to a given
114  *      region of an object should be writeable.
115  *
116  *      Associated with each object is a list of all resident
117  *      memory pages belonging to that object; this list is
118  *      maintained by the "vm_page" module, and locked by the object's
119  *      lock.
120  *
121  *      Each object also records a "pager" routine which is
122  *      used to retrieve (and store) pages to the proper backing
123  *      storage.  In addition, objects may be backed by other
124  *      objects from which they were virtual-copied.
125  *
126  *      The only items within the object structure which are
127  *      modified after time of creation are:
128  *              reference count         locked by object's lock
129  *              pager routine           locked by object's lock
130  *
131  */
132
133 struct object_q vm_object_list;
134 static struct lwkt_token vm_object_list_token;
135 static long vm_object_count;            /* count of all objects */
136 vm_object_t kernel_object;
137 vm_object_t kmem_object;
138 static struct vm_object kernel_object_store;
139 static struct vm_object kmem_object_store;
140 extern int vm_pageout_page_count;
141
142 static long object_collapses;
143 static long object_bypasses;
144 static int next_index;
145 static vm_zone_t obj_zone;
146 static struct vm_zone obj_zone_store;
147 static int object_hash_rand;
148 #define VM_OBJECTS_INIT 256
149 static struct vm_object vm_objects_init[VM_OBJECTS_INIT];
150
151 void
152 _vm_object_allocate(objtype_t type, vm_size_t size, vm_object_t object)
153 {
154         int incr;
155         TAILQ_INIT(&object->memq);
156         LIST_INIT(&object->shadow_head);
157
158         object->type = type;
159         object->size = size;
160         object->ref_count = 1;
161         object->flags = 0;
162         if ((object->type == OBJT_DEFAULT) || (object->type == OBJT_SWAP))
163                 vm_object_set_flag(object, OBJ_ONEMAPPING);
164         object->paging_in_progress = 0;
165         object->resident_page_count = 0;
166         object->shadow_count = 0;
167         object->pg_color = next_index;
168         if ( size > (PQ_L2_SIZE / 3 + PQ_PRIME1))
169                 incr = PQ_L2_SIZE / 3 + PQ_PRIME1;
170         else
171                 incr = size;
172         next_index = (next_index + incr) & PQ_L2_MASK;
173         object->handle = NULL;
174         object->backing_object = NULL;
175         object->backing_object_offset = (vm_ooffset_t) 0;
176         /*
177          * Try to generate a number that will spread objects out in the
178          * hash table.  We 'wipe' new objects across the hash in 128 page
179          * increments plus 1 more to offset it a little more by the time
180          * it wraps around.
181          */
182         object->hash_rand = object_hash_rand - 129;
183
184         object->generation++;
185
186         TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
187         vm_object_count++;
188         object_hash_rand = object->hash_rand;
189 }
190
191 /*
192  *      vm_object_init:
193  *
194  *      Initialize the VM objects module.
195  */
196 void
197 vm_object_init(void)
198 {
199         TAILQ_INIT(&vm_object_list);
200         lwkt_token_init(&vm_object_list_token);
201         vm_object_count = 0;
202         
203         kernel_object = &kernel_object_store;
204         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
205             kernel_object);
206
207         kmem_object = &kmem_object_store;
208         _vm_object_allocate(OBJT_DEFAULT, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS),
209             kmem_object);
210
211         obj_zone = &obj_zone_store;
212         zbootinit(obj_zone, "VM OBJECT", sizeof (struct vm_object),
213                 vm_objects_init, VM_OBJECTS_INIT);
214 }
215
216 void
217 vm_object_init2(void)
218 {
219         zinitna(obj_zone, NULL, NULL, 0, 0, ZONE_PANICFAIL, 1);
220 }
221
222 /*
223  *      vm_object_allocate:
224  *
225  *      Returns a new object with the given size.
226  */
227
228 vm_object_t
229 vm_object_allocate(objtype_t type, vm_size_t size)
230 {
231         vm_object_t result;
232
233         result = (vm_object_t) zalloc(obj_zone);
234
235         _vm_object_allocate(type, size, result);
236
237         return (result);
238 }
239
240
241 /*
242  *      vm_object_reference:
243  *
244  *      Gets another reference to the given object.
245  */
246 void
247 vm_object_reference(vm_object_t object)
248 {
249         if (object == NULL)
250                 return;
251
252 #if 0
253         /* object can be re-referenced during final cleaning */
254         KASSERT(!(object->flags & OBJ_DEAD),
255             ("vm_object_reference: attempting to reference dead obj"));
256 #endif
257
258         object->ref_count++;
259         if (object->type == OBJT_VNODE) {
260                 while (vget((struct vnode *) object->handle, NULL,
261                     LK_RETRY|LK_NOOBJ, curthread)) {
262                         printf("vm_object_reference: delay in getting object\n");
263                 }
264         }
265 }
266
267 void
268 vm_object_vndeallocate(vm_object_t object)
269 {
270         struct vnode *vp = (struct vnode *) object->handle;
271
272         KASSERT(object->type == OBJT_VNODE,
273             ("vm_object_vndeallocate: not a vnode object"));
274         KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp"));
275 #ifdef INVARIANTS
276         if (object->ref_count == 0) {
277                 vprint("vm_object_vndeallocate", vp);
278                 panic("vm_object_vndeallocate: bad object reference count");
279         }
280 #endif
281
282         object->ref_count--;
283         if (object->ref_count == 0) {
284                 vp->v_flag &= ~VTEXT;
285                 vm_object_clear_flag(object, OBJ_OPT);
286         }
287         vrele(vp);
288 }
289
290 /*
291  *      vm_object_deallocate:
292  *
293  *      Release a reference to the specified object,
294  *      gained either through a vm_object_allocate
295  *      or a vm_object_reference call.  When all references
296  *      are gone, storage associated with this object
297  *      may be relinquished.
298  *
299  *      No object may be locked.
300  */
301 void
302 vm_object_deallocate(vm_object_t object)
303 {
304         vm_object_t temp;
305
306         while (object != NULL) {
307
308                 if (object->type == OBJT_VNODE) {
309                         vm_object_vndeallocate(object);
310                         return;
311                 }
312
313                 if (object->ref_count == 0) {
314                         panic("vm_object_deallocate: object deallocated too many times: %d", object->type);
315                 } else if (object->ref_count > 2) {
316                         object->ref_count--;
317                         return;
318                 }
319
320                 /*
321                  * Here on ref_count of one or two, which are special cases for
322                  * objects.
323                  */
324                 if ((object->ref_count == 2) && (object->shadow_count == 0)) {
325                         vm_object_set_flag(object, OBJ_ONEMAPPING);
326                         object->ref_count--;
327                         return;
328                 } else if ((object->ref_count == 2) && (object->shadow_count == 1)) {
329                         object->ref_count--;
330                         if ((object->handle == NULL) &&
331                             (object->type == OBJT_DEFAULT ||
332                              object->type == OBJT_SWAP)) {
333                                 vm_object_t robject;
334
335                                 robject = LIST_FIRST(&object->shadow_head);
336                                 KASSERT(robject != NULL,
337                                     ("vm_object_deallocate: ref_count: %d, shadow_count: %d",
338                                          object->ref_count,
339                                          object->shadow_count));
340                                 if ((robject->handle == NULL) &&
341                                     (robject->type == OBJT_DEFAULT ||
342                                      robject->type == OBJT_SWAP)) {
343
344                                         robject->ref_count++;
345
346                                         while (
347                                                 robject->paging_in_progress ||
348                                                 object->paging_in_progress
349                                         ) {
350                                                 vm_object_pip_sleep(robject, "objde1");
351                                                 vm_object_pip_sleep(object, "objde2");
352                                         }
353
354                                         if (robject->ref_count == 1) {
355                                                 robject->ref_count--;
356                                                 object = robject;
357                                                 goto doterm;
358                                         }
359
360                                         object = robject;
361                                         vm_object_collapse(object);
362                                         continue;
363                                 }
364                         }
365
366                         return;
367
368                 } else {
369                         object->ref_count--;
370                         if (object->ref_count != 0)
371                                 return;
372                 }
373
374 doterm:
375
376                 temp = object->backing_object;
377                 if (temp) {
378                         LIST_REMOVE(object, shadow_list);
379                         temp->shadow_count--;
380                         if (temp->ref_count == 0)
381                                 vm_object_clear_flag(temp, OBJ_OPT);
382                         temp->generation++;
383                         object->backing_object = NULL;
384                 }
385
386                 /*
387                  * Don't double-terminate, we could be in a termination
388                  * recursion due to the terminate having to sync data
389                  * to disk.
390                  */
391                 if ((object->flags & OBJ_DEAD) == 0)
392                         vm_object_terminate(object);
393                 object = temp;
394         }
395 }
396
397 /*
398  *      vm_object_terminate actually destroys the specified object, freeing
399  *      up all previously used resources.
400  *
401  *      The object must be locked.
402  *      This routine may block.
403  */
404 void
405 vm_object_terminate(vm_object_t object)
406 {
407         lwkt_tokref ilock;
408         vm_page_t p;
409         int s;
410
411         /*
412          * Make sure no one uses us.
413          */
414         vm_object_set_flag(object, OBJ_DEAD);
415
416         /*
417          * wait for the pageout daemon to be done with the object
418          */
419         vm_object_pip_wait(object, "objtrm");
420
421         KASSERT(!object->paging_in_progress,
422                 ("vm_object_terminate: pageout in progress"));
423
424         /*
425          * Clean and free the pages, as appropriate. All references to the
426          * object are gone, so we don't need to lock it.
427          */
428         if (object->type == OBJT_VNODE) {
429                 struct vnode *vp;
430
431                 /*
432                  * Freeze optimized copies.
433                  */
434                 vm_freeze_copyopts(object, 0, object->size);
435
436                 /*
437                  * Clean pages and flush buffers.
438                  */
439                 vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
440
441                 vp = (struct vnode *) object->handle;
442                 vinvalbuf(vp, V_SAVE, NULL, 0, 0);
443         }
444
445         /*
446          * Wait for any I/O to complete, after which there had better not
447          * be any references left on the object.
448          */
449         vm_object_pip_wait(object, "objtrm");
450
451         if (object->ref_count != 0)
452                 panic("vm_object_terminate: object with references, ref_count=%d", object->ref_count);
453
454         /*
455          * Now free any remaining pages. For internal objects, this also
456          * removes them from paging queues. Don't free wired pages, just
457          * remove them from the object. 
458          */
459         s = splvm();
460         while ((p = TAILQ_FIRST(&object->memq)) != NULL) {
461                 if (p->busy || (p->flags & PG_BUSY))
462                         panic("vm_object_terminate: freeing busy page %p", p);
463                 if (p->wire_count == 0) {
464                         vm_page_busy(p);
465                         vm_page_free(p);
466                         mycpu->gd_cnt.v_pfree++;
467                 } else {
468                         vm_page_busy(p);
469                         vm_page_remove(p);
470                 }
471         }
472         splx(s);
473
474         /*
475          * Let the pager know object is dead.
476          */
477         vm_pager_deallocate(object);
478
479         /*
480          * Remove the object from the global object list.
481          */
482         lwkt_gettoken(&ilock, &vm_object_list_token);
483         TAILQ_REMOVE(&vm_object_list, object, object_list);
484         lwkt_reltoken(&ilock);
485
486         wakeup(object);
487
488         /*
489          * Free the space for the object.
490          */
491         zfree(obj_zone, object);
492 }
493
494 /*
495  *      vm_object_page_clean
496  *
497  *      Clean all dirty pages in the specified range of object.  Leaves page 
498  *      on whatever queue it is currently on.   If NOSYNC is set then do not
499  *      write out pages with PG_NOSYNC set (originally comes from MAP_NOSYNC),
500  *      leaving the object dirty.
501  *
502  *      When stuffing pages asynchronously, allow clustering.  XXX we need a
503  *      synchronous clustering mode implementation.
504  *
505  *      Odd semantics: if start == end, we clean everything.
506  */
507
508 void
509 vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
510     int flags)
511 {
512         vm_page_t p, np;
513         vm_offset_t tstart, tend;
514         vm_pindex_t pi;
515         struct vnode *vp;
516         int clearobjflags;
517         int pagerflags;
518         int curgeneration;
519         lwkt_tokref vlock;
520         int s;
521
522         if (object->type != OBJT_VNODE ||
523                 (object->flags & OBJ_MIGHTBEDIRTY) == 0)
524                 return;
525
526         pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
527         pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
528
529         vp = object->handle;
530
531         vm_object_set_flag(object, OBJ_CLEANING);
532
533         /*
534          * Handle 'entire object' case
535          */
536         tstart = start;
537         if (end == 0) {
538                 tend = object->size;
539         } else {
540                 tend = end;
541         }
542
543         /*
544          * If the caller is smart and only msync()s a range he knows is
545          * dirty, we may be able to avoid an object scan.  This results in
546          * a phenominal improvement in performance.  We cannot do this
547          * as a matter of course because the object may be huge - e.g.
548          * the size might be in the gigabytes or terrabytes.
549          */
550         if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) {
551                 vm_offset_t tscan;
552                 int scanlimit;
553                 int scanreset;
554
555                 scanreset = object->resident_page_count / EASY_SCAN_FACTOR;
556                 if (scanreset < 16)
557                         scanreset = 16;
558                 pagerflags |= VM_PAGER_IGNORE_CLEANCHK;
559
560                 scanlimit = scanreset;
561                 tscan = tstart;
562
563                 /*
564                  * spl protection is required despite the obj generation
565                  * tracking because we cannot safely call vm_page_test_dirty()
566                  * or avoid page field tests against an interrupt unbusy/free
567                  * race that might occur prior to the busy check in
568                  * vm_object_page_collect_flush().
569                  */
570                 s = splvm();
571                 while (tscan < tend) {
572                         curgeneration = object->generation;
573                         p = vm_page_lookup(object, tscan);
574                         if (p == NULL || p->valid == 0 ||
575                             (p->queue - p->pc) == PQ_CACHE) {
576                                 if (--scanlimit == 0)
577                                         break;
578                                 ++tscan;
579                                 continue;
580                         }
581                         vm_page_test_dirty(p);
582                         if ((p->dirty & p->valid) == 0) {
583                                 if (--scanlimit == 0)
584                                         break;
585                                 ++tscan;
586                                 continue;
587                         }
588                         /*
589                          * If we have been asked to skip nosync pages and 
590                          * this is a nosync page, we can't continue.
591                          */
592                         if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
593                                 if (--scanlimit == 0)
594                                         break;
595                                 ++tscan;
596                                 continue;
597                         }
598                         scanlimit = scanreset;
599
600                         /*
601                          * This returns 0 if it was unable to busy the first
602                          * page (i.e. had to sleep).
603                          */
604                         tscan += vm_object_page_collect_flush(object, p, 
605                                                 curgeneration, pagerflags);
606                 }
607                 splx(s);
608
609                 /*
610                  * If everything was dirty and we flushed it successfully,
611                  * and the requested range is not the entire object, we
612                  * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can
613                  * return immediately.
614                  */
615                 if (tscan >= tend && (tstart || tend < object->size)) {
616                         vm_object_clear_flag(object, OBJ_CLEANING);
617                         return;
618                 }
619                 pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK;
620         }
621
622         /*
623          * Generally set CLEANCHK interlock and make the page read-only so
624          * we can then clear the object flags.
625          *
626          * However, if this is a nosync mmap then the object is likely to 
627          * stay dirty so do not mess with the page and do not clear the
628          * object flags.
629          *
630          * spl protection is required because an interrupt can remove page
631          * from the object.
632          */
633         clearobjflags = 1;
634
635         s = splvm();
636         for (p = TAILQ_FIRST(&object->memq); p; p = TAILQ_NEXT(p, listq)) {
637                 vm_page_flag_set(p, PG_CLEANCHK);
638                 if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC))
639                         clearobjflags = 0;
640                 else
641                         vm_page_protect(p, VM_PROT_READ);
642         }
643         splx(s);
644
645         if (clearobjflags && (tstart == 0) && (tend == object->size)) {
646                 struct vnode *vp;
647
648                 vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
649                 if (object->type == OBJT_VNODE &&
650                     (vp = (struct vnode *)object->handle) != NULL) {
651                         if (vp->v_flag & VOBJDIRTY) {
652                                 lwkt_gettoken(&vlock, vp->v_interlock);
653                                 vp->v_flag &= ~VOBJDIRTY;
654                                 lwkt_reltoken(&vlock);
655                         }
656                 }
657         }
658
659         /*
660          * spl protection is required both to avoid an interrupt unbusy/free
661          * race against a vm_page_lookup(), and also to ensure that the
662          * memq is consistent.  We do not want a busy page to be ripped out
663          * from under us.
664          */
665         s = splvm();
666 rescan:
667         splx(s);        /* give interrupts a chance */
668         s = splvm();
669         curgeneration = object->generation;
670
671         for (p = TAILQ_FIRST(&object->memq); p; p = np) {
672                 int n;
673
674                 np = TAILQ_NEXT(p, listq);
675
676 again:
677                 pi = p->pindex;
678                 if (((p->flags & PG_CLEANCHK) == 0) ||
679                         (pi < tstart) || (pi >= tend) ||
680                         (p->valid == 0) ||
681                         ((p->queue - p->pc) == PQ_CACHE)) {
682                         vm_page_flag_clear(p, PG_CLEANCHK);
683                         continue;
684                 }
685
686                 vm_page_test_dirty(p);
687                 if ((p->dirty & p->valid) == 0) {
688                         vm_page_flag_clear(p, PG_CLEANCHK);
689                         continue;
690                 }
691
692                 /*
693                  * If we have been asked to skip nosync pages and this is a
694                  * nosync page, skip it.  Note that the object flags were
695                  * not cleared in this case so we do not have to set them.
696                  */
697                 if ((flags & OBJPC_NOSYNC) && (p->flags & PG_NOSYNC)) {
698                         vm_page_flag_clear(p, PG_CLEANCHK);
699                         continue;
700                 }
701
702                 n = vm_object_page_collect_flush(object, p,
703                         curgeneration, pagerflags);
704                 if (n == 0)
705                         goto rescan;
706                 if (object->generation != curgeneration)
707                         goto rescan;
708
709                 /*
710                  * Try to optimize the next page.  If we can't we pick up
711                  * our (random) scan where we left off.
712                  */
713                 if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) {
714                         if ((p = vm_page_lookup(object, pi + n)) != NULL)
715                                 goto again;
716                 }
717         }
718         splx(s);
719
720 #if 0
721         VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
722 #endif
723
724         vm_object_clear_flag(object, OBJ_CLEANING);
725         return;
726 }
727
728 /*
729  * This routine must be called at splvm() to properly avoid an interrupt
730  * unbusy/free race that can occur prior to the busy check.
731  *
732  * Using the object generation number here to detect page ripout is not
733  * the best idea in the world. XXX
734  *
735  * NOTE: we operate under the assumption that a page found to not be busy
736  * will not be ripped out from under us by an interrupt.  XXX we should
737  * recode this to explicitly busy the pages.
738  */
739 static int
740 vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags)
741 {
742         int runlen;
743         int maxf;
744         int chkb;
745         int maxb;
746         int i;
747         vm_pindex_t pi;
748         vm_page_t maf[vm_pageout_page_count];
749         vm_page_t mab[vm_pageout_page_count];
750         vm_page_t ma[vm_pageout_page_count];
751
752         pi = p->pindex;
753         while (vm_page_sleep_busy(p, TRUE, "vpcwai")) {
754                 if (object->generation != curgeneration) {
755                         return(0);
756                 }
757         }
758
759         maxf = 0;
760         for(i = 1; i < vm_pageout_page_count; i++) {
761                 vm_page_t tp;
762
763                 if ((tp = vm_page_lookup(object, pi + i)) != NULL) {
764                         if ((tp->flags & PG_BUSY) ||
765                                 ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 
766                                  (tp->flags & PG_CLEANCHK) == 0) ||
767                                 (tp->busy != 0))
768                                 break;
769                         if((tp->queue - tp->pc) == PQ_CACHE) {
770                                 vm_page_flag_clear(tp, PG_CLEANCHK);
771                                 break;
772                         }
773                         vm_page_test_dirty(tp);
774                         if ((tp->dirty & tp->valid) == 0) {
775                                 vm_page_flag_clear(tp, PG_CLEANCHK);
776                                 break;
777                         }
778                         maf[ i - 1 ] = tp;
779                         maxf++;
780                         continue;
781                 }
782                 break;
783         }
784
785         maxb = 0;
786         chkb = vm_pageout_page_count -  maxf;
787         if (chkb) {
788                 for(i = 1; i < chkb;i++) {
789                         vm_page_t tp;
790
791                         if ((tp = vm_page_lookup(object, pi - i)) != NULL) {
792                                 if ((tp->flags & PG_BUSY) ||
793                                         ((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 && 
794                                          (tp->flags & PG_CLEANCHK) == 0) ||
795                                         (tp->busy != 0))
796                                         break;
797                                 if((tp->queue - tp->pc) == PQ_CACHE) {
798                                         vm_page_flag_clear(tp, PG_CLEANCHK);
799                                         break;
800                                 }
801                                 vm_page_test_dirty(tp);
802                                 if ((tp->dirty & tp->valid) == 0) {
803                                         vm_page_flag_clear(tp, PG_CLEANCHK);
804                                         break;
805                                 }
806                                 mab[ i - 1 ] = tp;
807                                 maxb++;
808                                 continue;
809                         }
810                         break;
811                 }
812         }
813
814         for(i = 0; i < maxb; i++) {
815                 int index = (maxb - i) - 1;
816                 ma[index] = mab[i];
817                 vm_page_flag_clear(ma[index], PG_CLEANCHK);
818         }
819         vm_page_flag_clear(p, PG_CLEANCHK);
820         ma[maxb] = p;
821         for(i = 0; i < maxf; i++) {
822                 int index = (maxb + i) + 1;
823                 ma[index] = maf[i];
824                 vm_page_flag_clear(ma[index], PG_CLEANCHK);
825         }
826         runlen = maxb + maxf + 1;
827
828         vm_pageout_flush(ma, runlen, pagerflags);
829         for (i = 0; i < runlen; i++) {
830                 if (ma[i]->valid & ma[i]->dirty) {
831                         vm_page_protect(ma[i], VM_PROT_READ);
832                         vm_page_flag_set(ma[i], PG_CLEANCHK);
833
834                         /*
835                          * maxf will end up being the actual number of pages
836                          * we wrote out contiguously, non-inclusive of the
837                          * first page.  We do not count look-behind pages.
838                          */
839                         if (i >= maxb + 1 && (maxf > i - maxb - 1))
840                                 maxf = i - maxb - 1;
841                 }
842         }
843         return(maxf + 1);
844 }
845
846 #ifdef not_used
847 /* XXX I cannot tell if this should be an exported symbol */
848 /*
849  *      vm_object_deactivate_pages
850  *
851  *      Deactivate all pages in the specified object.  (Keep its pages
852  *      in memory even though it is no longer referenced.)
853  *
854  *      The object must be locked.
855  */
856 static void
857 vm_object_deactivate_pages(vm_object_t object)
858 {
859         vm_page_t p, next;
860         int s;
861
862         s = splvm();
863         for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
864                 next = TAILQ_NEXT(p, listq);
865                 vm_page_deactivate(p);
866         }
867         splx(s);
868 }
869 #endif
870
871 /*
872  * Same as vm_object_pmap_copy, except range checking really
873  * works, and is meant for small sections of an object.
874  *
875  * This code protects resident pages by making them read-only
876  * and is typically called on a fork or split when a page
877  * is converted to copy-on-write.  
878  *
879  * NOTE: If the page is already at VM_PROT_NONE, calling
880  * vm_page_protect will have no effect.
881  */
882 void
883 vm_object_pmap_copy_1(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
884 {
885         vm_pindex_t idx;
886         vm_page_t p;
887         int s;
888
889         if (object == NULL || (object->flags & OBJ_WRITEABLE) == 0)
890                 return;
891
892         /*
893          * spl protection needed to prevent races between the lookup,
894          * an interrupt unbusy/free, and our protect call.
895          */
896         s = splvm();
897         for (idx = start; idx < end; idx++) {
898                 p = vm_page_lookup(object, idx);
899                 if (p == NULL)
900                         continue;
901                 vm_page_protect(p, VM_PROT_READ);
902         }
903         splx(s);
904 }
905
906 /*
907  *      vm_object_pmap_remove:
908  *
909  *      Removes all physical pages in the specified
910  *      object range from all physical maps.
911  *
912  *      The object must *not* be locked.
913  */
914 void
915 vm_object_pmap_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
916 {
917         vm_page_t p;
918         int s;
919
920         if (object == NULL)
921                 return;
922
923         /*
924          * spl protection is required because an interrupt can unbusy/free
925          * a page.
926          */
927         s = splvm();
928         for (p = TAILQ_FIRST(&object->memq);
929             p != NULL;
930             p = TAILQ_NEXT(p, listq)
931         ) {
932                 if (p->pindex >= start && p->pindex < end)
933                         vm_page_protect(p, VM_PROT_NONE);
934         }
935         splx(s);
936         if ((start == 0) && (object->size == end))
937                 vm_object_clear_flag(object, OBJ_WRITEABLE);
938 }
939
940 /*
941  *      vm_object_madvise:
942  *
943  *      Implements the madvise function at the object/page level.
944  *
945  *      MADV_WILLNEED   (any object)
946  *
947  *          Activate the specified pages if they are resident.
948  *
949  *      MADV_DONTNEED   (any object)
950  *
951  *          Deactivate the specified pages if they are resident.
952  *
953  *      MADV_FREE       (OBJT_DEFAULT/OBJT_SWAP objects,
954  *                       OBJ_ONEMAPPING only)
955  *
956  *          Deactivate and clean the specified pages if they are
957  *          resident.  This permits the process to reuse the pages
958  *          without faulting or the kernel to reclaim the pages
959  *          without I/O.
960  */
961 void
962 vm_object_madvise(vm_object_t object, vm_pindex_t pindex, int count, int advise)
963 {
964         vm_pindex_t end, tpindex;
965         vm_object_t tobject;
966         vm_page_t m;
967         int s;
968
969         if (object == NULL)
970                 return;
971
972         end = pindex + count;
973
974         /*
975          * Locate and adjust resident pages
976          */
977
978         for (; pindex < end; pindex += 1) {
979 relookup:
980                 tobject = object;
981                 tpindex = pindex;
982 shadowlookup:
983                 /*
984                  * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages
985                  * and those pages must be OBJ_ONEMAPPING.
986                  */
987                 if (advise == MADV_FREE) {
988                         if ((tobject->type != OBJT_DEFAULT &&
989                              tobject->type != OBJT_SWAP) ||
990                             (tobject->flags & OBJ_ONEMAPPING) == 0) {
991                                 continue;
992                         }
993                 }
994
995                 /*
996                  * spl protection is required to avoid a race between the
997                  * lookup, an interrupt unbusy/free, and our busy check.
998                  */
999
1000                 s = splvm();
1001                 m = vm_page_lookup(tobject, tpindex);
1002
1003                 if (m == NULL) {
1004                         /*
1005                          * There may be swap even if there is no backing page
1006                          */
1007                         if (advise == MADV_FREE && tobject->type == OBJT_SWAP)
1008                                 swap_pager_freespace(tobject, tpindex, 1);
1009
1010                         /*
1011                          * next object
1012                          */
1013                         splx(s);
1014                         if (tobject->backing_object == NULL)
1015                                 continue;
1016                         tpindex += OFF_TO_IDX(tobject->backing_object_offset);
1017                         tobject = tobject->backing_object;
1018                         goto shadowlookup;
1019                 }
1020
1021                 /*
1022                  * If the page is busy or not in a normal active state,
1023                  * we skip it.  If the page is not managed there are no
1024                  * page queues to mess with.  Things can break if we mess
1025                  * with pages in any of the below states.
1026                  */
1027                 if (
1028                     m->hold_count ||
1029                     m->wire_count ||
1030                     (m->flags & PG_UNMANAGED) ||
1031                     m->valid != VM_PAGE_BITS_ALL
1032                 ) {
1033                         splx(s);
1034                         continue;
1035                 }
1036
1037                 if (vm_page_sleep_busy(m, TRUE, "madvpo")) {
1038                         splx(s);
1039                         goto relookup;
1040                 }
1041                 splx(s);
1042
1043                 /*
1044                  * Theoretically once a page is known not to be busy, an
1045                  * interrupt cannot come along and rip it out from under us.
1046                  */
1047
1048                 if (advise == MADV_WILLNEED) {
1049                         vm_page_activate(m);
1050                 } else if (advise == MADV_DONTNEED) {
1051                         vm_page_dontneed(m);
1052                 } else if (advise == MADV_FREE) {
1053                         /*
1054                          * Mark the page clean.  This will allow the page
1055                          * to be freed up by the system.  However, such pages
1056                          * are often reused quickly by malloc()/free()
1057                          * so we do not do anything that would cause
1058                          * a page fault if we can help it.
1059                          *
1060                          * Specifically, we do not try to actually free
1061                          * the page now nor do we try to put it in the
1062                          * cache (which would cause a page fault on reuse).
1063                          *
1064                          * But we do make the page is freeable as we
1065                          * can without actually taking the step of unmapping
1066                          * it.
1067                          */
1068                         pmap_clear_modify(m);
1069                         m->dirty = 0;
1070                         m->act_count = 0;
1071                         vm_page_dontneed(m);
1072                         if (tobject->type == OBJT_SWAP)
1073                                 swap_pager_freespace(tobject, tpindex, 1);
1074                 }
1075         }       
1076 }
1077
1078 /*
1079  *      vm_object_shadow:
1080  *
1081  *      Create a new object which is backed by the
1082  *      specified existing object range.  The source
1083  *      object reference is deallocated.
1084  *
1085  *      The new object and offset into that object
1086  *      are returned in the source parameters.
1087  */
1088
1089 void
1090 vm_object_shadow(vm_object_t *object,   /* IN/OUT */
1091                  vm_ooffset_t *offset,  /* IN/OUT */
1092                  vm_size_t length)
1093 {
1094         vm_object_t source;
1095         vm_object_t result;
1096
1097         source = *object;
1098
1099         /*
1100          * Don't create the new object if the old object isn't shared.
1101          */
1102
1103         if (source != NULL &&
1104             source->ref_count == 1 &&
1105             source->handle == NULL &&
1106             (source->type == OBJT_DEFAULT ||
1107              source->type == OBJT_SWAP))
1108                 return;
1109
1110         /*
1111          * Allocate a new object with the given length
1112          */
1113
1114         if ((result = vm_object_allocate(OBJT_DEFAULT, length)) == NULL)
1115                 panic("vm_object_shadow: no object for shadowing");
1116
1117         /*
1118          * The new object shadows the source object, adding a reference to it.
1119          * Our caller changes his reference to point to the new object,
1120          * removing a reference to the source object.  Net result: no change
1121          * of reference count.
1122          *
1123          * Try to optimize the result object's page color when shadowing
1124          * in order to maintain page coloring consistency in the combined 
1125          * shadowed object.
1126          */
1127         result->backing_object = source;
1128         if (source) {
1129                 LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list);
1130                 source->shadow_count++;
1131                 source->generation++;
1132                 result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & PQ_L2_MASK;
1133         }
1134
1135         /*
1136          * Store the offset into the source object, and fix up the offset into
1137          * the new object.
1138          */
1139
1140         result->backing_object_offset = *offset;
1141
1142         /*
1143          * Return the new things
1144          */
1145
1146         *offset = 0;
1147         *object = result;
1148 }
1149
1150 #define OBSC_TEST_ALL_SHADOWED  0x0001
1151 #define OBSC_COLLAPSE_NOWAIT    0x0002
1152 #define OBSC_COLLAPSE_WAIT      0x0004
1153
1154 static __inline int
1155 vm_object_backing_scan(vm_object_t object, int op)
1156 {
1157         int s;
1158         int r = 1;
1159         vm_page_t p;
1160         vm_object_t backing_object;
1161         vm_pindex_t backing_offset_index;
1162
1163         /*
1164          * spl protection is required to avoid races between the memq/lookup,
1165          * an interrupt doing an unbusy/free, and our busy check.  Amoung
1166          * other things.
1167          */
1168         s = splvm();
1169
1170         backing_object = object->backing_object;
1171         backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1172
1173         /*
1174          * Initial conditions
1175          */
1176
1177         if (op & OBSC_TEST_ALL_SHADOWED) {
1178                 /*
1179                  * We do not want to have to test for the existence of
1180                  * swap pages in the backing object.  XXX but with the
1181                  * new swapper this would be pretty easy to do.
1182                  *
1183                  * XXX what about anonymous MAP_SHARED memory that hasn't
1184                  * been ZFOD faulted yet?  If we do not test for this, the
1185                  * shadow test may succeed! XXX
1186                  */
1187                 if (backing_object->type != OBJT_DEFAULT) {
1188                         splx(s);
1189                         return(0);
1190                 }
1191         }
1192         if (op & OBSC_COLLAPSE_WAIT) {
1193                 vm_object_set_flag(backing_object, OBJ_DEAD);
1194         }
1195
1196         /*
1197          * Our scan
1198          */
1199
1200         p = TAILQ_FIRST(&backing_object->memq);
1201         while (p) {
1202                 vm_page_t next = TAILQ_NEXT(p, listq);
1203                 vm_pindex_t new_pindex = p->pindex - backing_offset_index;
1204
1205                 if (op & OBSC_TEST_ALL_SHADOWED) {
1206                         vm_page_t pp;
1207
1208                         /*
1209                          * Ignore pages outside the parent object's range
1210                          * and outside the parent object's mapping of the 
1211                          * backing object.
1212                          *
1213                          * note that we do not busy the backing object's
1214                          * page.
1215                          */
1216
1217                         if (
1218                             p->pindex < backing_offset_index ||
1219                             new_pindex >= object->size
1220                         ) {
1221                                 p = next;
1222                                 continue;
1223                         }
1224
1225                         /*
1226                          * See if the parent has the page or if the parent's
1227                          * object pager has the page.  If the parent has the
1228                          * page but the page is not valid, the parent's
1229                          * object pager must have the page.
1230                          *
1231                          * If this fails, the parent does not completely shadow
1232                          * the object and we might as well give up now.
1233                          */
1234
1235                         pp = vm_page_lookup(object, new_pindex);
1236                         if (
1237                             (pp == NULL || pp->valid == 0) &&
1238                             !vm_pager_has_page(object, new_pindex, NULL, NULL)
1239                         ) {
1240                                 r = 0;
1241                                 break;
1242                         }
1243                 }
1244
1245                 /*
1246                  * Check for busy page
1247                  */
1248
1249                 if (op & (OBSC_COLLAPSE_WAIT | OBSC_COLLAPSE_NOWAIT)) {
1250                         vm_page_t pp;
1251
1252                         if (op & OBSC_COLLAPSE_NOWAIT) {
1253                                 if (
1254                                     (p->flags & PG_BUSY) ||
1255                                     !p->valid || 
1256                                     p->hold_count || 
1257                                     p->wire_count ||
1258                                     p->busy
1259                                 ) {
1260                                         p = next;
1261                                         continue;
1262                                 }
1263                         } else if (op & OBSC_COLLAPSE_WAIT) {
1264                                 if (vm_page_sleep_busy(p, TRUE, "vmocol")) {
1265                                         /*
1266                                          * If we slept, anything could have
1267                                          * happened.  Since the object is
1268                                          * marked dead, the backing offset
1269                                          * should not have changed so we
1270                                          * just restart our scan.
1271                                          */
1272                                         p = TAILQ_FIRST(&backing_object->memq);
1273                                         continue;
1274                                 }
1275                         }
1276
1277                         /* 
1278                          * Busy the page
1279                          */
1280                         vm_page_busy(p);
1281
1282                         KASSERT(
1283                             p->object == backing_object,
1284                             ("vm_object_qcollapse(): object mismatch")
1285                         );
1286
1287                         /*
1288                          * Destroy any associated swap
1289                          */
1290                         if (backing_object->type == OBJT_SWAP) {
1291                                 swap_pager_freespace(
1292                                     backing_object, 
1293                                     p->pindex,
1294                                     1
1295                                 );
1296                         }
1297
1298                         if (
1299                             p->pindex < backing_offset_index ||
1300                             new_pindex >= object->size
1301                         ) {
1302                                 /*
1303                                  * Page is out of the parent object's range, we 
1304                                  * can simply destroy it. 
1305                                  */
1306                                 vm_page_protect(p, VM_PROT_NONE);
1307                                 vm_page_free(p);
1308                                 p = next;
1309                                 continue;
1310                         }
1311
1312                         pp = vm_page_lookup(object, new_pindex);
1313                         if (
1314                             pp != NULL ||
1315                             vm_pager_has_page(object, new_pindex, NULL, NULL)
1316                         ) {
1317                                 /*
1318                                  * page already exists in parent OR swap exists
1319                                  * for this location in the parent.  Destroy 
1320                                  * the original page from the backing object.
1321                                  *
1322                                  * Leave the parent's page alone
1323                                  */
1324                                 vm_page_protect(p, VM_PROT_NONE);
1325                                 vm_page_free(p);
1326                                 p = next;
1327                                 continue;
1328                         }
1329
1330                         /*
1331                          * Page does not exist in parent, rename the
1332                          * page from the backing object to the main object. 
1333                          *
1334                          * If the page was mapped to a process, it can remain 
1335                          * mapped through the rename.
1336                          */
1337                         if ((p->queue - p->pc) == PQ_CACHE)
1338                                 vm_page_deactivate(p);
1339
1340                         vm_page_rename(p, object, new_pindex);
1341                         /* page automatically made dirty by rename */
1342                 }
1343                 p = next;
1344         }
1345         splx(s);
1346         return(r);
1347 }
1348
1349
1350 /*
1351  * this version of collapse allows the operation to occur earlier and
1352  * when paging_in_progress is true for an object...  This is not a complete
1353  * operation, but should plug 99.9% of the rest of the leaks.
1354  */
1355 static void
1356 vm_object_qcollapse(vm_object_t object)
1357 {
1358         vm_object_t backing_object = object->backing_object;
1359
1360         if (backing_object->ref_count != 1)
1361                 return;
1362
1363         backing_object->ref_count += 2;
1364
1365         vm_object_backing_scan(object, OBSC_COLLAPSE_NOWAIT);
1366
1367         backing_object->ref_count -= 2;
1368 }
1369
1370 /*
1371  *      vm_object_collapse:
1372  *
1373  *      Collapse an object with the object backing it.
1374  *      Pages in the backing object are moved into the
1375  *      parent, and the backing object is deallocated.
1376  */
1377 void
1378 vm_object_collapse(vm_object_t object)
1379 {
1380         while (TRUE) {
1381                 vm_object_t backing_object;
1382
1383                 /*
1384                  * Verify that the conditions are right for collapse:
1385                  *
1386                  * The object exists and the backing object exists.
1387                  */
1388                 if (object == NULL)
1389                         break;
1390
1391                 if ((backing_object = object->backing_object) == NULL)
1392                         break;
1393
1394                 /*
1395                  * we check the backing object first, because it is most likely
1396                  * not collapsable.
1397                  */
1398                 if (backing_object->handle != NULL ||
1399                     (backing_object->type != OBJT_DEFAULT &&
1400                      backing_object->type != OBJT_SWAP) ||
1401                     (backing_object->flags & OBJ_DEAD) ||
1402                     object->handle != NULL ||
1403                     (object->type != OBJT_DEFAULT &&
1404                      object->type != OBJT_SWAP) ||
1405                     (object->flags & OBJ_DEAD)) {
1406                         break;
1407                 }
1408
1409                 if (
1410                     object->paging_in_progress != 0 ||
1411                     backing_object->paging_in_progress != 0
1412                 ) {
1413                         vm_object_qcollapse(object);
1414                         break;
1415                 }
1416
1417                 /*
1418                  * We know that we can either collapse the backing object (if
1419                  * the parent is the only reference to it) or (perhaps) have
1420                  * the parent bypass the object if the parent happens to shadow
1421                  * all the resident pages in the entire backing object.
1422                  *
1423                  * This is ignoring pager-backed pages such as swap pages.
1424                  * vm_object_backing_scan fails the shadowing test in this
1425                  * case.
1426                  */
1427
1428                 if (backing_object->ref_count == 1) {
1429                         /*
1430                          * If there is exactly one reference to the backing
1431                          * object, we can collapse it into the parent.  
1432                          */
1433
1434                         vm_object_backing_scan(object, OBSC_COLLAPSE_WAIT);
1435
1436                         /*
1437                          * Move the pager from backing_object to object.
1438                          */
1439
1440                         if (backing_object->type == OBJT_SWAP) {
1441                                 vm_object_pip_add(backing_object, 1);
1442
1443                                 /*
1444                                  * scrap the paging_offset junk and do a 
1445                                  * discrete copy.  This also removes major 
1446                                  * assumptions about how the swap-pager 
1447                                  * works from where it doesn't belong.  The
1448                                  * new swapper is able to optimize the
1449                                  * destroy-source case.
1450                                  */
1451
1452                                 vm_object_pip_add(object, 1);
1453                                 swap_pager_copy(
1454                                     backing_object,
1455                                     object,
1456                                     OFF_TO_IDX(object->backing_object_offset), TRUE);
1457                                 vm_object_pip_wakeup(object);
1458
1459                                 vm_object_pip_wakeup(backing_object);
1460                         }
1461                         /*
1462                          * Object now shadows whatever backing_object did.
1463                          * Note that the reference to 
1464                          * backing_object->backing_object moves from within 
1465                          * backing_object to within object.
1466                          */
1467
1468                         LIST_REMOVE(object, shadow_list);
1469                         object->backing_object->shadow_count--;
1470                         object->backing_object->generation++;
1471                         if (backing_object->backing_object) {
1472                                 LIST_REMOVE(backing_object, shadow_list);
1473                                 backing_object->backing_object->shadow_count--;
1474                                 backing_object->backing_object->generation++;
1475                         }
1476                         object->backing_object = backing_object->backing_object;
1477                         if (object->backing_object) {
1478                                 LIST_INSERT_HEAD(
1479                                     &object->backing_object->shadow_head,
1480                                     object, 
1481                                     shadow_list
1482                                 );
1483                                 object->backing_object->shadow_count++;
1484                                 object->backing_object->generation++;
1485                         }
1486
1487                         object->backing_object_offset +=
1488                             backing_object->backing_object_offset;
1489
1490                         /*
1491                          * Discard backing_object.
1492                          *
1493                          * Since the backing object has no pages, no pager left,
1494                          * and no object references within it, all that is
1495                          * necessary is to dispose of it.
1496                          */
1497
1498                         KASSERT(backing_object->ref_count == 1, ("backing_object %p was somehow re-referenced during collapse!", backing_object));
1499                         KASSERT(TAILQ_FIRST(&backing_object->memq) == NULL, ("backing_object %p somehow has left over pages during collapse!", backing_object));
1500                         TAILQ_REMOVE(
1501                             &vm_object_list, 
1502                             backing_object,
1503                             object_list
1504                         );
1505                         vm_object_count--;
1506
1507                         zfree(obj_zone, backing_object);
1508
1509                         object_collapses++;
1510                 } else {
1511                         vm_object_t new_backing_object;
1512
1513                         /*
1514                          * If we do not entirely shadow the backing object,
1515                          * there is nothing we can do so we give up.
1516                          */
1517
1518                         if (vm_object_backing_scan(object, OBSC_TEST_ALL_SHADOWED) == 0) {
1519                                 break;
1520                         }
1521
1522                         /*
1523                          * Make the parent shadow the next object in the
1524                          * chain.  Deallocating backing_object will not remove
1525                          * it, since its reference count is at least 2.
1526                          */
1527
1528                         LIST_REMOVE(object, shadow_list);
1529                         backing_object->shadow_count--;
1530                         backing_object->generation++;
1531
1532                         new_backing_object = backing_object->backing_object;
1533                         if ((object->backing_object = new_backing_object) != NULL) {
1534                                 vm_object_reference(new_backing_object);
1535                                 LIST_INSERT_HEAD(
1536                                     &new_backing_object->shadow_head,
1537                                     object,
1538                                     shadow_list
1539                                 );
1540                                 new_backing_object->shadow_count++;
1541                                 new_backing_object->generation++;
1542                                 object->backing_object_offset +=
1543                                         backing_object->backing_object_offset;
1544                         }
1545
1546                         /*
1547                          * Drop the reference count on backing_object. Since
1548                          * its ref_count was at least 2, it will not vanish;
1549                          * so we don't need to call vm_object_deallocate, but
1550                          * we do anyway.
1551                          */
1552                         vm_object_deallocate(backing_object);
1553                         object_bypasses++;
1554                 }
1555
1556                 /*
1557                  * Try again with this object's new backing object.
1558                  */
1559         }
1560 }
1561
1562 /*
1563  *      vm_object_page_remove: [internal]
1564  *
1565  *      Removes all physical pages in the specified
1566  *      object range from the object's list of pages.
1567  */
1568 void
1569 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
1570     boolean_t clean_only)
1571 {
1572         vm_page_t p, next;
1573         unsigned int size;
1574         int all;
1575         int s;
1576
1577         if (object == NULL || object->resident_page_count == 0)
1578                 return;
1579
1580         all = ((end == 0) && (start == 0));
1581
1582         /*
1583          * Since physically-backed objects do not use managed pages, we can't
1584          * remove pages from the object (we must instead remove the page
1585          * references, and then destroy the object).
1586          */
1587         KASSERT(object->type != OBJT_PHYS, 
1588                 ("attempt to remove pages from a physical object"));
1589
1590         /*
1591          * Indicating that the object is undergoing paging.
1592          *
1593          * spl protection is required to avoid a race between the memq scan,
1594          * an interrupt unbusy/free, and the busy check.
1595          */
1596         vm_object_pip_add(object, 1);
1597         s = splvm();
1598 again:
1599         size = end - start;
1600         if (all || size > object->resident_page_count / 4) {
1601                 for (p = TAILQ_FIRST(&object->memq); p != NULL; p = next) {
1602                         next = TAILQ_NEXT(p, listq);
1603                         if (all || ((start <= p->pindex) && (p->pindex < end))) {
1604                                 if (p->wire_count != 0) {
1605                                         vm_page_protect(p, VM_PROT_NONE);
1606                                         if (!clean_only)
1607                                                 p->valid = 0;
1608                                         continue;
1609                                 }
1610
1611                                 /*
1612                                  * The busy flags are only cleared at
1613                                  * interrupt -- minimize the spl transitions
1614                                  */
1615
1616                                 if (vm_page_sleep_busy(p, TRUE, "vmopar"))
1617                                         goto again;
1618
1619                                 if (clean_only && p->valid) {
1620                                         vm_page_test_dirty(p);
1621                                         if (p->valid & p->dirty)
1622                                                 continue;
1623                                 }
1624
1625                                 vm_page_busy(p);
1626                                 vm_page_protect(p, VM_PROT_NONE);
1627                                 vm_page_free(p);
1628                         }
1629                 }
1630         } else {
1631                 while (size > 0) {
1632                         if ((p = vm_page_lookup(object, start)) != 0) {
1633                                 if (p->wire_count != 0) {
1634                                         vm_page_protect(p, VM_PROT_NONE);
1635                                         if (!clean_only)
1636                                                 p->valid = 0;
1637                                         start += 1;
1638                                         size -= 1;
1639                                         continue;
1640                                 }
1641
1642                                 /*
1643                                  * The busy flags are only cleared at
1644                                  * interrupt -- minimize the spl transitions
1645                                  */
1646                                 if (vm_page_sleep_busy(p, TRUE, "vmopar"))
1647                                         goto again;
1648
1649                                 if (clean_only && p->valid) {
1650                                         vm_page_test_dirty(p);
1651                                         if (p->valid & p->dirty) {
1652                                                 start += 1;
1653                                                 size -= 1;
1654                                                 continue;
1655                                         }
1656                                 }
1657
1658                                 vm_page_busy(p);
1659                                 vm_page_protect(p, VM_PROT_NONE);
1660                                 vm_page_free(p);
1661                         }
1662                         start += 1;
1663                         size -= 1;
1664                 }
1665         }
1666         splx(s);
1667         vm_object_pip_wakeup(object);
1668 }
1669
1670 /*
1671  *      Routine:        vm_object_coalesce
1672  *      Function:       Coalesces two objects backing up adjoining
1673  *                      regions of memory into a single object.
1674  *
1675  *      returns TRUE if objects were combined.
1676  *
1677  *      NOTE:   Only works at the moment if the second object is NULL -
1678  *              if it's not, which object do we lock first?
1679  *
1680  *      Parameters:
1681  *              prev_object     First object to coalesce
1682  *              prev_offset     Offset into prev_object
1683  *              next_object     Second object into coalesce
1684  *              next_offset     Offset into next_object
1685  *
1686  *              prev_size       Size of reference to prev_object
1687  *              next_size       Size of reference to next_object
1688  *
1689  *      Conditions:
1690  *      The object must *not* be locked.
1691  */
1692 boolean_t
1693 vm_object_coalesce(vm_object_t prev_object, vm_pindex_t prev_pindex,
1694     vm_size_t prev_size, vm_size_t next_size)
1695 {
1696         vm_pindex_t next_pindex;
1697
1698         if (prev_object == NULL) {
1699                 return (TRUE);
1700         }
1701
1702         if (prev_object->type != OBJT_DEFAULT &&
1703             prev_object->type != OBJT_SWAP) {
1704                 return (FALSE);
1705         }
1706
1707         /*
1708          * Try to collapse the object first
1709          */
1710         vm_object_collapse(prev_object);
1711
1712         /*
1713          * Can't coalesce if: . more than one reference . paged out . shadows
1714          * another object . has a copy elsewhere (any of which mean that the
1715          * pages not mapped to prev_entry may be in use anyway)
1716          */
1717
1718         if (prev_object->backing_object != NULL) {
1719                 return (FALSE);
1720         }
1721
1722         prev_size >>= PAGE_SHIFT;
1723         next_size >>= PAGE_SHIFT;
1724         next_pindex = prev_pindex + prev_size;
1725
1726         if ((prev_object->ref_count > 1) &&
1727             (prev_object->size != next_pindex)) {
1728                 return (FALSE);
1729         }
1730
1731         /*
1732          * Remove any pages that may still be in the object from a previous
1733          * deallocation.
1734          */
1735         if (next_pindex < prev_object->size) {
1736                 vm_object_page_remove(prev_object,
1737                                       next_pindex,
1738                                       next_pindex + next_size, FALSE);
1739                 if (prev_object->type == OBJT_SWAP)
1740                         swap_pager_freespace(prev_object,
1741                                              next_pindex, next_size);
1742         }
1743
1744         /*
1745          * Extend the object if necessary.
1746          */
1747         if (next_pindex + next_size > prev_object->size)
1748                 prev_object->size = next_pindex + next_size;
1749
1750         return (TRUE);
1751 }
1752
1753 void
1754 vm_object_set_writeable_dirty(vm_object_t object)
1755 {
1756         struct vnode *vp;
1757         lwkt_tokref vlock;
1758
1759         vm_object_set_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
1760         if (object->type == OBJT_VNODE &&
1761             (vp = (struct vnode *)object->handle) != NULL) {
1762                 if ((vp->v_flag & VOBJDIRTY) == 0) {
1763                         lwkt_gettoken(&vlock, vp->v_interlock);
1764                         vp->v_flag |= VOBJDIRTY;
1765                         lwkt_reltoken(&vlock);
1766                 }
1767         }
1768 }
1769
1770
1771
1772 #include "opt_ddb.h"
1773 #ifdef DDB
1774 #include <sys/kernel.h>
1775
1776 #include <sys/cons.h>
1777
1778 #include <ddb/ddb.h>
1779
1780 static int      _vm_object_in_map (vm_map_t map, vm_object_t object,
1781                                        vm_map_entry_t entry);
1782 static int      vm_object_in_map (vm_object_t object);
1783
1784 static int
1785 _vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
1786 {
1787         vm_map_t tmpm;
1788         vm_map_entry_t tmpe;
1789         vm_object_t obj;
1790         int entcount;
1791
1792         if (map == 0)
1793                 return 0;
1794
1795         if (entry == 0) {
1796                 tmpe = map->header.next;
1797                 entcount = map->nentries;
1798                 while (entcount-- && (tmpe != &map->header)) {
1799                         if( _vm_object_in_map(map, object, tmpe)) {
1800                                 return 1;
1801                         }
1802                         tmpe = tmpe->next;
1803                 }
1804         } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
1805                 tmpm = entry->object.sub_map;
1806                 tmpe = tmpm->header.next;
1807                 entcount = tmpm->nentries;
1808                 while (entcount-- && tmpe != &tmpm->header) {
1809                         if( _vm_object_in_map(tmpm, object, tmpe)) {
1810                                 return 1;
1811                         }
1812                         tmpe = tmpe->next;
1813                 }
1814         } else if ((obj = entry->object.vm_object) != NULL) {
1815                 for(; obj; obj=obj->backing_object)
1816                         if( obj == object) {
1817                                 return 1;
1818                         }
1819         }
1820         return 0;
1821 }
1822
1823 static int
1824 vm_object_in_map(vm_object_t object)
1825 {
1826         struct proc *p;
1827         for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) {
1828                 if( !p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
1829                         continue;
1830                 if( _vm_object_in_map(&p->p_vmspace->vm_map, object, 0))
1831                         return 1;
1832         }
1833         if( _vm_object_in_map( kernel_map, object, 0))
1834                 return 1;
1835         if( _vm_object_in_map( pager_map, object, 0))
1836                 return 1;
1837         if( _vm_object_in_map( buffer_map, object, 0))
1838                 return 1;
1839         return 0;
1840 }
1841
1842 DB_SHOW_COMMAND(vmochk, vm_object_check)
1843 {
1844         vm_object_t object;
1845
1846         /*
1847          * make sure that internal objs are in a map somewhere
1848          * and none have zero ref counts.
1849          */
1850         for (object = TAILQ_FIRST(&vm_object_list);
1851                         object != NULL;
1852                         object = TAILQ_NEXT(object, object_list)) {
1853                 if (object->handle == NULL &&
1854                     (object->type == OBJT_DEFAULT || object->type == OBJT_SWAP)) {
1855                         if (object->ref_count == 0) {
1856                                 db_printf("vmochk: internal obj has zero ref count: %ld\n",
1857                                         (long)object->size);
1858                         }
1859                         if (!vm_object_in_map(object)) {
1860                                 db_printf(
1861                         "vmochk: internal obj is not in a map: "
1862                         "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
1863                                     object->ref_count, (u_long)object->size, 
1864                                     (u_long)object->size,
1865                                     (void *)object->backing_object);
1866                         }
1867                 }
1868         }
1869 }
1870
1871 /*
1872  *      vm_object_print:        [ debug ]
1873  */
1874 DB_SHOW_COMMAND(object, vm_object_print_static)
1875 {
1876         /* XXX convert args. */
1877         vm_object_t object = (vm_object_t)addr;
1878         boolean_t full = have_addr;
1879
1880         vm_page_t p;
1881
1882         /* XXX count is an (unused) arg.  Avoid shadowing it. */
1883 #define count   was_count
1884
1885         int count;
1886
1887         if (object == NULL)
1888                 return;
1889
1890         db_iprintf(
1891             "Object %p: type=%d, size=0x%lx, res=%d, ref=%d, flags=0x%x\n",
1892             object, (int)object->type, (u_long)object->size,
1893             object->resident_page_count, object->ref_count, object->flags);
1894         /*
1895          * XXX no %qd in kernel.  Truncate object->backing_object_offset.
1896          */
1897         db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%lx\n",
1898             object->shadow_count, 
1899             object->backing_object ? object->backing_object->ref_count : 0,
1900             object->backing_object, (long)object->backing_object_offset);
1901
1902         if (!full)
1903                 return;
1904
1905         db_indent += 2;
1906         count = 0;
1907         for (p = TAILQ_FIRST(&object->memq); p != NULL; p = TAILQ_NEXT(p, listq)) {
1908                 if (count == 0)
1909                         db_iprintf("memory:=");
1910                 else if (count == 6) {
1911                         db_printf("\n");
1912                         db_iprintf(" ...");
1913                         count = 0;
1914                 } else
1915                         db_printf(",");
1916                 count++;
1917
1918                 db_printf("(off=0x%lx,page=0x%lx)",
1919                     (u_long) p->pindex, (u_long) VM_PAGE_TO_PHYS(p));
1920         }
1921         if (count != 0)
1922                 db_printf("\n");
1923         db_indent -= 2;
1924 }
1925
1926 /* XXX. */
1927 #undef count
1928
1929 /* XXX need this non-static entry for calling from vm_map_print. */
1930 void
1931 vm_object_print(/* db_expr_t */ long addr,
1932                 boolean_t have_addr,
1933                 /* db_expr_t */ long count,
1934                 char *modif)
1935 {
1936         vm_object_print_static(addr, have_addr, count, modif);
1937 }
1938
1939 DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
1940 {
1941         vm_object_t object;
1942         int nl = 0;
1943         int c;
1944         for (object = TAILQ_FIRST(&vm_object_list);
1945                         object != NULL;
1946                         object = TAILQ_NEXT(object, object_list)) {
1947                 vm_pindex_t idx, fidx;
1948                 vm_pindex_t osize;
1949                 vm_paddr_t pa = -1, padiff;
1950                 int rcount;
1951                 vm_page_t m;
1952
1953                 db_printf("new object: %p\n", (void *)object);
1954                 if ( nl > 18) {
1955                         c = cngetc();
1956                         if (c != ' ')
1957                                 return;
1958                         nl = 0;
1959                 }
1960                 nl++;
1961                 rcount = 0;
1962                 fidx = 0;
1963                 osize = object->size;
1964                 if (osize > 128)
1965                         osize = 128;
1966                 for (idx = 0; idx < osize; idx++) {
1967                         m = vm_page_lookup(object, idx);
1968                         if (m == NULL) {
1969                                 if (rcount) {
1970                                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
1971                                                 (long)fidx, rcount, (long)pa);
1972                                         if ( nl > 18) {
1973                                                 c = cngetc();
1974                                                 if (c != ' ')
1975                                                         return;
1976                                                 nl = 0;
1977                                         }
1978                                         nl++;
1979                                         rcount = 0;
1980                                 }
1981                                 continue;
1982                         }
1983
1984                                 
1985                         if (rcount &&
1986                                 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
1987                                 ++rcount;
1988                                 continue;
1989                         }
1990                         if (rcount) {
1991                                 padiff = pa + rcount * PAGE_SIZE - VM_PAGE_TO_PHYS(m);
1992                                 padiff >>= PAGE_SHIFT;
1993                                 padiff &= PQ_L2_MASK;
1994                                 if (padiff == 0) {
1995                                         pa = VM_PAGE_TO_PHYS(m) - rcount * PAGE_SIZE;
1996                                         ++rcount;
1997                                         continue;
1998                                 }
1999                                 db_printf(" index(%ld)run(%d)pa(0x%lx)",
2000                                         (long)fidx, rcount, (long)pa);
2001                                 db_printf("pd(%ld)\n", (long)padiff);
2002                                 if ( nl > 18) {
2003                                         c = cngetc();
2004                                         if (c != ' ')
2005                                                 return;
2006                                         nl = 0;
2007                                 }
2008                                 nl++;
2009                         }
2010                         fidx = idx;
2011                         pa = VM_PAGE_TO_PHYS(m);
2012                         rcount = 1;
2013                 }
2014                 if (rcount) {
2015                         db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2016                                 (long)fidx, rcount, (long)pa);
2017                         if ( nl > 18) {
2018                                 c = cngetc();
2019                                 if (c != ' ')
2020                                         return;
2021                                 nl = 0;
2022                         }
2023                         nl++;
2024                 }
2025         }
2026 }
2027 #endif /* DDB */