2 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/vfs_lock.c,v 1.30 2008/06/30 03:57:41 dillon Exp $
38 * External virtual filesystem routines
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/malloc.h>
46 #include <sys/mount.h>
48 #include <sys/vnode.h>
50 #include <sys/sysctl.h>
52 #include <machine/limits.h>
55 #include <vm/vm_object.h>
58 #include <sys/thread2.h>
59 #include <sys/sysref2.h>
61 static void vnode_terminate(struct vnode *vp);
62 static boolean_t vnode_ctor(void *obj, void *private, int ocflags);
63 static void vnode_dtor(void *obj, void *private);
65 static MALLOC_DEFINE(M_VNODE, "vnodes", "vnode structures");
66 static struct sysref_class vnode_sysref_class = {
69 .proto = SYSREF_PROTO_VNODE,
70 .offset = offsetof(struct vnode, v_sysref),
71 .objsize = sizeof(struct vnode),
73 .flags = SRC_MANAGEDINIT,
77 .terminate = (sysref_terminate_func_t)vnode_terminate
82 * The vnode free list hold inactive vnodes. Aged inactive vnodes
83 * are inserted prior to the mid point, and otherwise inserted
86 static TAILQ_HEAD(freelst, vnode) vnode_free_list;
87 static struct vnode vnode_free_mid;
90 SYSCTL_INT(_debug, OID_AUTO, freevnodes, CTLFLAG_RD,
92 static int wantfreevnodes = 25;
93 SYSCTL_INT(_debug, OID_AUTO, wantfreevnodes, CTLFLAG_RW,
94 &wantfreevnodes, 0, "");
96 static ulong trackvnode;
97 SYSCTL_ULONG(_debug, OID_AUTO, trackvnode, CTLFLAG_RW,
102 * Called from vfsinit()
107 TAILQ_INIT(&vnode_free_list);
108 TAILQ_INSERT_HEAD(&vnode_free_list, &vnode_free_mid, v_freelist);
112 * Inline helper functions. vbusy() and vfree() must be called while in a
115 * Warning: must be callable if the caller holds a read spinlock to something
116 * else, meaning we can't use read spinlocks here.
120 __vbusy(struct vnode *vp)
123 if ((ulong)vp == trackvnode)
124 kprintf("__vbusy %p %08x\n", vp, vp->v_flag);
126 TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
128 vp->v_flag &= ~VFREE;
133 __vfree(struct vnode *vp)
136 if ((ulong)vp == trackvnode) {
137 kprintf("__vfree %p %08x\n", vp, vp->v_flag);
141 if (vp->v_flag & VRECLAIMED)
142 TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
143 else if (vp->v_flag & (VAGE0 | VAGE1))
144 TAILQ_INSERT_BEFORE(&vnode_free_mid, vp, v_freelist);
146 TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
153 __vfreetail(struct vnode *vp)
156 if ((ulong)vp == trackvnode)
157 kprintf("__vfreetail %p %08x\n", vp, vp->v_flag);
159 TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
165 * Return a C boolean if we should put the vnode on the freelist (VFREE),
166 * or leave it / mark it as VCACHED.
168 * This routine is only valid if the vnode is already either VFREE or
169 * VCACHED, or if it can become VFREE or VCACHED via vnode_terminate().
171 static __inline boolean_t
172 vshouldfree(struct vnode *vp)
174 return (vp->v_auxrefs == 0 &&
175 (vp->v_object == NULL || vp->v_object->resident_page_count == 0));
179 * Add a ref to an active vnode. This function should never be called
180 * with an inactive vnode (use vget() instead).
183 vref(struct vnode *vp)
185 KKASSERT(vp->v_sysref.refcnt > 0 &&
186 (vp->v_flag & (VFREE|VINACTIVE)) == 0);
187 sysref_get(&vp->v_sysref);
191 * Release a ref on an active or inactive vnode. The sysref termination
192 * function will be called when the active last active reference is released,
193 * and the vnode is returned to the objcache when the last inactive
194 * reference is released.
197 vrele(struct vnode *vp)
199 sysref_put(&vp->v_sysref);
203 * Add an auxiliary data structure reference to the vnode. Auxiliary
204 * references do not change the state of the vnode or prevent them
205 * from being deactivated, reclaimed, or placed on the free list.
207 * An auxiliary reference DOES prevent the vnode from being destroyed,
208 * allowing you to vx_lock() it, test state, etc.
210 * An auxiliary reference DOES NOT move a vnode out of the VFREE state
211 * once it has entered it.
216 vhold(struct vnode *vp)
218 KKASSERT(vp->v_sysref.refcnt != 0);
219 atomic_add_int(&vp->v_auxrefs, 1);
223 * Remove an auxiliary reference from the vnode.
225 * vdrop needs to check for a VCACHE->VFREE transition to catch cases
226 * where a vnode is held past its reclamation.
229 vdrop(struct vnode *vp)
231 KKASSERT(vp->v_sysref.refcnt != 0 && vp->v_auxrefs > 0);
232 atomic_subtract_int(&vp->v_auxrefs, 1);
233 if ((vp->v_flag & VCACHED) && vshouldfree(vp)) {
234 vp->v_flag &= ~VCACHED;
240 * This function is called when the last active reference on the vnode
241 * is released, typically via vrele(). SYSREF will give the vnode a
242 * negative ref count, indicating that it is undergoing termination or
243 * is being set aside for the cache, and one final sysref_put() is
244 * required to actually return it to the memory subsystem.
246 * However, because vnodes may have auxiliary structural references via
247 * v_auxrefs, we must interlock auxiliary references against termination
248 * via the VX lock mechanism. It is possible for a vnode to be reactivated
249 * while we were blocked on the lock.
252 vnode_terminate(struct vnode *vp)
255 if (sysref_isinactive(&vp->v_sysref)) {
257 * Deactivate the vnode by marking it VFREE or VCACHED.
258 * The vnode can be reactivated from either state until
259 * reclaimed. These states inherit the 'last' sysref on the
262 * NOTE: There may be additional inactive references from
263 * other entities blocking on the VX lock while we hold it,
264 * but this does not prevent us from changing the vnode's
267 * NOTE: The vnode could already be marked inactive. XXX
270 * NOTE: The vnode may be marked inactive with dirty buffers
271 * or dirty pages in its cached VM object still present.
273 if ((vp->v_flag & VINACTIVE) == 0) {
274 vp->v_flag |= VINACTIVE;
277 KKASSERT((vp->v_flag & (VFREE|VCACHED)) == 0);
281 vp->v_flag |= VCACHED; /* inactive but not yet free */
285 * Someone reactivated the vnode while were blocked on the
286 * VX lock. Release the VX lock and release the (now active)
287 * last reference which is no longer last.
295 * Physical vnode constructor / destructor. These are only executed on
296 * the backend of the objcache. They are NOT executed on every vnode
297 * allocation or deallocation.
300 vnode_ctor(void *obj, void *private, int ocflags)
302 struct vnode *vp = obj;
304 lwkt_token_init(&vp->v_token);
305 lockinit(&vp->v_lock, "vnode", 0, 0);
306 ccms_dataspace_init(&vp->v_ccms);
307 TAILQ_INIT(&vp->v_namecache);
308 RB_INIT(&vp->v_rbclean_tree);
309 RB_INIT(&vp->v_rbdirty_tree);
310 RB_INIT(&vp->v_rbhash_tree);
315 vnode_dtor(void *obj, void *private)
317 struct vnode *vp = obj;
319 ccms_dataspace_destroy(&vp->v_ccms);
322 /****************************************************************
323 * VX LOCKING FUNCTIONS *
324 ****************************************************************
326 * These functions lock vnodes for reclamation and deactivation related
327 * activities. The caller must already be holding some sort of reference
332 vx_lock(struct vnode *vp)
334 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
338 vx_lock_nonblock(struct vnode *vp)
340 return(lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT));
344 vx_unlock(struct vnode *vp)
346 lockmgr(&vp->v_lock, LK_RELEASE);
349 /****************************************************************
350 * VNODE ACQUISITION FUNCTIONS *
351 ****************************************************************
353 * These functions must be used when accessing a vnode via an auxiliary
354 * reference such as the namecache or free list, or when you wish to
355 * do a combo ref+lock sequence.
357 * These functions are MANDATORY for any code chain accessing a vnode
358 * whos activation state is not known.
360 * vget()/vput() are used when reactivation is desired.
362 * vx_get() and vx_put() are used when reactivation is not desired.
365 vget(struct vnode *vp, int flags)
370 * A lock type must be passed
372 if ((flags & LK_TYPE_MASK) == 0) {
373 panic("vget() called with no lock specified!");
378 * Reference the structure and then acquire the lock. 0->1
379 * transitions and refs during termination are allowed here so
380 * call sysref directly.
383 sysref_get(&vp->v_sysref);
384 if ((error = vn_lock(vp, flags)) != 0) {
386 * The lock failed, undo and return an error.
388 sysref_put(&vp->v_sysref);
389 } else if (vp->v_flag & VRECLAIMED) {
391 * The node is being reclaimed and cannot be reactivated
392 * any more, undo and return ENOENT.
399 * If the vnode is marked VFREE or VCACHED it needs to be
400 * reactivated, otherwise it had better already be active.
401 * VINACTIVE must also be cleared.
403 * In the VFREE/VCACHED case we have to throw away the
404 * sysref that was earmarking those cases and preventing
405 * the vnode from being destroyed. Our sysref is still held.
407 if (vp->v_flag & VFREE) {
409 sysref_put(&vp->v_sysref);
410 sysref_activate(&vp->v_sysref);
411 } else if (vp->v_flag & VCACHED) {
412 vp->v_flag &= ~VCACHED;
413 sysref_put(&vp->v_sysref);
414 sysref_activate(&vp->v_sysref);
416 KKASSERT(sysref_isactive(&vp->v_sysref));
418 vp->v_flag &= ~VINACTIVE;
425 vput(struct vnode *vp)
432 * XXX The vx_*() locks should use auxrefs, not the main reference counter.
435 vx_get(struct vnode *vp)
437 sysref_get(&vp->v_sysref);
438 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
442 vx_get_nonblock(struct vnode *vp)
446 sysref_get(&vp->v_sysref);
447 error = lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT);
449 sysref_put(&vp->v_sysref);
454 * Relase a VX lock that also held a ref on the vnode.
456 * vx_put needs to check for a VCACHE->VFREE transition to catch the
457 * case where e.g. vnlru issues a vgone*().
460 vx_put(struct vnode *vp)
462 if ((vp->v_flag & VCACHED) && vshouldfree(vp)) {
463 vp->v_flag &= ~VCACHED;
466 lockmgr(&vp->v_lock, LK_RELEASE);
467 sysref_put(&vp->v_sysref);
475 vsetflags(struct vnode *vp, int flags)
483 vclrflags(struct vnode *vp, int flags)
486 vp->v_flag &= ~flags;
491 * Try to reuse a vnode from the free list. NOTE: The returned vnode
492 * is not completely initialized.
501 for (count = 0; count < freevnodes; count++) {
503 * Note that regardless of how we block in this loop,
504 * we only get here if freevnodes != 0 so there
505 * had better be something on the list.
507 * Try to lock the first vnode on the free list.
512 vp = TAILQ_FIRST(&vnode_free_list);
513 if (vp == &vnode_free_mid)
514 vp = TAILQ_NEXT(vp, v_freelist);
515 if (vx_lock_nonblock(vp)) {
516 KKASSERT(vp->v_flag & VFREE);
517 TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
518 TAILQ_INSERT_TAIL(&vnode_free_list,
523 if ((ulong)vp == trackvnode)
524 kprintf("allocfreevnode %p %08x\n", vp, vp->v_flag);
528 * With the vnode locked we can safely remove it
529 * from the free list. We inherit the reference
530 * that was previously associated with the vnode
531 * being on the free list.
533 KKASSERT((vp->v_flag & (VFREE|VINACTIVE)) ==
535 KKASSERT(sysref_isinactive(&vp->v_sysref));
539 * Holding the VX lock on an inactive vnode prevents it
540 * from being reactivated or reused. New namecache
541 * associations can only be made using active vnodes.
543 * Another thread may be blocked on our vnode lock while
544 * holding a namecache lock. We can only reuse this vnode
545 * if we can clear all namecache associations without
548 if ((vp->v_flag & VRECLAIMED) == 0) {
549 if (cache_inval_vp_nonblock(vp)) {
555 /* vnode is still VX locked */
559 * We can reuse the vnode if no primary or auxiliary
560 * references remain other then ours, else put it
561 * back on the free list and keep looking.
563 * Either the free list inherits the last reference
564 * or we fall through and sysref_activate() the last
567 * Since the vnode is in a VRECLAIMED state, no new
568 * namecache associations could have been made.
570 KKASSERT(TAILQ_EMPTY(&vp->v_namecache));
572 !sysref_islastdeactivation(&vp->v_sysref)) {
579 * Return a VX locked vnode suitable for reuse. The caller
580 * inherits the sysref.
588 * Obtain a new vnode from the freelist, allocating more if necessary.
589 * The returned vnode is VX locked & refd.
591 * All new vnodes set the VAGE flags. An open() of the vnode will
592 * decrement the (2-bit) flags. Vnodes which are opened several times
593 * are thus retained in the cache over vnodes which are merely stat()d.
596 allocvnode(int lktimeout, int lkflags)
601 * Try to reuse vnodes if we hit the max. This situation only
602 * occurs in certain large-memory (2G+) situations. We cannot
603 * attempt to directly reclaim vnodes due to nasty recursion
606 while (numvnodes - freevnodes > desiredvnodes)
610 * Try to build up as many vnodes as we can before reallocating
611 * from the free list. A vnode on the free list simply means
612 * that it is inactive with no resident pages. It may or may not
613 * have been reclaimed and could have valuable information associated
614 * with it that we shouldn't throw away unless we really need to.
616 * HAMMER NOTE: Re-establishing a vnode is a fairly expensive
617 * operation for HAMMER but this should benefit UFS as well.
619 if (freevnodes >= wantfreevnodes && numvnodes >= desiredvnodes)
620 vp = allocfreevnode();
624 vp = sysref_alloc(&vnode_sysref_class);
625 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
630 * We are using a managed sysref class, vnode fields are only
631 * zerod on initial allocation from the backing store, not
632 * on reallocation. Thus we have to clear these fields for both
633 * reallocation and reuse.
637 panic("cleaned vnode isn't");
638 if (bio_track_active(&vp->v_track_read) ||
639 bio_track_active(&vp->v_track_write)) {
640 panic("Clean vnode has pending I/O's");
642 if (vp->v_flag & VONWORKLST)
643 panic("Clean vnode still pending on syncer worklist!");
644 if (!RB_EMPTY(&vp->v_rbdirty_tree))
645 panic("Clean vnode still has dirty buffers!");
646 if (!RB_EMPTY(&vp->v_rbclean_tree))
647 panic("Clean vnode still has clean buffers!");
648 if (!RB_EMPTY(&vp->v_rbhash_tree))
649 panic("Clean vnode still on hash tree!");
650 KKASSERT(vp->v_mount == NULL);
652 vp->v_flag = VAGE0 | VAGE1;
659 vp->v_writecount = 0; /* XXX */
662 * lktimeout only applies when LK_TIMELOCK is used, and only
663 * the pageout daemon uses it. The timeout may not be zero
664 * or the pageout daemon can deadlock in low-VM situations.
668 lockreinit(&vp->v_lock, "vnode", lktimeout, lkflags);
669 KKASSERT(TAILQ_FIRST(&vp->v_namecache) == NULL);
670 /* exclusive lock still held */
673 * Note: sysref needs to be activated to convert -0x40000000 to +1.
674 * The -0x40000000 comes from the last ref on reuse, and from
675 * sysref_init() on allocate.
677 sysref_activate(&vp->v_sysref);
678 vp->v_filesize = NOOFFSET;
683 KKASSERT(vp->v_mount == NULL);
689 freesomevnodes(int n)
696 if ((vp = allocfreevnode()) == NULL)