2 * Copyright (c) 2004 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * $DragonFly: src/sys/kern/vfs_lock.c,v 1.28 2007/06/09 19:46:02 dillon Exp $
38 * External virtual filesystem routines
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/malloc.h>
46 #include <sys/mount.h>
48 #include <sys/vnode.h>
50 #include <sys/sysctl.h>
52 #include <machine/limits.h>
55 #include <vm/vm_object.h>
58 #include <sys/thread2.h>
59 #include <sys/sysref2.h>
61 static void vnode_terminate(struct vnode *vp);
62 static boolean_t vnode_ctor(void *obj, void *private, int ocflags);
63 static void vnode_dtor(void *obj, void *private);
65 static MALLOC_DEFINE(M_VNODE, "vnodes", "vnode structures");
66 static struct sysref_class vnode_sysref_class = {
69 .proto = SYSREF_PROTO_VNODE,
70 .offset = offsetof(struct vnode, v_sysref),
71 .objsize = sizeof(struct vnode),
73 .flags = SRC_MANAGEDINIT,
77 .terminate = (sysref_terminate_func_t)vnode_terminate
81 static TAILQ_HEAD(freelst, vnode) vnode_free_list; /* vnode free list */
84 SYSCTL_INT(_debug, OID_AUTO, freevnodes, CTLFLAG_RD,
86 static int wantfreevnodes = 25;
87 SYSCTL_INT(_debug, OID_AUTO, wantfreevnodes, CTLFLAG_RW,
88 &wantfreevnodes, 0, "");
90 SYSCTL_INT(_kern, OID_AUTO, minvnodes, CTLFLAG_RW,
91 &minvnodes, 0, "Minimum number of vnodes");
94 * Called from vfsinit()
99 minvnodes = desiredvnodes / 4;
101 TAILQ_INIT(&vnode_free_list);
105 * Inline helper functions. vbusy() and vfree() must be called while in a
108 * Warning: must be callable if the caller holds a read spinlock to something
109 * else, meaning we can't use read spinlocks here.
113 __vbusy(struct vnode *vp)
115 TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
117 vp->v_flag &= ~(VFREE|VAGE);
122 __vfree(struct vnode *vp)
124 if (vp->v_flag & (VAGE|VRECLAIMED))
125 TAILQ_INSERT_HEAD(&vnode_free_list, vp, v_freelist);
127 TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
135 __vfreetail(struct vnode *vp)
137 TAILQ_INSERT_TAIL(&vnode_free_list, vp, v_freelist);
143 * Return a C boolean if we should put the vnode on the freelist (VFREE),
144 * or leave it / mark it as VCACHED.
146 * This routine is only valid if the vnode is already either VFREE or
147 * VCACHED, or if it can become VFREE or VCACHED via vnode_terminate().
149 static __inline boolean_t
150 vshouldfree(struct vnode *vp)
152 return (vp->v_auxrefs == 0 &&
153 (vp->v_object == NULL || vp->v_object->resident_page_count == 0));
157 * Add a ref to an active vnode. This function should never be called
158 * with an inactive vnode (use vget() instead).
161 vref(struct vnode *vp)
163 KKASSERT(vp->v_sysref.refcnt > 0 &&
164 (vp->v_flag & (VFREE|VINACTIVE)) == 0);
165 sysref_get(&vp->v_sysref);
169 * Release a ref on an active or inactive vnode. The sysref termination
170 * function will be called when the active last active reference is released,
171 * and the vnode is returned to the objcache when the last inactive
172 * reference is released.
175 vrele(struct vnode *vp)
177 sysref_put(&vp->v_sysref);
181 * Add an auxiliary data structure reference to the vnode. Auxiliary
182 * references do not change the state of the vnode or prevent them
183 * from being deactivated, reclaimed, or placed on the free list.
185 * An auxiliary reference DOES prevent the vnode from being destroyed,
186 * allowing you to vx_lock() it, test state, etc.
188 * An auxiliary reference DOES NOT move a vnode out of the VFREE state
189 * once it has entered it.
192 vhold(struct vnode *vp)
194 KKASSERT(vp->v_sysref.refcnt != 0);
195 atomic_add_int(&vp->v_auxrefs, 1);
199 * Remove an auxiliary reference from the vnode.
201 * vdrop needs to check for a VCACHE->VFREE transition to catch cases
202 * where a vnode is held past its reclamation.
205 vdrop(struct vnode *vp)
207 KKASSERT(vp->v_sysref.refcnt != 0 && vp->v_auxrefs > 0);
208 atomic_subtract_int(&vp->v_auxrefs, 1);
209 if ((vp->v_flag & VCACHED) && vshouldfree(vp)) {
211 vp->v_flag &= ~VCACHED;
217 * This function is called when the last active reference on the vnode
218 * is released, typically via vrele(). SYSREF will give the vnode a
219 * negative ref count, indicating that it is undergoing termination or
220 * is being set aside for the cache, and one final sysref_put() is
221 * required to actually return it to the memory subsystem.
223 * However, because vnodes may have auxiliary structural references via
224 * v_auxrefs, we must interlock auxiliary references against termination
225 * via the VX lock mechanism. It is possible for a vnode to be reactivated
226 * while we were blocked on the lock.
229 vnode_terminate(struct vnode *vp)
232 if (sysref_isinactive(&vp->v_sysref)) {
234 * Deactivate the vnode by marking it VFREE or VCACHED.
235 * The vnode can be reactivated from either state until
236 * reclaimed. These states inherit the 'last' sysref on the
239 * NOTE: There may be additional inactive references from
240 * other entities blocking on the VX lock while we hold it,
241 * but this does not prevent us from changing the vnode's
244 * NOTE: The vnode could already be marked inactive. XXX
247 * NOTE: The vnode may be marked inactive with dirty buffers
248 * or dirty pages in its cached VM object still present.
250 if ((vp->v_flag & VINACTIVE) == 0) {
251 vp->v_flag |= VINACTIVE;
254 KKASSERT((vp->v_flag & (VFREE|VCACHED)) == 0);
258 vp->v_flag |= VCACHED;
262 * Someone reactivated the vnode while were blocked on the
263 * VX lock. Release the VX lock and release the (now active)
264 * last reference which is no longer last.
272 * Physical vnode constructor / destructor. These are only executed on
273 * the backend of the objcache. They are NOT executed on every vnode
274 * allocation or deallocation.
277 vnode_ctor(void *obj, void *private, int ocflags)
279 struct vnode *vp = obj;
281 lwkt_token_init(&vp->v_pollinfo.vpi_token);
282 lockinit(&vp->v_lock, "vnode", 0, 0);
283 ccms_dataspace_init(&vp->v_ccms);
284 TAILQ_INIT(&vp->v_namecache);
285 RB_INIT(&vp->v_rbclean_tree);
286 RB_INIT(&vp->v_rbdirty_tree);
287 RB_INIT(&vp->v_rbhash_tree);
292 vnode_dtor(void *obj, void *private)
294 struct vnode *vp = obj;
296 ccms_dataspace_destroy(&vp->v_ccms);
299 /****************************************************************
300 * VX LOCKING FUNCTIONS *
301 ****************************************************************
303 * These functions lock vnodes for reclamation and deactivation related
304 * activities. The caller must already be holding some sort of reference
309 vx_lock(struct vnode *vp)
311 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
315 vx_lock_nonblock(struct vnode *vp)
317 return(lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT));
321 vx_unlock(struct vnode *vp)
323 lockmgr(&vp->v_lock, LK_RELEASE);
326 /****************************************************************
327 * VNODE ACQUISITION FUNCTIONS *
328 ****************************************************************
330 * These functions must be used when accessing a vnode via an auxiliary
331 * reference such as the namecache or free list, or when you wish to
332 * do a combo ref+lock sequence.
334 * These functions are MANDATORY for any code chain accessing a vnode
335 * whos activation state is not known.
337 * vget()/vput() are used when reactivation is desired.
339 * vx_get() and vx_put() are used when reactivation is not desired.
342 vget(struct vnode *vp, int flags)
347 * A lock type must be passed
349 if ((flags & LK_TYPE_MASK) == 0) {
350 panic("vget() called with no lock specified!");
355 * Reference the structure and then acquire the lock. 0->1
356 * transitions and refs during termination are allowed here so
357 * call sysref directly.
360 sysref_get(&vp->v_sysref);
361 if ((error = vn_lock(vp, flags)) != 0) {
363 * The lock failed, undo and return an error.
365 sysref_put(&vp->v_sysref);
366 } else if (vp->v_flag & VRECLAIMED) {
368 * The node is being reclaimed and cannot be reactivated
369 * any more, undo and return ENOENT.
376 * If the vnode is marked VFREE or VCACHED it needs to be
377 * reactivated, otherwise it had better already be active.
378 * VINACTIVE must also be cleared.
380 * In the VFREE/VCACHED case we have to throw away the
381 * sysref that was earmarking those cases and preventing
382 * the vnode from being destroyed. Our sysref is still held.
384 if (vp->v_flag & VFREE) {
386 sysref_put(&vp->v_sysref);
387 sysref_activate(&vp->v_sysref);
388 } else if (vp->v_flag & VCACHED) {
389 vp->v_flag &= ~VCACHED;
390 sysref_put(&vp->v_sysref);
391 sysref_activate(&vp->v_sysref);
393 KKASSERT(sysref_isactive(&vp->v_sysref));
395 vp->v_flag &= ~VINACTIVE;
402 vput(struct vnode *vp)
409 * XXX The vx_*() locks should use auxrefs, not the main reference counter.
412 vx_get(struct vnode *vp)
414 sysref_get(&vp->v_sysref);
415 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
419 vx_get_nonblock(struct vnode *vp)
423 sysref_get(&vp->v_sysref);
424 error = lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT);
426 sysref_put(&vp->v_sysref);
431 * Relase a VX lock that also held a ref on the vnode.
433 * vx_put needs to check for a VCACHE->VFREE transition to catch the
434 * case where e.g. vnlru issues a vgone*().
437 vx_put(struct vnode *vp)
439 if ((vp->v_flag & VCACHED) && vshouldfree(vp)) {
441 vp->v_flag &= ~VCACHED;
444 lockmgr(&vp->v_lock, LK_RELEASE);
445 sysref_put(&vp->v_sysref);
453 vsetflags(struct vnode *vp, int flags)
461 vclrflags(struct vnode *vp, int flags)
464 vp->v_flag &= ~flags;
469 * Try to reuse a vnode from the free list. NOTE: The returned vnode
470 * is not completely initialized.
479 for (count = 0; count < freevnodes; count++) {
481 * Note that regardless of how we block in this loop,
482 * we only get here if freevnodes != 0 so there
483 * had better be something on the list.
485 * Try to lock the first vnode on the free list.
490 vp = TAILQ_FIRST(&vnode_free_list);
491 if (vx_lock_nonblock(vp)) {
492 KKASSERT(vp->v_flag & VFREE);
493 TAILQ_REMOVE(&vnode_free_list, vp, v_freelist);
494 TAILQ_INSERT_TAIL(&vnode_free_list,
500 * With the vnode locked we can safely remove it
501 * from the free list. We inherit the reference
502 * that was previously associated with the vnode
503 * being on the free list.
505 KKASSERT((vp->v_flag & (VFREE|VINACTIVE)) ==
507 KKASSERT(sysref_isinactive(&vp->v_sysref));
511 * Holding the VX lock on an inactive vnode prevents it
512 * from being reactivated or reused. New namecache
513 * associations can only be made using active vnodes.
515 * Another thread may be blocked on our vnode lock while
516 * holding a namecache lock. We can only reuse this vnode
517 * if we can clear all namecache associations without
520 if ((vp->v_flag & VRECLAIMED) == 0) {
521 if (cache_inval_vp_nonblock(vp)) {
527 /* vnode is still VX locked */
531 * We can reuse the vnode if no primary or auxiliary
532 * references remain other then ours, else put it
533 * back on the free list and keep looking.
535 * Either the free list inherits the last reference
536 * or we fall through and sysref_activate() the last
539 * Since the vnode is in a VRECLAIMED state, no new
540 * namecache associations could have been made.
542 KKASSERT(TAILQ_EMPTY(&vp->v_namecache));
544 !sysref_islastdeactivation(&vp->v_sysref)) {
551 * Return a VX locked vnode suitable for reuse. The caller
552 * inherits the sysref.
560 * Obtain a new vnode from the freelist, allocating more if necessary.
561 * The returned vnode is VX locked & refd.
564 allocvnode(int lktimeout, int lkflags)
569 * Try to reuse vnodes if we hit the max. This situation only
570 * occurs in certain large-memory (2G+) situations. We cannot
571 * attempt to directly reclaim vnodes due to nasty recursion
574 while (numvnodes - freevnodes > desiredvnodes)
578 * Attempt to reuse a vnode already on the free list, allocating
579 * a new vnode if we can't find one or if we have not reached a
580 * good minimum for good LRU performance.
582 if (freevnodes >= wantfreevnodes && numvnodes >= minvnodes)
583 vp = allocfreevnode();
587 vp = sysref_alloc(&vnode_sysref_class);
588 lockmgr(&vp->v_lock, LK_EXCLUSIVE);
593 * We are using a managed sysref class, vnode fields are only
594 * zerod on initial allocation from the backing store, not
595 * on reallocation. Thus we have to clear these fields for both
596 * reallocation and reuse.
600 panic("cleaned vnode isn't");
601 if (vp->v_track_read.bk_active + vp->v_track_write.bk_active)
602 panic("Clean vnode has pending I/O's");
603 if (vp->v_flag & VONWORKLST)
604 panic("Clean vnode still pending on syncer worklist!");
605 if (!RB_EMPTY(&vp->v_rbdirty_tree))
606 panic("Clean vnode still has dirty buffers!");
607 if (!RB_EMPTY(&vp->v_rbclean_tree))
608 panic("Clean vnode still has clean buffers!");
609 if (!RB_EMPTY(&vp->v_rbhash_tree))
610 panic("Clean vnode still on hash tree!");
611 KKASSERT(vp->v_mount == NULL);
620 vp->v_writecount = 0; /* XXX */
621 lockreinit(&vp->v_lock, "vnode", lktimeout, lkflags);
622 KKASSERT(TAILQ_FIRST(&vp->v_namecache) == NULL);
623 /* exclusive lock still held */
626 * Note: sysref needs to be activated to convert -0x40000000 to +1.
627 * The -0x40000000 comes from the last ref on reuse, and from
628 * sysref_init() on allocate.
630 sysref_activate(&vp->v_sysref);
631 vp->v_filesize = NOOFFSET;
636 KKASSERT(vp->v_mount == NULL);
642 freesomevnodes(int n)
649 if ((vp = allocfreevnode()) == NULL)