4 * (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
7 #include <linux/hash.h>
8 #include <linux/slab.h>
9 #include <linux/file.h>
10 #include <linux/pagemap.h>
11 #include <linux/sched.h>
12 #include <linux/list_lru.h>
13 #include <linux/fsnotify_backend.h>
14 #include <linux/fsnotify.h>
15 #include <linux/seq_file.h>
16 #include <linux/rhashtable.h>
22 #include "filecache.h"
25 #define NFSD_LAUNDRETTE_DELAY (2 * HZ)
27 #define NFSD_FILE_CACHE_UP (0)
29 /* We only care about NFSD_MAY_READ/WRITE for this cache */
30 #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE)
32 static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
33 static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
34 static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
35 static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
36 static DEFINE_PER_CPU(unsigned long, nfsd_file_pages_flushed);
37 static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
39 struct nfsd_fcache_disposal {
40 struct work_struct work;
42 struct list_head freeme;
45 static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
47 static struct kmem_cache *nfsd_file_slab;
48 static struct kmem_cache *nfsd_file_mark_slab;
49 static struct list_lru nfsd_file_lru;
50 static unsigned long nfsd_file_flags;
51 static struct fsnotify_group *nfsd_file_fsnotify_group;
52 static struct delayed_work nfsd_filecache_laundrette;
53 static struct rhashtable nfsd_file_rhash_tbl
54 ____cacheline_aligned_in_smp;
56 enum nfsd_file_lookup_type {
61 struct nfsd_file_lookup_key {
64 const struct cred *cred;
66 enum nfsd_file_lookup_type type;
70 * The returned hash value is based solely on the address of an in-code
71 * inode, a pointer to a slab-allocated object. The entropy in such a
72 * pointer is concentrated in its middle bits.
74 static u32 nfsd_file_inode_hash(const struct inode *inode, u32 seed)
76 unsigned long ptr = (unsigned long)inode;
79 k = ptr >> L1_CACHE_SHIFT;
81 return jhash2(&k, 1, seed);
85 * nfsd_file_key_hashfn - Compute the hash value of a lookup key
86 * @data: key on which to compute the hash value
87 * @len: rhash table's key_len parameter (unused)
88 * @seed: rhash table's random seed of the day
91 * Computed 32-bit hash value
93 static u32 nfsd_file_key_hashfn(const void *data, u32 len, u32 seed)
95 const struct nfsd_file_lookup_key *key = data;
97 return nfsd_file_inode_hash(key->inode, seed);
101 * nfsd_file_obj_hashfn - Compute the hash value of an nfsd_file
102 * @data: object on which to compute the hash value
103 * @len: rhash table's key_len parameter (unused)
104 * @seed: rhash table's random seed of the day
107 * Computed 32-bit hash value
109 static u32 nfsd_file_obj_hashfn(const void *data, u32 len, u32 seed)
111 const struct nfsd_file *nf = data;
113 return nfsd_file_inode_hash(nf->nf_inode, seed);
117 nfsd_match_cred(const struct cred *c1, const struct cred *c2)
121 if (!uid_eq(c1->fsuid, c2->fsuid))
123 if (!gid_eq(c1->fsgid, c2->fsgid))
125 if (c1->group_info == NULL || c2->group_info == NULL)
126 return c1->group_info == c2->group_info;
127 if (c1->group_info->ngroups != c2->group_info->ngroups)
129 for (i = 0; i < c1->group_info->ngroups; i++) {
130 if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
137 * nfsd_file_obj_cmpfn - Match a cache item against search criteria
138 * @arg: search criteria
139 * @ptr: cache item to check
142 * %0 - Item matches search criteria
143 * %1 - Item does not match search criteria
145 static int nfsd_file_obj_cmpfn(struct rhashtable_compare_arg *arg,
148 const struct nfsd_file_lookup_key *key = arg->key;
149 const struct nfsd_file *nf = ptr;
152 case NFSD_FILE_KEY_INODE:
153 if (nf->nf_inode != key->inode)
156 case NFSD_FILE_KEY_FULL:
157 if (nf->nf_inode != key->inode)
159 if (nf->nf_may != key->need)
161 if (nf->nf_net != key->net)
163 if (!nfsd_match_cred(nf->nf_cred, key->cred))
165 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
172 static const struct rhashtable_params nfsd_file_rhash_params = {
173 .key_len = sizeof_field(struct nfsd_file, nf_inode),
174 .key_offset = offsetof(struct nfsd_file, nf_inode),
175 .head_offset = offsetof(struct nfsd_file, nf_rhash),
176 .hashfn = nfsd_file_key_hashfn,
177 .obj_hashfn = nfsd_file_obj_hashfn,
178 .obj_cmpfn = nfsd_file_obj_cmpfn,
179 /* Reduce resizing churn on light workloads */
180 .min_size = 512, /* buckets */
181 .automatic_shrinking = true,
185 nfsd_file_schedule_laundrette(void)
187 if ((atomic_read(&nfsd_file_rhash_tbl.nelems) == 0) ||
188 test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
191 queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
192 NFSD_LAUNDRETTE_DELAY);
196 nfsd_file_slab_free(struct rcu_head *rcu)
198 struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
200 put_cred(nf->nf_cred);
201 kmem_cache_free(nfsd_file_slab, nf);
205 nfsd_file_mark_free(struct fsnotify_mark *mark)
207 struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
210 kmem_cache_free(nfsd_file_mark_slab, nfm);
213 static struct nfsd_file_mark *
214 nfsd_file_mark_get(struct nfsd_file_mark *nfm)
216 if (!refcount_inc_not_zero(&nfm->nfm_ref))
222 nfsd_file_mark_put(struct nfsd_file_mark *nfm)
224 if (refcount_dec_and_test(&nfm->nfm_ref)) {
225 fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
226 fsnotify_put_mark(&nfm->nfm_mark);
230 static struct nfsd_file_mark *
231 nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
234 struct fsnotify_mark *mark;
235 struct nfsd_file_mark *nfm = NULL, *new;
238 fsnotify_group_lock(nfsd_file_fsnotify_group);
239 mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
240 nfsd_file_fsnotify_group);
242 nfm = nfsd_file_mark_get(container_of(mark,
243 struct nfsd_file_mark,
245 fsnotify_group_unlock(nfsd_file_fsnotify_group);
247 fsnotify_put_mark(mark);
250 /* Avoid soft lockup race with nfsd_file_mark_put() */
251 fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
252 fsnotify_put_mark(mark);
254 fsnotify_group_unlock(nfsd_file_fsnotify_group);
257 /* allocate a new nfm */
258 new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
261 fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
262 new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
263 refcount_set(&new->nfm_ref, 1);
265 err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
268 * If the add was successful, then return the object.
269 * Otherwise, we need to put the reference we hold on the
270 * nfm_mark. The fsnotify code will take a reference and put
271 * it on failure, so we can't just free it directly. It's also
272 * not safe to call fsnotify_destroy_mark on it as the
273 * mark->group will be NULL. Thus, we can't let the nfm_ref
274 * counter drive the destruction at this point.
279 fsnotify_put_mark(&new->nfm_mark);
280 } while (unlikely(err == -EEXIST));
285 static struct nfsd_file *
286 nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
288 struct nfsd_file *nf;
290 nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
292 INIT_LIST_HEAD(&nf->nf_lru);
293 nf->nf_birthtime = ktime_get();
295 nf->nf_cred = get_current_cred();
296 nf->nf_net = key->net;
298 __set_bit(NFSD_FILE_HASHED, &nf->nf_flags);
299 __set_bit(NFSD_FILE_PENDING, &nf->nf_flags);
300 nf->nf_inode = key->inode;
301 /* nf_ref is pre-incremented for hash table */
302 refcount_set(&nf->nf_ref, 2);
303 nf->nf_may = key->need;
310 nfsd_file_free(struct nfsd_file *nf)
312 s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
315 this_cpu_inc(nfsd_file_releases);
316 this_cpu_add(nfsd_file_total_age, age);
318 trace_nfsd_file_put_final(nf);
320 nfsd_file_mark_put(nf->nf_mark);
322 get_file(nf->nf_file);
323 filp_close(nf->nf_file, NULL);
329 * If this item is still linked via nf_lru, that's a bug.
330 * WARN and leak it to preserve system stability.
332 if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
335 call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
340 nfsd_file_check_writeback(struct nfsd_file *nf)
342 struct file *file = nf->nf_file;
343 struct address_space *mapping;
345 if (!file || !(file->f_mode & FMODE_WRITE))
347 mapping = file->f_mapping;
348 return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
349 mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
353 nfsd_file_check_write_error(struct nfsd_file *nf)
355 struct file *file = nf->nf_file;
357 if (!file || !(file->f_mode & FMODE_WRITE))
359 return filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err));
363 nfsd_file_flush(struct nfsd_file *nf)
365 struct file *file = nf->nf_file;
367 if (!file || !(file->f_mode & FMODE_WRITE))
369 this_cpu_add(nfsd_file_pages_flushed, file->f_mapping->nrpages);
370 if (vfs_fsync(file, 1) != 0)
371 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
374 static void nfsd_file_lru_add(struct nfsd_file *nf)
376 set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
377 if (list_lru_add(&nfsd_file_lru, &nf->nf_lru))
378 trace_nfsd_file_lru_add(nf);
381 static void nfsd_file_lru_remove(struct nfsd_file *nf)
383 if (list_lru_del(&nfsd_file_lru, &nf->nf_lru))
384 trace_nfsd_file_lru_del(nf);
388 nfsd_file_hash_remove(struct nfsd_file *nf)
390 trace_nfsd_file_unhash(nf);
392 if (nfsd_file_check_write_error(nf))
393 nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
394 rhashtable_remove_fast(&nfsd_file_rhash_tbl, &nf->nf_rhash,
395 nfsd_file_rhash_params);
399 nfsd_file_unhash(struct nfsd_file *nf)
401 if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
402 nfsd_file_hash_remove(nf);
409 * Return true if the file was unhashed.
412 nfsd_file_unhash_and_dispose(struct nfsd_file *nf, struct list_head *dispose)
414 trace_nfsd_file_unhash_and_dispose(nf);
415 if (!nfsd_file_unhash(nf))
417 /* keep final reference for nfsd_file_lru_dispose */
418 if (refcount_dec_not_one(&nf->nf_ref))
421 nfsd_file_lru_remove(nf);
422 list_add(&nf->nf_lru, dispose);
427 nfsd_file_put_noref(struct nfsd_file *nf)
429 trace_nfsd_file_put(nf);
431 if (refcount_dec_and_test(&nf->nf_ref)) {
432 WARN_ON(test_bit(NFSD_FILE_HASHED, &nf->nf_flags));
433 nfsd_file_lru_remove(nf);
439 nfsd_file_put(struct nfsd_file *nf)
443 nfsd_file_lru_add(nf);
444 if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0) {
446 nfsd_file_put_noref(nf);
447 } else if (nf->nf_file) {
448 nfsd_file_put_noref(nf);
449 nfsd_file_schedule_laundrette();
451 nfsd_file_put_noref(nf);
455 * nfsd_file_close - Close an nfsd_file
456 * @nf: nfsd_file to close
458 * If this is the final reference for @nf, free it immediately.
459 * This reflects an on-the-wire CLOSE or DELEGRETURN into the
460 * VFS and exported filesystem.
462 void nfsd_file_close(struct nfsd_file *nf)
465 if (refcount_dec_if_one(&nf->nf_ref)) {
466 nfsd_file_unhash(nf);
467 nfsd_file_lru_remove(nf);
473 nfsd_file_get(struct nfsd_file *nf)
475 if (likely(refcount_inc_not_zero(&nf->nf_ref)))
481 nfsd_file_dispose_list(struct list_head *dispose)
483 struct nfsd_file *nf;
485 while(!list_empty(dispose)) {
486 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
487 list_del_init(&nf->nf_lru);
489 nfsd_file_put_noref(nf);
494 nfsd_file_dispose_list_sync(struct list_head *dispose)
497 struct nfsd_file *nf;
499 while(!list_empty(dispose)) {
500 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
501 list_del_init(&nf->nf_lru);
503 if (!refcount_dec_and_test(&nf->nf_ref))
505 if (nfsd_file_free(nf))
509 flush_delayed_fput();
513 nfsd_file_list_remove_disposal(struct list_head *dst,
514 struct nfsd_fcache_disposal *l)
517 list_splice_init(&l->freeme, dst);
518 spin_unlock(&l->lock);
522 nfsd_file_list_add_disposal(struct list_head *files, struct net *net)
524 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
525 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
528 list_splice_tail_init(files, &l->freeme);
529 spin_unlock(&l->lock);
530 queue_work(nfsd_filecache_wq, &l->work);
534 nfsd_file_list_add_pernet(struct list_head *dst, struct list_head *src,
537 struct nfsd_file *nf, *tmp;
539 list_for_each_entry_safe(nf, tmp, src, nf_lru) {
540 if (nf->nf_net == net)
541 list_move_tail(&nf->nf_lru, dst);
546 nfsd_file_dispose_list_delayed(struct list_head *dispose)
549 struct nfsd_file *nf;
551 while(!list_empty(dispose)) {
552 nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
553 nfsd_file_list_add_pernet(&list, dispose, nf->nf_net);
554 nfsd_file_list_add_disposal(&list, nf->nf_net);
559 * nfsd_file_lru_cb - Examine an entry on the LRU list
560 * @item: LRU entry to examine
561 * @lru: controlling LRU
562 * @lock: LRU list lock (unused)
565 * Note this can deadlock with nfsd_file_cache_purge.
568 * %LRU_REMOVED: @item was removed from the LRU
569 * %LRU_ROTATE: @item is to be moved to the LRU tail
570 * %LRU_SKIP: @item cannot be evicted
572 static enum lru_status
573 nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
574 spinlock_t *lock, void *arg)
578 struct list_head *head = arg;
579 struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
582 * Do a lockless refcount check. The hashtable holds one reference, so
583 * we look to see if anything else has a reference, or if any have
584 * been put since the shrinker last ran. Those don't get unhashed and
587 * Note that in the put path, we set the flag and then decrement the
588 * counter. Here we check the counter and then test and clear the flag.
589 * That order is deliberate to ensure that we can do this locklessly.
591 if (refcount_read(&nf->nf_ref) > 1) {
592 list_lru_isolate(lru, &nf->nf_lru);
593 trace_nfsd_file_gc_in_use(nf);
598 * Don't throw out files that are still undergoing I/O or
599 * that have uncleared errors pending.
601 if (nfsd_file_check_writeback(nf)) {
602 trace_nfsd_file_gc_writeback(nf);
606 if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
607 trace_nfsd_file_gc_referenced(nf);
611 if (!test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
612 trace_nfsd_file_gc_hashed(nf);
616 list_lru_isolate_move(lru, &nf->nf_lru, head);
617 this_cpu_inc(nfsd_file_evictions);
618 trace_nfsd_file_gc_disposed(nf);
623 * Unhash items on @dispose immediately, then queue them on the
624 * disposal workqueue to finish releasing them in the background.
626 * cel: Note that between the time list_lru_shrink_walk runs and
627 * now, these items are in the hash table but marked unhashed.
628 * Why release these outside of lru_cb ? There's no lock ordering
629 * problem since lru_cb currently takes no lock.
631 static void nfsd_file_gc_dispose_list(struct list_head *dispose)
633 struct nfsd_file *nf;
635 list_for_each_entry(nf, dispose, nf_lru)
636 nfsd_file_hash_remove(nf);
637 nfsd_file_dispose_list_delayed(dispose);
646 ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
647 &dispose, list_lru_count(&nfsd_file_lru));
648 trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
649 nfsd_file_gc_dispose_list(&dispose);
653 nfsd_file_gc_worker(struct work_struct *work)
656 nfsd_file_schedule_laundrette();
660 nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
662 return list_lru_count(&nfsd_file_lru);
666 nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
671 ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
672 nfsd_file_lru_cb, &dispose);
673 trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
674 nfsd_file_gc_dispose_list(&dispose);
678 static struct shrinker nfsd_file_shrinker = {
679 .scan_objects = nfsd_file_lru_scan,
680 .count_objects = nfsd_file_lru_count,
685 * Find all cache items across all net namespaces that match @inode and
686 * move them to @dispose. The lookup is atomic wrt nfsd_file_acquire().
689 __nfsd_file_close_inode(struct inode *inode, struct list_head *dispose)
691 struct nfsd_file_lookup_key key = {
692 .type = NFSD_FILE_KEY_INODE,
695 unsigned int count = 0;
696 struct nfsd_file *nf;
700 nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
701 nfsd_file_rhash_params);
704 nfsd_file_unhash_and_dispose(nf, dispose);
712 * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
713 * @inode: inode of the file to attempt to remove
715 * Unhash and put, then flush and fput all cache items associated with @inode.
718 nfsd_file_close_inode_sync(struct inode *inode)
723 count = __nfsd_file_close_inode(inode, &dispose);
724 trace_nfsd_file_close_inode_sync(inode, count);
725 nfsd_file_dispose_list_sync(&dispose);
729 * nfsd_file_close_inode - attempt a delayed close of a nfsd_file
730 * @inode: inode of the file to attempt to remove
732 * Unhash and put all cache item associated with @inode.
735 nfsd_file_close_inode(struct inode *inode)
740 count = __nfsd_file_close_inode(inode, &dispose);
741 trace_nfsd_file_close_inode(inode, count);
742 nfsd_file_dispose_list_delayed(&dispose);
746 * nfsd_file_delayed_close - close unused nfsd_files
749 * Walk the LRU list and close any entries that have not been used since
752 * Note this can deadlock with nfsd_file_cache_purge.
755 nfsd_file_delayed_close(struct work_struct *work)
758 struct nfsd_fcache_disposal *l = container_of(work,
759 struct nfsd_fcache_disposal, work);
761 nfsd_file_list_remove_disposal(&head, l);
762 nfsd_file_dispose_list(&head);
766 nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
769 struct file_lock *fl = data;
771 /* Only close files for F_SETLEASE leases */
772 if (fl->fl_flags & FL_LEASE)
773 nfsd_file_close_inode_sync(file_inode(fl->fl_file));
777 static struct notifier_block nfsd_file_lease_notifier = {
778 .notifier_call = nfsd_file_lease_notifier_call,
782 nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
783 struct inode *inode, struct inode *dir,
784 const struct qstr *name, u32 cookie)
786 if (WARN_ON_ONCE(!inode))
789 trace_nfsd_file_fsnotify_handle_event(inode, mask);
791 /* Should be no marks on non-regular files */
792 if (!S_ISREG(inode->i_mode)) {
797 /* don't close files if this was not the last link */
798 if (mask & FS_ATTRIB) {
803 nfsd_file_close_inode(inode);
808 static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
809 .handle_inode_event = nfsd_file_fsnotify_handle_event,
810 .free_mark = nfsd_file_mark_free,
814 nfsd_file_cache_init(void)
818 lockdep_assert_held(&nfsd_mutex);
819 if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
822 ret = rhashtable_init(&nfsd_file_rhash_tbl, &nfsd_file_rhash_params);
827 nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
828 if (!nfsd_filecache_wq)
831 nfsd_file_slab = kmem_cache_create("nfsd_file",
832 sizeof(struct nfsd_file), 0, 0, NULL);
833 if (!nfsd_file_slab) {
834 pr_err("nfsd: unable to create nfsd_file_slab\n");
838 nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
839 sizeof(struct nfsd_file_mark), 0, 0, NULL);
840 if (!nfsd_file_mark_slab) {
841 pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
846 ret = list_lru_init(&nfsd_file_lru);
848 pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
852 ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
854 pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
858 ret = lease_register_notifier(&nfsd_file_lease_notifier);
860 pr_err("nfsd: unable to register lease notifier: %d\n", ret);
864 nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
865 FSNOTIFY_GROUP_NOFS);
866 if (IS_ERR(nfsd_file_fsnotify_group)) {
867 pr_err("nfsd: unable to create fsnotify group: %ld\n",
868 PTR_ERR(nfsd_file_fsnotify_group));
869 ret = PTR_ERR(nfsd_file_fsnotify_group);
870 nfsd_file_fsnotify_group = NULL;
874 INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
878 lease_unregister_notifier(&nfsd_file_lease_notifier);
880 unregister_shrinker(&nfsd_file_shrinker);
882 list_lru_destroy(&nfsd_file_lru);
884 kmem_cache_destroy(nfsd_file_slab);
885 nfsd_file_slab = NULL;
886 kmem_cache_destroy(nfsd_file_mark_slab);
887 nfsd_file_mark_slab = NULL;
888 destroy_workqueue(nfsd_filecache_wq);
889 nfsd_filecache_wq = NULL;
890 rhashtable_destroy(&nfsd_file_rhash_tbl);
895 * Note this can deadlock with nfsd_file_lru_cb.
898 __nfsd_file_cache_purge(struct net *net)
900 struct rhashtable_iter iter;
901 struct nfsd_file *nf;
905 rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
907 rhashtable_walk_start(&iter);
909 nf = rhashtable_walk_next(&iter);
910 while (!IS_ERR_OR_NULL(nf)) {
911 if (net && nf->nf_net != net)
913 del = nfsd_file_unhash_and_dispose(nf, &dispose);
916 * Deadlock detected! Something marked this entry as
917 * unhased, but hasn't removed it from the hash list.
921 nf = rhashtable_walk_next(&iter);
924 rhashtable_walk_stop(&iter);
925 } while (nf == ERR_PTR(-EAGAIN));
926 rhashtable_walk_exit(&iter);
928 nfsd_file_dispose_list(&dispose);
931 static struct nfsd_fcache_disposal *
932 nfsd_alloc_fcache_disposal(void)
934 struct nfsd_fcache_disposal *l;
936 l = kmalloc(sizeof(*l), GFP_KERNEL);
939 INIT_WORK(&l->work, nfsd_file_delayed_close);
940 spin_lock_init(&l->lock);
941 INIT_LIST_HEAD(&l->freeme);
946 nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
948 cancel_work_sync(&l->work);
949 nfsd_file_dispose_list(&l->freeme);
954 nfsd_free_fcache_disposal_net(struct net *net)
956 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
957 struct nfsd_fcache_disposal *l = nn->fcache_disposal;
959 nfsd_free_fcache_disposal(l);
963 nfsd_file_cache_start_net(struct net *net)
965 struct nfsd_net *nn = net_generic(net, nfsd_net_id);
967 nn->fcache_disposal = nfsd_alloc_fcache_disposal();
968 return nn->fcache_disposal ? 0 : -ENOMEM;
972 * nfsd_file_cache_purge - Remove all cache items associated with @net
973 * @net: target net namespace
977 nfsd_file_cache_purge(struct net *net)
979 lockdep_assert_held(&nfsd_mutex);
980 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
981 __nfsd_file_cache_purge(net);
985 nfsd_file_cache_shutdown_net(struct net *net)
987 nfsd_file_cache_purge(net);
988 nfsd_free_fcache_disposal_net(net);
992 nfsd_file_cache_shutdown(void)
996 lockdep_assert_held(&nfsd_mutex);
997 if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
1000 lease_unregister_notifier(&nfsd_file_lease_notifier);
1001 unregister_shrinker(&nfsd_file_shrinker);
1003 * make sure all callers of nfsd_file_lru_cb are done before
1004 * calling nfsd_file_cache_purge
1006 cancel_delayed_work_sync(&nfsd_filecache_laundrette);
1007 __nfsd_file_cache_purge(NULL);
1008 list_lru_destroy(&nfsd_file_lru);
1010 fsnotify_put_group(nfsd_file_fsnotify_group);
1011 nfsd_file_fsnotify_group = NULL;
1012 kmem_cache_destroy(nfsd_file_slab);
1013 nfsd_file_slab = NULL;
1014 fsnotify_wait_marks_destroyed();
1015 kmem_cache_destroy(nfsd_file_mark_slab);
1016 nfsd_file_mark_slab = NULL;
1017 destroy_workqueue(nfsd_filecache_wq);
1018 nfsd_filecache_wq = NULL;
1019 rhashtable_destroy(&nfsd_file_rhash_tbl);
1021 for_each_possible_cpu(i) {
1022 per_cpu(nfsd_file_cache_hits, i) = 0;
1023 per_cpu(nfsd_file_acquisitions, i) = 0;
1024 per_cpu(nfsd_file_releases, i) = 0;
1025 per_cpu(nfsd_file_total_age, i) = 0;
1026 per_cpu(nfsd_file_pages_flushed, i) = 0;
1027 per_cpu(nfsd_file_evictions, i) = 0;
1032 * nfsd_file_is_cached - are there any cached open files for this inode?
1033 * @inode: inode to check
1035 * The lookup matches inodes in all net namespaces and is atomic wrt
1036 * nfsd_file_acquire().
1039 * %true: filecache contains at least one file matching this inode
1040 * %false: filecache contains no files matching this inode
1043 nfsd_file_is_cached(struct inode *inode)
1045 struct nfsd_file_lookup_key key = {
1046 .type = NFSD_FILE_KEY_INODE,
1051 if (rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
1052 nfsd_file_rhash_params) != NULL)
1054 trace_nfsd_file_is_cached(inode, (int)ret);
1059 nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1060 unsigned int may_flags, struct nfsd_file **pnf, bool open)
1062 struct nfsd_file_lookup_key key = {
1063 .type = NFSD_FILE_KEY_FULL,
1064 .need = may_flags & NFSD_FILE_MAY_MASK,
1065 .net = SVC_NET(rqstp),
1067 struct nfsd_file *nf, *new;
1071 status = fh_verify(rqstp, fhp, S_IFREG,
1072 may_flags|NFSD_MAY_OWNER_OVERRIDE);
1073 if (status != nfs_ok)
1075 key.inode = d_inode(fhp->fh_dentry);
1076 key.cred = get_current_cred();
1079 /* Avoid allocation if the item is already in cache */
1080 nf = rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
1081 nfsd_file_rhash_params);
1083 nf = nfsd_file_get(nf);
1085 goto wait_for_construction;
1087 new = nfsd_file_alloc(&key, may_flags);
1089 status = nfserr_jukebox;
1093 nf = rhashtable_lookup_get_insert_key(&nfsd_file_rhash_tbl,
1094 &key, &new->nf_rhash,
1095 nfsd_file_rhash_params);
1102 nf = nfsd_file_get(nf);
1107 nfsd_file_slab_free(&new->nf_rcu);
1109 wait_for_construction:
1110 wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
1112 /* Did construction of this file fail? */
1113 if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
1114 trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
1116 status = nfserr_jukebox;
1120 nfsd_file_put_noref(nf);
1124 nfsd_file_lru_remove(nf);
1125 this_cpu_inc(nfsd_file_cache_hits);
1127 status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
1129 if (status == nfs_ok) {
1131 this_cpu_inc(nfsd_file_acquisitions);
1141 trace_nfsd_file_acquire(rqstp, key.inode, may_flags, nf, status);
1145 trace_nfsd_file_alloc(nf);
1146 nf->nf_mark = nfsd_file_mark_find_or_create(nf, key.inode);
1149 status = nfsd_open_verified(rqstp, fhp, may_flags,
1151 trace_nfsd_file_open(nf, status);
1155 status = nfserr_jukebox;
1157 * If construction failed, or we raced with a call to unlink()
1160 if (status != nfs_ok || key.inode->i_nlink == 0)
1161 if (nfsd_file_unhash(nf))
1162 nfsd_file_put_noref(nf);
1163 clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
1164 smp_mb__after_atomic();
1165 wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
1169 nfsd_file_slab_free(&new->nf_rcu);
1170 trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, PTR_ERR(nf));
1172 status = nfserr_jukebox;
1177 * nfsd_file_acquire - Get a struct nfsd_file with an open file
1178 * @rqstp: the RPC transaction being executed
1179 * @fhp: the NFS filehandle of the file to be opened
1180 * @may_flags: NFSD_MAY_ settings for the file
1181 * @pnf: OUT: new or found "struct nfsd_file" object
1183 * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1184 * network byte order is returned.
1187 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
1188 unsigned int may_flags, struct nfsd_file **pnf)
1190 return nfsd_file_do_acquire(rqstp, fhp, may_flags, pnf, true);
1194 * nfsd_file_create - Get a struct nfsd_file, do not open
1195 * @rqstp: the RPC transaction being executed
1196 * @fhp: the NFS filehandle of the file just created
1197 * @may_flags: NFSD_MAY_ settings for the file
1198 * @pnf: OUT: new or found "struct nfsd_file" object
1200 * Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
1201 * network byte order is returned.
1204 nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
1205 unsigned int may_flags, struct nfsd_file **pnf)
1207 return nfsd_file_do_acquire(rqstp, fhp, may_flags, pnf, false);
1211 * Note that fields may be added, removed or reordered in the future. Programs
1212 * scraping this file for info should test the labels to ensure they're
1213 * getting the correct field.
1215 static int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
1217 unsigned long releases = 0, pages_flushed = 0, evictions = 0;
1218 unsigned long hits = 0, acquisitions = 0;
1219 unsigned int i, count = 0, buckets = 0;
1220 unsigned long lru = 0, total_age = 0;
1222 /* Serialize with server shutdown */
1223 mutex_lock(&nfsd_mutex);
1224 if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
1225 struct bucket_table *tbl;
1226 struct rhashtable *ht;
1228 lru = list_lru_count(&nfsd_file_lru);
1231 ht = &nfsd_file_rhash_tbl;
1232 count = atomic_read(&ht->nelems);
1233 tbl = rht_dereference_rcu(ht->tbl, ht);
1234 buckets = tbl->size;
1237 mutex_unlock(&nfsd_mutex);
1239 for_each_possible_cpu(i) {
1240 hits += per_cpu(nfsd_file_cache_hits, i);
1241 acquisitions += per_cpu(nfsd_file_acquisitions, i);
1242 releases += per_cpu(nfsd_file_releases, i);
1243 total_age += per_cpu(nfsd_file_total_age, i);
1244 evictions += per_cpu(nfsd_file_evictions, i);
1245 pages_flushed += per_cpu(nfsd_file_pages_flushed, i);
1248 seq_printf(m, "total entries: %u\n", count);
1249 seq_printf(m, "hash buckets: %u\n", buckets);
1250 seq_printf(m, "lru entries: %lu\n", lru);
1251 seq_printf(m, "cache hits: %lu\n", hits);
1252 seq_printf(m, "acquisitions: %lu\n", acquisitions);
1253 seq_printf(m, "releases: %lu\n", releases);
1254 seq_printf(m, "evictions: %lu\n", evictions);
1256 seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
1258 seq_printf(m, "mean age (ms): -\n");
1259 seq_printf(m, "pages flushed: %lu\n", pages_flushed);
1263 int nfsd_file_cache_stats_open(struct inode *inode, struct file *file)
1265 return single_open(file, nfsd_file_cache_stats_show, NULL);