mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 19:00:25 +00:00
![Linus Torvalds](/assets/img/avatar_default.png)
Work on "courteous server", which was introduced in 5.19, continues apace. This release introduces a more flexible limit on the number of NFSv4 clients that NFSD allows, now that NFSv4 clients can remain in courtesy state long after the lease expiration timeout. The client limit is adjusted based on the physical memory size of the server. The NFSD filecache is a cache of files held open by NFSv4 clients or recently touched by NFSv2 or NFSv3 clients. This cache had some significant scalability constraints that have been relieved in this release. Thanks to all who contributed to this work. A data corruption bug found during the most recent NFS bake-a-thon that involves NFSv3 and NFSv4 clients writing the same file has been addressed in this release. This release includes several improvements in CPU scalability for NFSv4 operations. In addition, Neil Brown provided patches that simplify locking during file lookup, creation, rename, and removal that enables subsequent work on making these operations more scalable. We expect to see that work materialize in the next release. There are also numerous single-patch fixes, clean-ups, and the usual improvements in observability. -----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEKLLlsBKG3yQ88j7+M2qzM29mf5cFAmLujF0ACgkQM2qzM29m f5c14w/+Lsoryo5vdTXMAZBXBNvVdXQmHqLIEotJEVA3sECr+Kad2bF8rFaCWVzS Gf9KhTetmcDO9O73I5I/UtJ2qFT9B4I6baSGpOzIkjsM/aKeEEQpbpdzPYhKrCEv bQu3P54js7snH4YV8s0I39nBFOWdnahYXaw7peqE/2GOHxaR2mz88bkrQ+OybCxz KETqUxA6bKzkOT61S0nHcnQKd8HQzhocMDtrxtANHGsMM167ngI1dw4tUQAtfAUI s9R+GS6qwiKgwGz1oqhTR6LA/h4DROxPnc7AieuD9FvuAnR3kXw61bGMN5Biwv2T JZUTBbQvWhNasSV+7qOY9nBu+sHVC6Q7OZ5C9F/KjMyqCioDX0DnbxX9uKP20CDd EAAMS8n4Tdgd4KRBWdkLXPzizWYAjZQmFIJtcZne1JzGZ4IWRnikgM5qD6n1VviZ kcPRm5EN3DRHA+Hte4jG0EHIrE/7g5gnf+zr9dWl3uNhZtfTmumCfU16YYmKG8pP QN4kXBR2w7dAvp8nRaOsY6bBFLDAk/jHbpY8Q4xoUO4tsojfWayCTGVFOrecOjxv uSn0LhiidC5pLlkcPgwemhysVywDzr+gGXBRJXeUOHfdd05Q2gbFK8OpqDSvJ3dZ aC/RxFvHc8jaktUcuIjkE6Rsz6AVaAH3EZj84oMZ4hZhyGbEreg= =PEJ3 -----END PGP SIGNATURE----- Merge tag 'nfsd-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux Pull nfsd updates from Chuck Lever: "Work on 'courteous server', which was introduced in 5.19, continues apace. This release introduces a more flexible limit on the number of NFSv4 clients that NFSD allows, now that NFSv4 clients can remain in courtesy state long after the lease expiration timeout. The client limit is adjusted based on the physical memory size of the server. The NFSD filecache is a cache of files held open by NFSv4 clients or recently touched by NFSv2 or NFSv3 clients. This cache had some significant scalability constraints that have been relieved in this release. Thanks to all who contributed to this work. A data corruption bug found during the most recent NFS bake-a-thon that involves NFSv3 and NFSv4 clients writing the same file has been addressed in this release. This release includes several improvements in CPU scalability for NFSv4 operations. In addition, Neil Brown provided patches that simplify locking during file lookup, creation, rename, and removal that enables subsequent work on making these operations more scalable. We expect to see that work materialize in the next release. There are also numerous single-patch fixes, clean-ups, and the usual improvements in observability" * tag 'nfsd-6.0' of git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux: (78 commits) lockd: detect and reject lock arguments that overflow NFSD: discard fh_locked flag and fh_lock/fh_unlock NFSD: use (un)lock_inode instead of fh_(un)lock for file operations NFSD: use explicit lock/unlock for directory ops NFSD: reduce locking in nfsd_lookup() NFSD: only call fh_unlock() once in nfsd_link() NFSD: always drop directory lock in nfsd_unlink() NFSD: change nfsd_create()/nfsd_symlink() to unlock directory before returning. NFSD: add posix ACLs to struct nfsd_attrs NFSD: add security label to struct nfsd_attrs NFSD: set attributes when creating symlinks NFSD: introduce struct nfsd_attrs NFSD: verify the opened dentry after setting a delegation NFSD: drop fh argument from alloc_init_deleg NFSD: Move copy offload callback arguments into a separate structure NFSD: Add nfsd4_send_cb_offload() NFSD: Remove kmalloc from nfsd4_do_async_copy() NFSD: Refactor nfsd4_do_copy() NFSD: Refactor nfsd4_cleanup_inter_ssc() (2/2) NFSD: Refactor nfsd4_cleanup_inter_ssc() (1/2) ...
1266 lines
31 KiB
C
1266 lines
31 KiB
C
/*
|
|
* Open file cache.
|
|
*
|
|
* (c) 2015 - Jeff Layton <jeff.layton@primarydata.com>
|
|
*/
|
|
|
|
#include <linux/hash.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/file.h>
|
|
#include <linux/pagemap.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/list_lru.h>
|
|
#include <linux/fsnotify_backend.h>
|
|
#include <linux/fsnotify.h>
|
|
#include <linux/seq_file.h>
|
|
#include <linux/rhashtable.h>
|
|
|
|
#include "vfs.h"
|
|
#include "nfsd.h"
|
|
#include "nfsfh.h"
|
|
#include "netns.h"
|
|
#include "filecache.h"
|
|
#include "trace.h"
|
|
|
|
#define NFSD_LAUNDRETTE_DELAY (2 * HZ)
|
|
|
|
#define NFSD_FILE_CACHE_UP (0)
|
|
|
|
/* We only care about NFSD_MAY_READ/WRITE for this cache */
|
|
#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE)
|
|
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits);
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions);
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_releases);
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age);
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_pages_flushed);
|
|
static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions);
|
|
|
|
struct nfsd_fcache_disposal {
|
|
struct work_struct work;
|
|
spinlock_t lock;
|
|
struct list_head freeme;
|
|
};
|
|
|
|
static struct workqueue_struct *nfsd_filecache_wq __read_mostly;
|
|
|
|
static struct kmem_cache *nfsd_file_slab;
|
|
static struct kmem_cache *nfsd_file_mark_slab;
|
|
static struct list_lru nfsd_file_lru;
|
|
static unsigned long nfsd_file_flags;
|
|
static struct fsnotify_group *nfsd_file_fsnotify_group;
|
|
static struct delayed_work nfsd_filecache_laundrette;
|
|
static struct rhashtable nfsd_file_rhash_tbl
|
|
____cacheline_aligned_in_smp;
|
|
|
|
enum nfsd_file_lookup_type {
|
|
NFSD_FILE_KEY_INODE,
|
|
NFSD_FILE_KEY_FULL,
|
|
};
|
|
|
|
struct nfsd_file_lookup_key {
|
|
struct inode *inode;
|
|
struct net *net;
|
|
const struct cred *cred;
|
|
unsigned char need;
|
|
enum nfsd_file_lookup_type type;
|
|
};
|
|
|
|
/*
|
|
* The returned hash value is based solely on the address of an in-code
|
|
* inode, a pointer to a slab-allocated object. The entropy in such a
|
|
* pointer is concentrated in its middle bits.
|
|
*/
|
|
static u32 nfsd_file_inode_hash(const struct inode *inode, u32 seed)
|
|
{
|
|
unsigned long ptr = (unsigned long)inode;
|
|
u32 k;
|
|
|
|
k = ptr >> L1_CACHE_SHIFT;
|
|
k &= 0x00ffffff;
|
|
return jhash2(&k, 1, seed);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_key_hashfn - Compute the hash value of a lookup key
|
|
* @data: key on which to compute the hash value
|
|
* @len: rhash table's key_len parameter (unused)
|
|
* @seed: rhash table's random seed of the day
|
|
*
|
|
* Return value:
|
|
* Computed 32-bit hash value
|
|
*/
|
|
static u32 nfsd_file_key_hashfn(const void *data, u32 len, u32 seed)
|
|
{
|
|
const struct nfsd_file_lookup_key *key = data;
|
|
|
|
return nfsd_file_inode_hash(key->inode, seed);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_obj_hashfn - Compute the hash value of an nfsd_file
|
|
* @data: object on which to compute the hash value
|
|
* @len: rhash table's key_len parameter (unused)
|
|
* @seed: rhash table's random seed of the day
|
|
*
|
|
* Return value:
|
|
* Computed 32-bit hash value
|
|
*/
|
|
static u32 nfsd_file_obj_hashfn(const void *data, u32 len, u32 seed)
|
|
{
|
|
const struct nfsd_file *nf = data;
|
|
|
|
return nfsd_file_inode_hash(nf->nf_inode, seed);
|
|
}
|
|
|
|
static bool
|
|
nfsd_match_cred(const struct cred *c1, const struct cred *c2)
|
|
{
|
|
int i;
|
|
|
|
if (!uid_eq(c1->fsuid, c2->fsuid))
|
|
return false;
|
|
if (!gid_eq(c1->fsgid, c2->fsgid))
|
|
return false;
|
|
if (c1->group_info == NULL || c2->group_info == NULL)
|
|
return c1->group_info == c2->group_info;
|
|
if (c1->group_info->ngroups != c2->group_info->ngroups)
|
|
return false;
|
|
for (i = 0; i < c1->group_info->ngroups; i++) {
|
|
if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i]))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_obj_cmpfn - Match a cache item against search criteria
|
|
* @arg: search criteria
|
|
* @ptr: cache item to check
|
|
*
|
|
* Return values:
|
|
* %0 - Item matches search criteria
|
|
* %1 - Item does not match search criteria
|
|
*/
|
|
static int nfsd_file_obj_cmpfn(struct rhashtable_compare_arg *arg,
|
|
const void *ptr)
|
|
{
|
|
const struct nfsd_file_lookup_key *key = arg->key;
|
|
const struct nfsd_file *nf = ptr;
|
|
|
|
switch (key->type) {
|
|
case NFSD_FILE_KEY_INODE:
|
|
if (nf->nf_inode != key->inode)
|
|
return 1;
|
|
break;
|
|
case NFSD_FILE_KEY_FULL:
|
|
if (nf->nf_inode != key->inode)
|
|
return 1;
|
|
if (nf->nf_may != key->need)
|
|
return 1;
|
|
if (nf->nf_net != key->net)
|
|
return 1;
|
|
if (!nfsd_match_cred(nf->nf_cred, key->cred))
|
|
return 1;
|
|
if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0)
|
|
return 1;
|
|
break;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static const struct rhashtable_params nfsd_file_rhash_params = {
|
|
.key_len = sizeof_field(struct nfsd_file, nf_inode),
|
|
.key_offset = offsetof(struct nfsd_file, nf_inode),
|
|
.head_offset = offsetof(struct nfsd_file, nf_rhash),
|
|
.hashfn = nfsd_file_key_hashfn,
|
|
.obj_hashfn = nfsd_file_obj_hashfn,
|
|
.obj_cmpfn = nfsd_file_obj_cmpfn,
|
|
/* Reduce resizing churn on light workloads */
|
|
.min_size = 512, /* buckets */
|
|
.automatic_shrinking = true,
|
|
};
|
|
|
|
static void
|
|
nfsd_file_schedule_laundrette(void)
|
|
{
|
|
if ((atomic_read(&nfsd_file_rhash_tbl.nelems) == 0) ||
|
|
test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
|
|
return;
|
|
|
|
queue_delayed_work(system_wq, &nfsd_filecache_laundrette,
|
|
NFSD_LAUNDRETTE_DELAY);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_slab_free(struct rcu_head *rcu)
|
|
{
|
|
struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu);
|
|
|
|
put_cred(nf->nf_cred);
|
|
kmem_cache_free(nfsd_file_slab, nf);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_mark_free(struct fsnotify_mark *mark)
|
|
{
|
|
struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark,
|
|
nfm_mark);
|
|
|
|
kmem_cache_free(nfsd_file_mark_slab, nfm);
|
|
}
|
|
|
|
static struct nfsd_file_mark *
|
|
nfsd_file_mark_get(struct nfsd_file_mark *nfm)
|
|
{
|
|
if (!refcount_inc_not_zero(&nfm->nfm_ref))
|
|
return NULL;
|
|
return nfm;
|
|
}
|
|
|
|
static void
|
|
nfsd_file_mark_put(struct nfsd_file_mark *nfm)
|
|
{
|
|
if (refcount_dec_and_test(&nfm->nfm_ref)) {
|
|
fsnotify_destroy_mark(&nfm->nfm_mark, nfsd_file_fsnotify_group);
|
|
fsnotify_put_mark(&nfm->nfm_mark);
|
|
}
|
|
}
|
|
|
|
static struct nfsd_file_mark *
|
|
nfsd_file_mark_find_or_create(struct nfsd_file *nf, struct inode *inode)
|
|
{
|
|
int err;
|
|
struct fsnotify_mark *mark;
|
|
struct nfsd_file_mark *nfm = NULL, *new;
|
|
|
|
do {
|
|
fsnotify_group_lock(nfsd_file_fsnotify_group);
|
|
mark = fsnotify_find_mark(&inode->i_fsnotify_marks,
|
|
nfsd_file_fsnotify_group);
|
|
if (mark) {
|
|
nfm = nfsd_file_mark_get(container_of(mark,
|
|
struct nfsd_file_mark,
|
|
nfm_mark));
|
|
fsnotify_group_unlock(nfsd_file_fsnotify_group);
|
|
if (nfm) {
|
|
fsnotify_put_mark(mark);
|
|
break;
|
|
}
|
|
/* Avoid soft lockup race with nfsd_file_mark_put() */
|
|
fsnotify_destroy_mark(mark, nfsd_file_fsnotify_group);
|
|
fsnotify_put_mark(mark);
|
|
} else {
|
|
fsnotify_group_unlock(nfsd_file_fsnotify_group);
|
|
}
|
|
|
|
/* allocate a new nfm */
|
|
new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL);
|
|
if (!new)
|
|
return NULL;
|
|
fsnotify_init_mark(&new->nfm_mark, nfsd_file_fsnotify_group);
|
|
new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF;
|
|
refcount_set(&new->nfm_ref, 1);
|
|
|
|
err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0);
|
|
|
|
/*
|
|
* If the add was successful, then return the object.
|
|
* Otherwise, we need to put the reference we hold on the
|
|
* nfm_mark. The fsnotify code will take a reference and put
|
|
* it on failure, so we can't just free it directly. It's also
|
|
* not safe to call fsnotify_destroy_mark on it as the
|
|
* mark->group will be NULL. Thus, we can't let the nfm_ref
|
|
* counter drive the destruction at this point.
|
|
*/
|
|
if (likely(!err))
|
|
nfm = new;
|
|
else
|
|
fsnotify_put_mark(&new->nfm_mark);
|
|
} while (unlikely(err == -EEXIST));
|
|
|
|
return nfm;
|
|
}
|
|
|
|
static struct nfsd_file *
|
|
nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
|
|
{
|
|
struct nfsd_file *nf;
|
|
|
|
nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL);
|
|
if (nf) {
|
|
INIT_LIST_HEAD(&nf->nf_lru);
|
|
nf->nf_birthtime = ktime_get();
|
|
nf->nf_file = NULL;
|
|
nf->nf_cred = get_current_cred();
|
|
nf->nf_net = key->net;
|
|
nf->nf_flags = 0;
|
|
__set_bit(NFSD_FILE_HASHED, &nf->nf_flags);
|
|
__set_bit(NFSD_FILE_PENDING, &nf->nf_flags);
|
|
nf->nf_inode = key->inode;
|
|
/* nf_ref is pre-incremented for hash table */
|
|
refcount_set(&nf->nf_ref, 2);
|
|
nf->nf_may = key->need;
|
|
nf->nf_mark = NULL;
|
|
}
|
|
return nf;
|
|
}
|
|
|
|
static bool
|
|
nfsd_file_free(struct nfsd_file *nf)
|
|
{
|
|
s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
|
|
bool flush = false;
|
|
|
|
this_cpu_inc(nfsd_file_releases);
|
|
this_cpu_add(nfsd_file_total_age, age);
|
|
|
|
trace_nfsd_file_put_final(nf);
|
|
if (nf->nf_mark)
|
|
nfsd_file_mark_put(nf->nf_mark);
|
|
if (nf->nf_file) {
|
|
get_file(nf->nf_file);
|
|
filp_close(nf->nf_file, NULL);
|
|
fput(nf->nf_file);
|
|
flush = true;
|
|
}
|
|
|
|
/*
|
|
* If this item is still linked via nf_lru, that's a bug.
|
|
* WARN and leak it to preserve system stability.
|
|
*/
|
|
if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
|
|
return flush;
|
|
|
|
call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
|
|
return flush;
|
|
}
|
|
|
|
static bool
|
|
nfsd_file_check_writeback(struct nfsd_file *nf)
|
|
{
|
|
struct file *file = nf->nf_file;
|
|
struct address_space *mapping;
|
|
|
|
if (!file || !(file->f_mode & FMODE_WRITE))
|
|
return false;
|
|
mapping = file->f_mapping;
|
|
return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) ||
|
|
mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
|
|
}
|
|
|
|
static int
|
|
nfsd_file_check_write_error(struct nfsd_file *nf)
|
|
{
|
|
struct file *file = nf->nf_file;
|
|
|
|
if (!file || !(file->f_mode & FMODE_WRITE))
|
|
return 0;
|
|
return filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err));
|
|
}
|
|
|
|
static void
|
|
nfsd_file_flush(struct nfsd_file *nf)
|
|
{
|
|
struct file *file = nf->nf_file;
|
|
|
|
if (!file || !(file->f_mode & FMODE_WRITE))
|
|
return;
|
|
this_cpu_add(nfsd_file_pages_flushed, file->f_mapping->nrpages);
|
|
if (vfs_fsync(file, 1) != 0)
|
|
nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
|
|
}
|
|
|
|
static void nfsd_file_lru_add(struct nfsd_file *nf)
|
|
{
|
|
set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
|
|
if (list_lru_add(&nfsd_file_lru, &nf->nf_lru))
|
|
trace_nfsd_file_lru_add(nf);
|
|
}
|
|
|
|
static void nfsd_file_lru_remove(struct nfsd_file *nf)
|
|
{
|
|
if (list_lru_del(&nfsd_file_lru, &nf->nf_lru))
|
|
trace_nfsd_file_lru_del(nf);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_hash_remove(struct nfsd_file *nf)
|
|
{
|
|
trace_nfsd_file_unhash(nf);
|
|
|
|
if (nfsd_file_check_write_error(nf))
|
|
nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id));
|
|
rhashtable_remove_fast(&nfsd_file_rhash_tbl, &nf->nf_rhash,
|
|
nfsd_file_rhash_params);
|
|
}
|
|
|
|
static bool
|
|
nfsd_file_unhash(struct nfsd_file *nf)
|
|
{
|
|
if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
|
|
nfsd_file_hash_remove(nf);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* Return true if the file was unhashed.
|
|
*/
|
|
static bool
|
|
nfsd_file_unhash_and_dispose(struct nfsd_file *nf, struct list_head *dispose)
|
|
{
|
|
trace_nfsd_file_unhash_and_dispose(nf);
|
|
if (!nfsd_file_unhash(nf))
|
|
return false;
|
|
/* keep final reference for nfsd_file_lru_dispose */
|
|
if (refcount_dec_not_one(&nf->nf_ref))
|
|
return true;
|
|
|
|
nfsd_file_lru_remove(nf);
|
|
list_add(&nf->nf_lru, dispose);
|
|
return true;
|
|
}
|
|
|
|
static void
|
|
nfsd_file_put_noref(struct nfsd_file *nf)
|
|
{
|
|
trace_nfsd_file_put(nf);
|
|
|
|
if (refcount_dec_and_test(&nf->nf_ref)) {
|
|
WARN_ON(test_bit(NFSD_FILE_HASHED, &nf->nf_flags));
|
|
nfsd_file_lru_remove(nf);
|
|
nfsd_file_free(nf);
|
|
}
|
|
}
|
|
|
|
void
|
|
nfsd_file_put(struct nfsd_file *nf)
|
|
{
|
|
might_sleep();
|
|
|
|
nfsd_file_lru_add(nf);
|
|
if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0) {
|
|
nfsd_file_flush(nf);
|
|
nfsd_file_put_noref(nf);
|
|
} else if (nf->nf_file) {
|
|
nfsd_file_put_noref(nf);
|
|
nfsd_file_schedule_laundrette();
|
|
} else
|
|
nfsd_file_put_noref(nf);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_close - Close an nfsd_file
|
|
* @nf: nfsd_file to close
|
|
*
|
|
* If this is the final reference for @nf, free it immediately.
|
|
* This reflects an on-the-wire CLOSE or DELEGRETURN into the
|
|
* VFS and exported filesystem.
|
|
*/
|
|
void nfsd_file_close(struct nfsd_file *nf)
|
|
{
|
|
nfsd_file_put(nf);
|
|
if (refcount_dec_if_one(&nf->nf_ref)) {
|
|
nfsd_file_unhash(nf);
|
|
nfsd_file_lru_remove(nf);
|
|
nfsd_file_free(nf);
|
|
}
|
|
}
|
|
|
|
struct nfsd_file *
|
|
nfsd_file_get(struct nfsd_file *nf)
|
|
{
|
|
if (likely(refcount_inc_not_zero(&nf->nf_ref)))
|
|
return nf;
|
|
return NULL;
|
|
}
|
|
|
|
static void
|
|
nfsd_file_dispose_list(struct list_head *dispose)
|
|
{
|
|
struct nfsd_file *nf;
|
|
|
|
while(!list_empty(dispose)) {
|
|
nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
|
|
list_del_init(&nf->nf_lru);
|
|
nfsd_file_flush(nf);
|
|
nfsd_file_put_noref(nf);
|
|
}
|
|
}
|
|
|
|
static void
|
|
nfsd_file_dispose_list_sync(struct list_head *dispose)
|
|
{
|
|
bool flush = false;
|
|
struct nfsd_file *nf;
|
|
|
|
while(!list_empty(dispose)) {
|
|
nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
|
|
list_del_init(&nf->nf_lru);
|
|
nfsd_file_flush(nf);
|
|
if (!refcount_dec_and_test(&nf->nf_ref))
|
|
continue;
|
|
if (nfsd_file_free(nf))
|
|
flush = true;
|
|
}
|
|
if (flush)
|
|
flush_delayed_fput();
|
|
}
|
|
|
|
static void
|
|
nfsd_file_list_remove_disposal(struct list_head *dst,
|
|
struct nfsd_fcache_disposal *l)
|
|
{
|
|
spin_lock(&l->lock);
|
|
list_splice_init(&l->freeme, dst);
|
|
spin_unlock(&l->lock);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_list_add_disposal(struct list_head *files, struct net *net)
|
|
{
|
|
struct nfsd_net *nn = net_generic(net, nfsd_net_id);
|
|
struct nfsd_fcache_disposal *l = nn->fcache_disposal;
|
|
|
|
spin_lock(&l->lock);
|
|
list_splice_tail_init(files, &l->freeme);
|
|
spin_unlock(&l->lock);
|
|
queue_work(nfsd_filecache_wq, &l->work);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_list_add_pernet(struct list_head *dst, struct list_head *src,
|
|
struct net *net)
|
|
{
|
|
struct nfsd_file *nf, *tmp;
|
|
|
|
list_for_each_entry_safe(nf, tmp, src, nf_lru) {
|
|
if (nf->nf_net == net)
|
|
list_move_tail(&nf->nf_lru, dst);
|
|
}
|
|
}
|
|
|
|
static void
|
|
nfsd_file_dispose_list_delayed(struct list_head *dispose)
|
|
{
|
|
LIST_HEAD(list);
|
|
struct nfsd_file *nf;
|
|
|
|
while(!list_empty(dispose)) {
|
|
nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
|
|
nfsd_file_list_add_pernet(&list, dispose, nf->nf_net);
|
|
nfsd_file_list_add_disposal(&list, nf->nf_net);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_lru_cb - Examine an entry on the LRU list
|
|
* @item: LRU entry to examine
|
|
* @lru: controlling LRU
|
|
* @lock: LRU list lock (unused)
|
|
* @arg: dispose list
|
|
*
|
|
* Note this can deadlock with nfsd_file_cache_purge.
|
|
*
|
|
* Return values:
|
|
* %LRU_REMOVED: @item was removed from the LRU
|
|
* %LRU_ROTATE: @item is to be moved to the LRU tail
|
|
* %LRU_SKIP: @item cannot be evicted
|
|
*/
|
|
static enum lru_status
|
|
nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
|
|
spinlock_t *lock, void *arg)
|
|
__releases(lock)
|
|
__acquires(lock)
|
|
{
|
|
struct list_head *head = arg;
|
|
struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
|
|
|
|
/*
|
|
* Do a lockless refcount check. The hashtable holds one reference, so
|
|
* we look to see if anything else has a reference, or if any have
|
|
* been put since the shrinker last ran. Those don't get unhashed and
|
|
* released.
|
|
*
|
|
* Note that in the put path, we set the flag and then decrement the
|
|
* counter. Here we check the counter and then test and clear the flag.
|
|
* That order is deliberate to ensure that we can do this locklessly.
|
|
*/
|
|
if (refcount_read(&nf->nf_ref) > 1) {
|
|
list_lru_isolate(lru, &nf->nf_lru);
|
|
trace_nfsd_file_gc_in_use(nf);
|
|
return LRU_REMOVED;
|
|
}
|
|
|
|
/*
|
|
* Don't throw out files that are still undergoing I/O or
|
|
* that have uncleared errors pending.
|
|
*/
|
|
if (nfsd_file_check_writeback(nf)) {
|
|
trace_nfsd_file_gc_writeback(nf);
|
|
return LRU_SKIP;
|
|
}
|
|
|
|
if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
|
|
trace_nfsd_file_gc_referenced(nf);
|
|
return LRU_ROTATE;
|
|
}
|
|
|
|
if (!test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
|
|
trace_nfsd_file_gc_hashed(nf);
|
|
return LRU_SKIP;
|
|
}
|
|
|
|
list_lru_isolate_move(lru, &nf->nf_lru, head);
|
|
this_cpu_inc(nfsd_file_evictions);
|
|
trace_nfsd_file_gc_disposed(nf);
|
|
return LRU_REMOVED;
|
|
}
|
|
|
|
/*
|
|
* Unhash items on @dispose immediately, then queue them on the
|
|
* disposal workqueue to finish releasing them in the background.
|
|
*
|
|
* cel: Note that between the time list_lru_shrink_walk runs and
|
|
* now, these items are in the hash table but marked unhashed.
|
|
* Why release these outside of lru_cb ? There's no lock ordering
|
|
* problem since lru_cb currently takes no lock.
|
|
*/
|
|
static void nfsd_file_gc_dispose_list(struct list_head *dispose)
|
|
{
|
|
struct nfsd_file *nf;
|
|
|
|
list_for_each_entry(nf, dispose, nf_lru)
|
|
nfsd_file_hash_remove(nf);
|
|
nfsd_file_dispose_list_delayed(dispose);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_gc(void)
|
|
{
|
|
LIST_HEAD(dispose);
|
|
unsigned long ret;
|
|
|
|
ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
|
|
&dispose, list_lru_count(&nfsd_file_lru));
|
|
trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
|
|
nfsd_file_gc_dispose_list(&dispose);
|
|
}
|
|
|
|
static void
|
|
nfsd_file_gc_worker(struct work_struct *work)
|
|
{
|
|
nfsd_file_gc();
|
|
nfsd_file_schedule_laundrette();
|
|
}
|
|
|
|
static unsigned long
|
|
nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc)
|
|
{
|
|
return list_lru_count(&nfsd_file_lru);
|
|
}
|
|
|
|
static unsigned long
|
|
nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
|
|
{
|
|
LIST_HEAD(dispose);
|
|
unsigned long ret;
|
|
|
|
ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
|
|
nfsd_file_lru_cb, &dispose);
|
|
trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
|
|
nfsd_file_gc_dispose_list(&dispose);
|
|
return ret;
|
|
}
|
|
|
|
static struct shrinker nfsd_file_shrinker = {
|
|
.scan_objects = nfsd_file_lru_scan,
|
|
.count_objects = nfsd_file_lru_count,
|
|
.seeks = 1,
|
|
};
|
|
|
|
/*
|
|
* Find all cache items across all net namespaces that match @inode and
|
|
* move them to @dispose. The lookup is atomic wrt nfsd_file_acquire().
|
|
*/
|
|
static unsigned int
|
|
__nfsd_file_close_inode(struct inode *inode, struct list_head *dispose)
|
|
{
|
|
struct nfsd_file_lookup_key key = {
|
|
.type = NFSD_FILE_KEY_INODE,
|
|
.inode = inode,
|
|
};
|
|
unsigned int count = 0;
|
|
struct nfsd_file *nf;
|
|
|
|
rcu_read_lock();
|
|
do {
|
|
nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
|
|
nfsd_file_rhash_params);
|
|
if (!nf)
|
|
break;
|
|
nfsd_file_unhash_and_dispose(nf, dispose);
|
|
count++;
|
|
} while (1);
|
|
rcu_read_unlock();
|
|
return count;
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file
|
|
* @inode: inode of the file to attempt to remove
|
|
*
|
|
* Unhash and put, then flush and fput all cache items associated with @inode.
|
|
*/
|
|
void
|
|
nfsd_file_close_inode_sync(struct inode *inode)
|
|
{
|
|
LIST_HEAD(dispose);
|
|
unsigned int count;
|
|
|
|
count = __nfsd_file_close_inode(inode, &dispose);
|
|
trace_nfsd_file_close_inode_sync(inode, count);
|
|
nfsd_file_dispose_list_sync(&dispose);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_close_inode - attempt a delayed close of a nfsd_file
|
|
* @inode: inode of the file to attempt to remove
|
|
*
|
|
* Unhash and put all cache item associated with @inode.
|
|
*/
|
|
static void
|
|
nfsd_file_close_inode(struct inode *inode)
|
|
{
|
|
LIST_HEAD(dispose);
|
|
unsigned int count;
|
|
|
|
count = __nfsd_file_close_inode(inode, &dispose);
|
|
trace_nfsd_file_close_inode(inode, count);
|
|
nfsd_file_dispose_list_delayed(&dispose);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_delayed_close - close unused nfsd_files
|
|
* @work: dummy
|
|
*
|
|
* Walk the LRU list and close any entries that have not been used since
|
|
* the last scan.
|
|
*
|
|
* Note this can deadlock with nfsd_file_cache_purge.
|
|
*/
|
|
static void
|
|
nfsd_file_delayed_close(struct work_struct *work)
|
|
{
|
|
LIST_HEAD(head);
|
|
struct nfsd_fcache_disposal *l = container_of(work,
|
|
struct nfsd_fcache_disposal, work);
|
|
|
|
nfsd_file_list_remove_disposal(&head, l);
|
|
nfsd_file_dispose_list(&head);
|
|
}
|
|
|
|
static int
|
|
nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg,
|
|
void *data)
|
|
{
|
|
struct file_lock *fl = data;
|
|
|
|
/* Only close files for F_SETLEASE leases */
|
|
if (fl->fl_flags & FL_LEASE)
|
|
nfsd_file_close_inode_sync(file_inode(fl->fl_file));
|
|
return 0;
|
|
}
|
|
|
|
static struct notifier_block nfsd_file_lease_notifier = {
|
|
.notifier_call = nfsd_file_lease_notifier_call,
|
|
};
|
|
|
|
static int
|
|
nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask,
|
|
struct inode *inode, struct inode *dir,
|
|
const struct qstr *name, u32 cookie)
|
|
{
|
|
if (WARN_ON_ONCE(!inode))
|
|
return 0;
|
|
|
|
trace_nfsd_file_fsnotify_handle_event(inode, mask);
|
|
|
|
/* Should be no marks on non-regular files */
|
|
if (!S_ISREG(inode->i_mode)) {
|
|
WARN_ON_ONCE(1);
|
|
return 0;
|
|
}
|
|
|
|
/* don't close files if this was not the last link */
|
|
if (mask & FS_ATTRIB) {
|
|
if (inode->i_nlink)
|
|
return 0;
|
|
}
|
|
|
|
nfsd_file_close_inode(inode);
|
|
return 0;
|
|
}
|
|
|
|
|
|
static const struct fsnotify_ops nfsd_file_fsnotify_ops = {
|
|
.handle_inode_event = nfsd_file_fsnotify_handle_event,
|
|
.free_mark = nfsd_file_mark_free,
|
|
};
|
|
|
|
int
|
|
nfsd_file_cache_init(void)
|
|
{
|
|
int ret;
|
|
|
|
lockdep_assert_held(&nfsd_mutex);
|
|
if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
|
|
return 0;
|
|
|
|
ret = rhashtable_init(&nfsd_file_rhash_tbl, &nfsd_file_rhash_params);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = -ENOMEM;
|
|
nfsd_filecache_wq = alloc_workqueue("nfsd_filecache", 0, 0);
|
|
if (!nfsd_filecache_wq)
|
|
goto out;
|
|
|
|
nfsd_file_slab = kmem_cache_create("nfsd_file",
|
|
sizeof(struct nfsd_file), 0, 0, NULL);
|
|
if (!nfsd_file_slab) {
|
|
pr_err("nfsd: unable to create nfsd_file_slab\n");
|
|
goto out_err;
|
|
}
|
|
|
|
nfsd_file_mark_slab = kmem_cache_create("nfsd_file_mark",
|
|
sizeof(struct nfsd_file_mark), 0, 0, NULL);
|
|
if (!nfsd_file_mark_slab) {
|
|
pr_err("nfsd: unable to create nfsd_file_mark_slab\n");
|
|
goto out_err;
|
|
}
|
|
|
|
|
|
ret = list_lru_init(&nfsd_file_lru);
|
|
if (ret) {
|
|
pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret);
|
|
goto out_err;
|
|
}
|
|
|
|
ret = register_shrinker(&nfsd_file_shrinker, "nfsd-filecache");
|
|
if (ret) {
|
|
pr_err("nfsd: failed to register nfsd_file_shrinker: %d\n", ret);
|
|
goto out_lru;
|
|
}
|
|
|
|
ret = lease_register_notifier(&nfsd_file_lease_notifier);
|
|
if (ret) {
|
|
pr_err("nfsd: unable to register lease notifier: %d\n", ret);
|
|
goto out_shrinker;
|
|
}
|
|
|
|
nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops,
|
|
FSNOTIFY_GROUP_NOFS);
|
|
if (IS_ERR(nfsd_file_fsnotify_group)) {
|
|
pr_err("nfsd: unable to create fsnotify group: %ld\n",
|
|
PTR_ERR(nfsd_file_fsnotify_group));
|
|
ret = PTR_ERR(nfsd_file_fsnotify_group);
|
|
nfsd_file_fsnotify_group = NULL;
|
|
goto out_notifier;
|
|
}
|
|
|
|
INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker);
|
|
out:
|
|
return ret;
|
|
out_notifier:
|
|
lease_unregister_notifier(&nfsd_file_lease_notifier);
|
|
out_shrinker:
|
|
unregister_shrinker(&nfsd_file_shrinker);
|
|
out_lru:
|
|
list_lru_destroy(&nfsd_file_lru);
|
|
out_err:
|
|
kmem_cache_destroy(nfsd_file_slab);
|
|
nfsd_file_slab = NULL;
|
|
kmem_cache_destroy(nfsd_file_mark_slab);
|
|
nfsd_file_mark_slab = NULL;
|
|
destroy_workqueue(nfsd_filecache_wq);
|
|
nfsd_filecache_wq = NULL;
|
|
rhashtable_destroy(&nfsd_file_rhash_tbl);
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Note this can deadlock with nfsd_file_lru_cb.
|
|
*/
|
|
static void
|
|
__nfsd_file_cache_purge(struct net *net)
|
|
{
|
|
struct rhashtable_iter iter;
|
|
struct nfsd_file *nf;
|
|
LIST_HEAD(dispose);
|
|
bool del;
|
|
|
|
rhashtable_walk_enter(&nfsd_file_rhash_tbl, &iter);
|
|
do {
|
|
rhashtable_walk_start(&iter);
|
|
|
|
nf = rhashtable_walk_next(&iter);
|
|
while (!IS_ERR_OR_NULL(nf)) {
|
|
if (net && nf->nf_net != net)
|
|
continue;
|
|
del = nfsd_file_unhash_and_dispose(nf, &dispose);
|
|
|
|
/*
|
|
* Deadlock detected! Something marked this entry as
|
|
* unhased, but hasn't removed it from the hash list.
|
|
*/
|
|
WARN_ON_ONCE(!del);
|
|
|
|
nf = rhashtable_walk_next(&iter);
|
|
}
|
|
|
|
rhashtable_walk_stop(&iter);
|
|
} while (nf == ERR_PTR(-EAGAIN));
|
|
rhashtable_walk_exit(&iter);
|
|
|
|
nfsd_file_dispose_list(&dispose);
|
|
}
|
|
|
|
static struct nfsd_fcache_disposal *
|
|
nfsd_alloc_fcache_disposal(void)
|
|
{
|
|
struct nfsd_fcache_disposal *l;
|
|
|
|
l = kmalloc(sizeof(*l), GFP_KERNEL);
|
|
if (!l)
|
|
return NULL;
|
|
INIT_WORK(&l->work, nfsd_file_delayed_close);
|
|
spin_lock_init(&l->lock);
|
|
INIT_LIST_HEAD(&l->freeme);
|
|
return l;
|
|
}
|
|
|
|
static void
|
|
nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l)
|
|
{
|
|
cancel_work_sync(&l->work);
|
|
nfsd_file_dispose_list(&l->freeme);
|
|
kfree(l);
|
|
}
|
|
|
|
static void
|
|
nfsd_free_fcache_disposal_net(struct net *net)
|
|
{
|
|
struct nfsd_net *nn = net_generic(net, nfsd_net_id);
|
|
struct nfsd_fcache_disposal *l = nn->fcache_disposal;
|
|
|
|
nfsd_free_fcache_disposal(l);
|
|
}
|
|
|
|
int
|
|
nfsd_file_cache_start_net(struct net *net)
|
|
{
|
|
struct nfsd_net *nn = net_generic(net, nfsd_net_id);
|
|
|
|
nn->fcache_disposal = nfsd_alloc_fcache_disposal();
|
|
return nn->fcache_disposal ? 0 : -ENOMEM;
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_cache_purge - Remove all cache items associated with @net
|
|
* @net: target net namespace
|
|
*
|
|
*/
|
|
void
|
|
nfsd_file_cache_purge(struct net *net)
|
|
{
|
|
lockdep_assert_held(&nfsd_mutex);
|
|
if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1)
|
|
__nfsd_file_cache_purge(net);
|
|
}
|
|
|
|
void
|
|
nfsd_file_cache_shutdown_net(struct net *net)
|
|
{
|
|
nfsd_file_cache_purge(net);
|
|
nfsd_free_fcache_disposal_net(net);
|
|
}
|
|
|
|
void
|
|
nfsd_file_cache_shutdown(void)
|
|
{
|
|
int i;
|
|
|
|
lockdep_assert_held(&nfsd_mutex);
|
|
if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0)
|
|
return;
|
|
|
|
lease_unregister_notifier(&nfsd_file_lease_notifier);
|
|
unregister_shrinker(&nfsd_file_shrinker);
|
|
/*
|
|
* make sure all callers of nfsd_file_lru_cb are done before
|
|
* calling nfsd_file_cache_purge
|
|
*/
|
|
cancel_delayed_work_sync(&nfsd_filecache_laundrette);
|
|
__nfsd_file_cache_purge(NULL);
|
|
list_lru_destroy(&nfsd_file_lru);
|
|
rcu_barrier();
|
|
fsnotify_put_group(nfsd_file_fsnotify_group);
|
|
nfsd_file_fsnotify_group = NULL;
|
|
kmem_cache_destroy(nfsd_file_slab);
|
|
nfsd_file_slab = NULL;
|
|
fsnotify_wait_marks_destroyed();
|
|
kmem_cache_destroy(nfsd_file_mark_slab);
|
|
nfsd_file_mark_slab = NULL;
|
|
destroy_workqueue(nfsd_filecache_wq);
|
|
nfsd_filecache_wq = NULL;
|
|
rhashtable_destroy(&nfsd_file_rhash_tbl);
|
|
|
|
for_each_possible_cpu(i) {
|
|
per_cpu(nfsd_file_cache_hits, i) = 0;
|
|
per_cpu(nfsd_file_acquisitions, i) = 0;
|
|
per_cpu(nfsd_file_releases, i) = 0;
|
|
per_cpu(nfsd_file_total_age, i) = 0;
|
|
per_cpu(nfsd_file_pages_flushed, i) = 0;
|
|
per_cpu(nfsd_file_evictions, i) = 0;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_is_cached - are there any cached open files for this inode?
|
|
* @inode: inode to check
|
|
*
|
|
* The lookup matches inodes in all net namespaces and is atomic wrt
|
|
* nfsd_file_acquire().
|
|
*
|
|
* Return values:
|
|
* %true: filecache contains at least one file matching this inode
|
|
* %false: filecache contains no files matching this inode
|
|
*/
|
|
bool
|
|
nfsd_file_is_cached(struct inode *inode)
|
|
{
|
|
struct nfsd_file_lookup_key key = {
|
|
.type = NFSD_FILE_KEY_INODE,
|
|
.inode = inode,
|
|
};
|
|
bool ret = false;
|
|
|
|
if (rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
|
|
nfsd_file_rhash_params) != NULL)
|
|
ret = true;
|
|
trace_nfsd_file_is_cached(inode, (int)ret);
|
|
return ret;
|
|
}
|
|
|
|
static __be32
|
|
nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
|
|
unsigned int may_flags, struct nfsd_file **pnf, bool open)
|
|
{
|
|
struct nfsd_file_lookup_key key = {
|
|
.type = NFSD_FILE_KEY_FULL,
|
|
.need = may_flags & NFSD_FILE_MAY_MASK,
|
|
.net = SVC_NET(rqstp),
|
|
};
|
|
struct nfsd_file *nf, *new;
|
|
bool retry = true;
|
|
__be32 status;
|
|
|
|
status = fh_verify(rqstp, fhp, S_IFREG,
|
|
may_flags|NFSD_MAY_OWNER_OVERRIDE);
|
|
if (status != nfs_ok)
|
|
return status;
|
|
key.inode = d_inode(fhp->fh_dentry);
|
|
key.cred = get_current_cred();
|
|
|
|
retry:
|
|
/* Avoid allocation if the item is already in cache */
|
|
nf = rhashtable_lookup_fast(&nfsd_file_rhash_tbl, &key,
|
|
nfsd_file_rhash_params);
|
|
if (nf)
|
|
nf = nfsd_file_get(nf);
|
|
if (nf)
|
|
goto wait_for_construction;
|
|
|
|
new = nfsd_file_alloc(&key, may_flags);
|
|
if (!new) {
|
|
status = nfserr_jukebox;
|
|
goto out_status;
|
|
}
|
|
|
|
nf = rhashtable_lookup_get_insert_key(&nfsd_file_rhash_tbl,
|
|
&key, &new->nf_rhash,
|
|
nfsd_file_rhash_params);
|
|
if (!nf) {
|
|
nf = new;
|
|
goto open_file;
|
|
}
|
|
if (IS_ERR(nf))
|
|
goto insert_err;
|
|
nf = nfsd_file_get(nf);
|
|
if (nf == NULL) {
|
|
nf = new;
|
|
goto open_file;
|
|
}
|
|
nfsd_file_slab_free(&new->nf_rcu);
|
|
|
|
wait_for_construction:
|
|
wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE);
|
|
|
|
/* Did construction of this file fail? */
|
|
if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
|
|
trace_nfsd_file_cons_err(rqstp, key.inode, may_flags, nf);
|
|
if (!retry) {
|
|
status = nfserr_jukebox;
|
|
goto out;
|
|
}
|
|
retry = false;
|
|
nfsd_file_put_noref(nf);
|
|
goto retry;
|
|
}
|
|
|
|
nfsd_file_lru_remove(nf);
|
|
this_cpu_inc(nfsd_file_cache_hits);
|
|
|
|
status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
|
|
out:
|
|
if (status == nfs_ok) {
|
|
if (open)
|
|
this_cpu_inc(nfsd_file_acquisitions);
|
|
*pnf = nf;
|
|
} else {
|
|
nfsd_file_put(nf);
|
|
nf = NULL;
|
|
}
|
|
|
|
out_status:
|
|
put_cred(key.cred);
|
|
if (open)
|
|
trace_nfsd_file_acquire(rqstp, key.inode, may_flags, nf, status);
|
|
return status;
|
|
|
|
open_file:
|
|
trace_nfsd_file_alloc(nf);
|
|
nf->nf_mark = nfsd_file_mark_find_or_create(nf, key.inode);
|
|
if (nf->nf_mark) {
|
|
if (open) {
|
|
status = nfsd_open_verified(rqstp, fhp, may_flags,
|
|
&nf->nf_file);
|
|
trace_nfsd_file_open(nf, status);
|
|
} else
|
|
status = nfs_ok;
|
|
} else
|
|
status = nfserr_jukebox;
|
|
/*
|
|
* If construction failed, or we raced with a call to unlink()
|
|
* then unhash.
|
|
*/
|
|
if (status != nfs_ok || key.inode->i_nlink == 0)
|
|
if (nfsd_file_unhash(nf))
|
|
nfsd_file_put_noref(nf);
|
|
clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
|
|
smp_mb__after_atomic();
|
|
wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
|
|
goto out;
|
|
|
|
insert_err:
|
|
nfsd_file_slab_free(&new->nf_rcu);
|
|
trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, PTR_ERR(nf));
|
|
nf = NULL;
|
|
status = nfserr_jukebox;
|
|
goto out_status;
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_acquire - Get a struct nfsd_file with an open file
|
|
* @rqstp: the RPC transaction being executed
|
|
* @fhp: the NFS filehandle of the file to be opened
|
|
* @may_flags: NFSD_MAY_ settings for the file
|
|
* @pnf: OUT: new or found "struct nfsd_file" object
|
|
*
|
|
* Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
|
|
* network byte order is returned.
|
|
*/
|
|
__be32
|
|
nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
|
|
unsigned int may_flags, struct nfsd_file **pnf)
|
|
{
|
|
return nfsd_file_do_acquire(rqstp, fhp, may_flags, pnf, true);
|
|
}
|
|
|
|
/**
|
|
* nfsd_file_create - Get a struct nfsd_file, do not open
|
|
* @rqstp: the RPC transaction being executed
|
|
* @fhp: the NFS filehandle of the file just created
|
|
* @may_flags: NFSD_MAY_ settings for the file
|
|
* @pnf: OUT: new or found "struct nfsd_file" object
|
|
*
|
|
* Returns nfs_ok and sets @pnf on success; otherwise an nfsstat in
|
|
* network byte order is returned.
|
|
*/
|
|
__be32
|
|
nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
|
|
unsigned int may_flags, struct nfsd_file **pnf)
|
|
{
|
|
return nfsd_file_do_acquire(rqstp, fhp, may_flags, pnf, false);
|
|
}
|
|
|
|
/*
|
|
* Note that fields may be added, removed or reordered in the future. Programs
|
|
* scraping this file for info should test the labels to ensure they're
|
|
* getting the correct field.
|
|
*/
|
|
static int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
|
|
{
|
|
unsigned long releases = 0, pages_flushed = 0, evictions = 0;
|
|
unsigned long hits = 0, acquisitions = 0;
|
|
unsigned int i, count = 0, buckets = 0;
|
|
unsigned long lru = 0, total_age = 0;
|
|
|
|
/* Serialize with server shutdown */
|
|
mutex_lock(&nfsd_mutex);
|
|
if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) {
|
|
struct bucket_table *tbl;
|
|
struct rhashtable *ht;
|
|
|
|
lru = list_lru_count(&nfsd_file_lru);
|
|
|
|
rcu_read_lock();
|
|
ht = &nfsd_file_rhash_tbl;
|
|
count = atomic_read(&ht->nelems);
|
|
tbl = rht_dereference_rcu(ht->tbl, ht);
|
|
buckets = tbl->size;
|
|
rcu_read_unlock();
|
|
}
|
|
mutex_unlock(&nfsd_mutex);
|
|
|
|
for_each_possible_cpu(i) {
|
|
hits += per_cpu(nfsd_file_cache_hits, i);
|
|
acquisitions += per_cpu(nfsd_file_acquisitions, i);
|
|
releases += per_cpu(nfsd_file_releases, i);
|
|
total_age += per_cpu(nfsd_file_total_age, i);
|
|
evictions += per_cpu(nfsd_file_evictions, i);
|
|
pages_flushed += per_cpu(nfsd_file_pages_flushed, i);
|
|
}
|
|
|
|
seq_printf(m, "total entries: %u\n", count);
|
|
seq_printf(m, "hash buckets: %u\n", buckets);
|
|
seq_printf(m, "lru entries: %lu\n", lru);
|
|
seq_printf(m, "cache hits: %lu\n", hits);
|
|
seq_printf(m, "acquisitions: %lu\n", acquisitions);
|
|
seq_printf(m, "releases: %lu\n", releases);
|
|
seq_printf(m, "evictions: %lu\n", evictions);
|
|
if (releases)
|
|
seq_printf(m, "mean age (ms): %ld\n", total_age / releases);
|
|
else
|
|
seq_printf(m, "mean age (ms): -\n");
|
|
seq_printf(m, "pages flushed: %lu\n", pages_flushed);
|
|
return 0;
|
|
}
|
|
|
|
int nfsd_file_cache_stats_open(struct inode *inode, struct file *file)
|
|
{
|
|
return single_open(file, nfsd_file_cache_stats_show, NULL);
|
|
}
|