mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-21 08:19:28 +00:00
![Johannes Weiner](/assets/img/avatar_default.png)
The page cache and most shrinkable slab caches hold data that has been read from disk, but there are some caches that only cache CPU work, such as the dentry and inode caches of procfs and sysfs, as well as the subset of radix tree nodes that track non-resident page cache. Currently, all these are shrunk at the same rate: using DEFAULT_SEEKS for the shrinker's seeks setting tells the reclaim algorithm that for every two page cache pages scanned it should scan one slab object. This is a bogus setting. A virtual inode that required no IO to create is not twice as valuable as a page cache page; shadow cache entries with eviction distances beyond the size of memory aren't either. In most cases, the behavior in practice is still fine. Such virtual caches don't tend to grow and assert themselves aggressively, and usually get picked up before they cause problems. But there are scenarios where that's not true. Our database workloads suffer from two of those. For one, their file workingset is several times bigger than available memory, which has the kernel aggressively create shadow page cache entries for the non-resident parts of it. The workingset code does tell the VM that most of these are expendable, but the VM ends up balancing them 2:1 to cache pages as per the seeks setting. This is a huge waste of memory. These workloads also deal with tens of thousands of open files and use /proc for introspection, which ends up growing the proc_inode_cache to absurdly large sizes - again at the cost of valuable cache space, which isn't a reasonable trade-off, given that proc inodes can be re-created without involving the disk. This patch implements a "zero-seek" setting for shrinkers that results in a target ratio of 0:1 between their objects and IO-backed caches. This allows such virtual caches to grow when memory is available (they do cache/avoid CPU work after all), but effectively disables them as soon as IO-backed objects are under pressure. It then switches the shrinkers for procfs and sysfs metadata, as well as excess page cache shadow nodes, to the new zero-seek setting. Link: http://lkml.kernel.org/r/20181009184732.762-5-hannes@cmpxchg.org Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reported-by: Domas Mituzas <dmituzas@fb.com> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Reviewed-by: Rik van Riel <riel@surriel.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
540 lines
13 KiB
C
540 lines
13 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* linux/fs/proc/inode.c
|
|
*
|
|
* Copyright (C) 1991, 1992 Linus Torvalds
|
|
*/
|
|
|
|
#include <linux/cache.h>
|
|
#include <linux/time.h>
|
|
#include <linux/proc_fs.h>
|
|
#include <linux/kernel.h>
|
|
#include <linux/pid_namespace.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/string.h>
|
|
#include <linux/stat.h>
|
|
#include <linux/completion.h>
|
|
#include <linux/poll.h>
|
|
#include <linux/printk.h>
|
|
#include <linux/file.h>
|
|
#include <linux/limits.h>
|
|
#include <linux/init.h>
|
|
#include <linux/module.h>
|
|
#include <linux/sysctl.h>
|
|
#include <linux/seq_file.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/mount.h>
|
|
#include <linux/magic.h>
|
|
|
|
#include <linux/uaccess.h>
|
|
|
|
#include "internal.h"
|
|
|
|
static void proc_evict_inode(struct inode *inode)
|
|
{
|
|
struct proc_dir_entry *de;
|
|
struct ctl_table_header *head;
|
|
|
|
truncate_inode_pages_final(&inode->i_data);
|
|
clear_inode(inode);
|
|
|
|
/* Stop tracking associated processes */
|
|
put_pid(PROC_I(inode)->pid);
|
|
|
|
/* Let go of any associated proc directory entry */
|
|
de = PDE(inode);
|
|
if (de)
|
|
pde_put(de);
|
|
|
|
head = PROC_I(inode)->sysctl;
|
|
if (head) {
|
|
RCU_INIT_POINTER(PROC_I(inode)->sysctl, NULL);
|
|
proc_sys_evict_inode(inode, head);
|
|
}
|
|
}
|
|
|
|
static struct kmem_cache *proc_inode_cachep __ro_after_init;
|
|
static struct kmem_cache *pde_opener_cache __ro_after_init;
|
|
|
|
static struct inode *proc_alloc_inode(struct super_block *sb)
|
|
{
|
|
struct proc_inode *ei;
|
|
struct inode *inode;
|
|
|
|
ei = kmem_cache_alloc(proc_inode_cachep, GFP_KERNEL);
|
|
if (!ei)
|
|
return NULL;
|
|
ei->pid = NULL;
|
|
ei->fd = 0;
|
|
ei->op.proc_get_link = NULL;
|
|
ei->pde = NULL;
|
|
ei->sysctl = NULL;
|
|
ei->sysctl_entry = NULL;
|
|
ei->ns_ops = NULL;
|
|
inode = &ei->vfs_inode;
|
|
return inode;
|
|
}
|
|
|
|
static void proc_i_callback(struct rcu_head *head)
|
|
{
|
|
struct inode *inode = container_of(head, struct inode, i_rcu);
|
|
kmem_cache_free(proc_inode_cachep, PROC_I(inode));
|
|
}
|
|
|
|
static void proc_destroy_inode(struct inode *inode)
|
|
{
|
|
call_rcu(&inode->i_rcu, proc_i_callback);
|
|
}
|
|
|
|
static void init_once(void *foo)
|
|
{
|
|
struct proc_inode *ei = (struct proc_inode *) foo;
|
|
|
|
inode_init_once(&ei->vfs_inode);
|
|
}
|
|
|
|
void __init proc_init_kmemcache(void)
|
|
{
|
|
proc_inode_cachep = kmem_cache_create("proc_inode_cache",
|
|
sizeof(struct proc_inode),
|
|
0, (SLAB_RECLAIM_ACCOUNT|
|
|
SLAB_MEM_SPREAD|SLAB_ACCOUNT|
|
|
SLAB_PANIC),
|
|
init_once);
|
|
pde_opener_cache =
|
|
kmem_cache_create("pde_opener", sizeof(struct pde_opener), 0,
|
|
SLAB_ACCOUNT|SLAB_PANIC, NULL);
|
|
proc_dir_entry_cache = kmem_cache_create_usercopy(
|
|
"proc_dir_entry", SIZEOF_PDE, 0, SLAB_PANIC,
|
|
offsetof(struct proc_dir_entry, inline_name),
|
|
SIZEOF_PDE_INLINE_NAME, NULL);
|
|
BUILD_BUG_ON(sizeof(struct proc_dir_entry) >= SIZEOF_PDE);
|
|
}
|
|
|
|
static int proc_show_options(struct seq_file *seq, struct dentry *root)
|
|
{
|
|
struct super_block *sb = root->d_sb;
|
|
struct pid_namespace *pid = sb->s_fs_info;
|
|
|
|
if (!gid_eq(pid->pid_gid, GLOBAL_ROOT_GID))
|
|
seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, pid->pid_gid));
|
|
if (pid->hide_pid != HIDEPID_OFF)
|
|
seq_printf(seq, ",hidepid=%u", pid->hide_pid);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static const struct super_operations proc_sops = {
|
|
.alloc_inode = proc_alloc_inode,
|
|
.destroy_inode = proc_destroy_inode,
|
|
.drop_inode = generic_delete_inode,
|
|
.evict_inode = proc_evict_inode,
|
|
.statfs = simple_statfs,
|
|
.remount_fs = proc_remount,
|
|
.show_options = proc_show_options,
|
|
};
|
|
|
|
enum {BIAS = -1U<<31};
|
|
|
|
static inline int use_pde(struct proc_dir_entry *pde)
|
|
{
|
|
return likely(atomic_inc_unless_negative(&pde->in_use));
|
|
}
|
|
|
|
static void unuse_pde(struct proc_dir_entry *pde)
|
|
{
|
|
if (unlikely(atomic_dec_return(&pde->in_use) == BIAS))
|
|
complete(pde->pde_unload_completion);
|
|
}
|
|
|
|
/* pde is locked on entry, unlocked on exit */
|
|
static void close_pdeo(struct proc_dir_entry *pde, struct pde_opener *pdeo)
|
|
{
|
|
/*
|
|
* close() (proc_reg_release()) can't delete an entry and proceed:
|
|
* ->release hook needs to be available at the right moment.
|
|
*
|
|
* rmmod (remove_proc_entry() et al) can't delete an entry and proceed:
|
|
* "struct file" needs to be available at the right moment.
|
|
*
|
|
* Therefore, first process to enter this function does ->release() and
|
|
* signals its completion to the other process which does nothing.
|
|
*/
|
|
if (pdeo->closing) {
|
|
/* somebody else is doing that, just wait */
|
|
DECLARE_COMPLETION_ONSTACK(c);
|
|
pdeo->c = &c;
|
|
spin_unlock(&pde->pde_unload_lock);
|
|
wait_for_completion(&c);
|
|
} else {
|
|
struct file *file;
|
|
struct completion *c;
|
|
|
|
pdeo->closing = true;
|
|
spin_unlock(&pde->pde_unload_lock);
|
|
file = pdeo->file;
|
|
pde->proc_fops->release(file_inode(file), file);
|
|
spin_lock(&pde->pde_unload_lock);
|
|
/* After ->release. */
|
|
list_del(&pdeo->lh);
|
|
c = pdeo->c;
|
|
spin_unlock(&pde->pde_unload_lock);
|
|
if (unlikely(c))
|
|
complete(c);
|
|
kmem_cache_free(pde_opener_cache, pdeo);
|
|
}
|
|
}
|
|
|
|
void proc_entry_rundown(struct proc_dir_entry *de)
|
|
{
|
|
DECLARE_COMPLETION_ONSTACK(c);
|
|
/* Wait until all existing callers into module are done. */
|
|
de->pde_unload_completion = &c;
|
|
if (atomic_add_return(BIAS, &de->in_use) != BIAS)
|
|
wait_for_completion(&c);
|
|
|
|
/* ->pde_openers list can't grow from now on. */
|
|
|
|
spin_lock(&de->pde_unload_lock);
|
|
while (!list_empty(&de->pde_openers)) {
|
|
struct pde_opener *pdeo;
|
|
pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh);
|
|
close_pdeo(de, pdeo);
|
|
spin_lock(&de->pde_unload_lock);
|
|
}
|
|
spin_unlock(&de->pde_unload_lock);
|
|
}
|
|
|
|
static loff_t proc_reg_llseek(struct file *file, loff_t offset, int whence)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
loff_t rv = -EINVAL;
|
|
if (use_pde(pde)) {
|
|
loff_t (*llseek)(struct file *, loff_t, int);
|
|
llseek = pde->proc_fops->llseek;
|
|
if (!llseek)
|
|
llseek = default_llseek;
|
|
rv = llseek(file, offset, whence);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static ssize_t proc_reg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
|
|
{
|
|
ssize_t (*read)(struct file *, char __user *, size_t, loff_t *);
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
ssize_t rv = -EIO;
|
|
if (use_pde(pde)) {
|
|
read = pde->proc_fops->read;
|
|
if (read)
|
|
rv = read(file, buf, count, ppos);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static ssize_t proc_reg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
|
|
{
|
|
ssize_t (*write)(struct file *, const char __user *, size_t, loff_t *);
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
ssize_t rv = -EIO;
|
|
if (use_pde(pde)) {
|
|
write = pde->proc_fops->write;
|
|
if (write)
|
|
rv = write(file, buf, count, ppos);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static __poll_t proc_reg_poll(struct file *file, struct poll_table_struct *pts)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
__poll_t rv = DEFAULT_POLLMASK;
|
|
__poll_t (*poll)(struct file *, struct poll_table_struct *);
|
|
if (use_pde(pde)) {
|
|
poll = pde->proc_fops->poll;
|
|
if (poll)
|
|
rv = poll(file, pts);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static long proc_reg_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
long rv = -ENOTTY;
|
|
long (*ioctl)(struct file *, unsigned int, unsigned long);
|
|
if (use_pde(pde)) {
|
|
ioctl = pde->proc_fops->unlocked_ioctl;
|
|
if (ioctl)
|
|
rv = ioctl(file, cmd, arg);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
static long proc_reg_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
long rv = -ENOTTY;
|
|
long (*compat_ioctl)(struct file *, unsigned int, unsigned long);
|
|
if (use_pde(pde)) {
|
|
compat_ioctl = pde->proc_fops->compat_ioctl;
|
|
if (compat_ioctl)
|
|
rv = compat_ioctl(file, cmd, arg);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
#endif
|
|
|
|
static int proc_reg_mmap(struct file *file, struct vm_area_struct *vma)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
int rv = -EIO;
|
|
int (*mmap)(struct file *, struct vm_area_struct *);
|
|
if (use_pde(pde)) {
|
|
mmap = pde->proc_fops->mmap;
|
|
if (mmap)
|
|
rv = mmap(file, vma);
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static unsigned long
|
|
proc_reg_get_unmapped_area(struct file *file, unsigned long orig_addr,
|
|
unsigned long len, unsigned long pgoff,
|
|
unsigned long flags)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(file_inode(file));
|
|
unsigned long rv = -EIO;
|
|
|
|
if (use_pde(pde)) {
|
|
typeof(proc_reg_get_unmapped_area) *get_area;
|
|
|
|
get_area = pde->proc_fops->get_unmapped_area;
|
|
#ifdef CONFIG_MMU
|
|
if (!get_area)
|
|
get_area = current->mm->get_unmapped_area;
|
|
#endif
|
|
|
|
if (get_area)
|
|
rv = get_area(file, orig_addr, len, pgoff, flags);
|
|
else
|
|
rv = orig_addr;
|
|
unuse_pde(pde);
|
|
}
|
|
return rv;
|
|
}
|
|
|
|
static int proc_reg_open(struct inode *inode, struct file *file)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(inode);
|
|
int rv = 0;
|
|
int (*open)(struct inode *, struct file *);
|
|
int (*release)(struct inode *, struct file *);
|
|
struct pde_opener *pdeo;
|
|
|
|
/*
|
|
* Ensure that
|
|
* 1) PDE's ->release hook will be called no matter what
|
|
* either normally by close()/->release, or forcefully by
|
|
* rmmod/remove_proc_entry.
|
|
*
|
|
* 2) rmmod isn't blocked by opening file in /proc and sitting on
|
|
* the descriptor (including "rmmod foo </proc/foo" scenario).
|
|
*
|
|
* Save every "struct file" with custom ->release hook.
|
|
*/
|
|
if (!use_pde(pde))
|
|
return -ENOENT;
|
|
|
|
release = pde->proc_fops->release;
|
|
if (release) {
|
|
pdeo = kmem_cache_alloc(pde_opener_cache, GFP_KERNEL);
|
|
if (!pdeo) {
|
|
rv = -ENOMEM;
|
|
goto out_unuse;
|
|
}
|
|
}
|
|
|
|
open = pde->proc_fops->open;
|
|
if (open)
|
|
rv = open(inode, file);
|
|
|
|
if (release) {
|
|
if (rv == 0) {
|
|
/* To know what to release. */
|
|
pdeo->file = file;
|
|
pdeo->closing = false;
|
|
pdeo->c = NULL;
|
|
spin_lock(&pde->pde_unload_lock);
|
|
list_add(&pdeo->lh, &pde->pde_openers);
|
|
spin_unlock(&pde->pde_unload_lock);
|
|
} else
|
|
kmem_cache_free(pde_opener_cache, pdeo);
|
|
}
|
|
|
|
out_unuse:
|
|
unuse_pde(pde);
|
|
return rv;
|
|
}
|
|
|
|
static int proc_reg_release(struct inode *inode, struct file *file)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(inode);
|
|
struct pde_opener *pdeo;
|
|
spin_lock(&pde->pde_unload_lock);
|
|
list_for_each_entry(pdeo, &pde->pde_openers, lh) {
|
|
if (pdeo->file == file) {
|
|
close_pdeo(pde, pdeo);
|
|
return 0;
|
|
}
|
|
}
|
|
spin_unlock(&pde->pde_unload_lock);
|
|
return 0;
|
|
}
|
|
|
|
static const struct file_operations proc_reg_file_ops = {
|
|
.llseek = proc_reg_llseek,
|
|
.read = proc_reg_read,
|
|
.write = proc_reg_write,
|
|
.poll = proc_reg_poll,
|
|
.unlocked_ioctl = proc_reg_unlocked_ioctl,
|
|
#ifdef CONFIG_COMPAT
|
|
.compat_ioctl = proc_reg_compat_ioctl,
|
|
#endif
|
|
.mmap = proc_reg_mmap,
|
|
.get_unmapped_area = proc_reg_get_unmapped_area,
|
|
.open = proc_reg_open,
|
|
.release = proc_reg_release,
|
|
};
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
static const struct file_operations proc_reg_file_ops_no_compat = {
|
|
.llseek = proc_reg_llseek,
|
|
.read = proc_reg_read,
|
|
.write = proc_reg_write,
|
|
.poll = proc_reg_poll,
|
|
.unlocked_ioctl = proc_reg_unlocked_ioctl,
|
|
.mmap = proc_reg_mmap,
|
|
.get_unmapped_area = proc_reg_get_unmapped_area,
|
|
.open = proc_reg_open,
|
|
.release = proc_reg_release,
|
|
};
|
|
#endif
|
|
|
|
static void proc_put_link(void *p)
|
|
{
|
|
unuse_pde(p);
|
|
}
|
|
|
|
static const char *proc_get_link(struct dentry *dentry,
|
|
struct inode *inode,
|
|
struct delayed_call *done)
|
|
{
|
|
struct proc_dir_entry *pde = PDE(inode);
|
|
if (!use_pde(pde))
|
|
return ERR_PTR(-EINVAL);
|
|
set_delayed_call(done, proc_put_link, pde);
|
|
return pde->data;
|
|
}
|
|
|
|
const struct inode_operations proc_link_inode_operations = {
|
|
.get_link = proc_get_link,
|
|
};
|
|
|
|
struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de)
|
|
{
|
|
struct inode *inode = new_inode_pseudo(sb);
|
|
|
|
if (inode) {
|
|
inode->i_ino = de->low_ino;
|
|
inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode);
|
|
PROC_I(inode)->pde = de;
|
|
|
|
if (is_empty_pde(de)) {
|
|
make_empty_dir_inode(inode);
|
|
return inode;
|
|
}
|
|
if (de->mode) {
|
|
inode->i_mode = de->mode;
|
|
inode->i_uid = de->uid;
|
|
inode->i_gid = de->gid;
|
|
}
|
|
if (de->size)
|
|
inode->i_size = de->size;
|
|
if (de->nlink)
|
|
set_nlink(inode, de->nlink);
|
|
WARN_ON(!de->proc_iops);
|
|
inode->i_op = de->proc_iops;
|
|
if (de->proc_fops) {
|
|
if (S_ISREG(inode->i_mode)) {
|
|
#ifdef CONFIG_COMPAT
|
|
if (!de->proc_fops->compat_ioctl)
|
|
inode->i_fop =
|
|
&proc_reg_file_ops_no_compat;
|
|
else
|
|
#endif
|
|
inode->i_fop = &proc_reg_file_ops;
|
|
} else {
|
|
inode->i_fop = de->proc_fops;
|
|
}
|
|
}
|
|
} else
|
|
pde_put(de);
|
|
return inode;
|
|
}
|
|
|
|
int proc_fill_super(struct super_block *s, void *data, int silent)
|
|
{
|
|
struct pid_namespace *ns = get_pid_ns(s->s_fs_info);
|
|
struct inode *root_inode;
|
|
int ret;
|
|
|
|
if (!proc_parse_options(data, ns))
|
|
return -EINVAL;
|
|
|
|
/* User space would break if executables or devices appear on proc */
|
|
s->s_iflags |= SB_I_USERNS_VISIBLE | SB_I_NOEXEC | SB_I_NODEV;
|
|
s->s_flags |= SB_NODIRATIME | SB_NOSUID | SB_NOEXEC;
|
|
s->s_blocksize = 1024;
|
|
s->s_blocksize_bits = 10;
|
|
s->s_magic = PROC_SUPER_MAGIC;
|
|
s->s_op = &proc_sops;
|
|
s->s_time_gran = 1;
|
|
|
|
/*
|
|
* procfs isn't actually a stacking filesystem; however, there is
|
|
* too much magic going on inside it to permit stacking things on
|
|
* top of it
|
|
*/
|
|
s->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
|
|
|
|
/* procfs dentries and inodes don't require IO to create */
|
|
s->s_shrink.seeks = 0;
|
|
|
|
pde_get(&proc_root);
|
|
root_inode = proc_get_inode(s, &proc_root);
|
|
if (!root_inode) {
|
|
pr_err("proc_fill_super: get root inode failed\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
s->s_root = d_make_root(root_inode);
|
|
if (!s->s_root) {
|
|
pr_err("proc_fill_super: allocate dentry failed\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
ret = proc_setup_self(s);
|
|
if (ret) {
|
|
return ret;
|
|
}
|
|
return proc_setup_thread_self(s);
|
|
}
|