mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 13:55:32 +00:00
![Michal Hocko](/assets/img/avatar_default.png)
Nils Holland and Klaus Ethgen have reported unexpected OOM killer invocations with 32b kernel starting with 4.8 kernels kworker/u4:5 invoked oom-killer: gfp_mask=0x2400840(GFP_NOFS|__GFP_NOFAIL), nodemask=0, order=0, oom_score_adj=0 kworker/u4:5 cpuset=/ mems_allowed=0 CPU: 1 PID: 2603 Comm: kworker/u4:5 Not tainted 4.9.0-gentoo #2 [...] Mem-Info: active_anon:58685 inactive_anon:90 isolated_anon:0 active_file:274324 inactive_file:281962 isolated_file:0 unevictable:0 dirty:649 writeback:0 unstable:0 slab_reclaimable:40662 slab_unreclaimable:17754 mapped:7382 shmem:202 pagetables:351 bounce:0 free:206736 free_pcp:332 free_cma:0 Node 0 active_anon:234740kB inactive_anon:360kB active_file:1097296kB inactive_file:1127848kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:29528kB dirty:2596kB writeback:0kB shmem:0kB shmem_thp: 0kB shmem_pmdmapped: 184320kB anon_thp: 808kB writeback_tmp:0kB unstable:0kB pages_scanned:0 all_unreclaimable? no DMA free:3952kB min:788kB low:984kB high:1180kB active_anon:0kB inactive_anon:0kB active_file:7316kB inactive_file:0kB unevictable:0kB writepending:96kB present:15992kB managed:15916kB mlocked:0kB slab_reclaimable:3200kB slab_unreclaimable:1408kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB lowmem_reserve[]: 0 813 3474 3474 Normal free:41332kB min:41368kB low:51708kB high:62048kB active_anon:0kB inactive_anon:0kB active_file:532748kB inactive_file:44kB unevictable:0kB writepending:24kB present:897016kB managed:836248kB mlocked:0kB slab_reclaimable:159448kB slab_unreclaimable:69608kB kernel_stack:1112kB pagetables:1404kB bounce:0kB free_pcp:528kB local_pcp:340kB free_cma:0kB lowmem_reserve[]: 0 0 21292 21292 HighMem free:781660kB min:512kB low:34356kB high:68200kB active_anon:234740kB inactive_anon:360kB active_file:557232kB inactive_file:1127804kB unevictable:0kB writepending:2592kB present:2725384kB managed:2725384kB mlocked:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB bounce:0kB free_pcp:800kB local_pcp:608kB free_cma:0kB the oom killer is clearly pre-mature because there there is still a lot of page cache in the zone Normal which should satisfy this lowmem request. Further debugging has shown that the reclaim cannot make any forward progress because the page cache is hidden in the active list which doesn't get rotated because inactive_list_is_low is not memcg aware. The code simply subtracts per-zone highmem counters from the respective memcg's lru sizes which doesn't make any sense. We can simply end up always seeing the resulting active and inactive counts 0 and return false. This issue is not limited to 32b kernels but in practice the effect on systems without CONFIG_HIGHMEM would be much harder to notice because we do not invoke the OOM killer for allocations requests targeting < ZONE_NORMAL. Fix the issue by tracking per zone lru page counts in mem_cgroup_per_node and subtract per-memcg highmem counts when memcg is enabled. Introduce helper lruvec_zone_lru_size which redirects to either zone counters or mem_cgroup_get_zone_lru_size when appropriate. We are losing empty LRU but non-zero lru size detection introduced byca707239e8
("mm: update_lru_size warn and reset bad lru_size") because of the inherent zone vs. node discrepancy. Fixes:f8d1a31163
("mm: consider whether to decivate based on eligible zones inactive ratio") Link: http://lkml.kernel.org/r/20170104100825.3729-1-mhocko@kernel.org Signed-off-by: Michal Hocko <mhocko@suse.com> Reported-by: Nils Holland <nholland@tisys.org> Tested-by: Nils Holland <nholland@tisys.org> Reported-by: Klaus Ethgen <Klaus@Ethgen.de> Acked-by: Minchan Kim <minchan@kernel.org> Acked-by: Mel Gorman <mgorman@suse.de> Acked-by: Johannes Weiner <hannes@cmpxchg.org> Reviewed-by: Vladimir Davydov <vdavydov.dev@gmail.com> Cc: <stable@vger.kernel.org> [4.8+] Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
901 lines
22 KiB
C
901 lines
22 KiB
C
/* memcontrol.h - Memory Controller
|
|
*
|
|
* Copyright IBM Corporation, 2007
|
|
* Author Balbir Singh <balbir@linux.vnet.ibm.com>
|
|
*
|
|
* Copyright 2007 OpenVZ SWsoft Inc
|
|
* Author: Pavel Emelianov <xemul@openvz.org>
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*/
|
|
|
|
#ifndef _LINUX_MEMCONTROL_H
|
|
#define _LINUX_MEMCONTROL_H
|
|
#include <linux/cgroup.h>
|
|
#include <linux/vm_event_item.h>
|
|
#include <linux/hardirq.h>
|
|
#include <linux/jump_label.h>
|
|
#include <linux/page_counter.h>
|
|
#include <linux/vmpressure.h>
|
|
#include <linux/eventfd.h>
|
|
#include <linux/mmzone.h>
|
|
#include <linux/writeback.h>
|
|
#include <linux/page-flags.h>
|
|
|
|
struct mem_cgroup;
|
|
struct page;
|
|
struct mm_struct;
|
|
struct kmem_cache;
|
|
|
|
/*
|
|
* The corresponding mem_cgroup_stat_names is defined in mm/memcontrol.c,
|
|
* These two lists should keep in accord with each other.
|
|
*/
|
|
enum mem_cgroup_stat_index {
|
|
/*
|
|
* For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
|
|
*/
|
|
MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
|
|
MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
|
|
MEM_CGROUP_STAT_RSS_HUGE, /* # of pages charged as anon huge */
|
|
MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
|
|
MEM_CGROUP_STAT_DIRTY, /* # of dirty pages in page cache */
|
|
MEM_CGROUP_STAT_WRITEBACK, /* # of pages under writeback */
|
|
MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
|
|
MEM_CGROUP_STAT_NSTATS,
|
|
/* default hierarchy stats */
|
|
MEMCG_KERNEL_STACK_KB = MEM_CGROUP_STAT_NSTATS,
|
|
MEMCG_SLAB_RECLAIMABLE,
|
|
MEMCG_SLAB_UNRECLAIMABLE,
|
|
MEMCG_SOCK,
|
|
MEMCG_NR_STAT,
|
|
};
|
|
|
|
struct mem_cgroup_reclaim_cookie {
|
|
pg_data_t *pgdat;
|
|
int priority;
|
|
unsigned int generation;
|
|
};
|
|
|
|
enum mem_cgroup_events_index {
|
|
MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
|
|
MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
|
|
MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
|
|
MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
|
|
MEM_CGROUP_EVENTS_NSTATS,
|
|
/* default hierarchy events */
|
|
MEMCG_LOW = MEM_CGROUP_EVENTS_NSTATS,
|
|
MEMCG_HIGH,
|
|
MEMCG_MAX,
|
|
MEMCG_OOM,
|
|
MEMCG_NR_EVENTS,
|
|
};
|
|
|
|
/*
|
|
* Per memcg event counter is incremented at every pagein/pageout. With THP,
|
|
* it will be incremated by the number of pages. This counter is used for
|
|
* for trigger some periodic events. This is straightforward and better
|
|
* than using jiffies etc. to handle periodic memcg event.
|
|
*/
|
|
enum mem_cgroup_events_target {
|
|
MEM_CGROUP_TARGET_THRESH,
|
|
MEM_CGROUP_TARGET_SOFTLIMIT,
|
|
MEM_CGROUP_TARGET_NUMAINFO,
|
|
MEM_CGROUP_NTARGETS,
|
|
};
|
|
|
|
#ifdef CONFIG_MEMCG
|
|
|
|
#define MEM_CGROUP_ID_SHIFT 16
|
|
#define MEM_CGROUP_ID_MAX USHRT_MAX
|
|
|
|
struct mem_cgroup_id {
|
|
int id;
|
|
atomic_t ref;
|
|
};
|
|
|
|
struct mem_cgroup_stat_cpu {
|
|
long count[MEMCG_NR_STAT];
|
|
unsigned long events[MEMCG_NR_EVENTS];
|
|
unsigned long nr_page_events;
|
|
unsigned long targets[MEM_CGROUP_NTARGETS];
|
|
};
|
|
|
|
struct mem_cgroup_reclaim_iter {
|
|
struct mem_cgroup *position;
|
|
/* scan generation, increased every round-trip */
|
|
unsigned int generation;
|
|
};
|
|
|
|
/*
|
|
* per-zone information in memory controller.
|
|
*/
|
|
struct mem_cgroup_per_node {
|
|
struct lruvec lruvec;
|
|
unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
|
|
|
|
struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
|
|
|
|
struct rb_node tree_node; /* RB tree node */
|
|
unsigned long usage_in_excess;/* Set to the value by which */
|
|
/* the soft limit is exceeded*/
|
|
bool on_tree;
|
|
struct mem_cgroup *memcg; /* Back pointer, we cannot */
|
|
/* use container_of */
|
|
};
|
|
|
|
struct mem_cgroup_threshold {
|
|
struct eventfd_ctx *eventfd;
|
|
unsigned long threshold;
|
|
};
|
|
|
|
/* For threshold */
|
|
struct mem_cgroup_threshold_ary {
|
|
/* An array index points to threshold just below or equal to usage. */
|
|
int current_threshold;
|
|
/* Size of entries[] */
|
|
unsigned int size;
|
|
/* Array of thresholds */
|
|
struct mem_cgroup_threshold entries[0];
|
|
};
|
|
|
|
struct mem_cgroup_thresholds {
|
|
/* Primary thresholds array */
|
|
struct mem_cgroup_threshold_ary *primary;
|
|
/*
|
|
* Spare threshold array.
|
|
* This is needed to make mem_cgroup_unregister_event() "never fail".
|
|
* It must be able to store at least primary->size - 1 entries.
|
|
*/
|
|
struct mem_cgroup_threshold_ary *spare;
|
|
};
|
|
|
|
enum memcg_kmem_state {
|
|
KMEM_NONE,
|
|
KMEM_ALLOCATED,
|
|
KMEM_ONLINE,
|
|
};
|
|
|
|
/*
|
|
* The memory controller data structure. The memory controller controls both
|
|
* page cache and RSS per cgroup. We would eventually like to provide
|
|
* statistics based on the statistics developed by Rik Van Riel for clock-pro,
|
|
* to help the administrator determine what knobs to tune.
|
|
*/
|
|
struct mem_cgroup {
|
|
struct cgroup_subsys_state css;
|
|
|
|
/* Private memcg ID. Used to ID objects that outlive the cgroup */
|
|
struct mem_cgroup_id id;
|
|
|
|
/* Accounted resources */
|
|
struct page_counter memory;
|
|
struct page_counter swap;
|
|
|
|
/* Legacy consumer-oriented counters */
|
|
struct page_counter memsw;
|
|
struct page_counter kmem;
|
|
struct page_counter tcpmem;
|
|
|
|
/* Normal memory consumption range */
|
|
unsigned long low;
|
|
unsigned long high;
|
|
|
|
/* Range enforcement for interrupt charges */
|
|
struct work_struct high_work;
|
|
|
|
unsigned long soft_limit;
|
|
|
|
/* vmpressure notifications */
|
|
struct vmpressure vmpressure;
|
|
|
|
/*
|
|
* Should the accounting and control be hierarchical, per subtree?
|
|
*/
|
|
bool use_hierarchy;
|
|
|
|
/* protected by memcg_oom_lock */
|
|
bool oom_lock;
|
|
int under_oom;
|
|
|
|
int swappiness;
|
|
/* OOM-Killer disable */
|
|
int oom_kill_disable;
|
|
|
|
/* handle for "memory.events" */
|
|
struct cgroup_file events_file;
|
|
|
|
/* protect arrays of thresholds */
|
|
struct mutex thresholds_lock;
|
|
|
|
/* thresholds for memory usage. RCU-protected */
|
|
struct mem_cgroup_thresholds thresholds;
|
|
|
|
/* thresholds for mem+swap usage. RCU-protected */
|
|
struct mem_cgroup_thresholds memsw_thresholds;
|
|
|
|
/* For oom notifier event fd */
|
|
struct list_head oom_notify;
|
|
|
|
/*
|
|
* Should we move charges of a task when a task is moved into this
|
|
* mem_cgroup ? And what type of charges should we move ?
|
|
*/
|
|
unsigned long move_charge_at_immigrate;
|
|
/*
|
|
* set > 0 if pages under this cgroup are moving to other cgroup.
|
|
*/
|
|
atomic_t moving_account;
|
|
/* taken only while moving_account > 0 */
|
|
spinlock_t move_lock;
|
|
struct task_struct *move_lock_task;
|
|
unsigned long move_lock_flags;
|
|
/*
|
|
* percpu counter.
|
|
*/
|
|
struct mem_cgroup_stat_cpu __percpu *stat;
|
|
|
|
unsigned long socket_pressure;
|
|
|
|
/* Legacy tcp memory accounting */
|
|
bool tcpmem_active;
|
|
int tcpmem_pressure;
|
|
|
|
#ifndef CONFIG_SLOB
|
|
/* Index in the kmem_cache->memcg_params.memcg_caches array */
|
|
int kmemcg_id;
|
|
enum memcg_kmem_state kmem_state;
|
|
#endif
|
|
|
|
int last_scanned_node;
|
|
#if MAX_NUMNODES > 1
|
|
nodemask_t scan_nodes;
|
|
atomic_t numainfo_events;
|
|
atomic_t numainfo_updating;
|
|
#endif
|
|
|
|
#ifdef CONFIG_CGROUP_WRITEBACK
|
|
struct list_head cgwb_list;
|
|
struct wb_domain cgwb_domain;
|
|
#endif
|
|
|
|
/* List of events which userspace want to receive */
|
|
struct list_head event_list;
|
|
spinlock_t event_list_lock;
|
|
|
|
struct mem_cgroup_per_node *nodeinfo[0];
|
|
/* WARNING: nodeinfo must be the last member here */
|
|
};
|
|
|
|
extern struct mem_cgroup *root_mem_cgroup;
|
|
|
|
static inline bool mem_cgroup_disabled(void)
|
|
{
|
|
return !cgroup_subsys_enabled(memory_cgrp_subsys);
|
|
}
|
|
|
|
/**
|
|
* mem_cgroup_events - count memory events against a cgroup
|
|
* @memcg: the memory cgroup
|
|
* @idx: the event index
|
|
* @nr: the number of events to account for
|
|
*/
|
|
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
|
|
enum mem_cgroup_events_index idx,
|
|
unsigned int nr)
|
|
{
|
|
this_cpu_add(memcg->stat->events[idx], nr);
|
|
cgroup_file_notify(&memcg->events_file);
|
|
}
|
|
|
|
bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
|
|
|
|
int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
|
|
gfp_t gfp_mask, struct mem_cgroup **memcgp,
|
|
bool compound);
|
|
void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
|
|
bool lrucare, bool compound);
|
|
void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
|
|
bool compound);
|
|
void mem_cgroup_uncharge(struct page *page);
|
|
void mem_cgroup_uncharge_list(struct list_head *page_list);
|
|
|
|
void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
|
|
|
|
static struct mem_cgroup_per_node *
|
|
mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
|
|
{
|
|
return memcg->nodeinfo[nid];
|
|
}
|
|
|
|
/**
|
|
* mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
|
|
* @node: node of the wanted lruvec
|
|
* @memcg: memcg of the wanted lruvec
|
|
*
|
|
* Returns the lru list vector holding pages for a given @node or a given
|
|
* @memcg and @zone. This can be the node lruvec, if the memory controller
|
|
* is disabled.
|
|
*/
|
|
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
|
|
struct mem_cgroup *memcg)
|
|
{
|
|
struct mem_cgroup_per_node *mz;
|
|
struct lruvec *lruvec;
|
|
|
|
if (mem_cgroup_disabled()) {
|
|
lruvec = node_lruvec(pgdat);
|
|
goto out;
|
|
}
|
|
|
|
mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
|
|
lruvec = &mz->lruvec;
|
|
out:
|
|
/*
|
|
* Since a node can be onlined after the mem_cgroup was created,
|
|
* we have to be prepared to initialize lruvec->pgdat here;
|
|
* and if offlined then reonlined, we need to reinitialize it.
|
|
*/
|
|
if (unlikely(lruvec->pgdat != pgdat))
|
|
lruvec->pgdat = pgdat;
|
|
return lruvec;
|
|
}
|
|
|
|
struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
|
|
|
|
bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
|
|
struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
|
|
|
|
static inline
|
|
struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
|
|
return css ? container_of(css, struct mem_cgroup, css) : NULL;
|
|
}
|
|
|
|
#define mem_cgroup_from_counter(counter, member) \
|
|
container_of(counter, struct mem_cgroup, member)
|
|
|
|
struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
|
|
struct mem_cgroup *,
|
|
struct mem_cgroup_reclaim_cookie *);
|
|
void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
|
|
int mem_cgroup_scan_tasks(struct mem_cgroup *,
|
|
int (*)(struct task_struct *, void *), void *);
|
|
|
|
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
|
|
{
|
|
if (mem_cgroup_disabled())
|
|
return 0;
|
|
|
|
return memcg->id.id;
|
|
}
|
|
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
|
|
|
|
/**
|
|
* parent_mem_cgroup - find the accounting parent of a memcg
|
|
* @memcg: memcg whose parent to find
|
|
*
|
|
* Returns the parent memcg, or NULL if this is the root or the memory
|
|
* controller is in legacy no-hierarchy mode.
|
|
*/
|
|
static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
|
|
{
|
|
if (!memcg->memory.parent)
|
|
return NULL;
|
|
return mem_cgroup_from_counter(memcg->memory.parent, memory);
|
|
}
|
|
|
|
static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
|
|
struct mem_cgroup *root)
|
|
{
|
|
if (root == memcg)
|
|
return true;
|
|
if (!root->use_hierarchy)
|
|
return false;
|
|
return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
|
|
}
|
|
|
|
static inline bool mm_match_cgroup(struct mm_struct *mm,
|
|
struct mem_cgroup *memcg)
|
|
{
|
|
struct mem_cgroup *task_memcg;
|
|
bool match = false;
|
|
|
|
rcu_read_lock();
|
|
task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
|
|
if (task_memcg)
|
|
match = mem_cgroup_is_descendant(task_memcg, memcg);
|
|
rcu_read_unlock();
|
|
return match;
|
|
}
|
|
|
|
struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
|
|
ino_t page_cgroup_ino(struct page *page);
|
|
|
|
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
|
|
{
|
|
if (mem_cgroup_disabled())
|
|
return true;
|
|
return !!(memcg->css.flags & CSS_ONLINE);
|
|
}
|
|
|
|
/*
|
|
* For memory reclaim.
|
|
*/
|
|
int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
|
|
|
|
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
|
|
int zid, int nr_pages);
|
|
|
|
unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
|
|
int nid, unsigned int lru_mask);
|
|
|
|
static inline
|
|
unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
|
|
{
|
|
struct mem_cgroup_per_node *mz;
|
|
unsigned long nr_pages = 0;
|
|
int zid;
|
|
|
|
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
|
|
for (zid = 0; zid < MAX_NR_ZONES; zid++)
|
|
nr_pages += mz->lru_zone_size[zid][lru];
|
|
return nr_pages;
|
|
}
|
|
|
|
static inline
|
|
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
|
|
enum lru_list lru, int zone_idx)
|
|
{
|
|
struct mem_cgroup_per_node *mz;
|
|
|
|
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
|
|
return mz->lru_zone_size[zone_idx][lru];
|
|
}
|
|
|
|
void mem_cgroup_handle_over_high(void);
|
|
|
|
unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg);
|
|
|
|
void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
|
|
struct task_struct *p);
|
|
|
|
static inline void mem_cgroup_oom_enable(void)
|
|
{
|
|
WARN_ON(current->memcg_may_oom);
|
|
current->memcg_may_oom = 1;
|
|
}
|
|
|
|
static inline void mem_cgroup_oom_disable(void)
|
|
{
|
|
WARN_ON(!current->memcg_may_oom);
|
|
current->memcg_may_oom = 0;
|
|
}
|
|
|
|
static inline bool task_in_memcg_oom(struct task_struct *p)
|
|
{
|
|
return p->memcg_in_oom;
|
|
}
|
|
|
|
bool mem_cgroup_oom_synchronize(bool wait);
|
|
|
|
#ifdef CONFIG_MEMCG_SWAP
|
|
extern int do_swap_account;
|
|
#endif
|
|
|
|
void lock_page_memcg(struct page *page);
|
|
void unlock_page_memcg(struct page *page);
|
|
|
|
/**
|
|
* mem_cgroup_update_page_stat - update page state statistics
|
|
* @page: the page
|
|
* @idx: page state item to account
|
|
* @val: number of pages (positive or negative)
|
|
*
|
|
* The @page must be locked or the caller must use lock_page_memcg()
|
|
* to prevent double accounting when the page is concurrently being
|
|
* moved to another memcg:
|
|
*
|
|
* lock_page(page) or lock_page_memcg(page)
|
|
* if (TestClearPageState(page))
|
|
* mem_cgroup_update_page_stat(page, state, -1);
|
|
* unlock_page(page) or unlock_page_memcg(page)
|
|
*/
|
|
static inline void mem_cgroup_update_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx, int val)
|
|
{
|
|
VM_BUG_ON(!(rcu_read_lock_held() || PageLocked(page)));
|
|
|
|
if (page->mem_cgroup)
|
|
this_cpu_add(page->mem_cgroup->stat->count[idx], val);
|
|
}
|
|
|
|
static inline void mem_cgroup_inc_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx)
|
|
{
|
|
mem_cgroup_update_page_stat(page, idx, 1);
|
|
}
|
|
|
|
static inline void mem_cgroup_dec_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx)
|
|
{
|
|
mem_cgroup_update_page_stat(page, idx, -1);
|
|
}
|
|
|
|
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
|
|
gfp_t gfp_mask,
|
|
unsigned long *total_scanned);
|
|
|
|
static inline void mem_cgroup_count_vm_event(struct mm_struct *mm,
|
|
enum vm_event_item idx)
|
|
{
|
|
struct mem_cgroup *memcg;
|
|
|
|
if (mem_cgroup_disabled())
|
|
return;
|
|
|
|
rcu_read_lock();
|
|
memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
|
|
if (unlikely(!memcg))
|
|
goto out;
|
|
|
|
switch (idx) {
|
|
case PGFAULT:
|
|
this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
|
|
break;
|
|
case PGMAJFAULT:
|
|
this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
|
|
break;
|
|
default:
|
|
BUG();
|
|
}
|
|
out:
|
|
rcu_read_unlock();
|
|
}
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
|
void mem_cgroup_split_huge_fixup(struct page *head);
|
|
#endif
|
|
|
|
#else /* CONFIG_MEMCG */
|
|
|
|
#define MEM_CGROUP_ID_SHIFT 0
|
|
#define MEM_CGROUP_ID_MAX 0
|
|
|
|
struct mem_cgroup;
|
|
|
|
static inline bool mem_cgroup_disabled(void)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline void mem_cgroup_events(struct mem_cgroup *memcg,
|
|
enum mem_cgroup_events_index idx,
|
|
unsigned int nr)
|
|
{
|
|
}
|
|
|
|
static inline bool mem_cgroup_low(struct mem_cgroup *root,
|
|
struct mem_cgroup *memcg)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
|
|
gfp_t gfp_mask,
|
|
struct mem_cgroup **memcgp,
|
|
bool compound)
|
|
{
|
|
*memcgp = NULL;
|
|
return 0;
|
|
}
|
|
|
|
static inline void mem_cgroup_commit_charge(struct page *page,
|
|
struct mem_cgroup *memcg,
|
|
bool lrucare, bool compound)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_cancel_charge(struct page *page,
|
|
struct mem_cgroup *memcg,
|
|
bool compound)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_uncharge(struct page *page)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_migrate(struct page *old, struct page *new)
|
|
{
|
|
}
|
|
|
|
static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
|
|
struct mem_cgroup *memcg)
|
|
{
|
|
return node_lruvec(pgdat);
|
|
}
|
|
|
|
static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
|
|
struct pglist_data *pgdat)
|
|
{
|
|
return &pgdat->lruvec;
|
|
}
|
|
|
|
static inline bool mm_match_cgroup(struct mm_struct *mm,
|
|
struct mem_cgroup *memcg)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline bool task_in_mem_cgroup(struct task_struct *task,
|
|
const struct mem_cgroup *memcg)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline struct mem_cgroup *
|
|
mem_cgroup_iter(struct mem_cgroup *root,
|
|
struct mem_cgroup *prev,
|
|
struct mem_cgroup_reclaim_cookie *reclaim)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
|
|
struct mem_cgroup *prev)
|
|
{
|
|
}
|
|
|
|
static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
|
|
int (*fn)(struct task_struct *, void *), void *arg)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
|
|
{
|
|
WARN_ON_ONCE(id);
|
|
/* XXX: This should always return root_mem_cgroup */
|
|
return NULL;
|
|
}
|
|
|
|
static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
static inline unsigned long
|
|
mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline
|
|
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
|
|
enum lru_list lru, int zone_idx)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline unsigned long
|
|
mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
|
|
int nid, unsigned int lru_mask)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void
|
|
mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
|
|
{
|
|
}
|
|
|
|
static inline void lock_page_memcg(struct page *page)
|
|
{
|
|
}
|
|
|
|
static inline void unlock_page_memcg(struct page *page)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_handle_over_high(void)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_oom_enable(void)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_oom_disable(void)
|
|
{
|
|
}
|
|
|
|
static inline bool task_in_memcg_oom(struct task_struct *p)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline bool mem_cgroup_oom_synchronize(bool wait)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline void mem_cgroup_inc_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx)
|
|
{
|
|
}
|
|
|
|
static inline void mem_cgroup_dec_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx)
|
|
{
|
|
}
|
|
|
|
static inline
|
|
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
|
|
gfp_t gfp_mask,
|
|
unsigned long *total_scanned)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void mem_cgroup_split_huge_fixup(struct page *head)
|
|
{
|
|
}
|
|
|
|
static inline
|
|
void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
|
|
{
|
|
}
|
|
#endif /* CONFIG_MEMCG */
|
|
|
|
#ifdef CONFIG_CGROUP_WRITEBACK
|
|
|
|
struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
|
|
struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
|
|
void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
|
|
unsigned long *pheadroom, unsigned long *pdirty,
|
|
unsigned long *pwriteback);
|
|
|
|
#else /* CONFIG_CGROUP_WRITEBACK */
|
|
|
|
static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
|
|
unsigned long *pfilepages,
|
|
unsigned long *pheadroom,
|
|
unsigned long *pdirty,
|
|
unsigned long *pwriteback)
|
|
{
|
|
}
|
|
|
|
#endif /* CONFIG_CGROUP_WRITEBACK */
|
|
|
|
struct sock;
|
|
bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
|
|
void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
|
|
#ifdef CONFIG_MEMCG
|
|
extern struct static_key_false memcg_sockets_enabled_key;
|
|
#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
|
|
void mem_cgroup_sk_alloc(struct sock *sk);
|
|
void mem_cgroup_sk_free(struct sock *sk);
|
|
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
|
|
{
|
|
if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
|
|
return true;
|
|
do {
|
|
if (time_before(jiffies, memcg->socket_pressure))
|
|
return true;
|
|
} while ((memcg = parent_mem_cgroup(memcg)));
|
|
return false;
|
|
}
|
|
#else
|
|
#define mem_cgroup_sockets_enabled 0
|
|
static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
|
|
static inline void mem_cgroup_sk_free(struct sock *sk) { };
|
|
static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
|
|
{
|
|
return false;
|
|
}
|
|
#endif
|
|
|
|
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
|
|
void memcg_kmem_put_cache(struct kmem_cache *cachep);
|
|
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
|
|
struct mem_cgroup *memcg);
|
|
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
|
|
void memcg_kmem_uncharge(struct page *page, int order);
|
|
|
|
#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
|
|
extern struct static_key_false memcg_kmem_enabled_key;
|
|
|
|
extern int memcg_nr_cache_ids;
|
|
void memcg_get_cache_ids(void);
|
|
void memcg_put_cache_ids(void);
|
|
|
|
/*
|
|
* Helper macro to loop through all memcg-specific caches. Callers must still
|
|
* check if the cache is valid (it is either valid or NULL).
|
|
* the slab_mutex must be held when looping through those caches
|
|
*/
|
|
#define for_each_memcg_cache_index(_idx) \
|
|
for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
|
|
|
|
static inline bool memcg_kmem_enabled(void)
|
|
{
|
|
return static_branch_unlikely(&memcg_kmem_enabled_key);
|
|
}
|
|
|
|
/*
|
|
* helper for accessing a memcg's index. It will be used as an index in the
|
|
* child cache array in kmem_cache, and also to derive its name. This function
|
|
* will return -1 when this is not a kmem-limited memcg.
|
|
*/
|
|
static inline int memcg_cache_id(struct mem_cgroup *memcg)
|
|
{
|
|
return memcg ? memcg->kmemcg_id : -1;
|
|
}
|
|
|
|
/**
|
|
* memcg_kmem_update_page_stat - update kmem page state statistics
|
|
* @page: the page
|
|
* @idx: page state item to account
|
|
* @val: number of pages (positive or negative)
|
|
*/
|
|
static inline void memcg_kmem_update_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx, int val)
|
|
{
|
|
if (memcg_kmem_enabled() && page->mem_cgroup)
|
|
this_cpu_add(page->mem_cgroup->stat->count[idx], val);
|
|
}
|
|
|
|
#else
|
|
#define for_each_memcg_cache_index(_idx) \
|
|
for (; NULL; )
|
|
|
|
static inline bool memcg_kmem_enabled(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline int memcg_cache_id(struct mem_cgroup *memcg)
|
|
{
|
|
return -1;
|
|
}
|
|
|
|
static inline void memcg_get_cache_ids(void)
|
|
{
|
|
}
|
|
|
|
static inline void memcg_put_cache_ids(void)
|
|
{
|
|
}
|
|
|
|
static inline void memcg_kmem_update_page_stat(struct page *page,
|
|
enum mem_cgroup_stat_index idx, int val)
|
|
{
|
|
}
|
|
#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
|
|
|
|
#endif /* _LINUX_MEMCONTROL_H */
|