mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-28 07:13:34 +00:00
286e04b8ed
Allocate and release memory to store obj_cgroup pointers for each non-root slab page. Reuse page->mem_cgroup pointer to store a pointer to the allocated space. This commit temporarily increases the memory footprint of the kernel memory accounting. To store obj_cgroup pointers we'll need a place for an objcg_pointer for each allocated object. However, the following patches in the series will enable sharing of slab pages between memory cgroups, which will dramatically increase the total slab utilization. And the final memory footprint will be significantly smaller than before. To distinguish between obj_cgroups and memcg pointers in case when it's not obvious which one is used (as in page_cgroup_ino()), let's always set the lowest bit in the obj_cgroup case. The original obj_cgroups pointer is marked to be ignored by kmemleak, which otherwise would report a memory leak for each allocated vector. Signed-off-by: Roman Gushchin <guro@fb.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Reviewed-by: Vlastimil Babka <vbabka@suse.cz> Reviewed-by: Shakeel Butt <shakeelb@google.com> Cc: Christoph Lameter <cl@linux.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Michal Hocko <mhocko@kernel.org> Cc: Tejun Heo <tj@kernel.org> Link: http://lkml.kernel.org/r/20200623174037.3951353-8-guro@fb.com Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
123 lines
3 KiB
C
123 lines
3 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_SLAB_DEF_H
|
|
#define _LINUX_SLAB_DEF_H
|
|
|
|
#include <linux/reciprocal_div.h>
|
|
|
|
/*
|
|
* Definitions unique to the original Linux SLAB allocator.
|
|
*/
|
|
|
|
struct kmem_cache {
|
|
struct array_cache __percpu *cpu_cache;
|
|
|
|
/* 1) Cache tunables. Protected by slab_mutex */
|
|
unsigned int batchcount;
|
|
unsigned int limit;
|
|
unsigned int shared;
|
|
|
|
unsigned int size;
|
|
struct reciprocal_value reciprocal_buffer_size;
|
|
/* 2) touched by every alloc & free from the backend */
|
|
|
|
slab_flags_t flags; /* constant flags */
|
|
unsigned int num; /* # of objs per slab */
|
|
|
|
/* 3) cache_grow/shrink */
|
|
/* order of pgs per slab (2^n) */
|
|
unsigned int gfporder;
|
|
|
|
/* force GFP flags, e.g. GFP_DMA */
|
|
gfp_t allocflags;
|
|
|
|
size_t colour; /* cache colouring range */
|
|
unsigned int colour_off; /* colour offset */
|
|
struct kmem_cache *freelist_cache;
|
|
unsigned int freelist_size;
|
|
|
|
/* constructor func */
|
|
void (*ctor)(void *obj);
|
|
|
|
/* 4) cache creation/removal */
|
|
const char *name;
|
|
struct list_head list;
|
|
int refcount;
|
|
int object_size;
|
|
int align;
|
|
|
|
/* 5) statistics */
|
|
#ifdef CONFIG_DEBUG_SLAB
|
|
unsigned long num_active;
|
|
unsigned long num_allocations;
|
|
unsigned long high_mark;
|
|
unsigned long grown;
|
|
unsigned long reaped;
|
|
unsigned long errors;
|
|
unsigned long max_freeable;
|
|
unsigned long node_allocs;
|
|
unsigned long node_frees;
|
|
unsigned long node_overflow;
|
|
atomic_t allochit;
|
|
atomic_t allocmiss;
|
|
atomic_t freehit;
|
|
atomic_t freemiss;
|
|
|
|
/*
|
|
* If debugging is enabled, then the allocator can add additional
|
|
* fields and/or padding to every object. 'size' contains the total
|
|
* object size including these internal fields, while 'obj_offset'
|
|
* and 'object_size' contain the offset to the user object and its
|
|
* size.
|
|
*/
|
|
int obj_offset;
|
|
#endif /* CONFIG_DEBUG_SLAB */
|
|
|
|
#ifdef CONFIG_MEMCG
|
|
struct memcg_cache_params memcg_params;
|
|
#endif
|
|
#ifdef CONFIG_KASAN
|
|
struct kasan_cache kasan_info;
|
|
#endif
|
|
|
|
#ifdef CONFIG_SLAB_FREELIST_RANDOM
|
|
unsigned int *random_seq;
|
|
#endif
|
|
|
|
unsigned int useroffset; /* Usercopy region offset */
|
|
unsigned int usersize; /* Usercopy region size */
|
|
|
|
struct kmem_cache_node *node[MAX_NUMNODES];
|
|
};
|
|
|
|
static inline void *nearest_obj(struct kmem_cache *cache, struct page *page,
|
|
void *x)
|
|
{
|
|
void *object = x - (x - page->s_mem) % cache->size;
|
|
void *last_object = page->s_mem + (cache->num - 1) * cache->size;
|
|
|
|
if (unlikely(object > last_object))
|
|
return last_object;
|
|
else
|
|
return object;
|
|
}
|
|
|
|
/*
|
|
* We want to avoid an expensive divide : (offset / cache->size)
|
|
* Using the fact that size is a constant for a particular cache,
|
|
* we can replace (offset / cache->size) by
|
|
* reciprocal_divide(offset, cache->reciprocal_buffer_size)
|
|
*/
|
|
static inline unsigned int obj_to_index(const struct kmem_cache *cache,
|
|
const struct page *page, void *obj)
|
|
{
|
|
u32 offset = (obj - page->s_mem);
|
|
return reciprocal_divide(offset, cache->reciprocal_buffer_size);
|
|
}
|
|
|
|
static inline int objs_per_slab_page(const struct kmem_cache *cache,
|
|
const struct page *page)
|
|
{
|
|
return cache->num;
|
|
}
|
|
|
|
#endif /* _LINUX_SLAB_DEF_H */
|