mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-23 02:51:47 +00:00
Merge branch 'slab/for-6.1/common_kmalloc' into slab/for-next
The "common kmalloc v4" series [1] by Hyeonggon Yoo.
- Improves the mm/slab_common.c wrappers to allow deleting duplicated
code between SLAB and SLUB.
- Large kmalloc() allocations in SLAB are passed to page allocator like
in SLUB, reducing number of kmalloc caches.
- Removes the {kmem_cache_alloc,kmalloc}_node variants of tracepoints,
node id parameter added to non-_node variants.
- 8 files changed, 341 insertions(+), 651 deletions(-)
[1] https://lore.kernel.org/all/20220817101826.236819-1-42.hyeyoo@gmail.com/
--
Merge resolves trivial conflict in mm/slub.c with commit 5373b8a09d
("kasan: call kasan_malloc() from __kmalloc_*track_caller()")
This commit is contained in:
commit
3662c13ec6
8 changed files with 366 additions and 680 deletions
|
@ -187,7 +187,6 @@ int kmem_cache_shrink(struct kmem_cache *s);
|
|||
void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __alloc_size(2);
|
||||
void kfree(const void *objp);
|
||||
void kfree_sensitive(const void *objp);
|
||||
size_t __ksize(const void *objp);
|
||||
size_t ksize(const void *objp);
|
||||
#ifdef CONFIG_PRINTK
|
||||
bool kmem_valid_obj(void *object);
|
||||
|
@ -243,27 +242,17 @@ static inline unsigned int arch_slab_minalign(void)
|
|||
|
||||
#ifdef CONFIG_SLAB
|
||||
/*
|
||||
* The largest kmalloc size supported by the SLAB allocators is
|
||||
* 32 megabyte (2^25) or the maximum allocatable page order if that is
|
||||
* less than 32 MB.
|
||||
*
|
||||
* WARNING: Its not easy to increase this value since the allocators have
|
||||
* to do various tricks to work around compiler limitations in order to
|
||||
* ensure proper constant folding.
|
||||
* SLAB and SLUB directly allocates requests fitting in to an order-1 page
|
||||
* (PAGE_SIZE*2). Larger requests are passed to the page allocator.
|
||||
*/
|
||||
#define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \
|
||||
(MAX_ORDER + PAGE_SHIFT - 1) : 25)
|
||||
#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH
|
||||
#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
|
||||
#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)
|
||||
#ifndef KMALLOC_SHIFT_LOW
|
||||
#define KMALLOC_SHIFT_LOW 5
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SLUB
|
||||
/*
|
||||
* SLUB directly allocates requests fitting in to an order-1 page
|
||||
* (PAGE_SIZE*2). Larger requests are passed to the page allocator.
|
||||
*/
|
||||
#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1)
|
||||
#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1)
|
||||
#ifndef KMALLOC_SHIFT_LOW
|
||||
|
@ -415,10 +404,6 @@ static __always_inline unsigned int __kmalloc_index(size_t size,
|
|||
if (size <= 512 * 1024) return 19;
|
||||
if (size <= 1024 * 1024) return 20;
|
||||
if (size <= 2 * 1024 * 1024) return 21;
|
||||
if (size <= 4 * 1024 * 1024) return 22;
|
||||
if (size <= 8 * 1024 * 1024) return 23;
|
||||
if (size <= 16 * 1024 * 1024) return 24;
|
||||
if (size <= 32 * 1024 * 1024) return 25;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
|
||||
BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
|
||||
|
@ -428,6 +413,7 @@ static __always_inline unsigned int __kmalloc_index(size_t size,
|
|||
/* Will never be reached. Needed because the compiler may complain */
|
||||
return -1;
|
||||
}
|
||||
static_assert(PAGE_SHIFT <= 20);
|
||||
#define kmalloc_index(s) __kmalloc_index(s, true)
|
||||
#endif /* !CONFIG_SLOB */
|
||||
|
||||
|
@ -456,42 +442,22 @@ static __always_inline void kfree_bulk(size_t size, void **p)
|
|||
kmem_cache_free_bulk(NULL, size, p);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment
|
||||
__alloc_size(1);
|
||||
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment
|
||||
__malloc;
|
||||
#else
|
||||
static __always_inline __alloc_size(1) void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
return __kmalloc(size, flags);
|
||||
}
|
||||
|
||||
static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node)
|
||||
{
|
||||
return kmem_cache_alloc(s, flags);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
extern void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)
|
||||
__assume_slab_alignment __alloc_size(3);
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t size) __assume_slab_alignment
|
||||
__alloc_size(4);
|
||||
#else
|
||||
static __always_inline __alloc_size(4) void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
|
||||
gfp_t gfpflags, int node, size_t size)
|
||||
{
|
||||
return kmem_cache_alloc_trace(s, gfpflags, size);
|
||||
}
|
||||
#endif /* CONFIG_NUMA */
|
||||
void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)
|
||||
__assume_kmalloc_alignment __alloc_size(3);
|
||||
|
||||
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t size) __assume_kmalloc_alignment
|
||||
__alloc_size(4);
|
||||
#else /* CONFIG_TRACING */
|
||||
static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_cache *s,
|
||||
gfp_t flags, size_t size)
|
||||
/* Save a function call when CONFIG_TRACING=n */
|
||||
static __always_inline __alloc_size(3)
|
||||
void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size)
|
||||
{
|
||||
void *ret = kmem_cache_alloc(s, flags);
|
||||
|
||||
|
@ -499,8 +465,9 @@ static __always_inline __alloc_size(3) void *kmem_cache_alloc_trace(struct kmem_
|
|||
return ret;
|
||||
}
|
||||
|
||||
static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t size)
|
||||
static __always_inline __alloc_size(4)
|
||||
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t size)
|
||||
{
|
||||
void *ret = kmem_cache_alloc_node(s, gfpflags, node);
|
||||
|
||||
|
@ -509,25 +476,11 @@ static __always_inline void *kmem_cache_alloc_node_trace(struct kmem_cache *s, g
|
|||
}
|
||||
#endif /* CONFIG_TRACING */
|
||||
|
||||
extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment
|
||||
__alloc_size(1);
|
||||
void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment
|
||||
__alloc_size(1);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
|
||||
__assume_page_alignment __alloc_size(1);
|
||||
#else
|
||||
static __always_inline __alloc_size(1) void *kmalloc_order_trace(size_t size, gfp_t flags,
|
||||
unsigned int order)
|
||||
{
|
||||
return kmalloc_order(size, flags, order);
|
||||
}
|
||||
#endif
|
||||
|
||||
static __always_inline __alloc_size(1) void *kmalloc_large(size_t size, gfp_t flags)
|
||||
{
|
||||
unsigned int order = get_order(size);
|
||||
return kmalloc_order_trace(size, flags, order);
|
||||
}
|
||||
void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment
|
||||
__alloc_size(1);
|
||||
|
||||
/**
|
||||
* kmalloc - allocate memory
|
||||
|
@ -597,7 +550,7 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
|
|||
if (!index)
|
||||
return ZERO_SIZE_PTR;
|
||||
|
||||
return kmem_cache_alloc_trace(
|
||||
return kmalloc_trace(
|
||||
kmalloc_caches[kmalloc_type(flags)][index],
|
||||
flags, size);
|
||||
#endif
|
||||
|
@ -605,23 +558,35 @@ static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags)
|
|||
return __kmalloc(size, flags);
|
||||
}
|
||||
|
||||
#ifndef CONFIG_SLOB
|
||||
static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
#ifndef CONFIG_SLOB
|
||||
if (__builtin_constant_p(size) &&
|
||||
size <= KMALLOC_MAX_CACHE_SIZE) {
|
||||
unsigned int i = kmalloc_index(size);
|
||||
if (__builtin_constant_p(size)) {
|
||||
unsigned int index;
|
||||
|
||||
if (!i)
|
||||
if (size > KMALLOC_MAX_CACHE_SIZE)
|
||||
return kmalloc_large_node(size, flags, node);
|
||||
|
||||
index = kmalloc_index(size);
|
||||
|
||||
if (!index)
|
||||
return ZERO_SIZE_PTR;
|
||||
|
||||
return kmem_cache_alloc_node_trace(
|
||||
kmalloc_caches[kmalloc_type(flags)][i],
|
||||
flags, node, size);
|
||||
return kmalloc_node_trace(
|
||||
kmalloc_caches[kmalloc_type(flags)][index],
|
||||
flags, node, size);
|
||||
}
|
||||
#endif
|
||||
return __kmalloc_node(size, flags, node);
|
||||
}
|
||||
#else
|
||||
static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
if (__builtin_constant_p(size) && size > KMALLOC_MAX_CACHE_SIZE)
|
||||
return kmalloc_large_node(size, flags, node);
|
||||
|
||||
return __kmalloc_node(size, flags, node);
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* kmalloc_array - allocate memory for an array.
|
||||
|
@ -671,6 +636,12 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag
|
|||
return kmalloc_array(n, size, flags | __GFP_ZERO);
|
||||
}
|
||||
|
||||
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
|
||||
unsigned long caller) __alloc_size(1);
|
||||
#define kmalloc_node_track_caller(size, flags, node) \
|
||||
__kmalloc_node_track_caller(size, flags, node, \
|
||||
_RET_IP_)
|
||||
|
||||
/*
|
||||
* kmalloc_track_caller is a special version of kmalloc that records the
|
||||
* calling function of the routine calling it for slab leak tracking instead
|
||||
|
@ -679,9 +650,9 @@ static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flag
|
|||
* allocator where we care about the real place the memory allocation
|
||||
* request comes from.
|
||||
*/
|
||||
extern void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller);
|
||||
#define kmalloc_track_caller(size, flags) \
|
||||
__kmalloc_track_caller(size, flags, _RET_IP_)
|
||||
__kmalloc_node_track_caller(size, flags, \
|
||||
NUMA_NO_NODE, _RET_IP_)
|
||||
|
||||
static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags,
|
||||
int node)
|
||||
|
@ -700,21 +671,6 @@ static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t
|
|||
return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
|
||||
}
|
||||
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
extern void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
|
||||
unsigned long caller) __alloc_size(1);
|
||||
#define kmalloc_node_track_caller(size, flags, node) \
|
||||
__kmalloc_node_track_caller(size, flags, node, \
|
||||
_RET_IP_)
|
||||
|
||||
#else /* CONFIG_NUMA */
|
||||
|
||||
#define kmalloc_node_track_caller(size, flags, node) \
|
||||
kmalloc_track_caller(size, flags)
|
||||
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
/*
|
||||
* Shortcuts
|
||||
*/
|
||||
|
|
|
@ -9,73 +9,15 @@
|
|||
#include <linux/tracepoint.h>
|
||||
#include <trace/events/mmflags.h>
|
||||
|
||||
DECLARE_EVENT_CLASS(kmem_alloc,
|
||||
TRACE_EVENT(kmem_cache_alloc,
|
||||
|
||||
TP_PROTO(unsigned long call_site,
|
||||
const void *ptr,
|
||||
struct kmem_cache *s,
|
||||
size_t bytes_req,
|
||||
size_t bytes_alloc,
|
||||
gfp_t gfp_flags),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field( unsigned long, call_site )
|
||||
__field( const void *, ptr )
|
||||
__field( size_t, bytes_req )
|
||||
__field( size_t, bytes_alloc )
|
||||
__field( unsigned long, gfp_flags )
|
||||
__field( bool, accounted )
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->call_site = call_site;
|
||||
__entry->ptr = ptr;
|
||||
__entry->bytes_req = bytes_req;
|
||||
__entry->bytes_alloc = bytes_alloc;
|
||||
__entry->gfp_flags = (__force unsigned long)gfp_flags;
|
||||
__entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ?
|
||||
((gfp_flags & __GFP_ACCOUNT) ||
|
||||
(s && s->flags & SLAB_ACCOUNT)) : false;
|
||||
),
|
||||
|
||||
TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s accounted=%s",
|
||||
(void *)__entry->call_site,
|
||||
__entry->ptr,
|
||||
__entry->bytes_req,
|
||||
__entry->bytes_alloc,
|
||||
show_gfp_flags(__entry->gfp_flags),
|
||||
__entry->accounted ? "true" : "false")
|
||||
);
|
||||
|
||||
DEFINE_EVENT(kmem_alloc, kmalloc,
|
||||
|
||||
TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,
|
||||
size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)
|
||||
);
|
||||
|
||||
DEFINE_EVENT(kmem_alloc, kmem_cache_alloc,
|
||||
|
||||
TP_PROTO(unsigned long call_site, const void *ptr, struct kmem_cache *s,
|
||||
size_t bytes_req, size_t bytes_alloc, gfp_t gfp_flags),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags)
|
||||
);
|
||||
|
||||
DECLARE_EVENT_CLASS(kmem_alloc_node,
|
||||
|
||||
TP_PROTO(unsigned long call_site,
|
||||
const void *ptr,
|
||||
struct kmem_cache *s,
|
||||
size_t bytes_req,
|
||||
size_t bytes_alloc,
|
||||
gfp_t gfp_flags,
|
||||
int node),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node),
|
||||
TP_ARGS(call_site, ptr, s, gfp_flags, node),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field( unsigned long, call_site )
|
||||
|
@ -90,13 +32,13 @@ DECLARE_EVENT_CLASS(kmem_alloc_node,
|
|||
TP_fast_assign(
|
||||
__entry->call_site = call_site;
|
||||
__entry->ptr = ptr;
|
||||
__entry->bytes_req = bytes_req;
|
||||
__entry->bytes_alloc = bytes_alloc;
|
||||
__entry->bytes_req = s->object_size;
|
||||
__entry->bytes_alloc = s->size;
|
||||
__entry->gfp_flags = (__force unsigned long)gfp_flags;
|
||||
__entry->node = node;
|
||||
__entry->accounted = IS_ENABLED(CONFIG_MEMCG_KMEM) ?
|
||||
((gfp_flags & __GFP_ACCOUNT) ||
|
||||
(s && s->flags & SLAB_ACCOUNT)) : false;
|
||||
(s->flags & SLAB_ACCOUNT)) : false;
|
||||
),
|
||||
|
||||
TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s",
|
||||
|
@ -109,22 +51,44 @@ DECLARE_EVENT_CLASS(kmem_alloc_node,
|
|||
__entry->accounted ? "true" : "false")
|
||||
);
|
||||
|
||||
DEFINE_EVENT(kmem_alloc_node, kmalloc_node,
|
||||
TRACE_EVENT(kmalloc,
|
||||
|
||||
TP_PROTO(unsigned long call_site, const void *ptr,
|
||||
struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,
|
||||
gfp_t gfp_flags, int node),
|
||||
TP_PROTO(unsigned long call_site,
|
||||
const void *ptr,
|
||||
size_t bytes_req,
|
||||
size_t bytes_alloc,
|
||||
gfp_t gfp_flags,
|
||||
int node),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)
|
||||
);
|
||||
TP_ARGS(call_site, ptr, bytes_req, bytes_alloc, gfp_flags, node),
|
||||
|
||||
DEFINE_EVENT(kmem_alloc_node, kmem_cache_alloc_node,
|
||||
TP_STRUCT__entry(
|
||||
__field( unsigned long, call_site )
|
||||
__field( const void *, ptr )
|
||||
__field( size_t, bytes_req )
|
||||
__field( size_t, bytes_alloc )
|
||||
__field( unsigned long, gfp_flags )
|
||||
__field( int, node )
|
||||
),
|
||||
|
||||
TP_PROTO(unsigned long call_site, const void *ptr,
|
||||
struct kmem_cache *s, size_t bytes_req, size_t bytes_alloc,
|
||||
gfp_t gfp_flags, int node),
|
||||
TP_fast_assign(
|
||||
__entry->call_site = call_site;
|
||||
__entry->ptr = ptr;
|
||||
__entry->bytes_req = bytes_req;
|
||||
__entry->bytes_alloc = bytes_alloc;
|
||||
__entry->gfp_flags = (__force unsigned long)gfp_flags;
|
||||
__entry->node = node;
|
||||
),
|
||||
|
||||
TP_ARGS(call_site, ptr, s, bytes_req, bytes_alloc, gfp_flags, node)
|
||||
TP_printk("call_site=%pS ptr=%p bytes_req=%zu bytes_alloc=%zu gfp_flags=%s node=%d accounted=%s",
|
||||
(void *)__entry->call_site,
|
||||
__entry->ptr,
|
||||
__entry->bytes_req,
|
||||
__entry->bytes_alloc,
|
||||
show_gfp_flags(__entry->gfp_flags),
|
||||
__entry->node,
|
||||
(IS_ENABLED(CONFIG_MEMCG_KMEM) &&
|
||||
(__entry->gfp_flags & (__force unsigned long)__GFP_ACCOUNT)) ? "true" : "false")
|
||||
);
|
||||
|
||||
TRACE_EVENT(kfree,
|
||||
|
@ -149,20 +113,20 @@ TRACE_EVENT(kfree,
|
|||
|
||||
TRACE_EVENT(kmem_cache_free,
|
||||
|
||||
TP_PROTO(unsigned long call_site, const void *ptr, const char *name),
|
||||
TP_PROTO(unsigned long call_site, const void *ptr, const struct kmem_cache *s),
|
||||
|
||||
TP_ARGS(call_site, ptr, name),
|
||||
TP_ARGS(call_site, ptr, s),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field( unsigned long, call_site )
|
||||
__field( const void *, ptr )
|
||||
__string( name, name )
|
||||
__string( name, s->name )
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->call_site = call_site;
|
||||
__entry->ptr = ptr;
|
||||
__assign_str(name, name);
|
||||
__assign_str(name, s->name);
|
||||
),
|
||||
|
||||
TP_printk("call_site=%pS ptr=%p name=%s",
|
||||
|
|
|
@ -86,6 +86,7 @@ static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries
|
|||
/* Also the *_bulk() variants by only checking prefixes. */
|
||||
if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") ||
|
||||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") ||
|
||||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") ||
|
||||
str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") ||
|
||||
str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc"))
|
||||
goto found;
|
||||
|
|
305
mm/slab.c
305
mm/slab.c
|
@ -3181,84 +3181,46 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
|
|||
}
|
||||
|
||||
static __always_inline void *
|
||||
slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
|
||||
unsigned long caller)
|
||||
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid)
|
||||
{
|
||||
unsigned long save_flags;
|
||||
void *ptr;
|
||||
void *objp = NULL;
|
||||
int slab_node = numa_mem_id();
|
||||
struct obj_cgroup *objcg = NULL;
|
||||
bool init = false;
|
||||
|
||||
flags &= gfp_allowed_mask;
|
||||
cachep = slab_pre_alloc_hook(cachep, NULL, &objcg, 1, flags);
|
||||
if (unlikely(!cachep))
|
||||
return NULL;
|
||||
|
||||
ptr = kfence_alloc(cachep, orig_size, flags);
|
||||
if (unlikely(ptr))
|
||||
goto out_hooks;
|
||||
|
||||
local_irq_save(save_flags);
|
||||
|
||||
if (nodeid == NUMA_NO_NODE)
|
||||
nodeid = slab_node;
|
||||
|
||||
if (unlikely(!get_node(cachep, nodeid))) {
|
||||
/* Node not bootstrapped yet */
|
||||
ptr = fallback_alloc(cachep, flags);
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (nodeid == slab_node) {
|
||||
if (nodeid == NUMA_NO_NODE) {
|
||||
if (current->mempolicy || cpuset_do_slab_mem_spread()) {
|
||||
objp = alternate_node_alloc(cachep, flags);
|
||||
if (objp)
|
||||
goto out;
|
||||
}
|
||||
/*
|
||||
* Use the locally cached objects if possible.
|
||||
* However ____cache_alloc does not allow fallback
|
||||
* to other nodes. It may fail while we still have
|
||||
* objects on other nodes available.
|
||||
*/
|
||||
ptr = ____cache_alloc(cachep, flags);
|
||||
if (ptr)
|
||||
goto out;
|
||||
objp = ____cache_alloc(cachep, flags);
|
||||
nodeid = slab_node;
|
||||
} else if (nodeid == slab_node) {
|
||||
objp = ____cache_alloc(cachep, flags);
|
||||
} else if (!get_node(cachep, nodeid)) {
|
||||
/* Node not bootstrapped yet */
|
||||
objp = fallback_alloc(cachep, flags);
|
||||
goto out;
|
||||
}
|
||||
/* ___cache_alloc_node can fall back to other nodes */
|
||||
ptr = ____cache_alloc_node(cachep, flags, nodeid);
|
||||
out:
|
||||
local_irq_restore(save_flags);
|
||||
ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
|
||||
init = slab_want_init_on_alloc(flags, cachep);
|
||||
|
||||
out_hooks:
|
||||
slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static __always_inline void *
|
||||
__do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
|
||||
{
|
||||
void *objp;
|
||||
|
||||
if (current->mempolicy || cpuset_do_slab_mem_spread()) {
|
||||
objp = alternate_node_alloc(cache, flags);
|
||||
if (objp)
|
||||
goto out;
|
||||
}
|
||||
objp = ____cache_alloc(cache, flags);
|
||||
|
||||
/*
|
||||
* We may just have run out of memory on the local node.
|
||||
* ____cache_alloc_node() knows how to locate memory on other nodes
|
||||
*/
|
||||
if (!objp)
|
||||
objp = ____cache_alloc_node(cache, flags, numa_mem_id());
|
||||
|
||||
objp = ____cache_alloc_node(cachep, flags, nodeid);
|
||||
out:
|
||||
return objp;
|
||||
}
|
||||
#else
|
||||
|
||||
static __always_inline void *
|
||||
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
||||
__do_cache_alloc(struct kmem_cache *cachep, gfp_t flags, int nodeid __maybe_unused)
|
||||
{
|
||||
return ____cache_alloc(cachep, flags);
|
||||
}
|
||||
|
@ -3266,8 +3228,8 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
|
|||
#endif /* CONFIG_NUMA */
|
||||
|
||||
static __always_inline void *
|
||||
slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,
|
||||
size_t orig_size, unsigned long caller)
|
||||
slab_alloc_node(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,
|
||||
int nodeid, size_t orig_size, unsigned long caller)
|
||||
{
|
||||
unsigned long save_flags;
|
||||
void *objp;
|
||||
|
@ -3284,7 +3246,7 @@ slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,
|
|||
goto out;
|
||||
|
||||
local_irq_save(save_flags);
|
||||
objp = __do_cache_alloc(cachep, flags);
|
||||
objp = __do_cache_alloc(cachep, flags, nodeid);
|
||||
local_irq_restore(save_flags);
|
||||
objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
|
||||
prefetchw(objp);
|
||||
|
@ -3295,6 +3257,14 @@ slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,
|
|||
return objp;
|
||||
}
|
||||
|
||||
static __always_inline void *
|
||||
slab_alloc(struct kmem_cache *cachep, struct list_lru *lru, gfp_t flags,
|
||||
size_t orig_size, unsigned long caller)
|
||||
{
|
||||
return slab_alloc_node(cachep, lru, flags, NUMA_NO_NODE, orig_size,
|
||||
caller);
|
||||
}
|
||||
|
||||
/*
|
||||
* Caller needs to acquire correct kmem_cache_node's list_lock
|
||||
* @list: List of detached free slabs should be freed by caller
|
||||
|
@ -3470,8 +3440,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru,
|
|||
{
|
||||
void *ret = slab_alloc(cachep, lru, flags, cachep->object_size, _RET_IP_);
|
||||
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, cachep,
|
||||
cachep->object_size, cachep->size, flags);
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, NUMA_NO_NODE);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
@ -3521,7 +3490,8 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
|
|||
|
||||
local_irq_disable();
|
||||
for (i = 0; i < size; i++) {
|
||||
void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
|
||||
void *objp = kfence_alloc(s, s->object_size, flags) ?:
|
||||
__do_cache_alloc(s, flags, NUMA_NO_NODE);
|
||||
|
||||
if (unlikely(!objp))
|
||||
goto error;
|
||||
|
@ -3548,23 +3518,6 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
|
|||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_bulk);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *
|
||||
kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
ret = slab_alloc(cachep, NULL, flags, size, _RET_IP_);
|
||||
|
||||
ret = kasan_kmalloc(cachep, ret, size, flags);
|
||||
trace_kmalloc(_RET_IP_, ret, cachep,
|
||||
size, cachep->size, flags);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_trace);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
/**
|
||||
* kmem_cache_alloc_node - Allocate an object on the specified node
|
||||
* @cachep: The cache to allocate from.
|
||||
|
@ -3580,65 +3533,21 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace);
|
|||
*/
|
||||
void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
|
||||
{
|
||||
void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
|
||||
void *ret = slab_alloc_node(cachep, NULL, flags, nodeid, cachep->object_size, _RET_IP_);
|
||||
|
||||
trace_kmem_cache_alloc_node(_RET_IP_, ret, cachep,
|
||||
cachep->object_size, cachep->size,
|
||||
flags, nodeid);
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, cachep, flags, nodeid);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_node);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
|
||||
gfp_t flags,
|
||||
int nodeid,
|
||||
size_t size)
|
||||
void *__kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
|
||||
int nodeid, size_t orig_size,
|
||||
unsigned long caller)
|
||||
{
|
||||
void *ret;
|
||||
|
||||
ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
|
||||
|
||||
ret = kasan_kmalloc(cachep, ret, size, flags);
|
||||
trace_kmalloc_node(_RET_IP_, ret, cachep,
|
||||
size, cachep->size,
|
||||
flags, nodeid);
|
||||
return ret;
|
||||
return slab_alloc_node(cachep, NULL, flags, nodeid,
|
||||
orig_size, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
|
||||
#endif
|
||||
|
||||
static __always_inline void *
|
||||
__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
|
||||
{
|
||||
struct kmem_cache *cachep;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
|
||||
return NULL;
|
||||
cachep = kmalloc_slab(size, flags);
|
||||
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
||||
return cachep;
|
||||
ret = kmem_cache_alloc_node_trace(cachep, flags, node, size);
|
||||
ret = kasan_kmalloc(cachep, ret, size, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
return __do_kmalloc_node(size, flags, node, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node);
|
||||
|
||||
void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
|
||||
int node, unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc_node(size, flags, node, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
#ifdef CONFIG_PRINTK
|
||||
void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
|
||||
|
@ -3662,45 +3571,25 @@ void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
|
|||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* __do_kmalloc - allocate memory
|
||||
* @size: how many bytes of memory are required.
|
||||
* @flags: the type of memory to allocate (see kmalloc).
|
||||
* @caller: function caller for debug tracking of the caller
|
||||
*
|
||||
* Return: pointer to the allocated memory or %NULL in case of error
|
||||
*/
|
||||
static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
|
||||
unsigned long caller)
|
||||
static __always_inline
|
||||
void __do_kmem_cache_free(struct kmem_cache *cachep, void *objp,
|
||||
unsigned long caller)
|
||||
{
|
||||
struct kmem_cache *cachep;
|
||||
void *ret;
|
||||
unsigned long flags;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
|
||||
return NULL;
|
||||
cachep = kmalloc_slab(size, flags);
|
||||
if (unlikely(ZERO_OR_NULL_PTR(cachep)))
|
||||
return cachep;
|
||||
ret = slab_alloc(cachep, NULL, flags, size, caller);
|
||||
|
||||
ret = kasan_kmalloc(cachep, ret, size, flags);
|
||||
trace_kmalloc(caller, ret, cachep,
|
||||
size, cachep->size, flags);
|
||||
|
||||
return ret;
|
||||
local_irq_save(flags);
|
||||
debug_check_no_locks_freed(objp, cachep->object_size);
|
||||
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
|
||||
debug_check_no_obj_freed(objp, cachep->object_size);
|
||||
__cache_free(cachep, objp, caller);
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
|
||||
void *__kmalloc(size_t size, gfp_t flags)
|
||||
void __kmem_cache_free(struct kmem_cache *cachep, void *objp,
|
||||
unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc(size, flags, _RET_IP_);
|
||||
__do_kmem_cache_free(cachep, objp, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc);
|
||||
|
||||
void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc(size, flags, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_track_caller);
|
||||
|
||||
/**
|
||||
* kmem_cache_free - Deallocate an object
|
||||
|
@ -3712,34 +3601,38 @@ EXPORT_SYMBOL(__kmalloc_track_caller);
|
|||
*/
|
||||
void kmem_cache_free(struct kmem_cache *cachep, void *objp)
|
||||
{
|
||||
unsigned long flags;
|
||||
cachep = cache_from_obj(cachep, objp);
|
||||
if (!cachep)
|
||||
return;
|
||||
|
||||
trace_kmem_cache_free(_RET_IP_, objp, cachep->name);
|
||||
local_irq_save(flags);
|
||||
debug_check_no_locks_freed(objp, cachep->object_size);
|
||||
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
|
||||
debug_check_no_obj_freed(objp, cachep->object_size);
|
||||
__cache_free(cachep, objp, _RET_IP_);
|
||||
local_irq_restore(flags);
|
||||
trace_kmem_cache_free(_RET_IP_, objp, cachep);
|
||||
__do_kmem_cache_free(cachep, objp, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_free);
|
||||
|
||||
void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
size_t i;
|
||||
|
||||
local_irq_disable();
|
||||
for (i = 0; i < size; i++) {
|
||||
for (int i = 0; i < size; i++) {
|
||||
void *objp = p[i];
|
||||
struct kmem_cache *s;
|
||||
|
||||
if (!orig_s) /* called via kfree_bulk */
|
||||
s = virt_to_cache(objp);
|
||||
else
|
||||
if (!orig_s) {
|
||||
struct folio *folio = virt_to_folio(objp);
|
||||
|
||||
/* called via kfree_bulk */
|
||||
if (!folio_test_slab(folio)) {
|
||||
local_irq_enable();
|
||||
free_large_kmalloc(folio, objp);
|
||||
local_irq_disable();
|
||||
continue;
|
||||
}
|
||||
s = folio_slab(folio)->slab_cache;
|
||||
} else {
|
||||
s = cache_from_obj(orig_s, objp);
|
||||
}
|
||||
|
||||
if (!s)
|
||||
continue;
|
||||
|
||||
|
@ -3755,39 +3648,6 @@ void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
|
|||
}
|
||||
EXPORT_SYMBOL(kmem_cache_free_bulk);
|
||||
|
||||
/**
|
||||
* kfree - free previously allocated memory
|
||||
* @objp: pointer returned by kmalloc.
|
||||
*
|
||||
* If @objp is NULL, no operation is performed.
|
||||
*
|
||||
* Don't free memory not originally allocated by kmalloc()
|
||||
* or you will run into trouble.
|
||||
*/
|
||||
void kfree(const void *objp)
|
||||
{
|
||||
struct kmem_cache *c;
|
||||
unsigned long flags;
|
||||
|
||||
trace_kfree(_RET_IP_, objp);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(objp)))
|
||||
return;
|
||||
local_irq_save(flags);
|
||||
kfree_debugcheck(objp);
|
||||
c = virt_to_cache(objp);
|
||||
if (!c) {
|
||||
local_irq_restore(flags);
|
||||
return;
|
||||
}
|
||||
debug_check_no_locks_freed(objp, c->object_size);
|
||||
|
||||
debug_check_no_obj_freed(objp, c->object_size);
|
||||
__cache_free(c, (void *)objp, _RET_IP_);
|
||||
local_irq_restore(flags);
|
||||
}
|
||||
EXPORT_SYMBOL(kfree);
|
||||
|
||||
/*
|
||||
* This initializes kmem_cache_node or resizes various caches for all nodes.
|
||||
*/
|
||||
|
@ -4190,28 +4050,3 @@ void __check_heap_object(const void *ptr, unsigned long n,
|
|||
usercopy_abort("SLAB object", cachep->name, to_user, offset, n);
|
||||
}
|
||||
#endif /* CONFIG_HARDENED_USERCOPY */
|
||||
|
||||
/**
|
||||
* __ksize -- Uninstrumented ksize.
|
||||
* @objp: pointer to the object
|
||||
*
|
||||
* Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
|
||||
* safety checks as ksize() with KASAN instrumentation enabled.
|
||||
*
|
||||
* Return: size of the actual memory used by @objp in bytes
|
||||
*/
|
||||
size_t __ksize(const void *objp)
|
||||
{
|
||||
struct kmem_cache *c;
|
||||
size_t size;
|
||||
|
||||
BUG_ON(!objp);
|
||||
if (unlikely(objp == ZERO_SIZE_PTR))
|
||||
return 0;
|
||||
|
||||
c = virt_to_cache(objp);
|
||||
size = c ? c->object_size : 0;
|
||||
|
||||
return size;
|
||||
}
|
||||
EXPORT_SYMBOL(__ksize);
|
||||
|
|
10
mm/slab.h
10
mm/slab.h
|
@ -273,6 +273,11 @@ void create_kmalloc_caches(slab_flags_t);
|
|||
|
||||
/* Find the kmalloc slab corresponding for a certain size */
|
||||
struct kmem_cache *kmalloc_slab(size_t, gfp_t);
|
||||
|
||||
void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t orig_size,
|
||||
unsigned long caller);
|
||||
void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller);
|
||||
#endif
|
||||
|
||||
gfp_t kmalloc_fix_flags(gfp_t flags);
|
||||
|
@ -658,8 +663,13 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
|
|||
print_tracking(cachep, x);
|
||||
return cachep;
|
||||
}
|
||||
|
||||
void free_large_kmalloc(struct folio *folio, void *object);
|
||||
|
||||
#endif /* CONFIG_SLOB */
|
||||
|
||||
size_t __ksize(const void *objp);
|
||||
|
||||
static inline size_t slab_ksize(const struct kmem_cache *s)
|
||||
{
|
||||
#ifndef CONFIG_SLUB
|
||||
|
|
197
mm/slab_common.c
197
mm/slab_common.c
|
@ -756,8 +756,8 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
|
|||
|
||||
/*
|
||||
* kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time.
|
||||
* kmalloc_index() supports up to 2^25=32MB, so the final entry of the table is
|
||||
* kmalloc-32M.
|
||||
* kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is
|
||||
* kmalloc-2M.
|
||||
*/
|
||||
const struct kmalloc_info_struct kmalloc_info[] __initconst = {
|
||||
INIT_KMALLOC_INFO(0, 0),
|
||||
|
@ -781,11 +781,7 @@ const struct kmalloc_info_struct kmalloc_info[] __initconst = {
|
|||
INIT_KMALLOC_INFO(262144, 256k),
|
||||
INIT_KMALLOC_INFO(524288, 512k),
|
||||
INIT_KMALLOC_INFO(1048576, 1M),
|
||||
INIT_KMALLOC_INFO(2097152, 2M),
|
||||
INIT_KMALLOC_INFO(4194304, 4M),
|
||||
INIT_KMALLOC_INFO(8388608, 8M),
|
||||
INIT_KMALLOC_INFO(16777216, 16M),
|
||||
INIT_KMALLOC_INFO(33554432, 32M)
|
||||
INIT_KMALLOC_INFO(2097152, 2M)
|
||||
};
|
||||
|
||||
/*
|
||||
|
@ -898,6 +894,144 @@ void __init create_kmalloc_caches(slab_flags_t flags)
|
|||
/* Kmalloc array is now usable */
|
||||
slab_state = UP;
|
||||
}
|
||||
|
||||
void free_large_kmalloc(struct folio *folio, void *object)
|
||||
{
|
||||
unsigned int order = folio_order(folio);
|
||||
|
||||
if (WARN_ON_ONCE(order == 0))
|
||||
pr_warn_once("object pointer: 0x%p\n", object);
|
||||
|
||||
kmemleak_free(object);
|
||||
kasan_kfree_large(object);
|
||||
|
||||
mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
|
||||
-(PAGE_SIZE << order));
|
||||
__free_pages(folio_page(folio, 0), order);
|
||||
}
|
||||
|
||||
static void *__kmalloc_large_node(size_t size, gfp_t flags, int node);
|
||||
static __always_inline
|
||||
void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
|
||||
ret = __kmalloc_large_node(size, flags, node);
|
||||
trace_kmalloc(_RET_IP_, ret, size,
|
||||
PAGE_SIZE << get_order(size), flags, node);
|
||||
return ret;
|
||||
}
|
||||
|
||||
s = kmalloc_slab(size, flags);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(s)))
|
||||
return s;
|
||||
|
||||
ret = __kmem_cache_alloc_node(s, flags, node, size, caller);
|
||||
ret = kasan_kmalloc(s, ret, size, flags);
|
||||
trace_kmalloc(_RET_IP_, ret, size, s->size, flags, node);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
return __do_kmalloc_node(size, flags, node, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node);
|
||||
|
||||
void *__kmalloc(size_t size, gfp_t flags)
|
||||
{
|
||||
return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc);
|
||||
|
||||
void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
|
||||
int node, unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc_node(size, flags, node, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
||||
|
||||
/**
|
||||
* kfree - free previously allocated memory
|
||||
* @object: pointer returned by kmalloc.
|
||||
*
|
||||
* If @object is NULL, no operation is performed.
|
||||
*
|
||||
* Don't free memory not originally allocated by kmalloc()
|
||||
* or you will run into trouble.
|
||||
*/
|
||||
void kfree(const void *object)
|
||||
{
|
||||
struct folio *folio;
|
||||
struct slab *slab;
|
||||
struct kmem_cache *s;
|
||||
|
||||
trace_kfree(_RET_IP_, object);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(object)))
|
||||
return;
|
||||
|
||||
folio = virt_to_folio(object);
|
||||
if (unlikely(!folio_test_slab(folio))) {
|
||||
free_large_kmalloc(folio, (void *)object);
|
||||
return;
|
||||
}
|
||||
|
||||
slab = folio_slab(folio);
|
||||
s = slab->slab_cache;
|
||||
__kmem_cache_free(s, (void *)object, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(kfree);
|
||||
|
||||
/* Uninstrumented ksize. Only called by KASAN. */
|
||||
size_t __ksize(const void *object)
|
||||
{
|
||||
struct folio *folio;
|
||||
|
||||
if (unlikely(object == ZERO_SIZE_PTR))
|
||||
return 0;
|
||||
|
||||
folio = virt_to_folio(object);
|
||||
|
||||
if (unlikely(!folio_test_slab(folio))) {
|
||||
if (WARN_ON(folio_size(folio) <= KMALLOC_MAX_CACHE_SIZE))
|
||||
return 0;
|
||||
if (WARN_ON(object != folio_address(folio)))
|
||||
return 0;
|
||||
return folio_size(folio);
|
||||
}
|
||||
|
||||
return slab_ksize(folio_slab(folio)->slab_cache);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
|
||||
{
|
||||
void *ret = __kmem_cache_alloc_node(s, gfpflags, NUMA_NO_NODE,
|
||||
size, _RET_IP_);
|
||||
|
||||
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmalloc_trace);
|
||||
|
||||
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t size)
|
||||
{
|
||||
void *ret = __kmem_cache_alloc_node(s, gfpflags, node, size, _RET_IP_);
|
||||
|
||||
trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmalloc_node_trace);
|
||||
#endif /* !CONFIG_TRACING */
|
||||
#endif /* !CONFIG_SLOB */
|
||||
|
||||
gfp_t kmalloc_fix_flags(gfp_t flags)
|
||||
|
@ -917,37 +1051,50 @@ gfp_t kmalloc_fix_flags(gfp_t flags)
|
|||
* directly to the page allocator. We use __GFP_COMP, because we will need to
|
||||
* know the allocation order to free the pages properly in kfree.
|
||||
*/
|
||||
void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
|
||||
|
||||
static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
void *ret = NULL;
|
||||
struct page *page;
|
||||
void *ptr = NULL;
|
||||
unsigned int order = get_order(size);
|
||||
|
||||
if (unlikely(flags & GFP_SLAB_BUG_MASK))
|
||||
flags = kmalloc_fix_flags(flags);
|
||||
|
||||
flags |= __GFP_COMP;
|
||||
page = alloc_pages(flags, order);
|
||||
if (likely(page)) {
|
||||
ret = page_address(page);
|
||||
page = alloc_pages_node(node, flags, order);
|
||||
if (page) {
|
||||
ptr = page_address(page);
|
||||
mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
|
||||
PAGE_SIZE << order);
|
||||
}
|
||||
ret = kasan_kmalloc_large(ret, size, flags);
|
||||
/* As ret might get tagged, call kmemleak hook after KASAN. */
|
||||
kmemleak_alloc(ret, size, 1, flags);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmalloc_order);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
|
||||
ptr = kasan_kmalloc_large(ptr, size, flags);
|
||||
/* As ptr might get tagged, call kmemleak hook after KASAN. */
|
||||
kmemleak_alloc(ptr, size, 1, flags);
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void *kmalloc_large(size_t size, gfp_t flags)
|
||||
{
|
||||
void *ret = kmalloc_order(size, flags, order);
|
||||
trace_kmalloc(_RET_IP_, ret, NULL, size, PAGE_SIZE << order, flags);
|
||||
void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
|
||||
|
||||
trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
|
||||
flags, NUMA_NO_NODE);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmalloc_order_trace);
|
||||
#endif
|
||||
EXPORT_SYMBOL(kmalloc_large);
|
||||
|
||||
void *kmalloc_large_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
void *ret = __kmalloc_large_node(size, flags, node);
|
||||
|
||||
trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << get_order(size),
|
||||
flags, node);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmalloc_large_node);
|
||||
|
||||
#ifdef CONFIG_SLAB_FREELIST_RANDOM
|
||||
/* Randomize a generic freelist */
|
||||
|
@ -1279,8 +1426,6 @@ EXPORT_SYMBOL(ksize);
|
|||
/* Tracepoints definitions. */
|
||||
EXPORT_TRACEPOINT_SYMBOL(kmalloc);
|
||||
EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
|
||||
EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
|
||||
EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
|
||||
EXPORT_TRACEPOINT_SYMBOL(kfree);
|
||||
EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
|
||||
|
||||
|
|
31
mm/slob.c
31
mm/slob.c
|
@ -507,8 +507,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
|
|||
*m = size;
|
||||
ret = (void *)m + minalign;
|
||||
|
||||
trace_kmalloc_node(caller, ret, NULL,
|
||||
size, size + minalign, gfp, node);
|
||||
trace_kmalloc(caller, ret, size, size + minalign, gfp, node);
|
||||
} else {
|
||||
unsigned int order = get_order(size);
|
||||
|
||||
|
@ -516,8 +515,7 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
|
|||
gfp |= __GFP_COMP;
|
||||
ret = slob_new_pages(gfp, order, node);
|
||||
|
||||
trace_kmalloc_node(caller, ret, NULL,
|
||||
size, PAGE_SIZE << order, gfp, node);
|
||||
trace_kmalloc(caller, ret, size, PAGE_SIZE << order, gfp, node);
|
||||
}
|
||||
|
||||
kmemleak_alloc(ret, size, 1, gfp);
|
||||
|
@ -530,20 +528,12 @@ void *__kmalloc(size_t size, gfp_t gfp)
|
|||
}
|
||||
EXPORT_SYMBOL(__kmalloc);
|
||||
|
||||
void *__kmalloc_track_caller(size_t size, gfp_t gfp, unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc_node(size, gfp, NUMA_NO_NODE, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_track_caller);
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
void *__kmalloc_node_track_caller(size_t size, gfp_t gfp,
|
||||
int node, unsigned long caller)
|
||||
{
|
||||
return __do_kmalloc_node(size, gfp, node, caller);
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
||||
#endif
|
||||
|
||||
void kfree(const void *block)
|
||||
{
|
||||
|
@ -594,7 +584,6 @@ size_t __ksize(const void *block)
|
|||
m = (unsigned int *)(block - align);
|
||||
return SLOB_UNITS(*m) * SLOB_UNIT;
|
||||
}
|
||||
EXPORT_SYMBOL(__ksize);
|
||||
|
||||
int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)
|
||||
{
|
||||
|
@ -602,6 +591,9 @@ int __kmem_cache_create(struct kmem_cache *c, slab_flags_t flags)
|
|||
/* leave room for rcu footer at the end of object */
|
||||
c->size += sizeof(struct slob_rcu);
|
||||
}
|
||||
|
||||
/* Actual size allocated */
|
||||
c->size = SLOB_UNITS(c->size) * SLOB_UNIT;
|
||||
c->flags = flags;
|
||||
return 0;
|
||||
}
|
||||
|
@ -616,14 +608,10 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
|
|||
|
||||
if (c->size < PAGE_SIZE) {
|
||||
b = slob_alloc(c->size, flags, c->align, node, 0);
|
||||
trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size,
|
||||
SLOB_UNITS(c->size) * SLOB_UNIT,
|
||||
flags, node);
|
||||
trace_kmem_cache_alloc(_RET_IP_, b, c, flags, node);
|
||||
} else {
|
||||
b = slob_new_pages(flags, get_order(c->size), node);
|
||||
trace_kmem_cache_alloc_node(_RET_IP_, b, NULL, c->object_size,
|
||||
PAGE_SIZE << get_order(c->size),
|
||||
flags, node);
|
||||
trace_kmem_cache_alloc(_RET_IP_, b, c, flags, node);
|
||||
}
|
||||
|
||||
if (b && c->ctor) {
|
||||
|
@ -647,7 +635,7 @@ void *kmem_cache_alloc_lru(struct kmem_cache *cachep, struct list_lru *lru, gfp_
|
|||
return slob_alloc_node(cachep, flags, NUMA_NO_NODE);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_lru);
|
||||
#ifdef CONFIG_NUMA
|
||||
|
||||
void *__kmalloc_node(size_t size, gfp_t gfp, int node)
|
||||
{
|
||||
return __do_kmalloc_node(size, gfp, node, _RET_IP_);
|
||||
|
@ -659,7 +647,6 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t gfp, int node)
|
|||
return slob_alloc_node(cachep, gfp, node);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_node);
|
||||
#endif
|
||||
|
||||
static void __kmem_cache_free(void *b, int size)
|
||||
{
|
||||
|
@ -680,7 +667,7 @@ static void kmem_rcu_free(struct rcu_head *head)
|
|||
void kmem_cache_free(struct kmem_cache *c, void *b)
|
||||
{
|
||||
kmemleak_free_recursive(b, c->flags);
|
||||
trace_kmem_cache_free(_RET_IP_, b, c->name);
|
||||
trace_kmem_cache_free(_RET_IP_, b, c);
|
||||
if (unlikely(c->flags & SLAB_TYPESAFE_BY_RCU)) {
|
||||
struct slob_rcu *slob_rcu;
|
||||
slob_rcu = b + (c->size - sizeof(struct slob_rcu));
|
||||
|
|
238
mm/slub.c
238
mm/slub.c
|
@ -1709,20 +1709,6 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
|
|||
* Hooks for other subsystems that check memory allocations. In a typical
|
||||
* production configuration these hooks all should produce no code at all.
|
||||
*/
|
||||
static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
|
||||
{
|
||||
ptr = kasan_kmalloc_large(ptr, size, flags);
|
||||
/* As ptr might get tagged, call kmemleak hook after KASAN. */
|
||||
kmemleak_alloc(ptr, size, 1, flags);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static __always_inline void kfree_hook(void *x)
|
||||
{
|
||||
kmemleak_free(x);
|
||||
kasan_kfree_large(x);
|
||||
}
|
||||
|
||||
static __always_inline bool slab_free_hook(struct kmem_cache *s,
|
||||
void *x, bool init)
|
||||
{
|
||||
|
@ -3262,8 +3248,7 @@ void *__kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
|
|||
{
|
||||
void *ret = slab_alloc(s, lru, gfpflags, _RET_IP_, s->object_size);
|
||||
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, s, s->object_size,
|
||||
s->size, gfpflags);
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
@ -3281,46 +3266,24 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
|
|||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_lru);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
|
||||
void *__kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags,
|
||||
int node, size_t orig_size,
|
||||
unsigned long caller)
|
||||
{
|
||||
void *ret = slab_alloc(s, NULL, gfpflags, _RET_IP_, size);
|
||||
trace_kmalloc(_RET_IP_, ret, s, size, s->size, gfpflags);
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
return ret;
|
||||
return slab_alloc_node(s, NULL, gfpflags, node,
|
||||
caller, orig_size);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_trace);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
|
||||
{
|
||||
void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
|
||||
|
||||
trace_kmem_cache_alloc_node(_RET_IP_, ret, s,
|
||||
s->object_size, s->size, gfpflags, node);
|
||||
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_node);
|
||||
|
||||
#ifdef CONFIG_TRACING
|
||||
void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
|
||||
gfp_t gfpflags,
|
||||
int node, size_t size)
|
||||
{
|
||||
void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
|
||||
|
||||
trace_kmalloc_node(_RET_IP_, ret, s,
|
||||
size, s->size, gfpflags, node);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
|
||||
#endif
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
/*
|
||||
* Slow path handling. This may still be called frequently since objects
|
||||
* have a longer lifetime than the cpu slabs in most processing loads.
|
||||
|
@ -3547,12 +3510,17 @@ void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
|
|||
}
|
||||
#endif
|
||||
|
||||
void __kmem_cache_free(struct kmem_cache *s, void *x, unsigned long caller)
|
||||
{
|
||||
slab_free(s, virt_to_slab(x), x, NULL, &x, 1, caller);
|
||||
}
|
||||
|
||||
void kmem_cache_free(struct kmem_cache *s, void *x)
|
||||
{
|
||||
s = cache_from_obj(s, x);
|
||||
if (!s)
|
||||
return;
|
||||
trace_kmem_cache_free(_RET_IP_, x, s->name);
|
||||
trace_kmem_cache_free(_RET_IP_, x, s);
|
||||
slab_free(s, virt_to_slab(x), x, NULL, &x, 1, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(kmem_cache_free);
|
||||
|
@ -3565,19 +3533,6 @@ struct detached_freelist {
|
|||
struct kmem_cache *s;
|
||||
};
|
||||
|
||||
static inline void free_large_kmalloc(struct folio *folio, void *object)
|
||||
{
|
||||
unsigned int order = folio_order(folio);
|
||||
|
||||
if (WARN_ON_ONCE(order == 0))
|
||||
pr_warn_once("object pointer: 0x%p\n", object);
|
||||
|
||||
kfree_hook(object);
|
||||
mod_lruvec_page_state(folio_page(folio, 0), NR_SLAB_UNRECLAIMABLE_B,
|
||||
-(PAGE_SIZE << order));
|
||||
__free_pages(folio_page(folio, 0), order);
|
||||
}
|
||||
|
||||
/*
|
||||
* This function progressively scans the array with free objects (with
|
||||
* a limited look ahead) and extract objects belonging to the same
|
||||
|
@ -4409,78 +4364,6 @@ static int __init setup_slub_min_objects(char *str)
|
|||
|
||||
__setup("slub_min_objects=", setup_slub_min_objects);
|
||||
|
||||
void *__kmalloc(size_t size, gfp_t flags)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
|
||||
return kmalloc_large(size, flags);
|
||||
|
||||
s = kmalloc_slab(size, flags);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(s)))
|
||||
return s;
|
||||
|
||||
ret = slab_alloc(s, NULL, flags, _RET_IP_, size);
|
||||
|
||||
trace_kmalloc(_RET_IP_, ret, s, size, s->size, flags);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc);
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
struct page *page;
|
||||
void *ptr = NULL;
|
||||
unsigned int order = get_order(size);
|
||||
|
||||
flags |= __GFP_COMP;
|
||||
page = alloc_pages_node(node, flags, order);
|
||||
if (page) {
|
||||
ptr = page_address(page);
|
||||
mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
|
||||
PAGE_SIZE << order);
|
||||
}
|
||||
|
||||
return kmalloc_large_node_hook(ptr, size, flags);
|
||||
}
|
||||
|
||||
void *__kmalloc_node(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
|
||||
ret = kmalloc_large_node(size, flags, node);
|
||||
|
||||
trace_kmalloc_node(_RET_IP_, ret, NULL,
|
||||
size, PAGE_SIZE << get_order(size),
|
||||
flags, node);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
s = kmalloc_slab(size, flags);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(s)))
|
||||
return s;
|
||||
|
||||
ret = slab_alloc_node(s, NULL, flags, node, _RET_IP_, size);
|
||||
|
||||
trace_kmalloc_node(_RET_IP_, ret, s, size, s->size, flags, node);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node);
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
#ifdef CONFIG_HARDENED_USERCOPY
|
||||
/*
|
||||
* Rejects incorrectly sized objects and objects that are to be copied
|
||||
|
@ -4531,43 +4414,6 @@ void __check_heap_object(const void *ptr, unsigned long n,
|
|||
}
|
||||
#endif /* CONFIG_HARDENED_USERCOPY */
|
||||
|
||||
size_t __ksize(const void *object)
|
||||
{
|
||||
struct folio *folio;
|
||||
|
||||
if (unlikely(object == ZERO_SIZE_PTR))
|
||||
return 0;
|
||||
|
||||
folio = virt_to_folio(object);
|
||||
|
||||
if (unlikely(!folio_test_slab(folio)))
|
||||
return folio_size(folio);
|
||||
|
||||
return slab_ksize(folio_slab(folio)->slab_cache);
|
||||
}
|
||||
EXPORT_SYMBOL(__ksize);
|
||||
|
||||
void kfree(const void *x)
|
||||
{
|
||||
struct folio *folio;
|
||||
struct slab *slab;
|
||||
void *object = (void *)x;
|
||||
|
||||
trace_kfree(_RET_IP_, x);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(x)))
|
||||
return;
|
||||
|
||||
folio = virt_to_folio(x);
|
||||
if (unlikely(!folio_test_slab(folio))) {
|
||||
free_large_kmalloc(folio, object);
|
||||
return;
|
||||
}
|
||||
slab = folio_slab(folio);
|
||||
slab_free(slab->slab_cache, slab, object, NULL, &object, 1, _RET_IP_);
|
||||
}
|
||||
EXPORT_SYMBOL(kfree);
|
||||
|
||||
#define SHRINK_PROMOTE_MAX 32
|
||||
|
||||
/*
|
||||
|
@ -4915,64 +4761,6 @@ int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
|
|||
return 0;
|
||||
}
|
||||
|
||||
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
|
||||
return kmalloc_large(size, gfpflags);
|
||||
|
||||
s = kmalloc_slab(size, gfpflags);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(s)))
|
||||
return s;
|
||||
|
||||
ret = slab_alloc(s, NULL, gfpflags, caller, size);
|
||||
|
||||
/* Honor the call site pointer we received. */
|
||||
trace_kmalloc(caller, ret, s, size, s->size, gfpflags);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_track_caller);
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
|
||||
int node, unsigned long caller)
|
||||
{
|
||||
struct kmem_cache *s;
|
||||
void *ret;
|
||||
|
||||
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
|
||||
ret = kmalloc_large_node(size, gfpflags, node);
|
||||
|
||||
trace_kmalloc_node(caller, ret, NULL,
|
||||
size, PAGE_SIZE << get_order(size),
|
||||
gfpflags, node);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
s = kmalloc_slab(size, gfpflags);
|
||||
|
||||
if (unlikely(ZERO_OR_NULL_PTR(s)))
|
||||
return s;
|
||||
|
||||
ret = slab_alloc_node(s, NULL, gfpflags, node, caller, size);
|
||||
|
||||
/* Honor the call site pointer we received. */
|
||||
trace_kmalloc_node(caller, ret, s, size, s->size, gfpflags, node);
|
||||
|
||||
ret = kasan_kmalloc(s, ret, size, gfpflags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL(__kmalloc_node_track_caller);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SYSFS
|
||||
static int count_inuse(struct slab *slab)
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue