mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-22 17:01:14 +00:00
![Christoph Lameter](/assets/img/avatar_default.png)
Remove the fields in struct kmem_cache_cpu that were used to cache data from struct kmem_cache when they were in different cachelines. The cacheline that holds the per cpu array pointer now also holds these values. We can cut down the struct kmem_cache_cpu size to almost half. The get_freepointer() and set_freepointer() functions that used to be only intended for the slow path now are also useful for the hot path since access to the size field does not require accessing an additional cacheline anymore. This results in consistent use of functions for setting the freepointer of objects throughout SLUB. Also we initialize all possible kmem_cache_cpu structures when a slab is created. No need to initialize them when a processor or node comes online. Signed-off-by: Christoph Lameter <cl@linux-foundation.org> Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>
302 lines
8.1 KiB
C
302 lines
8.1 KiB
C
#ifndef _LINUX_SLUB_DEF_H
|
|
#define _LINUX_SLUB_DEF_H
|
|
|
|
/*
|
|
* SLUB : A Slab allocator without object queues.
|
|
*
|
|
* (C) 2007 SGI, Christoph Lameter
|
|
*/
|
|
#include <linux/types.h>
|
|
#include <linux/gfp.h>
|
|
#include <linux/workqueue.h>
|
|
#include <linux/kobject.h>
|
|
#include <linux/kmemtrace.h>
|
|
#include <linux/kmemleak.h>
|
|
|
|
enum stat_item {
|
|
ALLOC_FASTPATH, /* Allocation from cpu slab */
|
|
ALLOC_SLOWPATH, /* Allocation by getting a new cpu slab */
|
|
FREE_FASTPATH, /* Free to cpu slub */
|
|
FREE_SLOWPATH, /* Freeing not to cpu slab */
|
|
FREE_FROZEN, /* Freeing to frozen slab */
|
|
FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */
|
|
FREE_REMOVE_PARTIAL, /* Freeing removes last object */
|
|
ALLOC_FROM_PARTIAL, /* Cpu slab acquired from partial list */
|
|
ALLOC_SLAB, /* Cpu slab acquired from page allocator */
|
|
ALLOC_REFILL, /* Refill cpu slab from slab freelist */
|
|
FREE_SLAB, /* Slab freed to the page allocator */
|
|
CPUSLAB_FLUSH, /* Abandoning of the cpu slab */
|
|
DEACTIVATE_FULL, /* Cpu slab was full when deactivated */
|
|
DEACTIVATE_EMPTY, /* Cpu slab was empty when deactivated */
|
|
DEACTIVATE_TO_HEAD, /* Cpu slab was moved to the head of partials */
|
|
DEACTIVATE_TO_TAIL, /* Cpu slab was moved to the tail of partials */
|
|
DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
|
|
ORDER_FALLBACK, /* Number of times fallback was necessary */
|
|
NR_SLUB_STAT_ITEMS };
|
|
|
|
struct kmem_cache_cpu {
|
|
void **freelist; /* Pointer to first free per cpu object */
|
|
struct page *page; /* The slab from which we are allocating */
|
|
int node; /* The node of the page (or -1 for debug) */
|
|
#ifdef CONFIG_SLUB_STATS
|
|
unsigned stat[NR_SLUB_STAT_ITEMS];
|
|
#endif
|
|
};
|
|
|
|
struct kmem_cache_node {
|
|
spinlock_t list_lock; /* Protect partial list and nr_partial */
|
|
unsigned long nr_partial;
|
|
struct list_head partial;
|
|
#ifdef CONFIG_SLUB_DEBUG
|
|
atomic_long_t nr_slabs;
|
|
atomic_long_t total_objects;
|
|
struct list_head full;
|
|
#endif
|
|
};
|
|
|
|
/*
|
|
* Word size structure that can be atomically updated or read and that
|
|
* contains both the order and the number of objects that a slab of the
|
|
* given order would contain.
|
|
*/
|
|
struct kmem_cache_order_objects {
|
|
unsigned long x;
|
|
};
|
|
|
|
/*
|
|
* Slab cache management.
|
|
*/
|
|
struct kmem_cache {
|
|
struct kmem_cache_cpu *cpu_slab;
|
|
/* Used for retriving partial slabs etc */
|
|
unsigned long flags;
|
|
int size; /* The size of an object including meta data */
|
|
int objsize; /* The size of an object without meta data */
|
|
int offset; /* Free pointer offset. */
|
|
struct kmem_cache_order_objects oo;
|
|
|
|
/*
|
|
* Avoid an extra cache line for UP, SMP and for the node local to
|
|
* struct kmem_cache.
|
|
*/
|
|
struct kmem_cache_node local_node;
|
|
|
|
/* Allocation and freeing of slabs */
|
|
struct kmem_cache_order_objects max;
|
|
struct kmem_cache_order_objects min;
|
|
gfp_t allocflags; /* gfp flags to use on each alloc */
|
|
int refcount; /* Refcount for slab cache destroy */
|
|
void (*ctor)(void *);
|
|
int inuse; /* Offset to metadata */
|
|
int align; /* Alignment */
|
|
unsigned long min_partial;
|
|
const char *name; /* Name (only for display!) */
|
|
struct list_head list; /* List of slab caches */
|
|
#ifdef CONFIG_SLUB_DEBUG
|
|
struct kobject kobj; /* For sysfs */
|
|
#endif
|
|
|
|
#ifdef CONFIG_NUMA
|
|
/*
|
|
* Defragmentation by allocating from a remote node.
|
|
*/
|
|
int remote_node_defrag_ratio;
|
|
struct kmem_cache_node *node[MAX_NUMNODES];
|
|
#endif
|
|
};
|
|
|
|
/*
|
|
* Kmalloc subsystem.
|
|
*/
|
|
#if defined(ARCH_KMALLOC_MINALIGN) && ARCH_KMALLOC_MINALIGN > 8
|
|
#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
|
|
#else
|
|
#define KMALLOC_MIN_SIZE 8
|
|
#endif
|
|
|
|
#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
|
|
|
|
/*
|
|
* Maximum kmalloc object size handled by SLUB. Larger object allocations
|
|
* are passed through to the page allocator. The page allocator "fastpath"
|
|
* is relatively slow so we need this value sufficiently high so that
|
|
* performance critical objects are allocated through the SLUB fastpath.
|
|
*
|
|
* This should be dropped to PAGE_SIZE / 2 once the page allocator
|
|
* "fastpath" becomes competitive with the slab allocator fastpaths.
|
|
*/
|
|
#define SLUB_MAX_SIZE (2 * PAGE_SIZE)
|
|
|
|
#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
|
|
|
|
#ifdef CONFIG_ZONE_DMA
|
|
#define SLUB_DMA __GFP_DMA
|
|
/* Reserve extra caches for potential DMA use */
|
|
#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT - 6)
|
|
#else
|
|
/* Disable DMA functionality */
|
|
#define SLUB_DMA (__force gfp_t)0
|
|
#define KMALLOC_CACHES SLUB_PAGE_SHIFT
|
|
#endif
|
|
|
|
/*
|
|
* We keep the general caches in an array of slab caches that are used for
|
|
* 2^x bytes of allocations.
|
|
*/
|
|
extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES];
|
|
|
|
/*
|
|
* Sorry that the following has to be that ugly but some versions of GCC
|
|
* have trouble with constant propagation and loops.
|
|
*/
|
|
static __always_inline int kmalloc_index(size_t size)
|
|
{
|
|
if (!size)
|
|
return 0;
|
|
|
|
if (size <= KMALLOC_MIN_SIZE)
|
|
return KMALLOC_SHIFT_LOW;
|
|
|
|
if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
|
|
return 1;
|
|
if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
|
|
return 2;
|
|
if (size <= 8) return 3;
|
|
if (size <= 16) return 4;
|
|
if (size <= 32) return 5;
|
|
if (size <= 64) return 6;
|
|
if (size <= 128) return 7;
|
|
if (size <= 256) return 8;
|
|
if (size <= 512) return 9;
|
|
if (size <= 1024) return 10;
|
|
if (size <= 2 * 1024) return 11;
|
|
if (size <= 4 * 1024) return 12;
|
|
/*
|
|
* The following is only needed to support architectures with a larger page
|
|
* size than 4k.
|
|
*/
|
|
if (size <= 8 * 1024) return 13;
|
|
if (size <= 16 * 1024) return 14;
|
|
if (size <= 32 * 1024) return 15;
|
|
if (size <= 64 * 1024) return 16;
|
|
if (size <= 128 * 1024) return 17;
|
|
if (size <= 256 * 1024) return 18;
|
|
if (size <= 512 * 1024) return 19;
|
|
if (size <= 1024 * 1024) return 20;
|
|
if (size <= 2 * 1024 * 1024) return 21;
|
|
return -1;
|
|
|
|
/*
|
|
* What we really wanted to do and cannot do because of compiler issues is:
|
|
* int i;
|
|
* for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
|
|
* if (size <= (1 << i))
|
|
* return i;
|
|
*/
|
|
}
|
|
|
|
/*
|
|
* Find the slab cache for a given combination of allocation flags and size.
|
|
*
|
|
* This ought to end up with a global pointer to the right cache
|
|
* in kmalloc_caches.
|
|
*/
|
|
static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
|
|
{
|
|
int index = kmalloc_index(size);
|
|
|
|
if (index == 0)
|
|
return NULL;
|
|
|
|
return &kmalloc_caches[index];
|
|
}
|
|
|
|
void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
|
|
void *__kmalloc(size_t size, gfp_t flags);
|
|
|
|
#ifdef CONFIG_TRACING
|
|
extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags);
|
|
#else
|
|
static __always_inline void *
|
|
kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
|
|
{
|
|
return kmem_cache_alloc(s, gfpflags);
|
|
}
|
|
#endif
|
|
|
|
static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
|
|
{
|
|
unsigned int order = get_order(size);
|
|
void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
|
|
|
|
kmemleak_alloc(ret, size, 1, flags);
|
|
trace_kmalloc(_THIS_IP_, ret, size, PAGE_SIZE << order, flags);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static __always_inline void *kmalloc(size_t size, gfp_t flags)
|
|
{
|
|
void *ret;
|
|
|
|
if (__builtin_constant_p(size)) {
|
|
if (size > SLUB_MAX_SIZE)
|
|
return kmalloc_large(size, flags);
|
|
|
|
if (!(flags & SLUB_DMA)) {
|
|
struct kmem_cache *s = kmalloc_slab(size);
|
|
|
|
if (!s)
|
|
return ZERO_SIZE_PTR;
|
|
|
|
ret = kmem_cache_alloc_notrace(s, flags);
|
|
|
|
trace_kmalloc(_THIS_IP_, ret, size, s->size, flags);
|
|
|
|
return ret;
|
|
}
|
|
}
|
|
return __kmalloc(size, flags);
|
|
}
|
|
|
|
#ifdef CONFIG_NUMA
|
|
void *__kmalloc_node(size_t size, gfp_t flags, int node);
|
|
void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
|
|
|
|
#ifdef CONFIG_TRACING
|
|
extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
|
|
gfp_t gfpflags,
|
|
int node);
|
|
#else
|
|
static __always_inline void *
|
|
kmem_cache_alloc_node_notrace(struct kmem_cache *s,
|
|
gfp_t gfpflags,
|
|
int node)
|
|
{
|
|
return kmem_cache_alloc_node(s, gfpflags, node);
|
|
}
|
|
#endif
|
|
|
|
static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
|
|
{
|
|
void *ret;
|
|
|
|
if (__builtin_constant_p(size) &&
|
|
size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
|
|
struct kmem_cache *s = kmalloc_slab(size);
|
|
|
|
if (!s)
|
|
return ZERO_SIZE_PTR;
|
|
|
|
ret = kmem_cache_alloc_node_notrace(s, flags, node);
|
|
|
|
trace_kmalloc_node(_THIS_IP_, ret,
|
|
size, s->size, flags, node);
|
|
|
|
return ret;
|
|
}
|
|
return __kmalloc_node(size, flags, node);
|
|
}
|
|
#endif
|
|
|
|
#endif /* _LINUX_SLUB_DEF_H */
|