mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-27 04:47:05 +00:00
Merge branch 'slab/for-6.9/optimize-get-freelist' into slab/for-linus
Merge a series from Chengming Zhou that optimizes cpu freelist loading when grabbing a cpu partial slab, and removes some unnecessary code.
This commit is contained in:
commit
466ed9eed6
1 changed files with 14 additions and 18 deletions
32
mm/slub.c
32
mm/slub.c
|
@ -2228,7 +2228,7 @@ static void __init init_freelist_randomization(void)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Get the next entry on the pre-computed freelist randomized */
|
/* Get the next entry on the pre-computed freelist randomized */
|
||||||
static void *next_freelist_entry(struct kmem_cache *s, struct slab *slab,
|
static void *next_freelist_entry(struct kmem_cache *s,
|
||||||
unsigned long *pos, void *start,
|
unsigned long *pos, void *start,
|
||||||
unsigned long page_limit,
|
unsigned long page_limit,
|
||||||
unsigned long freelist_count)
|
unsigned long freelist_count)
|
||||||
|
@ -2267,13 +2267,12 @@ static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
|
||||||
start = fixup_red_left(s, slab_address(slab));
|
start = fixup_red_left(s, slab_address(slab));
|
||||||
|
|
||||||
/* First entry is used as the base of the freelist */
|
/* First entry is used as the base of the freelist */
|
||||||
cur = next_freelist_entry(s, slab, &pos, start, page_limit,
|
cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
|
||||||
freelist_count);
|
|
||||||
cur = setup_object(s, cur);
|
cur = setup_object(s, cur);
|
||||||
slab->freelist = cur;
|
slab->freelist = cur;
|
||||||
|
|
||||||
for (idx = 1; idx < slab->objects; idx++) {
|
for (idx = 1; idx < slab->objects; idx++) {
|
||||||
next = next_freelist_entry(s, slab, &pos, start, page_limit,
|
next = next_freelist_entry(s, &pos, start, page_limit,
|
||||||
freelist_count);
|
freelist_count);
|
||||||
next = setup_object(s, next);
|
next = setup_object(s, next);
|
||||||
set_freepointer(s, cur, next);
|
set_freepointer(s, cur, next);
|
||||||
|
@ -3311,7 +3310,6 @@ static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
|
||||||
counters = slab->counters;
|
counters = slab->counters;
|
||||||
|
|
||||||
new.counters = counters;
|
new.counters = counters;
|
||||||
VM_BUG_ON(!new.frozen);
|
|
||||||
|
|
||||||
new.inuse = slab->objects;
|
new.inuse = slab->objects;
|
||||||
new.frozen = freelist != NULL;
|
new.frozen = freelist != NULL;
|
||||||
|
@ -3483,18 +3481,20 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
|
||||||
|
|
||||||
slab = slub_percpu_partial(c);
|
slab = slub_percpu_partial(c);
|
||||||
slub_set_percpu_partial(c, slab);
|
slub_set_percpu_partial(c, slab);
|
||||||
local_unlock_irqrestore(&s->cpu_slab->lock, flags);
|
|
||||||
stat(s, CPU_PARTIAL_ALLOC);
|
|
||||||
|
|
||||||
if (unlikely(!node_match(slab, node) ||
|
if (likely(node_match(slab, node) &&
|
||||||
!pfmemalloc_match(slab, gfpflags))) {
|
pfmemalloc_match(slab, gfpflags))) {
|
||||||
slab->next = NULL;
|
c->slab = slab;
|
||||||
__put_partials(s, slab);
|
freelist = get_freelist(s, slab);
|
||||||
continue;
|
VM_BUG_ON(!freelist);
|
||||||
|
stat(s, CPU_PARTIAL_ALLOC);
|
||||||
|
goto load_freelist;
|
||||||
}
|
}
|
||||||
|
|
||||||
freelist = freeze_slab(s, slab);
|
local_unlock_irqrestore(&s->cpu_slab->lock, flags);
|
||||||
goto retry_load_slab;
|
|
||||||
|
slab->next = NULL;
|
||||||
|
__put_partials(s, slab);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
@ -4172,7 +4172,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
|
||||||
* then add it.
|
* then add it.
|
||||||
*/
|
*/
|
||||||
if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
|
if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
|
||||||
remove_full(s, n, slab);
|
|
||||||
add_partial(n, slab, DEACTIVATE_TO_TAIL);
|
add_partial(n, slab, DEACTIVATE_TO_TAIL);
|
||||||
stat(s, FREE_ADD_PARTIAL);
|
stat(s, FREE_ADD_PARTIAL);
|
||||||
}
|
}
|
||||||
|
@ -4186,9 +4185,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
|
||||||
*/
|
*/
|
||||||
remove_partial(n, slab);
|
remove_partial(n, slab);
|
||||||
stat(s, FREE_REMOVE_PARTIAL);
|
stat(s, FREE_REMOVE_PARTIAL);
|
||||||
} else {
|
|
||||||
/* Slab must be on the full list */
|
|
||||||
remove_full(s, n, slab);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_unlock_irqrestore(&n->list_lock, flags);
|
spin_unlock_irqrestore(&n->list_lock, flags);
|
||||||
|
|
Loading…
Reference in a new issue