mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-28 05:12:49 +00:00
Code Issues Releases Wiki Activity

slub: Store max number of objects in the page struct.

Browse source 
Split the inuse field up to be able to store the number of objects in this
page in the page struct as well. Necessary if we want to have pages of
various orders for a slab. Also avoids touching struct kmem_cache cachelines in
__slab_alloc().

Update diagnostic code to check the number of objects and make sure that
the number of objects always stays within the bounds of a 16 bit unsigned
integer.

Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>
This commit is contained in:
Christoph Lameter 2008-04-14 19:11:30 +03:00 committed by Pekka Enberg
parent 33b12c3813
commit 39b264641a
2 changed files with 38 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/linux/mm_types.h
View file

@ -42,7 +42,10 @@ struct page {
* to show when page is mapped * to show when page is mapped
* & limit reverse map searches. * & limit reverse map searches.
*/ */
unsigned int inuse; /* SLUB: Nr of objects */ struct { /* SLUB */
u16 inuse;
u16 objects;
};
}; };
union { union {
struct { struct {

54
mm/slub.c
View file

@ -301,7 +301,7 @@ static inline int check_valid_pointer(struct kmem_cache *s,
return 1; return 1;
base = page_address(page); base = page_address(page);
if (object < base || object >= base + s->objects * s->size || if (object < base || object >= base + page->objects * s->size ||
(object - base) % s->size) { (object - base) % s->size) {
return 0; return 0;
} }
@ -451,8 +451,8 @@ static void print_tracking(struct kmem_cache *s, void *object)
static void print_page_info(struct page *page) static void print_page_info(struct page *page)
{ {
printk(KERN_ERR "INFO: Slab 0x%p used=%u fp=0x%p flags=0x%04lx\n", printk(KERN_ERR "INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
page, page->inuse, page->freelist, page->flags); page, page->objects, page->inuse, page->freelist, page->flags);
} }
@ -652,6 +652,7 @@ static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
p + off, POISON_INUSE, s->size - off); p + off, POISON_INUSE, s->size - off);
} }
/* Check the pad bytes at the end of a slab page */
static int slab_pad_check(struct kmem_cache *s, struct page *page) static int slab_pad_check(struct kmem_cache *s, struct page *page)
{ {
u8 *start; u8 *start;
@ -664,20 +665,20 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page)
return 1; return 1;
start = page_address(page); start = page_address(page);
end = start + (PAGE_SIZE << s->order); length = (PAGE_SIZE << s->order);
length = s->objects * s->size; end = start + length;
remainder = end - (start + length); remainder = length % s->size;
if (!remainder) if (!remainder)
return 1; return 1;
fault = check_bytes(start + length, POISON_INUSE, remainder); fault = check_bytes(end - remainder, POISON_INUSE, remainder);
if (!fault) if (!fault)
return 1; return 1;
while (end > fault && end[-1] == POISON_INUSE) while (end > fault && end[-1] == POISON_INUSE)
end--; end--;
slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1);
print_section("Padding", start, length); print_section("Padding", end - remainder, remainder);
restore_bytes(s, "slab padding", POISON_INUSE, start, end); restore_bytes(s, "slab padding", POISON_INUSE, start, end);
return 0; return 0;
@ -739,15 +740,24 @@ static int check_object(struct kmem_cache *s, struct page *page,
static int check_slab(struct kmem_cache *s, struct page *page) static int check_slab(struct kmem_cache *s, struct page *page)
{ {
int maxobj;
VM_BUG_ON(!irqs_disabled()); VM_BUG_ON(!irqs_disabled());
if (!PageSlab(page)) { if (!PageSlab(page)) {
slab_err(s, page, "Not a valid slab page"); slab_err(s, page, "Not a valid slab page");
return 0; return 0;
} }
if (page->inuse > s->objects) {
maxobj = (PAGE_SIZE << compound_order(page)) / s->size;
if (page->objects > maxobj) {
slab_err(s, page, "objects %u > max %u",
s->name, page->objects, maxobj);
return 0;
}
if (page->inuse > page->objects) {
slab_err(s, page, "inuse %u > max %u", slab_err(s, page, "inuse %u > max %u",
s->name, page->inuse, s->objects); s->name, page->inuse, page->objects);
return 0; return 0;
} }
/* Slab_pad_check fixes things up after itself */ /* Slab_pad_check fixes things up after itself */
@ -765,7 +775,7 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
void *fp = page->freelist; void *fp = page->freelist;
void *object = NULL; void *object = NULL;
while (fp && nr <= s->objects) { while (fp && nr <= page->objects) {
if (fp == search) if (fp == search)
return 1; return 1;
if (!check_valid_pointer(s, page, fp)) { if (!check_valid_pointer(s, page, fp)) {
@ -777,7 +787,7 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
} else { } else {
slab_err(s, page, "Freepointer corrupt"); slab_err(s, page, "Freepointer corrupt");
page->freelist = NULL; page->freelist = NULL;
page->inuse = s->objects; page->inuse = page->objects;
slab_fix(s, "Freelist cleared"); slab_fix(s, "Freelist cleared");
return 0; return 0;
} }
@ -788,10 +798,10 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
nr++; nr++;
} }
if (page->inuse != s->objects - nr) { if (page->inuse != page->objects - nr) {
slab_err(s, page, "Wrong object count. Counter is %d but " slab_err(s, page, "Wrong object count. Counter is %d but "
"counted were %d", page->inuse, s->objects - nr); "counted were %d", page->inuse, page->objects - nr);
page->inuse = s->objects - nr; page->inuse = page->objects - nr;
slab_fix(s, "Object count adjusted."); slab_fix(s, "Object count adjusted.");
} }
return search == NULL; return search == NULL;
@ -910,7 +920,7 @@ static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
* as used avoids touching the remaining objects. * as used avoids touching the remaining objects.
*/ */
slab_fix(s, "Marking all objects used"); slab_fix(s, "Marking all objects used");
page->inuse = s->objects; page->inuse = page->objects;
page->freelist = NULL; page->freelist = NULL;
} }
return 0; return 0;
@ -1081,6 +1091,7 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
if (!page) if (!page)
return NULL; return NULL;
page->objects = s->objects;
mod_zone_page_state(page_zone(page), mod_zone_page_state(page_zone(page),
(s->flags & SLAB_RECLAIM_ACCOUNT) ? (s->flags & SLAB_RECLAIM_ACCOUNT) ?
NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
@ -1519,7 +1530,7 @@ static void *__slab_alloc(struct kmem_cache *s,
goto debug; goto debug;
c->freelist = object[c->offset]; c->freelist = object[c->offset];
c->page->inuse = s->objects; c->page->inuse = c->page->objects;
c->page->freelist = NULL; c->page->freelist = NULL;
c->node = page_to_nid(c->page); c->node = page_to_nid(c->page);
unlock_out: unlock_out:
@ -1818,6 +1829,9 @@ static inline int slab_order(int size, int min_objects,
int rem; int rem;
int min_order = slub_min_order; int min_order = slub_min_order;
if ((PAGE_SIZE << min_order) / size > 65535)
return get_order(size * 65535) - 1;
for (order = max(min_order, for (order = max(min_order,
fls(min_objects * size - 1) - PAGE_SHIFT); fls(min_objects * size - 1) - PAGE_SHIFT);
order <= max_order; order++) { order <= max_order; order++) {
@ -3251,7 +3265,7 @@ static int validate_slab(struct kmem_cache *s, struct page *page,
return 0; return 0;
/* Now we know that a valid freelist exists */ /* Now we know that a valid freelist exists */
bitmap_zero(map, s->objects); bitmap_zero(map, page->objects);
for_each_free_object(p, s, page->freelist) { for_each_free_object(p, s, page->freelist) {
set_bit(slab_index(p, s, addr), map); set_bit(slab_index(p, s, addr), map);
@ -3528,10 +3542,10 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s,
struct page *page, enum track_item alloc) struct page *page, enum track_item alloc)
{ {
void *addr = page_address(page); void *addr = page_address(page);
DECLARE_BITMAP(map, s->objects); DECLARE_BITMAP(map, page->objects);
void *p; void *p;
bitmap_zero(map, s->objects); bitmap_zero(map, page->objects);
for_each_free_object(p, s, page->freelist) for_each_free_object(p, s, page->freelist)
set_bit(slab_index(p, s, addr), map); set_bit(slab_index(p, s, addr), map);