mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-28 21:33:52 +00:00
mm: combine LRU and main union in struct page
This gives us five words of space in a single union in struct page. The compound_mapcount moves position (from offset 24 to offset 20) on 64-bit systems, but that does not seem likely to cause any trouble. Link: http://lkml.kernel.org/r/20180518194519.3820-11-willy@infradead.org Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com> Acked-by: Vlastimil Babka <vbabka@suse.cz> Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Christoph Lameter <cl@linux.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Jérôme Glisse <jglisse@redhat.com> Cc: Lai Jiangshan <jiangshanlai@gmail.com> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: Randy Dunlap <rdunlap@infradead.org> Cc: Andrey Ryabinin <aryabinin@virtuozzo.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
parent
b7ccc7f8c6
commit
4da1984edb
2 changed files with 47 additions and 52 deletions
|
@ -73,59 +73,19 @@ struct page {
|
||||||
unsigned long flags; /* Atomic flags, some possibly
|
unsigned long flags; /* Atomic flags, some possibly
|
||||||
* updated asynchronously */
|
* updated asynchronously */
|
||||||
/*
|
/*
|
||||||
* WARNING: bit 0 of the first word encode PageTail(). That means
|
* Five words (20/40 bytes) are available in this union.
|
||||||
* the rest users of the storage space MUST NOT use the bit to
|
* WARNING: bit 0 of the first word is used for PageTail(). That
|
||||||
|
* means the other users of this union MUST NOT use the bit to
|
||||||
* avoid collision and false-positive PageTail().
|
* avoid collision and false-positive PageTail().
|
||||||
*/
|
*/
|
||||||
union {
|
|
||||||
struct list_head lru; /* Pageout list, eg. active_list
|
|
||||||
* protected by zone_lru_lock !
|
|
||||||
* Can be used as a generic list
|
|
||||||
* by the page owner.
|
|
||||||
*/
|
|
||||||
struct dev_pagemap *pgmap; /* ZONE_DEVICE pages are never on an
|
|
||||||
* lru or handled by a slab
|
|
||||||
* allocator, this points to the
|
|
||||||
* hosting device page map.
|
|
||||||
*/
|
|
||||||
struct { /* slub per cpu partial pages */
|
|
||||||
struct page *next; /* Next partial slab */
|
|
||||||
#ifdef CONFIG_64BIT
|
|
||||||
int pages; /* Nr of partial slabs left */
|
|
||||||
int pobjects; /* Approximate # of objects */
|
|
||||||
#else
|
|
||||||
short int pages;
|
|
||||||
short int pobjects;
|
|
||||||
#endif
|
|
||||||
};
|
|
||||||
|
|
||||||
struct rcu_head rcu_head; /* Used by SLAB
|
|
||||||
* when destroying via RCU
|
|
||||||
*/
|
|
||||||
/* Tail pages of compound page */
|
|
||||||
struct {
|
|
||||||
unsigned long compound_head; /* If bit zero is set */
|
|
||||||
|
|
||||||
/* First tail page only */
|
|
||||||
unsigned char compound_dtor;
|
|
||||||
unsigned char compound_order;
|
|
||||||
/* two/six bytes available here */
|
|
||||||
};
|
|
||||||
|
|
||||||
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
|
|
||||||
struct {
|
|
||||||
unsigned long __pad; /* do not overlay pmd_huge_pte
|
|
||||||
* with compound_head to avoid
|
|
||||||
* possible bit 0 collision.
|
|
||||||
*/
|
|
||||||
pgtable_t pmd_huge_pte; /* protected by page->ptl */
|
|
||||||
};
|
|
||||||
#endif
|
|
||||||
};
|
|
||||||
|
|
||||||
/* Three words (12/24 bytes) are available in this union. */
|
|
||||||
union {
|
union {
|
||||||
struct { /* Page cache and anonymous pages */
|
struct { /* Page cache and anonymous pages */
|
||||||
|
/**
|
||||||
|
* @lru: Pageout list, eg. active_list protected by
|
||||||
|
* zone_lru_lock. Sometimes used as a generic list
|
||||||
|
* by the page owner.
|
||||||
|
*/
|
||||||
|
struct list_head lru;
|
||||||
/* See page-flags.h for PAGE_MAPPING_FLAGS */
|
/* See page-flags.h for PAGE_MAPPING_FLAGS */
|
||||||
struct address_space *mapping;
|
struct address_space *mapping;
|
||||||
pgoff_t index; /* Our offset within mapping. */
|
pgoff_t index; /* Our offset within mapping. */
|
||||||
|
@ -138,6 +98,19 @@ struct page {
|
||||||
unsigned long private;
|
unsigned long private;
|
||||||
};
|
};
|
||||||
struct { /* slab, slob and slub */
|
struct { /* slab, slob and slub */
|
||||||
|
union {
|
||||||
|
struct list_head slab_list; /* uses lru */
|
||||||
|
struct { /* Partial pages */
|
||||||
|
struct page *next;
|
||||||
|
#ifdef CONFIG_64BIT
|
||||||
|
int pages; /* Nr of pages left */
|
||||||
|
int pobjects; /* Approximate count */
|
||||||
|
#else
|
||||||
|
short int pages;
|
||||||
|
short int pobjects;
|
||||||
|
#endif
|
||||||
|
};
|
||||||
|
};
|
||||||
struct kmem_cache *slab_cache; /* not slob */
|
struct kmem_cache *slab_cache; /* not slob */
|
||||||
/* Double-word boundary */
|
/* Double-word boundary */
|
||||||
void *freelist; /* first free object */
|
void *freelist; /* first free object */
|
||||||
|
@ -151,9 +124,22 @@ struct page {
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
atomic_t compound_mapcount; /* first tail page */
|
struct { /* Tail pages of compound page */
|
||||||
struct list_head deferred_list; /* second tail page */
|
unsigned long compound_head; /* Bit zero is set */
|
||||||
|
|
||||||
|
/* First tail page only */
|
||||||
|
unsigned char compound_dtor;
|
||||||
|
unsigned char compound_order;
|
||||||
|
atomic_t compound_mapcount;
|
||||||
|
};
|
||||||
|
struct { /* Second tail page of compound page */
|
||||||
|
unsigned long _compound_pad_1; /* compound_head */
|
||||||
|
unsigned long _compound_pad_2;
|
||||||
|
struct list_head deferred_list;
|
||||||
|
};
|
||||||
struct { /* Page table pages */
|
struct { /* Page table pages */
|
||||||
|
unsigned long _pt_pad_1; /* compound_head */
|
||||||
|
pgtable_t pmd_huge_pte; /* protected by page->ptl */
|
||||||
unsigned long _pt_pad_2; /* mapping */
|
unsigned long _pt_pad_2; /* mapping */
|
||||||
unsigned long _pt_pad_3;
|
unsigned long _pt_pad_3;
|
||||||
#if ALLOC_SPLIT_PTLOCKS
|
#if ALLOC_SPLIT_PTLOCKS
|
||||||
|
@ -162,6 +148,15 @@ struct page {
|
||||||
spinlock_t ptl;
|
spinlock_t ptl;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** @rcu_head: You can use this to free a page by RCU. */
|
||||||
|
struct rcu_head rcu_head;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @pgmap: For ZONE_DEVICE pages, this points to the hosting
|
||||||
|
* device page map.
|
||||||
|
*/
|
||||||
|
struct dev_pagemap *pgmap;
|
||||||
};
|
};
|
||||||
|
|
||||||
union { /* This union is 4 bytes in size. */
|
union { /* This union is 4 bytes in size. */
|
||||||
|
|
|
@ -943,7 +943,7 @@ static int free_tail_pages_check(struct page *head_page, struct page *page)
|
||||||
}
|
}
|
||||||
switch (page - head_page) {
|
switch (page - head_page) {
|
||||||
case 1:
|
case 1:
|
||||||
/* the first tail page: ->mapping is compound_mapcount() */
|
/* the first tail page: ->mapping may be compound_mapcount() */
|
||||||
if (unlikely(compound_mapcount(page))) {
|
if (unlikely(compound_mapcount(page))) {
|
||||||
bad_page(page, "nonzero compound_mapcount", 0);
|
bad_page(page, "nonzero compound_mapcount", 0);
|
||||||
goto out;
|
goto out;
|
||||||
|
|
Loading…
Reference in a new issue