mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 13:55:32 +00:00
mm: Add folio flag manipulation functions
These new functions are the folio analogues of the various PageFlags functions. If CONFIG_DEBUG_VM_PGFLAGS is enabled, we check the folio is not a tail page at every invocation. This will also catch the PagePoisoned case as a poisoned page has every bit set, which would include PageTail. This saves 1684 bytes of text with the distro-derived config that I'm testing due to removing a double call to compound_head() in PageSwapCache(). Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Reviewed-by: Christoph Hellwig <hch@lst.de> Acked-by: Jeff Layton <jlayton@kernel.org> Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Acked-by: Vlastimil Babka <vbabka@suse.cz> Reviewed-by: William Kucharski <william.kucharski@oracle.com> Reviewed-by: David Howells <dhowells@redhat.com> Acked-by: Mike Rapoport <rppt@linux.ibm.com>
This commit is contained in:
parent
020853b6f5
commit
d389a4a811
1 changed files with 160 additions and 67 deletions
|
@ -143,6 +143,8 @@ enum pageflags {
|
|||
#endif
|
||||
__NR_PAGEFLAGS,
|
||||
|
||||
PG_readahead = PG_reclaim,
|
||||
|
||||
/* Filesystems */
|
||||
PG_checked = PG_owner_priv_1,
|
||||
|
||||
|
@ -245,6 +247,15 @@ static inline void page_init_poison(struct page *page, size_t size)
|
|||
}
|
||||
#endif
|
||||
|
||||
static unsigned long *folio_flags(struct folio *folio, unsigned n)
|
||||
{
|
||||
struct page *page = &folio->page;
|
||||
|
||||
VM_BUG_ON_PGFLAGS(PageTail(page), page);
|
||||
VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags), page);
|
||||
return &page[n].flags;
|
||||
}
|
||||
|
||||
/*
|
||||
* Page flags policies wrt compound pages
|
||||
*
|
||||
|
@ -289,36 +300,64 @@ static inline void page_init_poison(struct page *page, size_t size)
|
|||
VM_BUG_ON_PGFLAGS(!PageHead(page), page); \
|
||||
PF_POISONED_CHECK(&page[1]); })
|
||||
|
||||
/* Which page is the flag stored in */
|
||||
#define FOLIO_PF_ANY 0
|
||||
#define FOLIO_PF_HEAD 0
|
||||
#define FOLIO_PF_ONLY_HEAD 0
|
||||
#define FOLIO_PF_NO_TAIL 0
|
||||
#define FOLIO_PF_NO_COMPOUND 0
|
||||
#define FOLIO_PF_SECOND 1
|
||||
|
||||
/*
|
||||
* Macros to create function definitions for page flags
|
||||
*/
|
||||
#define TESTPAGEFLAG(uname, lname, policy) \
|
||||
static __always_inline bool folio_test_##lname(struct folio *folio) \
|
||||
{ return test_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline int Page##uname(struct page *page) \
|
||||
{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
|
||||
{ return test_bit(PG_##lname, &policy(page, 0)->flags); }
|
||||
|
||||
#define SETPAGEFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
void folio_set_##lname(struct folio *folio) \
|
||||
{ set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline void SetPage##uname(struct page *page) \
|
||||
{ set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define CLEARPAGEFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
void folio_clear_##lname(struct folio *folio) \
|
||||
{ clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline void ClearPage##uname(struct page *page) \
|
||||
{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define __SETPAGEFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
void __folio_set_##lname(struct folio *folio) \
|
||||
{ __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline void __SetPage##uname(struct page *page) \
|
||||
{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define __CLEARPAGEFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
void __folio_clear_##lname(struct folio *folio) \
|
||||
{ __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline void __ClearPage##uname(struct page *page) \
|
||||
{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define TESTSETFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
bool folio_test_set_##lname(struct folio *folio) \
|
||||
{ return test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline int TestSetPage##uname(struct page *page) \
|
||||
{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define TESTCLEARFLAG(uname, lname, policy) \
|
||||
static __always_inline \
|
||||
bool folio_test_clear_##lname(struct folio *folio) \
|
||||
{ return test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
|
||||
static __always_inline int TestClearPage##uname(struct page *page) \
|
||||
{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
|
||||
|
||||
#define PAGEFLAG(uname, lname, policy) \
|
||||
TESTPAGEFLAG(uname, lname, policy) \
|
||||
|
@ -334,29 +373,37 @@ static __always_inline int TestClearPage##uname(struct page *page) \
|
|||
TESTSETFLAG(uname, lname, policy) \
|
||||
TESTCLEARFLAG(uname, lname, policy)
|
||||
|
||||
#define TESTPAGEFLAG_FALSE(uname) \
|
||||
#define TESTPAGEFLAG_FALSE(uname, lname) \
|
||||
static inline bool folio_test_##lname(const struct folio *folio) { return 0; } \
|
||||
static inline int Page##uname(const struct page *page) { return 0; }
|
||||
|
||||
#define SETPAGEFLAG_NOOP(uname) \
|
||||
#define SETPAGEFLAG_NOOP(uname, lname) \
|
||||
static inline void folio_set_##lname(struct folio *folio) { } \
|
||||
static inline void SetPage##uname(struct page *page) { }
|
||||
|
||||
#define CLEARPAGEFLAG_NOOP(uname) \
|
||||
#define CLEARPAGEFLAG_NOOP(uname, lname) \
|
||||
static inline void folio_clear_##lname(struct folio *folio) { } \
|
||||
static inline void ClearPage##uname(struct page *page) { }
|
||||
|
||||
#define __CLEARPAGEFLAG_NOOP(uname) \
|
||||
#define __CLEARPAGEFLAG_NOOP(uname, lname) \
|
||||
static inline void __folio_clear_##lname(struct folio *folio) { } \
|
||||
static inline void __ClearPage##uname(struct page *page) { }
|
||||
|
||||
#define TESTSETFLAG_FALSE(uname) \
|
||||
#define TESTSETFLAG_FALSE(uname, lname) \
|
||||
static inline bool folio_test_set_##lname(struct folio *folio) \
|
||||
{ return 0; } \
|
||||
static inline int TestSetPage##uname(struct page *page) { return 0; }
|
||||
|
||||
#define TESTCLEARFLAG_FALSE(uname) \
|
||||
#define TESTCLEARFLAG_FALSE(uname, lname) \
|
||||
static inline bool folio_test_clear_##lname(struct folio *folio) \
|
||||
{ return 0; } \
|
||||
static inline int TestClearPage##uname(struct page *page) { return 0; }
|
||||
|
||||
#define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \
|
||||
SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname)
|
||||
#define PAGEFLAG_FALSE(uname, lname) TESTPAGEFLAG_FALSE(uname, lname) \
|
||||
SETPAGEFLAG_NOOP(uname, lname) CLEARPAGEFLAG_NOOP(uname, lname)
|
||||
|
||||
#define TESTSCFLAG_FALSE(uname) \
|
||||
TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname)
|
||||
#define TESTSCFLAG_FALSE(uname, lname) \
|
||||
TESTSETFLAG_FALSE(uname, lname) TESTCLEARFLAG_FALSE(uname, lname)
|
||||
|
||||
__PAGEFLAG(Locked, locked, PF_NO_TAIL)
|
||||
PAGEFLAG(Waiters, waiters, PF_ONLY_HEAD) __CLEARPAGEFLAG(Waiters, waiters, PF_ONLY_HEAD)
|
||||
|
@ -412,8 +459,8 @@ PAGEFLAG(MappedToDisk, mappedtodisk, PF_NO_TAIL)
|
|||
/* PG_readahead is only used for reads; PG_reclaim is only for writes */
|
||||
PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL)
|
||||
TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL)
|
||||
PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
|
||||
TESTCLEARFLAG(Readahead, reclaim, PF_NO_COMPOUND)
|
||||
PAGEFLAG(Readahead, readahead, PF_NO_COMPOUND)
|
||||
TESTCLEARFLAG(Readahead, readahead, PF_NO_COMPOUND)
|
||||
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
/*
|
||||
|
@ -422,22 +469,25 @@ PAGEFLAG(Readahead, reclaim, PF_NO_COMPOUND)
|
|||
*/
|
||||
#define PageHighMem(__p) is_highmem_idx(page_zonenum(__p))
|
||||
#else
|
||||
PAGEFLAG_FALSE(HighMem)
|
||||
PAGEFLAG_FALSE(HighMem, highmem)
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SWAP
|
||||
static __always_inline int PageSwapCache(struct page *page)
|
||||
static __always_inline bool folio_test_swapcache(struct folio *folio)
|
||||
{
|
||||
#ifdef CONFIG_THP_SWAP
|
||||
page = compound_head(page);
|
||||
#endif
|
||||
return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags);
|
||||
|
||||
return folio_test_swapbacked(folio) &&
|
||||
test_bit(PG_swapcache, folio_flags(folio, 0));
|
||||
}
|
||||
|
||||
static __always_inline bool PageSwapCache(struct page *page)
|
||||
{
|
||||
return folio_test_swapcache(page_folio(page));
|
||||
}
|
||||
|
||||
SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
|
||||
CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL)
|
||||
#else
|
||||
PAGEFLAG_FALSE(SwapCache)
|
||||
PAGEFLAG_FALSE(SwapCache, swapcache)
|
||||
#endif
|
||||
|
||||
PAGEFLAG(Unevictable, unevictable, PF_HEAD)
|
||||
|
@ -449,14 +499,14 @@ PAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
|
|||
__CLEARPAGEFLAG(Mlocked, mlocked, PF_NO_TAIL)
|
||||
TESTSCFLAG(Mlocked, mlocked, PF_NO_TAIL)
|
||||
#else
|
||||
PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked)
|
||||
TESTSCFLAG_FALSE(Mlocked)
|
||||
PAGEFLAG_FALSE(Mlocked, mlocked) __CLEARPAGEFLAG_NOOP(Mlocked, mlocked)
|
||||
TESTSCFLAG_FALSE(Mlocked, mlocked)
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
|
||||
PAGEFLAG(Uncached, uncached, PF_NO_COMPOUND)
|
||||
#else
|
||||
PAGEFLAG_FALSE(Uncached)
|
||||
PAGEFLAG_FALSE(Uncached, uncached)
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_MEMORY_FAILURE
|
||||
|
@ -465,7 +515,7 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY)
|
|||
#define __PG_HWPOISON (1UL << PG_hwpoison)
|
||||
extern bool take_page_off_buddy(struct page *page);
|
||||
#else
|
||||
PAGEFLAG_FALSE(HWPoison)
|
||||
PAGEFLAG_FALSE(HWPoison, hwpoison)
|
||||
#define __PG_HWPOISON 0
|
||||
#endif
|
||||
|
||||
|
@ -479,7 +529,7 @@ PAGEFLAG(Idle, idle, PF_ANY)
|
|||
#ifdef CONFIG_KASAN_HW_TAGS
|
||||
PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD)
|
||||
#else
|
||||
PAGEFLAG_FALSE(SkipKASanPoison)
|
||||
PAGEFLAG_FALSE(SkipKASanPoison, skip_kasan_poison)
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
@ -517,10 +567,14 @@ static __always_inline int PageMappingFlags(struct page *page)
|
|||
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) != 0;
|
||||
}
|
||||
|
||||
static __always_inline int PageAnon(struct page *page)
|
||||
static __always_inline bool folio_test_anon(struct folio *folio)
|
||||
{
|
||||
page = compound_head(page);
|
||||
return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
|
||||
return ((unsigned long)folio->mapping & PAGE_MAPPING_ANON) != 0;
|
||||
}
|
||||
|
||||
static __always_inline bool PageAnon(struct page *page)
|
||||
{
|
||||
return folio_test_anon(page_folio(page));
|
||||
}
|
||||
|
||||
static __always_inline int __PageMovable(struct page *page)
|
||||
|
@ -536,30 +590,32 @@ static __always_inline int __PageMovable(struct page *page)
|
|||
* is found in VM_MERGEABLE vmas. It's a PageAnon page, pointing not to any
|
||||
* anon_vma, but to that page's node of the stable tree.
|
||||
*/
|
||||
static __always_inline int PageKsm(struct page *page)
|
||||
static __always_inline bool folio_test_ksm(struct folio *folio)
|
||||
{
|
||||
page = compound_head(page);
|
||||
return ((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) ==
|
||||
return ((unsigned long)folio->mapping & PAGE_MAPPING_FLAGS) ==
|
||||
PAGE_MAPPING_KSM;
|
||||
}
|
||||
|
||||
static __always_inline bool PageKsm(struct page *page)
|
||||
{
|
||||
return folio_test_ksm(page_folio(page));
|
||||
}
|
||||
#else
|
||||
TESTPAGEFLAG_FALSE(Ksm)
|
||||
TESTPAGEFLAG_FALSE(Ksm, ksm)
|
||||
#endif
|
||||
|
||||
u64 stable_page_flags(struct page *page);
|
||||
|
||||
static inline int PageUptodate(struct page *page)
|
||||
static inline bool folio_test_uptodate(struct folio *folio)
|
||||
{
|
||||
int ret;
|
||||
page = compound_head(page);
|
||||
ret = test_bit(PG_uptodate, &(page)->flags);
|
||||
bool ret = test_bit(PG_uptodate, folio_flags(folio, 0));
|
||||
/*
|
||||
* Must ensure that the data we read out of the page is loaded
|
||||
* _after_ we've loaded page->flags to check for PageUptodate.
|
||||
* We can skip the barrier if the page is not uptodate, because
|
||||
* Must ensure that the data we read out of the folio is loaded
|
||||
* _after_ we've loaded folio->flags to check the uptodate bit.
|
||||
* We can skip the barrier if the folio is not uptodate, because
|
||||
* we wouldn't be reading anything from it.
|
||||
*
|
||||
* See SetPageUptodate() for the other side of the story.
|
||||
* See folio_mark_uptodate() for the other side of the story.
|
||||
*/
|
||||
if (ret)
|
||||
smp_rmb();
|
||||
|
@ -567,23 +623,36 @@ static inline int PageUptodate(struct page *page)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static inline int PageUptodate(struct page *page)
|
||||
{
|
||||
return folio_test_uptodate(page_folio(page));
|
||||
}
|
||||
|
||||
static __always_inline void __folio_mark_uptodate(struct folio *folio)
|
||||
{
|
||||
smp_wmb();
|
||||
__set_bit(PG_uptodate, folio_flags(folio, 0));
|
||||
}
|
||||
|
||||
static __always_inline void folio_mark_uptodate(struct folio *folio)
|
||||
{
|
||||
/*
|
||||
* Memory barrier must be issued before setting the PG_uptodate bit,
|
||||
* so that all previous stores issued in order to bring the folio
|
||||
* uptodate are actually visible before folio_test_uptodate becomes true.
|
||||
*/
|
||||
smp_wmb();
|
||||
set_bit(PG_uptodate, folio_flags(folio, 0));
|
||||
}
|
||||
|
||||
static __always_inline void __SetPageUptodate(struct page *page)
|
||||
{
|
||||
VM_BUG_ON_PAGE(PageTail(page), page);
|
||||
smp_wmb();
|
||||
__set_bit(PG_uptodate, &page->flags);
|
||||
__folio_mark_uptodate((struct folio *)page);
|
||||
}
|
||||
|
||||
static __always_inline void SetPageUptodate(struct page *page)
|
||||
{
|
||||
VM_BUG_ON_PAGE(PageTail(page), page);
|
||||
/*
|
||||
* Memory barrier must be issued before setting the PG_uptodate bit,
|
||||
* so that all previous stores issued in order to bring the page
|
||||
* uptodate are actually visible before PageUptodate becomes true.
|
||||
*/
|
||||
smp_wmb();
|
||||
set_bit(PG_uptodate, &page->flags);
|
||||
folio_mark_uptodate((struct folio *)page);
|
||||
}
|
||||
|
||||
CLEARPAGEFLAG(Uptodate, uptodate, PF_NO_TAIL)
|
||||
|
@ -608,6 +677,17 @@ static inline void set_page_writeback_keepwrite(struct page *page)
|
|||
|
||||
__PAGEFLAG(Head, head, PF_ANY) CLEARPAGEFLAG(Head, head, PF_ANY)
|
||||
|
||||
/* Whether there are one or multiple pages in a folio */
|
||||
static inline bool folio_test_single(struct folio *folio)
|
||||
{
|
||||
return !folio_test_head(folio);
|
||||
}
|
||||
|
||||
static inline bool folio_test_multi(struct folio *folio)
|
||||
{
|
||||
return folio_test_head(folio);
|
||||
}
|
||||
|
||||
static __always_inline void set_compound_head(struct page *page, struct page *head)
|
||||
{
|
||||
WRITE_ONCE(page->compound_head, (unsigned long)head + 1);
|
||||
|
@ -631,12 +711,15 @@ static inline void ClearPageCompound(struct page *page)
|
|||
#ifdef CONFIG_HUGETLB_PAGE
|
||||
int PageHuge(struct page *page);
|
||||
int PageHeadHuge(struct page *page);
|
||||
static inline bool folio_test_hugetlb(struct folio *folio)
|
||||
{
|
||||
return PageHeadHuge(&folio->page);
|
||||
}
|
||||
#else
|
||||
TESTPAGEFLAG_FALSE(Huge)
|
||||
TESTPAGEFLAG_FALSE(HeadHuge)
|
||||
TESTPAGEFLAG_FALSE(Huge, hugetlb)
|
||||
TESTPAGEFLAG_FALSE(HeadHuge, headhuge)
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
/*
|
||||
* PageHuge() only returns true for hugetlbfs pages, but not for
|
||||
|
@ -652,6 +735,11 @@ static inline int PageTransHuge(struct page *page)
|
|||
return PageHead(page);
|
||||
}
|
||||
|
||||
static inline bool folio_test_transhuge(struct folio *folio)
|
||||
{
|
||||
return folio_test_head(folio);
|
||||
}
|
||||
|
||||
/*
|
||||
* PageTransCompound returns true for both transparent huge pages
|
||||
* and hugetlbfs pages, so it should only be called when it's known
|
||||
|
@ -688,12 +776,12 @@ static inline int PageTransTail(struct page *page)
|
|||
PAGEFLAG(DoubleMap, double_map, PF_SECOND)
|
||||
TESTSCFLAG(DoubleMap, double_map, PF_SECOND)
|
||||
#else
|
||||
TESTPAGEFLAG_FALSE(TransHuge)
|
||||
TESTPAGEFLAG_FALSE(TransCompound)
|
||||
TESTPAGEFLAG_FALSE(TransCompoundMap)
|
||||
TESTPAGEFLAG_FALSE(TransTail)
|
||||
PAGEFLAG_FALSE(DoubleMap)
|
||||
TESTSCFLAG_FALSE(DoubleMap)
|
||||
TESTPAGEFLAG_FALSE(TransHuge, transhuge)
|
||||
TESTPAGEFLAG_FALSE(TransCompound, transcompound)
|
||||
TESTPAGEFLAG_FALSE(TransCompoundMap, transcompoundmap)
|
||||
TESTPAGEFLAG_FALSE(TransTail, transtail)
|
||||
PAGEFLAG_FALSE(DoubleMap, double_map)
|
||||
TESTSCFLAG_FALSE(DoubleMap, double_map)
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
@ -877,6 +965,11 @@ static inline int page_has_private(struct page *page)
|
|||
return !!(page->flags & PAGE_FLAGS_PRIVATE);
|
||||
}
|
||||
|
||||
static inline bool folio_has_private(struct folio *folio)
|
||||
{
|
||||
return page_has_private(&folio->page);
|
||||
}
|
||||
|
||||
#undef PF_ANY
|
||||
#undef PF_HEAD
|
||||
#undef PF_ONLY_HEAD
|
||||
|
|
Loading…
Reference in a new issue