filemap: Allow __filemap_get_folio to allocate large folios
Allow callers of __filemap_get_folio() to specify a preferred folio order in the FGP flags. This is only honoured in the FGP_CREATE path; if there is already a folio in the page cache that covers the index, we will return it, no matter what its order is. No create-around is attempted; we will only create folios which start at the specified index. Unmodified callers will continue to allocate order 0 folios. Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Reviewed-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Darrick J. Wong <djwong@kernel.org>
This commit is contained in:
Matthew Wilcox (Oracle)
parent
ffc143db63
commit
4f66170119
|
|
@ -470,6 +470,19 @@ static inline void *detach_page_private(struct page *page)
|
|||
return folio_detach_private(page_folio(page));
|
||||
}
|
||||
|
||||
/*
|
||||
* There are some parts of the kernel which assume that PMD entries
|
||||
* are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then,
|
||||
* limit the maximum allocation order to PMD size. I'm not aware of any
|
||||
* assumptions about maximum order if THP are disabled, but 8 seems like
|
||||
* a good order (that's 1MB if you're using 4kB pages)
|
||||
*/
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER
|
||||
#else
|
||||
#define MAX_PAGECACHE_ORDER 8
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order);
|
||||
#else
|
||||
|
|
@ -535,9 +548,30 @@ typedef unsigned int __bitwise fgf_t;
|
|||
#define FGP_NOWAIT ((__force fgf_t)0x00000020)
|
||||
#define FGP_FOR_MMAP ((__force fgf_t)0x00000040)
|
||||
#define FGP_STABLE ((__force fgf_t)0x00000080)
|
||||
#define FGF_GET_ORDER(fgf) (((__force unsigned)fgf) >> 26) /* top 6 bits */
|
||||
|
||||
#define FGP_WRITEBEGIN (FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_STABLE)
|
||||
|
||||
/**
|
||||
* fgf_set_order - Encode a length in the fgf_t flags.
|
||||
* @size: The suggested size of the folio to create.
|
||||
*
|
||||
* The caller of __filemap_get_folio() can use this to suggest a preferred
|
||||
* size for the folio that is created. If there is already a folio at
|
||||
* the index, it will be returned, no matter what its size. If a folio
|
||||
* is freshly created, it may be of a different size than requested
|
||||
* due to alignment constraints, memory pressure, or the presence of
|
||||
* other folios at nearby indices.
|
||||
*/
|
||||
static inline fgf_t fgf_set_order(size_t size)
|
||||
{
|
||||
unsigned int shift = ilog2(size);
|
||||
|
||||
if (shift <= PAGE_SHIFT)
|
||||
return 0;
|
||||
return (__force fgf_t)((shift - PAGE_SHIFT) << 26);
|
||||
}
|
||||
|
||||
void *filemap_get_entry(struct address_space *mapping, pgoff_t index);
|
||||
struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
|
||||
fgf_t fgp_flags, gfp_t gfp);
|
||||
|
|
|
|||
46
mm/filemap.c
46
mm/filemap.c
|
|
@ -1905,7 +1905,9 @@ repeat:
|
|||
folio_wait_stable(folio);
|
||||
no_page:
|
||||
if (!folio && (fgp_flags & FGP_CREAT)) {
|
||||
unsigned order = FGF_GET_ORDER(fgp_flags);
|
||||
int err;
|
||||
|
||||
if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
|
||||
gfp |= __GFP_WRITE;
|
||||
if (fgp_flags & FGP_NOFS)
|
||||
|
|
@ -1914,26 +1916,44 @@ no_page:
|
|||
gfp &= ~GFP_KERNEL;
|
||||
gfp |= GFP_NOWAIT | __GFP_NOWARN;
|
||||
}
|
||||
|
||||
folio = filemap_alloc_folio(gfp, 0);
|
||||
if (!folio)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
if (WARN_ON_ONCE(!(fgp_flags & (FGP_LOCK | FGP_FOR_MMAP))))
|
||||
fgp_flags |= FGP_LOCK;
|
||||
|
||||
/* Init accessed so avoid atomic mark_page_accessed later */
|
||||
if (fgp_flags & FGP_ACCESSED)
|
||||
__folio_set_referenced(folio);
|
||||
if (!mapping_large_folio_support(mapping))
|
||||
order = 0;
|
||||
if (order > MAX_PAGECACHE_ORDER)
|
||||
order = MAX_PAGECACHE_ORDER;
|
||||
/* If we're not aligned, allocate a smaller folio */
|
||||
if (index & ((1UL << order) - 1))
|
||||
order = __ffs(index);
|
||||
|
||||
err = filemap_add_folio(mapping, folio, index, gfp);
|
||||
if (unlikely(err)) {
|
||||
do {
|
||||
gfp_t alloc_gfp = gfp;
|
||||
|
||||
err = -ENOMEM;
|
||||
if (order == 1)
|
||||
order = 0;
|
||||
if (order > 0)
|
||||
alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN;
|
||||
folio = filemap_alloc_folio(alloc_gfp, order);
|
||||
if (!folio)
|
||||
continue;
|
||||
|
||||
/* Init accessed so avoid atomic mark_page_accessed later */
|
||||
if (fgp_flags & FGP_ACCESSED)
|
||||
__folio_set_referenced(folio);
|
||||
|
||||
err = filemap_add_folio(mapping, folio, index, gfp);
|
||||
if (!err)
|
||||
break;
|
||||
folio_put(folio);
|
||||
folio = NULL;
|
||||
if (err == -EEXIST)
|
||||
goto repeat;
|
||||
}
|
||||
} while (order-- > 0);
|
||||
|
||||
if (err == -EEXIST)
|
||||
goto repeat;
|
||||
if (err)
|
||||
return ERR_PTR(err);
|
||||
/*
|
||||
* filemap_add_folio locks the page, and for mmap
|
||||
* we expect an unlocked page.
|
||||
|
|
|
|||
|
|
@ -461,19 +461,6 @@ static int try_context_readahead(struct address_space *mapping,
|
|||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* There are some parts of the kernel which assume that PMD entries
|
||||
* are exactly HPAGE_PMD_ORDER. Those should be fixed, but until then,
|
||||
* limit the maximum allocation order to PMD size. I'm not aware of any
|
||||
* assumptions about maximum order if THP are disabled, but 8 seems like
|
||||
* a good order (that's 1MB if you're using 4kB pages)
|
||||
*/
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define MAX_PAGECACHE_ORDER HPAGE_PMD_ORDER
|
||||
#else
|
||||
#define MAX_PAGECACHE_ORDER 8
|
||||
#endif
|
||||
|
||||
static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,
|
||||
pgoff_t mark, unsigned int order, gfp_t gfp)
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue