mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-28 21:33:52 +00:00
mm/rmap: Convert try_to_migrate() to folios
Convert the callers to pass a folio and the try_to_migrate_one() worker to use a folio throughout. Fixes an assumption that a folio must be <= PMD size. Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
This commit is contained in:
parent
869f7ee6f6
commit
4b8554c527
5 changed files with 42 additions and 35 deletions
|
@ -193,7 +193,7 @@ static inline void page_dup_rmap(struct page *page, bool compound)
|
||||||
int folio_referenced(struct folio *, int is_locked,
|
int folio_referenced(struct folio *, int is_locked,
|
||||||
struct mem_cgroup *memcg, unsigned long *vm_flags);
|
struct mem_cgroup *memcg, unsigned long *vm_flags);
|
||||||
|
|
||||||
void try_to_migrate(struct page *page, enum ttu_flags flags);
|
void try_to_migrate(struct folio *folio, enum ttu_flags flags);
|
||||||
void try_to_unmap(struct folio *, enum ttu_flags flags);
|
void try_to_unmap(struct folio *, enum ttu_flags flags);
|
||||||
|
|
||||||
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
|
int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
|
||||||
|
|
|
@ -2262,8 +2262,8 @@ static void unmap_page(struct page *page)
|
||||||
* pages can simply be left unmapped, then faulted back on demand.
|
* pages can simply be left unmapped, then faulted back on demand.
|
||||||
* If that is ever changed (perhaps for mlock), update remap_page().
|
* If that is ever changed (perhaps for mlock), update remap_page().
|
||||||
*/
|
*/
|
||||||
if (PageAnon(page))
|
if (folio_test_anon(folio))
|
||||||
try_to_migrate(page, ttu_flags);
|
try_to_migrate(folio, ttu_flags);
|
||||||
else
|
else
|
||||||
try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
|
try_to_unmap(folio, ttu_flags | TTU_IGNORE_MLOCK);
|
||||||
|
|
||||||
|
|
|
@ -912,6 +912,7 @@ static int move_to_new_page(struct page *newpage, struct page *page,
|
||||||
static int __unmap_and_move(struct page *page, struct page *newpage,
|
static int __unmap_and_move(struct page *page, struct page *newpage,
|
||||||
int force, enum migrate_mode mode)
|
int force, enum migrate_mode mode)
|
||||||
{
|
{
|
||||||
|
struct folio *folio = page_folio(page);
|
||||||
int rc = -EAGAIN;
|
int rc = -EAGAIN;
|
||||||
bool page_was_mapped = false;
|
bool page_was_mapped = false;
|
||||||
struct anon_vma *anon_vma = NULL;
|
struct anon_vma *anon_vma = NULL;
|
||||||
|
@ -1015,7 +1016,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
|
||||||
/* Establish migration ptes */
|
/* Establish migration ptes */
|
||||||
VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
|
VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
|
||||||
page);
|
page);
|
||||||
try_to_migrate(page, 0);
|
try_to_migrate(folio, 0);
|
||||||
page_was_mapped = true;
|
page_was_mapped = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1165,6 +1166,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
|
||||||
enum migrate_mode mode, int reason,
|
enum migrate_mode mode, int reason,
|
||||||
struct list_head *ret)
|
struct list_head *ret)
|
||||||
{
|
{
|
||||||
|
struct folio *src = page_folio(hpage);
|
||||||
int rc = -EAGAIN;
|
int rc = -EAGAIN;
|
||||||
int page_was_mapped = 0;
|
int page_was_mapped = 0;
|
||||||
struct page *new_hpage;
|
struct page *new_hpage;
|
||||||
|
@ -1241,7 +1243,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
|
||||||
ttu |= TTU_RMAP_LOCKED;
|
ttu |= TTU_RMAP_LOCKED;
|
||||||
}
|
}
|
||||||
|
|
||||||
try_to_migrate(hpage, ttu);
|
try_to_migrate(src, ttu);
|
||||||
page_was_mapped = 1;
|
page_was_mapped = 1;
|
||||||
|
|
||||||
if (mapping_locked)
|
if (mapping_locked)
|
||||||
|
|
|
@ -333,6 +333,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate)
|
||||||
|
|
||||||
for (i = 0; i < npages; i++) {
|
for (i = 0; i < npages; i++) {
|
||||||
struct page *page = migrate_pfn_to_page(migrate->src[i]);
|
struct page *page = migrate_pfn_to_page(migrate->src[i]);
|
||||||
|
struct folio *folio;
|
||||||
|
|
||||||
if (!page)
|
if (!page)
|
||||||
continue;
|
continue;
|
||||||
|
@ -356,8 +357,9 @@ static void migrate_vma_unmap(struct migrate_vma *migrate)
|
||||||
put_page(page);
|
put_page(page);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (page_mapped(page))
|
folio = page_folio(page);
|
||||||
try_to_migrate(page, 0);
|
if (folio_mapped(folio))
|
||||||
|
try_to_migrate(folio, 0);
|
||||||
|
|
||||||
if (page_mapped(page) || !migrate_vma_check_page(page)) {
|
if (page_mapped(page) || !migrate_vma_check_page(page)) {
|
||||||
if (!is_zone_device_page(page)) {
|
if (!is_zone_device_page(page)) {
|
||||||
|
|
59
mm/rmap.c
59
mm/rmap.c
|
@ -1706,7 +1706,7 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
{
|
{
|
||||||
struct folio *folio = page_folio(page);
|
struct folio *folio = page_folio(page);
|
||||||
struct mm_struct *mm = vma->vm_mm;
|
struct mm_struct *mm = vma->vm_mm;
|
||||||
DEFINE_PAGE_VMA_WALK(pvmw, page, vma, address, 0);
|
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
|
||||||
pte_t pteval;
|
pte_t pteval;
|
||||||
struct page *subpage;
|
struct page *subpage;
|
||||||
bool ret = true;
|
bool ret = true;
|
||||||
|
@ -1740,7 +1740,7 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
range.end = vma_address_end(&pvmw);
|
range.end = vma_address_end(&pvmw);
|
||||||
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
|
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
|
||||||
address, range.end);
|
address, range.end);
|
||||||
if (PageHuge(page)) {
|
if (folio_test_hugetlb(folio)) {
|
||||||
/*
|
/*
|
||||||
* If sharing is possible, start and end will be adjusted
|
* If sharing is possible, start and end will be adjusted
|
||||||
* accordingly.
|
* accordingly.
|
||||||
|
@ -1754,21 +1754,24 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
|
#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
|
||||||
/* PMD-mapped THP migration entry */
|
/* PMD-mapped THP migration entry */
|
||||||
if (!pvmw.pte) {
|
if (!pvmw.pte) {
|
||||||
VM_BUG_ON_PAGE(PageHuge(page) ||
|
subpage = folio_page(folio,
|
||||||
!PageTransCompound(page), page);
|
pmd_pfn(*pvmw.pmd) - folio_pfn(folio));
|
||||||
|
VM_BUG_ON_FOLIO(folio_test_hugetlb(folio) ||
|
||||||
|
!folio_test_pmd_mappable(folio), folio);
|
||||||
|
|
||||||
set_pmd_migration_entry(&pvmw, page);
|
set_pmd_migration_entry(&pvmw, subpage);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/* Unexpected PMD-mapped THP? */
|
/* Unexpected PMD-mapped THP? */
|
||||||
VM_BUG_ON_PAGE(!pvmw.pte, page);
|
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
|
||||||
|
|
||||||
subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
|
subpage = folio_page(folio,
|
||||||
|
pte_pfn(*pvmw.pte) - folio_pfn(folio));
|
||||||
address = pvmw.address;
|
address = pvmw.address;
|
||||||
|
|
||||||
if (PageHuge(page) && !PageAnon(page)) {
|
if (folio_test_hugetlb(folio) && !folio_test_anon(folio)) {
|
||||||
/*
|
/*
|
||||||
* To call huge_pmd_unshare, i_mmap_rwsem must be
|
* To call huge_pmd_unshare, i_mmap_rwsem must be
|
||||||
* held in write mode. Caller needs to explicitly
|
* held in write mode. Caller needs to explicitly
|
||||||
|
@ -1806,15 +1809,15 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
|
flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
|
||||||
pteval = ptep_clear_flush(vma, address, pvmw.pte);
|
pteval = ptep_clear_flush(vma, address, pvmw.pte);
|
||||||
|
|
||||||
/* Move the dirty bit to the page. Now the pte is gone. */
|
/* Set the dirty flag on the folio now the pte is gone. */
|
||||||
if (pte_dirty(pteval))
|
if (pte_dirty(pteval))
|
||||||
set_page_dirty(page);
|
folio_mark_dirty(folio);
|
||||||
|
|
||||||
/* Update high watermark before we lower rss */
|
/* Update high watermark before we lower rss */
|
||||||
update_hiwater_rss(mm);
|
update_hiwater_rss(mm);
|
||||||
|
|
||||||
if (is_zone_device_page(page)) {
|
if (folio_is_zone_device(folio)) {
|
||||||
unsigned long pfn = page_to_pfn(page);
|
unsigned long pfn = folio_pfn(folio);
|
||||||
swp_entry_t entry;
|
swp_entry_t entry;
|
||||||
pte_t swp_pte;
|
pte_t swp_pte;
|
||||||
|
|
||||||
|
@ -1850,16 +1853,16 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
* changed when hugepage migrations to device private
|
* changed when hugepage migrations to device private
|
||||||
* memory are supported.
|
* memory are supported.
|
||||||
*/
|
*/
|
||||||
subpage = page;
|
subpage = &folio->page;
|
||||||
} else if (PageHWPoison(page)) {
|
} else if (PageHWPoison(subpage)) {
|
||||||
pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
|
pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
|
||||||
if (PageHuge(page)) {
|
if (folio_test_hugetlb(folio)) {
|
||||||
hugetlb_count_sub(compound_nr(page), mm);
|
hugetlb_count_sub(folio_nr_pages(folio), mm);
|
||||||
set_huge_swap_pte_at(mm, address,
|
set_huge_swap_pte_at(mm, address,
|
||||||
pvmw.pte, pteval,
|
pvmw.pte, pteval,
|
||||||
vma_mmu_pagesize(vma));
|
vma_mmu_pagesize(vma));
|
||||||
} else {
|
} else {
|
||||||
dec_mm_counter(mm, mm_counter(page));
|
dec_mm_counter(mm, mm_counter(&folio->page));
|
||||||
set_pte_at(mm, address, pvmw.pte, pteval);
|
set_pte_at(mm, address, pvmw.pte, pteval);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1874,7 +1877,7 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
* migration) will not expect userfaults on already
|
* migration) will not expect userfaults on already
|
||||||
* copied pages.
|
* copied pages.
|
||||||
*/
|
*/
|
||||||
dec_mm_counter(mm, mm_counter(page));
|
dec_mm_counter(mm, mm_counter(&folio->page));
|
||||||
/* We have to invalidate as we cleared the pte */
|
/* We have to invalidate as we cleared the pte */
|
||||||
mmu_notifier_invalidate_range(mm, address,
|
mmu_notifier_invalidate_range(mm, address,
|
||||||
address + PAGE_SIZE);
|
address + PAGE_SIZE);
|
||||||
|
@ -1920,10 +1923,10 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
*
|
*
|
||||||
* See Documentation/vm/mmu_notifier.rst
|
* See Documentation/vm/mmu_notifier.rst
|
||||||
*/
|
*/
|
||||||
page_remove_rmap(subpage, vma, PageHuge(page));
|
page_remove_rmap(subpage, vma, folio_test_hugetlb(folio));
|
||||||
if (vma->vm_flags & VM_LOCKED)
|
if (vma->vm_flags & VM_LOCKED)
|
||||||
mlock_page_drain(smp_processor_id());
|
mlock_page_drain(smp_processor_id());
|
||||||
put_page(page);
|
folio_put(folio);
|
||||||
}
|
}
|
||||||
|
|
||||||
mmu_notifier_invalidate_range_end(&range);
|
mmu_notifier_invalidate_range_end(&range);
|
||||||
|
@ -1933,13 +1936,13 @@ static bool try_to_migrate_one(struct page *page, struct vm_area_struct *vma,
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* try_to_migrate - try to replace all page table mappings with swap entries
|
* try_to_migrate - try to replace all page table mappings with swap entries
|
||||||
* @page: the page to replace page table entries for
|
* @folio: the folio to replace page table entries for
|
||||||
* @flags: action and flags
|
* @flags: action and flags
|
||||||
*
|
*
|
||||||
* Tries to remove all the page table entries which are mapping this page and
|
* Tries to remove all the page table entries which are mapping this folio and
|
||||||
* replace them with special swap entries. Caller must hold the page lock.
|
* replace them with special swap entries. Caller must hold the folio lock.
|
||||||
*/
|
*/
|
||||||
void try_to_migrate(struct page *page, enum ttu_flags flags)
|
void try_to_migrate(struct folio *folio, enum ttu_flags flags)
|
||||||
{
|
{
|
||||||
struct rmap_walk_control rwc = {
|
struct rmap_walk_control rwc = {
|
||||||
.rmap_one = try_to_migrate_one,
|
.rmap_one = try_to_migrate_one,
|
||||||
|
@ -1956,7 +1959,7 @@ void try_to_migrate(struct page *page, enum ttu_flags flags)
|
||||||
TTU_SYNC)))
|
TTU_SYNC)))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
if (is_zone_device_page(page) && !is_device_private_page(page))
|
if (folio_is_zone_device(folio) && !folio_is_device_private(folio))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -1967,13 +1970,13 @@ void try_to_migrate(struct page *page, enum ttu_flags flags)
|
||||||
* locking requirements of exec(), migration skips
|
* locking requirements of exec(), migration skips
|
||||||
* temporary VMAs until after exec() completes.
|
* temporary VMAs until after exec() completes.
|
||||||
*/
|
*/
|
||||||
if (!PageKsm(page) && PageAnon(page))
|
if (!folio_test_ksm(folio) && folio_test_anon(folio))
|
||||||
rwc.invalid_vma = invalid_migration_vma;
|
rwc.invalid_vma = invalid_migration_vma;
|
||||||
|
|
||||||
if (flags & TTU_RMAP_LOCKED)
|
if (flags & TTU_RMAP_LOCKED)
|
||||||
rmap_walk_locked(page, &rwc);
|
rmap_walk_locked(&folio->page, &rwc);
|
||||||
else
|
else
|
||||||
rmap_walk(page, &rwc);
|
rmap_walk(&folio->page, &rwc);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_DEVICE_PRIVATE
|
#ifdef CONFIG_DEVICE_PRIVATE
|
||||||
|
|
Loading…
Reference in a new issue