mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

mm/rmap: Convert make_device_exclusive_range() to use folios 

Move the PageTail check earlier so we can avoid even taking the folio
lock on tail pages.  Otherwise, this is a straightforward use of
folios throughout.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
This commit is contained in:
Matthew Wilcox (Oracle) 2022-01-28 16:03:42 -05:00
parent 4b8554c527
commit 0d2514859c
1 changed files with 31 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
mm/rmap.c
View File

@ -1990,8 +1990,9 @@ struct make_exclusive_args {
static bool page_make_device_exclusive_one(struct page *page,
struct vm_area_struct *vma, unsigned long address, void *priv)
{
struct folio *folio = page_folio(page);
struct mm_struct *mm = vma->vm_mm;
DEFINE_PAGE_VMA_WALK(pvmw, page, vma, address, 0);
DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
struct make_exclusive_args *args = priv;
pte_t pteval;
struct page *subpage;
@ -2002,12 +2003,13 @@ static bool page_make_device_exclusive_one(struct page *page,
mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
vma->vm_mm, address, min(vma->vm_end,
address + page_size(page)), args->owner);
address + folio_size(folio)),
args->owner);
mmu_notifier_invalidate_range_start(&range);
while (page_vma_mapped_walk(&pvmw)) {
/* Unexpected PMD-mapped THP? */
VM_BUG_ON_PAGE(!pvmw.pte, page);
VM_BUG_ON_FOLIO(!pvmw.pte, folio);
if (!pte_present(*pvmw.pte)) {
ret = false;
@ -2015,16 +2017,17 @@ static bool page_make_device_exclusive_one(struct page *page,
break;
}
subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
subpage = folio_page(folio,
pte_pfn(*pvmw.pte) - folio_pfn(folio));
address = pvmw.address;
/* Nuke the page table entry. */
flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
pteval = ptep_clear_flush(vma, address, pvmw.pte);
/* Move the dirty bit to the page. Now the pte is gone. */
/* Set the dirty flag on the folio now the pte is gone. */
if (pte_dirty(pteval))
set_page_dirty(page);
folio_mark_dirty(folio);
/*
* Check that our target page is still mapped at the expected
@ -2066,21 +2069,22 @@ static bool page_make_device_exclusive_one(struct page *page,
}
/**
* page_make_device_exclusive - mark the page exclusively owned by a device
* @page: the page to replace page table entries for
* @mm: the mm_struct where the page is expected to be mapped
* @address: address where the page is expected to be mapped
* folio_make_device_exclusive - Mark the folio exclusively owned by a device.
* @folio: The folio to replace page table entries for.
* @mm: The mm_struct where the folio is expected to be mapped.
* @address: Address where the folio is expected to be mapped.
* @owner: passed to MMU_NOTIFY_EXCLUSIVE range notifier callbacks
*
* Tries to remove all the page table entries which are mapping this page and
* replace them with special device exclusive swap entries to grant a device
* exclusive access to the page. Caller must hold the page lock.
* Tries to remove all the page table entries which are mapping this
* folio and replace them with special device exclusive swap entries to
* grant a device exclusive access to the folio.
*
* Returns false if the page is still mapped, or if it could not be unmapped
* Context: Caller must hold the folio lock.
* Return: false if the page is still mapped, or if it could not be unmapped
* from the expected address. Otherwise returns true (success).
*/
static bool page_make_device_exclusive(struct page *page, struct mm_struct *mm,
unsigned long address, void *owner)
static bool folio_make_device_exclusive(struct folio *folio,
struct mm_struct *mm, unsigned long address, void *owner)
{
struct make_exclusive_args args = {
.mm = mm,
@ -2096,16 +2100,15 @@ static bool page_make_device_exclusive(struct page *page, struct mm_struct *mm,
};
/*
* Restrict to anonymous pages for now to avoid potential writeback
* issues. Also tail pages shouldn't be passed to rmap_walk so skip
* those.
* Restrict to anonymous folios for now to avoid potential writeback
* issues.
*/
if (!PageAnon(page) || PageTail(page))
if (!folio_test_anon(folio))
return false;
rmap_walk(page, &rwc);
rmap_walk(&folio->page, &rwc);
return args.valid && !page_mapcount(page);
return args.valid && !folio_mapcount(folio);
}
/**
@ -2143,15 +2146,16 @@ int make_device_exclusive_range(struct mm_struct *mm, unsigned long start,
return npages;
for (i = 0; i < npages; i++, start += PAGE_SIZE) {
if (!trylock_page(pages[i])) {
put_page(pages[i]);
struct folio *folio = page_folio(pages[i]);
if (PageTail(pages[i]) || !folio_trylock(folio)) {
folio_put(folio);
pages[i] = NULL;
continue;
}
if (!page_make_device_exclusive(pages[i], mm, start, owner)) {
unlock_page(pages[i]);
put_page(pages[i]);
if (!folio_make_device_exclusive(folio, mm, start, owner)) {
folio_unlock(folio);
folio_put(folio);
pages[i] = NULL;
}
}