2019-05-27 06:55:21 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2013-04-10 12:48:00 +00:00
|
|
|
/*
|
|
|
|
* arch/arm64/mm/hugetlbpage.c
|
|
|
|
*
|
|
|
|
* Copyright (C) 2013 Linaro Ltd.
|
|
|
|
*
|
|
|
|
* Based on arch/x86/mm/hugetlbpage.c.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/hugetlb.h>
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#include <asm/mman.h>
|
|
|
|
#include <asm/tlb.h>
|
|
|
|
#include <asm/tlbflush.h>
|
|
|
|
|
2020-07-01 04:42:01 +00:00
|
|
|
/*
|
|
|
|
* HugeTLB Support Matrix
|
|
|
|
*
|
|
|
|
* ---------------------------------------------------
|
|
|
|
* | Page Size | CONT PTE | PMD | CONT PMD | PUD |
|
|
|
|
* ---------------------------------------------------
|
|
|
|
* | 4K | 64K | 2M | 32M | 1G |
|
|
|
|
* | 16K | 2M | 32M | 1G | |
|
|
|
|
* | 64K | 2M | 512M | 16G | |
|
|
|
|
* ---------------------------------------------------
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reserve CMA areas for the largest supported gigantic
|
|
|
|
* huge page when requested. Any other smaller gigantic
|
|
|
|
* huge pages could still be served from those areas.
|
|
|
|
*/
|
|
|
|
#ifdef CONFIG_CMA
|
|
|
|
void __init arm64_hugetlb_cma_reserve(void)
|
|
|
|
{
|
|
|
|
int order;
|
|
|
|
|
2021-09-20 09:29:31 +00:00
|
|
|
if (pud_sect_supported())
|
|
|
|
order = PUD_SHIFT - PAGE_SHIFT;
|
|
|
|
else
|
2021-10-29 11:27:53 +00:00
|
|
|
order = CONT_PMD_SHIFT - PAGE_SHIFT;
|
|
|
|
|
2020-07-01 04:42:01 +00:00
|
|
|
/*
|
|
|
|
* HugeTLB CMA reservation is required for gigantic
|
|
|
|
* huge pages which could not be allocated via the
|
|
|
|
* page allocator. Just warn if there is any change
|
|
|
|
* breaking this assumption.
|
|
|
|
*/
|
2023-12-28 14:47:04 +00:00
|
|
|
WARN_ON(order <= MAX_PAGE_ORDER);
|
2020-07-01 04:42:01 +00:00
|
|
|
hugetlb_cma_reserve(order);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_CMA */
|
|
|
|
|
2022-02-17 04:52:37 +00:00
|
|
|
static bool __hugetlb_valid_size(unsigned long size)
|
2019-03-05 23:43:58 +00:00
|
|
|
{
|
2022-02-17 04:52:37 +00:00
|
|
|
switch (size) {
|
2021-09-20 09:29:31 +00:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2019-03-05 23:43:58 +00:00
|
|
|
case PUD_SIZE:
|
2021-09-20 09:29:31 +00:00
|
|
|
return pud_sect_supported();
|
2019-03-05 23:43:58 +00:00
|
|
|
#endif
|
|
|
|
case CONT_PMD_SIZE:
|
2022-02-17 04:52:37 +00:00
|
|
|
case PMD_SIZE:
|
2019-03-05 23:43:58 +00:00
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
return true;
|
|
|
|
}
|
2022-02-17 04:52:37 +00:00
|
|
|
|
2019-03-05 23:43:58 +00:00
|
|
|
return false;
|
|
|
|
}
|
2022-02-17 04:52:37 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
|
|
|
|
bool arch_hugetlb_migration_supported(struct hstate *h)
|
|
|
|
{
|
|
|
|
size_t pagesize = huge_page_size(h);
|
|
|
|
|
|
|
|
if (!__hugetlb_valid_size(pagesize)) {
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
return true;
|
|
|
|
}
|
2019-03-05 23:43:58 +00:00
|
|
|
#endif
|
|
|
|
|
2013-04-10 12:48:00 +00:00
|
|
|
int pmd_huge(pmd_t pmd)
|
|
|
|
{
|
2015-07-01 12:08:31 +00:00
|
|
|
return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
|
2013-04-10 12:48:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int pud_huge(pud_t pud)
|
|
|
|
{
|
2014-05-15 14:19:22 +00:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2015-07-01 12:08:31 +00:00
|
|
|
return pud_val(pud) && !(pud_val(pud) & PUD_TABLE_BIT);
|
2014-05-15 14:19:22 +00:00
|
|
|
#else
|
|
|
|
return 0;
|
|
|
|
#endif
|
2013-04-10 12:48:00 +00:00
|
|
|
}
|
|
|
|
|
2015-12-17 19:31:26 +00:00
|
|
|
static int find_num_contig(struct mm_struct *mm, unsigned long addr,
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-06 22:39:29 +00:00
|
|
|
pte_t *ptep, size_t *pgsize)
|
2015-12-17 19:31:26 +00:00
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
pgd_t *pgdp = pgd_offset(mm, addr);
|
2020-06-04 23:46:23 +00:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 11:14:56 +00:00
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
2015-12-17 19:31:26 +00:00
|
|
|
|
|
|
|
*pgsize = PAGE_SIZE;
|
2020-06-04 23:46:23 +00:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 11:14:56 +00:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
if ((pte_t *)pmdp == ptep) {
|
2015-12-17 19:31:26 +00:00
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
return CONT_PMDS;
|
|
|
|
}
|
|
|
|
return CONT_PTES;
|
|
|
|
}
|
|
|
|
|
2017-08-22 10:42:46 +00:00
|
|
|
static inline int num_contig_ptes(unsigned long size, size_t *pgsize)
|
|
|
|
{
|
|
|
|
int contig_ptes = 0;
|
|
|
|
|
|
|
|
*pgsize = size;
|
|
|
|
|
|
|
|
switch (size) {
|
2021-09-20 09:29:31 +00:00
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
2017-08-22 10:42:46 +00:00
|
|
|
case PUD_SIZE:
|
2021-09-20 09:29:31 +00:00
|
|
|
if (pud_sect_supported())
|
|
|
|
contig_ptes = 1;
|
|
|
|
break;
|
2017-08-22 10:42:46 +00:00
|
|
|
#endif
|
|
|
|
case PMD_SIZE:
|
|
|
|
contig_ptes = 1;
|
|
|
|
break;
|
|
|
|
case CONT_PMD_SIZE:
|
|
|
|
*pgsize = PMD_SIZE;
|
|
|
|
contig_ptes = CONT_PMDS;
|
|
|
|
break;
|
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
*pgsize = PAGE_SIZE;
|
|
|
|
contig_ptes = CONT_PTES;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return contig_ptes;
|
|
|
|
}
|
|
|
|
|
2022-05-16 00:55:58 +00:00
|
|
|
pte_t huge_ptep_get(pte_t *ptep)
|
|
|
|
{
|
|
|
|
int ncontig, i;
|
|
|
|
size_t pgsize;
|
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
|
|
|
|
|
|
|
if (!pte_present(orig_pte) || !pte_cont(orig_pte))
|
|
|
|
return orig_pte;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(page_size(pte_page(orig_pte)), &pgsize);
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++) {
|
|
|
|
pte_t pte = ptep_get(ptep);
|
|
|
|
|
|
|
|
if (pte_dirty(pte))
|
|
|
|
orig_pte = pte_mkdirty(orig_pte);
|
|
|
|
|
|
|
|
if (pte_young(pte))
|
|
|
|
orig_pte = pte_mkyoung(orig_pte);
|
|
|
|
}
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2017-08-22 10:42:44 +00:00
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step.
|
|
|
|
*/
|
2022-05-10 04:39:30 +00:00
|
|
|
static pte_t get_clear_contig(struct mm_struct *mm,
|
2017-08-22 10:42:44 +00:00
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
2022-05-16 00:55:57 +00:00
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
2022-05-10 04:39:30 +00:00
|
|
|
unsigned long i;
|
2017-08-22 10:42:44 +00:00
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) {
|
|
|
|
pte_t pte = ptep_get_and_clear(mm, addr, ptep);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If HW_AFDBM is enabled, then the HW could turn on
|
2018-09-21 15:34:04 +00:00
|
|
|
* the dirty or accessed bit for any page in the set,
|
|
|
|
* so check them all.
|
2017-08-22 10:42:44 +00:00
|
|
|
*/
|
|
|
|
if (pte_dirty(pte))
|
|
|
|
orig_pte = pte_mkdirty(orig_pte);
|
2018-09-21 15:34:04 +00:00
|
|
|
|
|
|
|
if (pte_young(pte))
|
|
|
|
orig_pte = pte_mkyoung(orig_pte);
|
2017-08-22 10:42:44 +00:00
|
|
|
}
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2022-06-29 09:53:49 +00:00
|
|
|
static pte_t get_clear_contig_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
|
|
|
pte_t orig_pte = get_clear_contig(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
|
|
|
|
|
|
|
flush_tlb_range(&vma, addr, addr + (pgsize * ncontig));
|
|
|
|
return orig_pte;
|
|
|
|
}
|
|
|
|
|
2017-08-22 10:42:44 +00:00
|
|
|
/*
|
|
|
|
* Changing some bits of contiguous entries requires us to follow a
|
|
|
|
* Break-Before-Make approach, breaking the whole contiguous set
|
|
|
|
* before we can change any entries. See ARM DDI 0487A.k_iss10775,
|
|
|
|
* "Misprogramming of the Contiguous bit", page D4-1762.
|
|
|
|
*
|
|
|
|
* This helper performs the break step for use cases where the
|
|
|
|
* original pte is not needed.
|
|
|
|
*/
|
|
|
|
static void clear_flush(struct mm_struct *mm,
|
|
|
|
unsigned long addr,
|
|
|
|
pte_t *ptep,
|
|
|
|
unsigned long pgsize,
|
|
|
|
unsigned long ncontig)
|
|
|
|
{
|
mm: do not initialize TLB stack vma's with vma_init()
Commit 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and
data segments") tried to initialize various left-over ad-hoc vma's
"properly", but actually made things worse for the temporary vma's used
for TLB flushing.
vma_init() doesn't actually initialize all of the vma, just a few
fields, so doing something like
- struct vm_area_struct vma = { .vm_mm = tlb->mm, };
+ struct vm_area_struct vma;
+
+ vma_init(&vma, tlb->mm);
was actually very bad: instead of having a nicely initialized vma with
every field but "vm_mm" zeroed, you'd have an entirely uninitialized vma
with only a couple of fields initialized. And they weren't even fields
that the code in question mostly cared about.
The flush_tlb_range() function takes a "struct vma" rather than a
"struct mm_struct", because a few architectures actually care about what
kind of range it is - being able to only do an ITLB flush if it's a
range that doesn't have data accesses enabled, for example. And all the
normal users already have the vma for doing the range invalidation.
But a few people want to call flush_tlb_range() with a range they just
made up, so they also end up using a made-up vma. x86 just has a
special "flush_tlb_mm_range()" function for this, but other
architectures (arm and ia64) do the "use fake vma" thing instead, and
thus got caught up in the vma_init() changes.
At the same time, the TLB flushing code really doesn't care about most
other fields in the vma, so vma_init() is just unnecessary and
pointless.
This fixes things by having an explicit "this is just an initializer for
the TLB flush" initializer macro, which is used by the arm/arm64/ia64
people who mis-use this interface with just a dummy vma.
Fixes: 2c4541e24c55 ("mm: use vma_init() to initialize VMAs on stack and data segments")
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: John Stultz <john.stultz@linaro.org>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2018-08-01 20:43:38 +00:00
|
|
|
struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
|
2017-08-22 10:42:44 +00:00
|
|
|
unsigned long i, saddr = addr;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
2023-08-10 09:32:41 +00:00
|
|
|
ptep_clear(mm, addr, ptep);
|
2017-08-22 10:42:44 +00:00
|
|
|
|
|
|
|
flush_tlb_range(&vma, saddr, addr);
|
|
|
|
}
|
|
|
|
|
2015-12-17 19:31:26 +00:00
|
|
|
void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
|
mm: hugetlb: add huge page size param to set_huge_pte_at()
Patch series "Fix set_huge_pte_at() panic on arm64", v2.
This series fixes a bug in arm64's implementation of set_huge_pte_at(),
which can result in an unprivileged user causing a kernel panic. The
problem was triggered when running the new uffd poison mm selftest for
HUGETLB memory. This test (and the uffd poison feature) was merged for
v6.5-rc7.
Ideally, I'd like to get this fix in for v6.6 and I've cc'ed stable
(correctly this time) to get it backported to v6.5, where the issue first
showed up.
Description of Bug
==================
arm64's huge pte implementation supports multiple huge page sizes, some of
which are implemented in the page table with multiple contiguous entries.
So set_huge_pte_at() needs to work out how big the logical pte is, so that
it can also work out how many physical ptes (or pmds) need to be written.
It previously did this by grabbing the folio out of the pte and querying
its size.
However, there are cases when the pte being set is actually a swap entry.
But this also used to work fine, because for huge ptes, we only ever saw
migration entries and hwpoison entries. And both of these types of swap
entries have a PFN embedded, so the code would grab that and everything
still worked out.
But over time, more calls to set_huge_pte_at() have been added that set
swap entry types that do not embed a PFN. And this causes the code to go
bang. The triggering case is for the uffd poison test, commit
99aa77215ad0 ("selftests/mm: add uffd unit test for UFFDIO_POISON"), which
causes a PTE_MARKER_POISONED swap entry to be set, coutesey of commit
8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs") -
added in v6.5-rc7. Although review shows that there are other call sites
that set PTE_MARKER_UFFD_WP (which also has no PFN), these don't trigger
on arm64 because arm64 doesn't support UFFD WP.
If CONFIG_DEBUG_VM is enabled, we do at least get a BUG(), but otherwise,
it will dereference a bad pointer in page_folio():
static inline struct folio *hugetlb_swap_entry_to_folio(swp_entry_t entry)
{
VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry));
return page_folio(pfn_to_page(swp_offset_pfn(entry)));
}
Fix
===
The simplest fix would have been to revert the dodgy cleanup commit
18f3962953e4 ("mm: hugetlb: kill set_huge_swap_pte_at()"), but since
things have moved on, this would have required an audit of all the new
set_huge_pte_at() call sites to see if they should be converted to
set_huge_swap_pte_at(). As per the original intent of the change, it
would also leave us open to future bugs when people invariably get it
wrong and call the wrong helper.
So instead, I've added a huge page size parameter to set_huge_pte_at().
This means that the arm64 code has the size in all cases. It's a bigger
change, due to needing to touch the arches that implement the function,
but it is entirely mechanical, so in my view, low risk.
I've compile-tested all touched arches; arm64, parisc, powerpc, riscv,
s390, sparc (and additionally x86_64). I've additionally booted and run
mm selftests against arm64, where I observe the uffd poison test is fixed,
and there are no other regressions.
This patch (of 2):
In order to fix a bug, arm64 needs to be told the size of the huge page
for which the pte is being set in set_huge_pte_at(). Provide for this by
adding an `unsigned long sz` parameter to the function. This follows the
same pattern as huge_pte_clear().
This commit makes the required interface modifications to the core mm as
well as all arches that implement this function (arm64, parisc, powerpc,
riscv, s390, sparc). The actual arm64 bug will be fixed in a separate
commit.
No behavioral changes intended.
Link: https://lkml.kernel.org/r/20230922115804.2043771-1-ryan.roberts@arm.com
Link: https://lkml.kernel.org/r/20230922115804.2043771-2-ryan.roberts@arm.com
Fixes: 8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs")
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu> [powerpc 8xx]
Reviewed-by: Lorenzo Stoakes <lstoakes@gmail.com> [vmalloc change]
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: David S. Miller <davem@davemloft.net>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Cc: <stable@vger.kernel.org> [6.5+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-09-22 11:58:03 +00:00
|
|
|
pte_t *ptep, pte_t pte, unsigned long sz)
|
2015-12-17 19:31:26 +00:00
|
|
|
{
|
|
|
|
size_t pgsize;
|
|
|
|
int i;
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-06 22:39:29 +00:00
|
|
|
int ncontig;
|
2017-08-22 10:42:43 +00:00
|
|
|
unsigned long pfn, dpfn;
|
2015-12-17 19:31:26 +00:00
|
|
|
pgprot_t hugeprot;
|
|
|
|
|
arm64: hugetlb: fix set_huge_pte_at() to work with all swap entries
When called with a swap entry that does not embed a PFN (e.g.
PTE_MARKER_POISONED or PTE_MARKER_UFFD_WP), the previous implementation of
set_huge_pte_at() would either cause a BUG() to fire (if CONFIG_DEBUG_VM
is enabled) or cause a dereference of an invalid address and subsequent
panic.
arm64's huge pte implementation supports multiple huge page sizes, some of
which are implemented in the page table with multiple contiguous entries.
So set_huge_pte_at() needs to work out how big the logical pte is, so that
it can also work out how many physical ptes (or pmds) need to be written.
It previously did this by grabbing the folio out of the pte and querying
its size.
However, there are cases when the pte being set is actually a swap entry.
But this also used to work fine, because for huge ptes, we only ever saw
migration entries and hwpoison entries. And both of these types of swap
entries have a PFN embedded, so the code would grab that and everything
still worked out.
But over time, more calls to set_huge_pte_at() have been added that set
swap entry types that do not embed a PFN. And this causes the code to go
bang. The triggering case is for the uffd poison test, commit
99aa77215ad0 ("selftests/mm: add uffd unit test for UFFDIO_POISON"), which
causes a PTE_MARKER_POISONED swap entry to be set, coutesey of commit
8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs") -
added in v6.5-rc7. Although review shows that there are other call sites
that set PTE_MARKER_UFFD_WP (which also has no PFN), these don't trigger
on arm64 because arm64 doesn't support UFFD WP.
Arguably, the root cause is really due to commit 18f3962953e4 ("mm:
hugetlb: kill set_huge_swap_pte_at()"), which aimed to simplify the
interface to the core code by removing set_huge_swap_pte_at() (which took
a page size parameter) and replacing it with calls to set_huge_pte_at()
where the size was inferred from the folio, as descibed above. While that
commit didn't break anything at the time, it did break the interface
because it couldn't handle swap entries without PFNs. And since then new
callers have come along which rely on this working. But given the
brokeness is only observable after commit 8a13897fb0da ("mm: userfaultfd:
support UFFDIO_POISON for hugetlbfs"), that one gets the Fixes tag.
Now that we have modified the set_huge_pte_at() interface to pass the huge
page size in the previous patch, we can trivially fix this issue.
Link: https://lkml.kernel.org/r/20230922115804.2043771-3-ryan.roberts@arm.com
Fixes: 8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs")
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Axel Rasmussen <axelrasmussen@google.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: David S. Miller <davem@davemloft.net>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Cc: <stable@vger.kernel.org> [6.5+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-09-22 11:58:04 +00:00
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
2022-06-26 14:57:17 +00:00
|
|
|
|
arm64: hugetlb: fix set_huge_pte_at() to work with all swap entries
When called with a swap entry that does not embed a PFN (e.g.
PTE_MARKER_POISONED or PTE_MARKER_UFFD_WP), the previous implementation of
set_huge_pte_at() would either cause a BUG() to fire (if CONFIG_DEBUG_VM
is enabled) or cause a dereference of an invalid address and subsequent
panic.
arm64's huge pte implementation supports multiple huge page sizes, some of
which are implemented in the page table with multiple contiguous entries.
So set_huge_pte_at() needs to work out how big the logical pte is, so that
it can also work out how many physical ptes (or pmds) need to be written.
It previously did this by grabbing the folio out of the pte and querying
its size.
However, there are cases when the pte being set is actually a swap entry.
But this also used to work fine, because for huge ptes, we only ever saw
migration entries and hwpoison entries. And both of these types of swap
entries have a PFN embedded, so the code would grab that and everything
still worked out.
But over time, more calls to set_huge_pte_at() have been added that set
swap entry types that do not embed a PFN. And this causes the code to go
bang. The triggering case is for the uffd poison test, commit
99aa77215ad0 ("selftests/mm: add uffd unit test for UFFDIO_POISON"), which
causes a PTE_MARKER_POISONED swap entry to be set, coutesey of commit
8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs") -
added in v6.5-rc7. Although review shows that there are other call sites
that set PTE_MARKER_UFFD_WP (which also has no PFN), these don't trigger
on arm64 because arm64 doesn't support UFFD WP.
Arguably, the root cause is really due to commit 18f3962953e4 ("mm:
hugetlb: kill set_huge_swap_pte_at()"), which aimed to simplify the
interface to the core code by removing set_huge_swap_pte_at() (which took
a page size parameter) and replacing it with calls to set_huge_pte_at()
where the size was inferred from the folio, as descibed above. While that
commit didn't break anything at the time, it did break the interface
because it couldn't handle swap entries without PFNs. And since then new
callers have come along which rely on this working. But given the
brokeness is only observable after commit 8a13897fb0da ("mm: userfaultfd:
support UFFDIO_POISON for hugetlbfs"), that one gets the Fixes tag.
Now that we have modified the set_huge_pte_at() interface to pass the huge
page size in the previous patch, we can trivially fix this issue.
Link: https://lkml.kernel.org/r/20230922115804.2043771-3-ryan.roberts@arm.com
Fixes: 8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs")
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Axel Rasmussen <axelrasmussen@google.com>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: David S. Miller <davem@davemloft.net>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Lorenzo Stoakes <lstoakes@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Cc: <stable@vger.kernel.org> [6.5+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-09-22 11:58:04 +00:00
|
|
|
if (!pte_present(pte)) {
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize)
|
2022-06-26 14:57:17 +00:00
|
|
|
set_pte_at(mm, addr, ptep, pte);
|
|
|
|
return;
|
|
|
|
}
|
2017-08-22 10:42:41 +00:00
|
|
|
|
arm64: hugetlb: refactor find_num_contig()
Patch series "Support for contiguous pte hugepages", v4.
This patchset updates the hugetlb code to fix issues arising from
contiguous pte hugepages (such as on arm64). Compared to v3, This
version addresses a build failure on arm64 by including two cleanup
patches. Other than the arm64 cleanups, the rest are generic code
changes. The remaining arm64 support based on these patches will be
posted separately. The patches are based on v4.12-rc2. Previous
related postings can be found at [0], [1], [2], and [3].
The patches fall into three categories -
* Patch 1-2 - arm64 cleanups required to greatly simplify changing
huge_pte_offset() prototype in Patch 5.
Catalin, Will - are you happy for these patches to go via mm?
* Patches 3-4 address issues with gup
* Patches 5-8 relate to passing a size argument to hugepage helpers to
disambiguate the size of the referred page. These changes are
required to enable arch code to properly handle swap entries for
contiguous pte hugepages.
The changes to huge_pte_offset() (patch 5) touch multiple
architectures but I've managed to minimise these changes for the
other affected functions - huge_pte_clear() and set_huge_pte_at().
These patches gate the enabling of contiguous hugepages support on arm64
which has been requested for systems using !4k page granule.
The ARM64 architecture supports two flavours of hugepages -
* Block mappings at the pud/pmd level
These are regular hugepages where a pmd or a pud page table entry
points to a block of memory. Depending on the PAGE_SIZE in use the
following size of block mappings are supported -
PMD PUD
--- ---
4K: 2M 1G
16K: 32M
64K: 512M
For certain applications/usecases such as HPC and large enterprise
workloads, folks are using 64k page size but the minimum hugepage size
of 512MB isn't very practical.
To overcome this ...
* Using the Contiguous bit
The architecture provides a contiguous bit in the translation table
entry which acts as a hint to the mmu to indicate that it is one of a
contiguous set of entries that can be cached in a single TLB entry.
We use the contiguous bit in Linux to increase the mapping size at the
pmd and pte (last) level.
The number of supported contiguous entries varies by page size and
level of the page table.
Using the contiguous bit allows additional hugepage sizes -
CONT PTE PMD CONT PMD PUD
-------- --- -------- ---
4K: 64K 2M 32M 1G
16K: 2M 32M 1G
64K: 2M 512M 16G
Of these, 64K with 4K and 2M with 64K pages have been explicitly
requested by a few different users.
Entries with the contiguous bit set are required to be modified all
together - which makes things like memory poisoning and migration
impossible to do correctly without knowing the size of hugepage being
dealt with - the reason for adding size parameter to a few of the
hugepage helpers in this series.
This patch (of 8):
As we regularly check for contiguous pte's in the huge accessors, remove
this extra check from find_num_contig.
[punit.agrawal@arm.com: resolve rebase conflicts due to patch re-ordering]
Link: http://lkml.kernel.org/r/20170524115409.31309-2-punit.agrawal@arm.com
Signed-off-by: Steve Capper <steve.capper@arm.com>
Signed-off-by: Punit Agrawal <punit.agrawal@arm.com>
Cc: David Woods <dwoods@mellanox.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Hillf Danton <hillf.zj@alibaba-inc.com>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-07-06 22:39:29 +00:00
|
|
|
if (!pte_cont(pte)) {
|
2015-12-17 19:31:26 +00:00
|
|
|
set_pte_at(mm, addr, ptep, pte);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
pfn = pte_pfn(pte);
|
2017-08-22 10:42:43 +00:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
2017-08-22 10:42:42 +00:00
|
|
|
hugeprot = pte_pgprot(pte);
|
2017-08-22 10:42:43 +00:00
|
|
|
|
2017-08-22 10:42:44 +00:00
|
|
|
clear_flush(mm, addr, ptep, pgsize, ncontig);
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
2015-12-17 19:31:26 +00:00
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
|
|
|
}
|
|
|
|
|
2021-05-05 01:33:00 +00:00
|
|
|
pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
|
2015-12-17 19:31:26 +00:00
|
|
|
unsigned long addr, unsigned long sz)
|
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
pgd_t *pgdp;
|
2020-06-04 23:46:23 +00:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 11:14:56 +00:00
|
|
|
pud_t *pudp;
|
|
|
|
pmd_t *pmdp;
|
|
|
|
pte_t *ptep = NULL;
|
|
|
|
|
|
|
|
pgdp = pgd_offset(mm, addr);
|
2020-06-04 23:46:23 +00:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
pudp = pud_alloc(mm, p4dp, addr);
|
2018-02-15 11:14:56 +00:00
|
|
|
if (!pudp)
|
2015-12-17 19:31:26 +00:00
|
|
|
return NULL;
|
|
|
|
|
|
|
|
if (sz == PUD_SIZE) {
|
2018-02-15 11:14:56 +00:00
|
|
|
ptep = (pte_t *)pudp;
|
2019-05-21 03:35:03 +00:00
|
|
|
} else if (sz == (CONT_PTE_SIZE)) {
|
2018-02-15 11:14:56 +00:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2020-05-05 12:59:30 +00:00
|
|
|
if (!pmdp)
|
|
|
|
return NULL;
|
2015-12-17 19:31:26 +00:00
|
|
|
|
|
|
|
WARN_ON(addr & (sz - 1));
|
2023-06-08 19:13:13 +00:00
|
|
|
ptep = pte_alloc_huge(mm, pmdp, addr);
|
2015-12-17 19:31:26 +00:00
|
|
|
} else if (sz == PMD_SIZE) {
|
2021-05-05 01:33:04 +00:00
|
|
|
if (want_pmd_share(vma, addr) && pud_none(READ_ONCE(*pudp)))
|
2021-05-05 01:33:00 +00:00
|
|
|
ptep = huge_pmd_share(mm, vma, addr, pudp);
|
2015-12-17 19:31:26 +00:00
|
|
|
else
|
2018-02-15 11:14:56 +00:00
|
|
|
ptep = (pte_t *)pmd_alloc(mm, pudp, addr);
|
2019-05-21 03:35:03 +00:00
|
|
|
} else if (sz == (CONT_PMD_SIZE)) {
|
2018-02-15 11:14:56 +00:00
|
|
|
pmdp = pmd_alloc(mm, pudp, addr);
|
2015-12-17 19:31:26 +00:00
|
|
|
WARN_ON(addr & (sz - 1));
|
2018-02-15 11:14:56 +00:00
|
|
|
return (pte_t *)pmdp;
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
return ptep;
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
2017-07-06 22:39:42 +00:00
|
|
|
pte_t *huge_pte_offset(struct mm_struct *mm,
|
|
|
|
unsigned long addr, unsigned long sz)
|
2015-12-17 19:31:26 +00:00
|
|
|
{
|
2018-02-15 11:14:56 +00:00
|
|
|
pgd_t *pgdp;
|
2020-06-04 23:46:23 +00:00
|
|
|
p4d_t *p4dp;
|
2018-02-15 11:14:56 +00:00
|
|
|
pud_t *pudp, pud;
|
|
|
|
pmd_t *pmdp, pmd;
|
2015-12-17 19:31:26 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
pgdp = pgd_offset(mm, addr);
|
|
|
|
if (!pgd_present(READ_ONCE(*pgdp)))
|
2015-12-17 19:31:26 +00:00
|
|
|
return NULL;
|
2017-06-08 17:25:26 +00:00
|
|
|
|
2020-06-04 23:46:23 +00:00
|
|
|
p4dp = p4d_offset(pgdp, addr);
|
|
|
|
if (!p4d_present(READ_ONCE(*p4dp)))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
pudp = pud_offset(p4dp, addr);
|
2018-02-15 11:14:56 +00:00
|
|
|
pud = READ_ONCE(*pudp);
|
|
|
|
if (sz != PUD_SIZE && pud_none(pud))
|
2015-12-17 19:31:26 +00:00
|
|
|
return NULL;
|
2017-08-22 10:42:45 +00:00
|
|
|
/* hugepage or swap? */
|
2018-02-15 11:14:56 +00:00
|
|
|
if (pud_huge(pud) || !pud_present(pud))
|
|
|
|
return (pte_t *)pudp;
|
2017-06-08 17:25:26 +00:00
|
|
|
/* table; check the next level */
|
|
|
|
|
2017-08-22 10:42:45 +00:00
|
|
|
if (sz == CONT_PMD_SIZE)
|
|
|
|
addr &= CONT_PMD_MASK;
|
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
pmdp = pmd_offset(pudp, addr);
|
|
|
|
pmd = READ_ONCE(*pmdp);
|
2017-08-22 10:42:45 +00:00
|
|
|
if (!(sz == PMD_SIZE || sz == CONT_PMD_SIZE) &&
|
2018-02-15 11:14:56 +00:00
|
|
|
pmd_none(pmd))
|
2015-12-17 19:31:26 +00:00
|
|
|
return NULL;
|
2018-02-15 11:14:56 +00:00
|
|
|
if (pmd_huge(pmd) || !pmd_present(pmd))
|
|
|
|
return (pte_t *)pmdp;
|
2017-06-08 17:25:26 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
if (sz == CONT_PTE_SIZE)
|
2023-06-08 19:13:13 +00:00
|
|
|
return pte_offset_huge(pmdp, (addr & CONT_PTE_MASK));
|
2017-08-22 10:42:45 +00:00
|
|
|
|
2015-12-17 19:31:26 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2022-06-21 23:56:18 +00:00
|
|
|
unsigned long hugetlb_mask_last_page(struct hstate *h)
|
|
|
|
{
|
|
|
|
unsigned long hp_size = huge_page_size(h);
|
|
|
|
|
|
|
|
switch (hp_size) {
|
|
|
|
#ifndef __PAGETABLE_PMD_FOLDED
|
|
|
|
case PUD_SIZE:
|
|
|
|
return PGDIR_SIZE - PUD_SIZE;
|
|
|
|
#endif
|
|
|
|
case CONT_PMD_SIZE:
|
|
|
|
return PUD_SIZE - CONT_PMD_SIZE;
|
|
|
|
case PMD_SIZE:
|
|
|
|
return PUD_SIZE - PMD_SIZE;
|
|
|
|
case CONT_PTE_SIZE:
|
|
|
|
return PMD_SIZE - CONT_PTE_SIZE;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0UL;
|
|
|
|
}
|
|
|
|
|
mm/hugetlb: change parameters of arch_make_huge_pte()
Patch series "Subject: [PATCH v2 0/5] Implement huge VMAP and VMALLOC on powerpc 8xx", v2.
This series implements huge VMAP and VMALLOC on powerpc 8xx.
Powerpc 8xx has 4 page sizes:
- 4k
- 16k
- 512k
- 8M
At the time being, vmalloc and vmap only support huge pages which are
leaf at PMD level.
Here the PMD level is 4M, it doesn't correspond to any supported
page size.
For now, implement use of 16k and 512k pages which is done
at PTE level.
Support of 8M pages will be implemented later, it requires use of
hugepd tables.
To allow this, the architecture provides two functions:
- arch_vmap_pte_range_map_size() which tells vmap_pte_range() what
page size to use. A stub returning PAGE_SIZE is provided when the
architecture doesn't provide this function.
- arch_vmap_pte_supported_shift() which tells __vmalloc_node_range()
what page shift to use for a given area size. A stub returning
PAGE_SHIFT is provided when the architecture doesn't provide this
function.
This patch (of 5):
At the time being, arch_make_huge_pte() has the following prototype:
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
struct page *page, int writable);
vma is used to get the pages shift or size.
vma is also used on Sparc to get vm_flags.
page is not used.
writable is not used.
In order to use this function without a vma, replace vma by shift and
flags. Also remove the used parameters.
Link: https://lkml.kernel.org/r/cover.1620795204.git.christophe.leroy@csgroup.eu
Link: https://lkml.kernel.org/r/f4633ac6a7da2f22f31a04a89e0a7026bb78b15b.1620795204.git.christophe.leroy@csgroup.eu
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Acked-by: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2021-07-01 01:48:00 +00:00
|
|
|
pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
|
2015-12-17 19:31:26 +00:00
|
|
|
{
|
mm/hugetlb: change parameters of arch_make_huge_pte()
Patch series "Subject: [PATCH v2 0/5] Implement huge VMAP and VMALLOC on powerpc 8xx", v2.
This series implements huge VMAP and VMALLOC on powerpc 8xx.
Powerpc 8xx has 4 page sizes:
- 4k
- 16k
- 512k
- 8M
At the time being, vmalloc and vmap only support huge pages which are
leaf at PMD level.
Here the PMD level is 4M, it doesn't correspond to any supported
page size.
For now, implement use of 16k and 512k pages which is done
at PTE level.
Support of 8M pages will be implemented later, it requires use of
hugepd tables.
To allow this, the architecture provides two functions:
- arch_vmap_pte_range_map_size() which tells vmap_pte_range() what
page size to use. A stub returning PAGE_SIZE is provided when the
architecture doesn't provide this function.
- arch_vmap_pte_supported_shift() which tells __vmalloc_node_range()
what page shift to use for a given area size. A stub returning
PAGE_SHIFT is provided when the architecture doesn't provide this
function.
This patch (of 5):
At the time being, arch_make_huge_pte() has the following prototype:
pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
struct page *page, int writable);
vma is used to get the pages shift or size.
vma is also used on Sparc to get vm_flags.
page is not used.
writable is not used.
In order to use this function without a vma, replace vma by shift and
flags. Also remove the used parameters.
Link: https://lkml.kernel.org/r/cover.1620795204.git.christophe.leroy@csgroup.eu
Link: https://lkml.kernel.org/r/f4633ac6a7da2f22f31a04a89e0a7026bb78b15b.1620795204.git.christophe.leroy@csgroup.eu
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Acked-by: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2021-07-01 01:48:00 +00:00
|
|
|
size_t pagesize = 1UL << shift;
|
2015-12-17 19:31:26 +00:00
|
|
|
|
2022-03-22 21:41:47 +00:00
|
|
|
entry = pte_mkhuge(entry);
|
2015-12-17 19:31:26 +00:00
|
|
|
if (pagesize == CONT_PTE_SIZE) {
|
|
|
|
entry = pte_mkcont(entry);
|
|
|
|
} else if (pagesize == CONT_PMD_SIZE) {
|
|
|
|
entry = pmd_pte(pmd_mkcont(pte_pmd(entry)));
|
|
|
|
} else if (pagesize != PUD_SIZE && pagesize != PMD_SIZE) {
|
|
|
|
pr_warn("%s: unrecognized huge page size 0x%lx\n",
|
|
|
|
__func__, pagesize);
|
|
|
|
}
|
|
|
|
return entry;
|
|
|
|
}
|
|
|
|
|
2017-08-22 10:42:46 +00:00
|
|
|
void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
|
|
|
|
pte_t *ptep, unsigned long sz)
|
|
|
|
{
|
|
|
|
int i, ncontig;
|
|
|
|
size_t pgsize;
|
|
|
|
|
|
|
|
ncontig = num_contig_ptes(sz, &pgsize);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
|
|
|
|
pte_clear(mm, addr, ptep);
|
|
|
|
}
|
|
|
|
|
2015-12-17 19:31:26 +00:00
|
|
|
pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 10:42:44 +00:00
|
|
|
int ncontig;
|
2017-08-22 10:42:43 +00:00
|
|
|
size_t pgsize;
|
2022-05-16 00:55:57 +00:00
|
|
|
pte_t orig_pte = ptep_get(ptep);
|
2017-08-22 10:42:43 +00:00
|
|
|
|
|
|
|
if (!pte_cont(orig_pte))
|
2015-12-17 19:31:26 +00:00
|
|
|
return ptep_get_and_clear(mm, addr, ptep);
|
2017-08-22 10:42:43 +00:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
|
|
|
|
2022-05-10 04:39:30 +00:00
|
|
|
return get_clear_contig(mm, addr, ptep, pgsize, ncontig);
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
2018-09-21 15:34:05 +00:00
|
|
|
/*
|
|
|
|
* huge_ptep_set_access_flags will update access flags (dirty, accesssed)
|
|
|
|
* and write permission.
|
|
|
|
*
|
|
|
|
* For a contiguous huge pte range we need to check whether or not write
|
|
|
|
* permission has to change only on the first pte in the set. Then for
|
|
|
|
* all the contiguous ptes we need to check whether or not there is a
|
|
|
|
* discrepancy between dirty or young.
|
|
|
|
*/
|
|
|
|
static int __cont_access_flags_changed(pte_t *ptep, pte_t pte, int ncontig)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2022-05-16 00:55:57 +00:00
|
|
|
if (pte_write(pte) != pte_write(ptep_get(ptep)))
|
2018-09-21 15:34:05 +00:00
|
|
|
return 1;
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++) {
|
2022-05-16 00:55:57 +00:00
|
|
|
pte_t orig_pte = ptep_get(ptep + i);
|
2018-09-21 15:34:05 +00:00
|
|
|
|
|
|
|
if (pte_dirty(pte) != pte_dirty(orig_pte))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
if (pte_young(pte) != pte_young(orig_pte))
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-12-17 19:31:26 +00:00
|
|
|
int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep,
|
|
|
|
pte_t pte, int dirty)
|
|
|
|
{
|
2018-09-21 15:34:05 +00:00
|
|
|
int ncontig, i;
|
2017-08-22 10:42:43 +00:00
|
|
|
size_t pgsize = 0;
|
|
|
|
unsigned long pfn = pte_pfn(pte), dpfn;
|
2022-06-29 09:53:49 +00:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2017-08-22 10:42:43 +00:00
|
|
|
pgprot_t hugeprot;
|
2017-08-22 10:42:44 +00:00
|
|
|
pte_t orig_pte;
|
2017-08-22 10:42:43 +00:00
|
|
|
|
|
|
|
if (!pte_cont(pte))
|
2015-12-17 19:31:26 +00:00
|
|
|
return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
|
2017-08-22 10:42:43 +00:00
|
|
|
|
2022-06-29 09:53:49 +00:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2017-08-22 10:42:43 +00:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
2018-09-21 15:34:05 +00:00
|
|
|
if (!__cont_access_flags_changed(ptep, pte, ncontig))
|
|
|
|
return 0;
|
|
|
|
|
2022-06-29 09:53:49 +00:00
|
|
|
orig_pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2017-08-22 10:42:44 +00:00
|
|
|
|
2018-09-21 15:34:04 +00:00
|
|
|
/* Make sure we don't lose the dirty or young state */
|
2017-08-22 10:42:44 +00:00
|
|
|
if (pte_dirty(orig_pte))
|
|
|
|
pte = pte_mkdirty(pte);
|
|
|
|
|
2018-09-21 15:34:04 +00:00
|
|
|
if (pte_young(orig_pte))
|
|
|
|
pte = pte_mkyoung(pte);
|
|
|
|
|
2017-08-22 10:42:44 +00:00
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
2022-06-29 09:53:49 +00:00
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2017-08-22 10:42:43 +00:00
|
|
|
|
2018-09-21 15:34:05 +00:00
|
|
|
return 1;
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void huge_ptep_set_wrprotect(struct mm_struct *mm,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
2017-08-22 10:42:44 +00:00
|
|
|
unsigned long pfn, dpfn;
|
|
|
|
pgprot_t hugeprot;
|
2017-08-22 10:42:43 +00:00
|
|
|
int ncontig, i;
|
|
|
|
size_t pgsize;
|
2017-08-22 10:42:44 +00:00
|
|
|
pte_t pte;
|
2015-12-17 19:31:26 +00:00
|
|
|
|
2018-02-15 11:14:56 +00:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep))) {
|
2015-12-17 19:31:26 +00:00
|
|
|
ptep_set_wrprotect(mm, addr, ptep);
|
2017-08-22 10:42:43 +00:00
|
|
|
return;
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
2017-08-22 10:42:43 +00:00
|
|
|
|
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
2017-08-22 10:42:44 +00:00
|
|
|
dpfn = pgsize >> PAGE_SHIFT;
|
|
|
|
|
2022-06-29 09:53:49 +00:00
|
|
|
pte = get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2017-08-22 10:42:44 +00:00
|
|
|
pte = pte_wrprotect(pte);
|
|
|
|
|
|
|
|
hugeprot = pte_pgprot(pte);
|
|
|
|
pfn = pte_pfn(pte);
|
|
|
|
|
|
|
|
for (i = 0; i < ncontig; i++, ptep++, addr += pgsize, pfn += dpfn)
|
|
|
|
set_pte_at(mm, addr, ptep, pfn_pte(pfn, hugeprot));
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
2022-05-13 23:48:55 +00:00
|
|
|
pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
|
|
|
|
unsigned long addr, pte_t *ptep)
|
2015-12-17 19:31:26 +00:00
|
|
|
{
|
2022-06-29 09:53:49 +00:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2017-08-22 10:42:43 +00:00
|
|
|
size_t pgsize;
|
2017-08-22 10:42:44 +00:00
|
|
|
int ncontig;
|
2017-08-22 10:42:43 +00:00
|
|
|
|
2022-05-13 23:48:55 +00:00
|
|
|
if (!pte_cont(READ_ONCE(*ptep)))
|
|
|
|
return ptep_clear_flush(vma, addr, ptep);
|
2017-08-22 10:42:43 +00:00
|
|
|
|
2022-06-29 09:53:49 +00:00
|
|
|
ncontig = find_num_contig(mm, addr, ptep, &pgsize);
|
|
|
|
return get_clear_contig_flush(mm, addr, ptep, pgsize, ncontig);
|
2015-12-17 19:31:26 +00:00
|
|
|
}
|
|
|
|
|
2018-10-23 01:06:57 +00:00
|
|
|
static int __init hugetlbpage_init(void)
|
|
|
|
{
|
2021-09-20 09:29:31 +00:00
|
|
|
if (pud_sect_supported())
|
|
|
|
hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
|
|
|
|
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 06:24:28 +00:00
|
|
|
hugetlb_add_hstate(CONT_PMD_SHIFT - PAGE_SHIFT);
|
2020-06-03 23:00:42 +00:00
|
|
|
hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
|
arm64/mm: Redefine CONT_{PTE, PMD}_SHIFT
Currently, the value of CONT_{PTE, PMD}_SHIFT is off from standard
{PAGE, PMD}_SHIFT. In turn, we have to consider adding {PAGE, PMD}_SHIFT
when using CONT_{PTE, PMD}_SHIFT in the function hugetlbpage_init().
It's a bit confusing.
This redefines CONT_{PTE, PMD}_SHIFT with {PAGE, PMD}_SHIFT included
so that the later values needn't be added when using the former ones
in function hugetlbpage_init(). Note that the values of CONT_{PTES, PMDS}
are unchanged.
Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: Anshuman Khandual <anshuman.khandual@arm.com>
Link: https://lkml.org/lkml/2020/5/6/190
Link: https://lore.kernel.org/r/20200630062428.194235-1-gshan@redhat.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2020-06-30 06:24:28 +00:00
|
|
|
hugetlb_add_hstate(CONT_PTE_SHIFT - PAGE_SHIFT);
|
2018-10-23 01:06:57 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
arch_initcall(hugetlbpage_init);
|
|
|
|
|
2020-06-03 23:00:34 +00:00
|
|
|
bool __init arch_hugetlb_valid_size(unsigned long size)
|
2013-04-10 12:48:00 +00:00
|
|
|
{
|
2022-02-17 04:52:37 +00:00
|
|
|
return __hugetlb_valid_size(size);
|
2020-06-03 23:00:34 +00:00
|
|
|
}
|
2023-01-02 06:16:51 +00:00
|
|
|
|
|
|
|
pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
|
|
|
|
{
|
arm64: Avoid cpus_have_const_cap() for ARM64_WORKAROUND_2645198
We use cpus_have_const_cap() to check for ARM64_WORKAROUND_2645198 but
this is not necessary and alternative_has_cap() would be preferable.
For historical reasons, cpus_have_const_cap() is more complicated than
it needs to be. Before cpucaps are finalized, it will perform a bitmap
test of the system_cpucaps bitmap, and once cpucaps are finalized it
will use an alternative branch. This used to be necessary to handle some
race conditions in the window between cpucap detection and the
subsequent patching of alternatives and static branches, where different
branches could be out-of-sync with one another (or w.r.t. alternative
sequences). Now that we use alternative branches instead of static
branches, these are all patched atomically w.r.t. one another, and there
are only a handful of cases that need special care in the window between
cpucap detection and alternative patching.
Due to the above, it would be nice to remove cpus_have_const_cap(), and
migrate callers over to alternative_has_cap_*(), cpus_have_final_cap(),
or cpus_have_cap() depending on when their requirements. This will
remove redundant instructions and improve code generation, and will make
it easier to determine how each callsite will behave before, during, and
after alternative patching.
The ARM64_WORKAROUND_2645198 cpucap is detected and patched before any
userspace translation table exist, and the workaround is only necessary
when manipulating usrspace translation tables which are in use. Thus it
is not necessary to use cpus_have_const_cap(), and alternative_has_cap()
is equivalent.
This patch replaces the use of cpus_have_const_cap() with
alternative_has_cap_unlikely(), which will avoid generating code to test
the system_cpucaps bitmap and should be better for all subsequent calls
at runtime. The ARM64_WORKAROUND_2645198 cpucap is added to
cpucap_is_possible() so that code can be elided entirely when this is
not possible, and redundant IS_ENABLED() checks are removed.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2023-10-16 10:24:57 +00:00
|
|
|
if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
|
2023-01-02 06:16:51 +00:00
|
|
|
/*
|
|
|
|
* Break-before-make (BBM) is required for all user space mappings
|
|
|
|
* when the permission changes from executable to non-executable
|
|
|
|
* in cases where cpu is affected with errata #2645198.
|
|
|
|
*/
|
|
|
|
if (pte_user_exec(READ_ONCE(*ptep)))
|
|
|
|
return huge_ptep_clear_flush(vma, addr, ptep);
|
|
|
|
}
|
|
|
|
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
|
|
|
|
}
|
|
|
|
|
|
|
|
void huge_ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
|
|
|
|
pte_t old_pte, pte_t pte)
|
|
|
|
{
|
mm: hugetlb: add huge page size param to set_huge_pte_at()
Patch series "Fix set_huge_pte_at() panic on arm64", v2.
This series fixes a bug in arm64's implementation of set_huge_pte_at(),
which can result in an unprivileged user causing a kernel panic. The
problem was triggered when running the new uffd poison mm selftest for
HUGETLB memory. This test (and the uffd poison feature) was merged for
v6.5-rc7.
Ideally, I'd like to get this fix in for v6.6 and I've cc'ed stable
(correctly this time) to get it backported to v6.5, where the issue first
showed up.
Description of Bug
==================
arm64's huge pte implementation supports multiple huge page sizes, some of
which are implemented in the page table with multiple contiguous entries.
So set_huge_pte_at() needs to work out how big the logical pte is, so that
it can also work out how many physical ptes (or pmds) need to be written.
It previously did this by grabbing the folio out of the pte and querying
its size.
However, there are cases when the pte being set is actually a swap entry.
But this also used to work fine, because for huge ptes, we only ever saw
migration entries and hwpoison entries. And both of these types of swap
entries have a PFN embedded, so the code would grab that and everything
still worked out.
But over time, more calls to set_huge_pte_at() have been added that set
swap entry types that do not embed a PFN. And this causes the code to go
bang. The triggering case is for the uffd poison test, commit
99aa77215ad0 ("selftests/mm: add uffd unit test for UFFDIO_POISON"), which
causes a PTE_MARKER_POISONED swap entry to be set, coutesey of commit
8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs") -
added in v6.5-rc7. Although review shows that there are other call sites
that set PTE_MARKER_UFFD_WP (which also has no PFN), these don't trigger
on arm64 because arm64 doesn't support UFFD WP.
If CONFIG_DEBUG_VM is enabled, we do at least get a BUG(), but otherwise,
it will dereference a bad pointer in page_folio():
static inline struct folio *hugetlb_swap_entry_to_folio(swp_entry_t entry)
{
VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry));
return page_folio(pfn_to_page(swp_offset_pfn(entry)));
}
Fix
===
The simplest fix would have been to revert the dodgy cleanup commit
18f3962953e4 ("mm: hugetlb: kill set_huge_swap_pte_at()"), but since
things have moved on, this would have required an audit of all the new
set_huge_pte_at() call sites to see if they should be converted to
set_huge_swap_pte_at(). As per the original intent of the change, it
would also leave us open to future bugs when people invariably get it
wrong and call the wrong helper.
So instead, I've added a huge page size parameter to set_huge_pte_at().
This means that the arm64 code has the size in all cases. It's a bigger
change, due to needing to touch the arches that implement the function,
but it is entirely mechanical, so in my view, low risk.
I've compile-tested all touched arches; arm64, parisc, powerpc, riscv,
s390, sparc (and additionally x86_64). I've additionally booted and run
mm selftests against arm64, where I observe the uffd poison test is fixed,
and there are no other regressions.
This patch (of 2):
In order to fix a bug, arm64 needs to be told the size of the huge page
for which the pte is being set in set_huge_pte_at(). Provide for this by
adding an `unsigned long sz` parameter to the function. This follows the
same pattern as huge_pte_clear().
This commit makes the required interface modifications to the core mm as
well as all arches that implement this function (arm64, parisc, powerpc,
riscv, s390, sparc). The actual arm64 bug will be fixed in a separate
commit.
No behavioral changes intended.
Link: https://lkml.kernel.org/r/20230922115804.2043771-1-ryan.roberts@arm.com
Link: https://lkml.kernel.org/r/20230922115804.2043771-2-ryan.roberts@arm.com
Fixes: 8a13897fb0da ("mm: userfaultfd: support UFFDIO_POISON for hugetlbfs")
Signed-off-by: Ryan Roberts <ryan.roberts@arm.com>
Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu> [powerpc 8xx]
Reviewed-by: Lorenzo Stoakes <lstoakes@gmail.com> [vmalloc change]
Cc: Alexandre Ghiti <alex@ghiti.fr>
Cc: Albert Ou <aou@eecs.berkeley.edu>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Anshuman Khandual <anshuman.khandual@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Axel Rasmussen <axelrasmussen@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: David S. Miller <davem@davemloft.net>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Helge Deller <deller@gmx.de>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <muchun.song@linux.dev>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Palmer Dabbelt <palmer@dabbelt.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Qi Zheng <zhengqi.arch@bytedance.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Uladzislau Rezki (Sony) <urezki@gmail.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Will Deacon <will@kernel.org>
Cc: <stable@vger.kernel.org> [6.5+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
2023-09-22 11:58:03 +00:00
|
|
|
unsigned long psize = huge_page_size(hstate_vma(vma));
|
|
|
|
|
|
|
|
set_huge_pte_at(vma->vm_mm, addr, ptep, pte, psize);
|
2023-01-02 06:16:51 +00:00
|
|
|
}
|