mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-26 10:49:33 +00:00
![Ryan Roberts](/assets/img/avatar_default.png)
Convert all instances of direct pte_t* dereferencing to instead use ptep_get() helper. This means that by default, the accesses change from a C dereference to a READ_ONCE(). This is technically the correct thing to do since where pgtables are modified by HW (for access/dirty) they are volatile and therefore we should always ensure READ_ONCE() semantics. But more importantly, by always using the helper, it can be overridden by the architecture to fully encapsulate the contents of the pte. Arch code is deliberately not converted, as the arch code knows best. It is intended that arch code (arm64) will override the default with its own implementation that can (e.g.) hide certain bits from the core code, or determine young/dirty status by mixing in state from another source. Conversion was done using Coccinelle: ---- // $ make coccicheck \ // COCCI=ptepget.cocci \ // SPFLAGS="--include-headers" \ // MODE=patch virtual patch @ depends on patch @ pte_t *v; @@ - *v + ptep_get(v) ---- Then reviewed and hand-edited to avoid multiple unnecessary calls to ptep_get(), instead opting to store the result of a single call in a variable, where it is correct to do so. This aims to negate any cost of READ_ONCE() and will benefit arch-overrides that may be more complex. Included is a fix for an issue in an earlier version of this patch that was pointed out by kernel test robot. The issue arose because config MMU=n elides definition of the ptep helper functions, including ptep_get(). HUGETLB_PAGE=n configs still define a simple huge_ptep_clear_flush() for linking purposes, which dereferences the ptep. So when both configs are disabled, this caused a build error because ptep_get() is not defined. Fix by continuing to do a direct dereference when MMU=n. This is safe because for this config the arch code cannot be trying to virtualize the ptes because none of the ptep helpers are defined. Link: https://lkml.kernel.org/r/20230612151545.3317766-4-ryan.roberts@arm.com Reported-by: kernel test robot <lkp@intel.com> Link: https://lore.kernel.org/oe-kbuild-all/202305120142.yXsNEo6H-lkp@intel.com/ Signed-off-by: Ryan Roberts <ryan.roberts@arm.com> Cc: Adrian Hunter <adrian.hunter@intel.com> Cc: Alexander Potapenko <glider@google.com> Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com> Cc: Alex Williamson <alex.williamson@redhat.com> Cc: Al Viro <viro@zeniv.linux.org.uk> Cc: Andrey Konovalov <andreyknvl@gmail.com> Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com> Cc: Christian Brauner <brauner@kernel.org> Cc: Christoph Hellwig <hch@infradead.org> Cc: Daniel Vetter <daniel@ffwll.ch> Cc: Dave Airlie <airlied@gmail.com> Cc: Dimitri Sivanich <dimitri.sivanich@hpe.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Ian Rogers <irogers@google.com> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: Jérôme Glisse <jglisse@redhat.com> Cc: Jiri Olsa <jolsa@kernel.org> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Lorenzo Stoakes <lstoakes@gmail.com> Cc: Mark Rutland <mark.rutland@arm.com> Cc: Matthew Wilcox <willy@infradead.org> Cc: Miaohe Lin <linmiaohe@huawei.com> Cc: Michal Hocko <mhocko@kernel.org> Cc: Mike Kravetz <mike.kravetz@oracle.com> Cc: Mike Rapoport (IBM) <rppt@kernel.org> Cc: Muchun Song <muchun.song@linux.dev> Cc: Namhyung Kim <namhyung@kernel.org> Cc: Naoya Horiguchi <naoya.horiguchi@nec.com> Cc: Oleksandr Tyshchenko <oleksandr_tyshchenko@epam.com> Cc: Pavel Tatashin <pasha.tatashin@soleen.com> Cc: Roman Gushchin <roman.gushchin@linux.dev> Cc: SeongJae Park <sj@kernel.org> Cc: Shakeel Butt <shakeelb@google.com> Cc: Uladzislau Rezki (Sony) <urezki@gmail.com> Cc: Vincenzo Frascino <vincenzo.frascino@arm.com> Cc: Yu Zhao <yuzhao@google.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
470 lines
12 KiB
C
470 lines
12 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Virtual Memory Map support
|
|
*
|
|
* (C) 2007 sgi. Christoph Lameter.
|
|
*
|
|
* Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
|
|
* virt_to_page, page_address() to be implemented as a base offset
|
|
* calculation without memory access.
|
|
*
|
|
* However, virtual mappings need a page table and TLBs. Many Linux
|
|
* architectures already map their physical space using 1-1 mappings
|
|
* via TLBs. For those arches the virtual memory map is essentially
|
|
* for free if we use the same page size as the 1-1 mappings. In that
|
|
* case the overhead consists of a few additional pages that are
|
|
* allocated to create a view of memory for vmemmap.
|
|
*
|
|
* The architecture is expected to provide a vmemmap_populate() function
|
|
* to instantiate the mapping.
|
|
*/
|
|
#include <linux/mm.h>
|
|
#include <linux/mmzone.h>
|
|
#include <linux/memblock.h>
|
|
#include <linux/memremap.h>
|
|
#include <linux/highmem.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/sched.h>
|
|
|
|
#include <asm/dma.h>
|
|
#include <asm/pgalloc.h>
|
|
|
|
/*
|
|
* Allocate a block of memory to be used to back the virtual memory map
|
|
* or to back the page tables that are used to create the mapping.
|
|
* Uses the main allocators if they are available, else bootmem.
|
|
*/
|
|
|
|
static void * __ref __earlyonly_bootmem_alloc(int node,
|
|
unsigned long size,
|
|
unsigned long align,
|
|
unsigned long goal)
|
|
{
|
|
return memblock_alloc_try_nid_raw(size, align, goal,
|
|
MEMBLOCK_ALLOC_ACCESSIBLE, node);
|
|
}
|
|
|
|
void * __meminit vmemmap_alloc_block(unsigned long size, int node)
|
|
{
|
|
/* If the main allocator is up use that, fallback to bootmem. */
|
|
if (slab_is_available()) {
|
|
gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
|
|
int order = get_order(size);
|
|
static bool warned;
|
|
struct page *page;
|
|
|
|
page = alloc_pages_node(node, gfp_mask, order);
|
|
if (page)
|
|
return page_address(page);
|
|
|
|
if (!warned) {
|
|
warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
|
|
"vmemmap alloc failure: order:%u", order);
|
|
warned = true;
|
|
}
|
|
return NULL;
|
|
} else
|
|
return __earlyonly_bootmem_alloc(node, size, size,
|
|
__pa(MAX_DMA_ADDRESS));
|
|
}
|
|
|
|
static void * __meminit altmap_alloc_block_buf(unsigned long size,
|
|
struct vmem_altmap *altmap);
|
|
|
|
/* need to make sure size is all the same during early stage */
|
|
void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
|
|
struct vmem_altmap *altmap)
|
|
{
|
|
void *ptr;
|
|
|
|
if (altmap)
|
|
return altmap_alloc_block_buf(size, altmap);
|
|
|
|
ptr = sparse_buffer_alloc(size);
|
|
if (!ptr)
|
|
ptr = vmemmap_alloc_block(size, node);
|
|
return ptr;
|
|
}
|
|
|
|
static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
|
|
{
|
|
return altmap->base_pfn + altmap->reserve + altmap->alloc
|
|
+ altmap->align;
|
|
}
|
|
|
|
static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
|
|
{
|
|
unsigned long allocated = altmap->alloc + altmap->align;
|
|
|
|
if (altmap->free > allocated)
|
|
return altmap->free - allocated;
|
|
return 0;
|
|
}
|
|
|
|
static void * __meminit altmap_alloc_block_buf(unsigned long size,
|
|
struct vmem_altmap *altmap)
|
|
{
|
|
unsigned long pfn, nr_pfns, nr_align;
|
|
|
|
if (size & ~PAGE_MASK) {
|
|
pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
|
|
__func__, size);
|
|
return NULL;
|
|
}
|
|
|
|
pfn = vmem_altmap_next_pfn(altmap);
|
|
nr_pfns = size >> PAGE_SHIFT;
|
|
nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
|
|
nr_align = ALIGN(pfn, nr_align) - pfn;
|
|
if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
|
|
return NULL;
|
|
|
|
altmap->alloc += nr_pfns;
|
|
altmap->align += nr_align;
|
|
pfn += nr_align;
|
|
|
|
pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
|
|
__func__, pfn, altmap->alloc, altmap->align, nr_pfns);
|
|
return __va(__pfn_to_phys(pfn));
|
|
}
|
|
|
|
void __meminit vmemmap_verify(pte_t *pte, int node,
|
|
unsigned long start, unsigned long end)
|
|
{
|
|
unsigned long pfn = pte_pfn(ptep_get(pte));
|
|
int actual_node = early_pfn_to_nid(pfn);
|
|
|
|
if (node_distance(actual_node, node) > LOCAL_DISTANCE)
|
|
pr_warn_once("[%lx-%lx] potential offnode page_structs\n",
|
|
start, end - 1);
|
|
}
|
|
|
|
pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
|
|
struct vmem_altmap *altmap,
|
|
struct page *reuse)
|
|
{
|
|
pte_t *pte = pte_offset_kernel(pmd, addr);
|
|
if (pte_none(ptep_get(pte))) {
|
|
pte_t entry;
|
|
void *p;
|
|
|
|
if (!reuse) {
|
|
p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
|
|
if (!p)
|
|
return NULL;
|
|
} else {
|
|
/*
|
|
* When a PTE/PMD entry is freed from the init_mm
|
|
* there's a free_pages() call to this page allocated
|
|
* above. Thus this get_page() is paired with the
|
|
* put_page_testzero() on the freeing path.
|
|
* This can only called by certain ZONE_DEVICE path,
|
|
* and through vmemmap_populate_compound_pages() when
|
|
* slab is available.
|
|
*/
|
|
get_page(reuse);
|
|
p = page_to_virt(reuse);
|
|
}
|
|
entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
|
|
set_pte_at(&init_mm, addr, pte, entry);
|
|
}
|
|
return pte;
|
|
}
|
|
|
|
static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
|
|
{
|
|
void *p = vmemmap_alloc_block(size, node);
|
|
|
|
if (!p)
|
|
return NULL;
|
|
memset(p, 0, size);
|
|
|
|
return p;
|
|
}
|
|
|
|
pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
|
|
{
|
|
pmd_t *pmd = pmd_offset(pud, addr);
|
|
if (pmd_none(*pmd)) {
|
|
void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
|
|
if (!p)
|
|
return NULL;
|
|
pmd_populate_kernel(&init_mm, pmd, p);
|
|
}
|
|
return pmd;
|
|
}
|
|
|
|
void __weak __meminit pmd_init(void *addr)
|
|
{
|
|
}
|
|
|
|
pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
|
|
{
|
|
pud_t *pud = pud_offset(p4d, addr);
|
|
if (pud_none(*pud)) {
|
|
void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
|
|
if (!p)
|
|
return NULL;
|
|
pmd_init(p);
|
|
pud_populate(&init_mm, pud, p);
|
|
}
|
|
return pud;
|
|
}
|
|
|
|
void __weak __meminit pud_init(void *addr)
|
|
{
|
|
}
|
|
|
|
p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
|
|
{
|
|
p4d_t *p4d = p4d_offset(pgd, addr);
|
|
if (p4d_none(*p4d)) {
|
|
void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
|
|
if (!p)
|
|
return NULL;
|
|
pud_init(p);
|
|
p4d_populate(&init_mm, p4d, p);
|
|
}
|
|
return p4d;
|
|
}
|
|
|
|
pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
|
|
{
|
|
pgd_t *pgd = pgd_offset_k(addr);
|
|
if (pgd_none(*pgd)) {
|
|
void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
|
|
if (!p)
|
|
return NULL;
|
|
pgd_populate(&init_mm, pgd, p);
|
|
}
|
|
return pgd;
|
|
}
|
|
|
|
static pte_t * __meminit vmemmap_populate_address(unsigned long addr, int node,
|
|
struct vmem_altmap *altmap,
|
|
struct page *reuse)
|
|
{
|
|
pgd_t *pgd;
|
|
p4d_t *p4d;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
pte_t *pte;
|
|
|
|
pgd = vmemmap_pgd_populate(addr, node);
|
|
if (!pgd)
|
|
return NULL;
|
|
p4d = vmemmap_p4d_populate(pgd, addr, node);
|
|
if (!p4d)
|
|
return NULL;
|
|
pud = vmemmap_pud_populate(p4d, addr, node);
|
|
if (!pud)
|
|
return NULL;
|
|
pmd = vmemmap_pmd_populate(pud, addr, node);
|
|
if (!pmd)
|
|
return NULL;
|
|
pte = vmemmap_pte_populate(pmd, addr, node, altmap, reuse);
|
|
if (!pte)
|
|
return NULL;
|
|
vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
|
|
|
|
return pte;
|
|
}
|
|
|
|
static int __meminit vmemmap_populate_range(unsigned long start,
|
|
unsigned long end, int node,
|
|
struct vmem_altmap *altmap,
|
|
struct page *reuse)
|
|
{
|
|
unsigned long addr = start;
|
|
pte_t *pte;
|
|
|
|
for (; addr < end; addr += PAGE_SIZE) {
|
|
pte = vmemmap_populate_address(addr, node, altmap, reuse);
|
|
if (!pte)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
|
|
int node, struct vmem_altmap *altmap)
|
|
{
|
|
return vmemmap_populate_range(start, end, node, altmap, NULL);
|
|
}
|
|
|
|
void __weak __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node,
|
|
unsigned long addr, unsigned long next)
|
|
{
|
|
}
|
|
|
|
int __weak __meminit vmemmap_check_pmd(pmd_t *pmd, int node,
|
|
unsigned long addr, unsigned long next)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
int __meminit vmemmap_populate_hugepages(unsigned long start, unsigned long end,
|
|
int node, struct vmem_altmap *altmap)
|
|
{
|
|
unsigned long addr;
|
|
unsigned long next;
|
|
pgd_t *pgd;
|
|
p4d_t *p4d;
|
|
pud_t *pud;
|
|
pmd_t *pmd;
|
|
|
|
for (addr = start; addr < end; addr = next) {
|
|
next = pmd_addr_end(addr, end);
|
|
|
|
pgd = vmemmap_pgd_populate(addr, node);
|
|
if (!pgd)
|
|
return -ENOMEM;
|
|
|
|
p4d = vmemmap_p4d_populate(pgd, addr, node);
|
|
if (!p4d)
|
|
return -ENOMEM;
|
|
|
|
pud = vmemmap_pud_populate(p4d, addr, node);
|
|
if (!pud)
|
|
return -ENOMEM;
|
|
|
|
pmd = pmd_offset(pud, addr);
|
|
if (pmd_none(READ_ONCE(*pmd))) {
|
|
void *p;
|
|
|
|
p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
|
|
if (p) {
|
|
vmemmap_set_pmd(pmd, p, node, addr, next);
|
|
continue;
|
|
} else if (altmap) {
|
|
/*
|
|
* No fallback: In any case we care about, the
|
|
* altmap should be reasonably sized and aligned
|
|
* such that vmemmap_alloc_block_buf() will always
|
|
* succeed. For consistency with the PTE case,
|
|
* return an error here as failure could indicate
|
|
* a configuration issue with the size of the altmap.
|
|
*/
|
|
return -ENOMEM;
|
|
}
|
|
} else if (vmemmap_check_pmd(pmd, node, addr, next))
|
|
continue;
|
|
if (vmemmap_populate_basepages(addr, next, node, altmap))
|
|
return -ENOMEM;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* For compound pages bigger than section size (e.g. x86 1G compound
|
|
* pages with 2M subsection size) fill the rest of sections as tail
|
|
* pages.
|
|
*
|
|
* Note that memremap_pages() resets @nr_range value and will increment
|
|
* it after each range successful onlining. Thus the value or @nr_range
|
|
* at section memmap populate corresponds to the in-progress range
|
|
* being onlined here.
|
|
*/
|
|
static bool __meminit reuse_compound_section(unsigned long start_pfn,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
unsigned long nr_pages = pgmap_vmemmap_nr(pgmap);
|
|
unsigned long offset = start_pfn -
|
|
PHYS_PFN(pgmap->ranges[pgmap->nr_range].start);
|
|
|
|
return !IS_ALIGNED(offset, nr_pages) && nr_pages > PAGES_PER_SUBSECTION;
|
|
}
|
|
|
|
static pte_t * __meminit compound_section_tail_page(unsigned long addr)
|
|
{
|
|
pte_t *pte;
|
|
|
|
addr -= PAGE_SIZE;
|
|
|
|
/*
|
|
* Assuming sections are populated sequentially, the previous section's
|
|
* page data can be reused.
|
|
*/
|
|
pte = pte_offset_kernel(pmd_off_k(addr), addr);
|
|
if (!pte)
|
|
return NULL;
|
|
|
|
return pte;
|
|
}
|
|
|
|
static int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn,
|
|
unsigned long start,
|
|
unsigned long end, int node,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
unsigned long size, addr;
|
|
pte_t *pte;
|
|
int rc;
|
|
|
|
if (reuse_compound_section(start_pfn, pgmap)) {
|
|
pte = compound_section_tail_page(start);
|
|
if (!pte)
|
|
return -ENOMEM;
|
|
|
|
/*
|
|
* Reuse the page that was populated in the prior iteration
|
|
* with just tail struct pages.
|
|
*/
|
|
return vmemmap_populate_range(start, end, node, NULL,
|
|
pte_page(ptep_get(pte)));
|
|
}
|
|
|
|
size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page));
|
|
for (addr = start; addr < end; addr += size) {
|
|
unsigned long next, last = addr + size;
|
|
|
|
/* Populate the head page vmemmap page */
|
|
pte = vmemmap_populate_address(addr, node, NULL, NULL);
|
|
if (!pte)
|
|
return -ENOMEM;
|
|
|
|
/* Populate the tail pages vmemmap page */
|
|
next = addr + PAGE_SIZE;
|
|
pte = vmemmap_populate_address(next, node, NULL, NULL);
|
|
if (!pte)
|
|
return -ENOMEM;
|
|
|
|
/*
|
|
* Reuse the previous page for the rest of tail pages
|
|
* See layout diagram in Documentation/mm/vmemmap_dedup.rst
|
|
*/
|
|
next += PAGE_SIZE;
|
|
rc = vmemmap_populate_range(next, last, node, NULL,
|
|
pte_page(ptep_get(pte)));
|
|
if (rc)
|
|
return -ENOMEM;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
struct page * __meminit __populate_section_memmap(unsigned long pfn,
|
|
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
unsigned long start = (unsigned long) pfn_to_page(pfn);
|
|
unsigned long end = start + nr_pages * sizeof(struct page);
|
|
int r;
|
|
|
|
if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
|
|
!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
|
|
return NULL;
|
|
|
|
if (vmemmap_can_optimize(altmap, pgmap))
|
|
r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap);
|
|
else
|
|
r = vmemmap_populate(start, end, nid, altmap);
|
|
|
|
if (r < 0)
|
|
return NULL;
|
|
|
|
return pfn_to_page(pfn);
|
|
}
|