mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-12 13:55:32 +00:00
![Joao Martins](/assets/img/avatar_default.png)
Add a new @vmemmap_shift property for struct dev_pagemap which specifies
that a devmap is composed of a set of compound pages of order
@vmemmap_shift, instead of base pages. When a compound page devmap is
requested, all but the first page are initialised as tail pages instead
of order-0 pages.
For certain ZONE_DEVICE users like device-dax which have a fixed page
size, this creates an opportunity to optimize GUP and GUP-fast walkers,
treating it the same way as THP or hugetlb pages.
Additionally, commit 7118fc2906
("hugetlb: address ref count racing in
prep_compound_gigantic_page") removed set_page_count() because the
setting of page ref count to zero was redundant. devmap pages don't
come from page allocator though and only head page refcount is used for
compound pages, hence initialize tail page count to zero.
Link: https://lkml.kernel.org/r/20211202204422.26777-5-joao.m.martins@oracle.com
Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Cc: Muchun Song <songmuchun@bytedance.com>
Cc: Naoya Horiguchi <naoya.horiguchi@nec.com>
Cc: Vishal Verma <vishal.l.verma@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
208 lines
6.4 KiB
C
208 lines
6.4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_MEMREMAP_H_
|
|
#define _LINUX_MEMREMAP_H_
|
|
#include <linux/range.h>
|
|
#include <linux/ioport.h>
|
|
#include <linux/percpu-refcount.h>
|
|
|
|
struct resource;
|
|
struct device;
|
|
|
|
/**
|
|
* struct vmem_altmap - pre-allocated storage for vmemmap_populate
|
|
* @base_pfn: base of the entire dev_pagemap mapping
|
|
* @reserve: pages mapped, but reserved for driver use (relative to @base)
|
|
* @free: free pages set aside in the mapping for memmap storage
|
|
* @align: pages reserved to meet allocation alignments
|
|
* @alloc: track pages consumed, private to vmemmap_populate()
|
|
*/
|
|
struct vmem_altmap {
|
|
unsigned long base_pfn;
|
|
const unsigned long end_pfn;
|
|
const unsigned long reserve;
|
|
unsigned long free;
|
|
unsigned long align;
|
|
unsigned long alloc;
|
|
};
|
|
|
|
/*
|
|
* Specialize ZONE_DEVICE memory into multiple types each has a different
|
|
* usage.
|
|
*
|
|
* MEMORY_DEVICE_PRIVATE:
|
|
* Device memory that is not directly addressable by the CPU: CPU can neither
|
|
* read nor write private memory. In this case, we do still have struct pages
|
|
* backing the device memory. Doing so simplifies the implementation, but it is
|
|
* important to remember that there are certain points at which the struct page
|
|
* must be treated as an opaque object, rather than a "normal" struct page.
|
|
*
|
|
* A more complete discussion of unaddressable memory may be found in
|
|
* include/linux/hmm.h and Documentation/vm/hmm.rst.
|
|
*
|
|
* MEMORY_DEVICE_FS_DAX:
|
|
* Host memory that has similar access semantics as System RAM i.e. DMA
|
|
* coherent and supports page pinning. In support of coordinating page
|
|
* pinning vs other operations MEMORY_DEVICE_FS_DAX arranges for a
|
|
* wakeup event whenever a page is unpinned and becomes idle. This
|
|
* wakeup is used to coordinate physical address space management (ex:
|
|
* fs truncate/hole punch) vs pinned pages (ex: device dma).
|
|
*
|
|
* MEMORY_DEVICE_GENERIC:
|
|
* Host memory that has similar access semantics as System RAM i.e. DMA
|
|
* coherent and supports page pinning. This is for example used by DAX devices
|
|
* that expose memory using a character device.
|
|
*
|
|
* MEMORY_DEVICE_PCI_P2PDMA:
|
|
* Device memory residing in a PCI BAR intended for use with Peer-to-Peer
|
|
* transactions.
|
|
*/
|
|
enum memory_type {
|
|
/* 0 is reserved to catch uninitialized type fields */
|
|
MEMORY_DEVICE_PRIVATE = 1,
|
|
MEMORY_DEVICE_FS_DAX,
|
|
MEMORY_DEVICE_GENERIC,
|
|
MEMORY_DEVICE_PCI_P2PDMA,
|
|
};
|
|
|
|
struct dev_pagemap_ops {
|
|
/*
|
|
* Called once the page refcount reaches 1. (ZONE_DEVICE pages never
|
|
* reach 0 refcount unless there is a refcount bug. This allows the
|
|
* device driver to implement its own memory management.)
|
|
*/
|
|
void (*page_free)(struct page *page);
|
|
|
|
/*
|
|
* Transition the refcount in struct dev_pagemap to the dead state.
|
|
*/
|
|
void (*kill)(struct dev_pagemap *pgmap);
|
|
|
|
/*
|
|
* Wait for refcount in struct dev_pagemap to be idle and reap it.
|
|
*/
|
|
void (*cleanup)(struct dev_pagemap *pgmap);
|
|
|
|
/*
|
|
* Used for private (un-addressable) device memory only. Must migrate
|
|
* the page back to a CPU accessible page.
|
|
*/
|
|
vm_fault_t (*migrate_to_ram)(struct vm_fault *vmf);
|
|
};
|
|
|
|
#define PGMAP_ALTMAP_VALID (1 << 0)
|
|
|
|
/**
|
|
* struct dev_pagemap - metadata for ZONE_DEVICE mappings
|
|
* @altmap: pre-allocated/reserved memory for vmemmap allocations
|
|
* @ref: reference count that pins the devm_memremap_pages() mapping
|
|
* @internal_ref: internal reference if @ref is not provided by the caller
|
|
* @done: completion for @internal_ref
|
|
* @type: memory type: see MEMORY_* in memory_hotplug.h
|
|
* @flags: PGMAP_* flags to specify defailed behavior
|
|
* @vmemmap_shift: structural definition of how the vmemmap page metadata
|
|
* is populated, specifically the metadata page order.
|
|
* A zero value (default) uses base pages as the vmemmap metadata
|
|
* representation. A bigger value will set up compound struct pages
|
|
* of the requested order value.
|
|
* @ops: method table
|
|
* @owner: an opaque pointer identifying the entity that manages this
|
|
* instance. Used by various helpers to make sure that no
|
|
* foreign ZONE_DEVICE memory is accessed.
|
|
* @nr_range: number of ranges to be mapped
|
|
* @range: range to be mapped when nr_range == 1
|
|
* @ranges: array of ranges to be mapped when nr_range > 1
|
|
*/
|
|
struct dev_pagemap {
|
|
struct vmem_altmap altmap;
|
|
struct percpu_ref *ref;
|
|
struct percpu_ref internal_ref;
|
|
struct completion done;
|
|
enum memory_type type;
|
|
unsigned int flags;
|
|
unsigned long vmemmap_shift;
|
|
const struct dev_pagemap_ops *ops;
|
|
void *owner;
|
|
int nr_range;
|
|
union {
|
|
struct range range;
|
|
struct range ranges[0];
|
|
};
|
|
};
|
|
|
|
static inline struct vmem_altmap *pgmap_altmap(struct dev_pagemap *pgmap)
|
|
{
|
|
if (pgmap->flags & PGMAP_ALTMAP_VALID)
|
|
return &pgmap->altmap;
|
|
return NULL;
|
|
}
|
|
|
|
static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap)
|
|
{
|
|
return 1 << pgmap->vmemmap_shift;
|
|
}
|
|
|
|
#ifdef CONFIG_ZONE_DEVICE
|
|
void *memremap_pages(struct dev_pagemap *pgmap, int nid);
|
|
void memunmap_pages(struct dev_pagemap *pgmap);
|
|
void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
|
|
void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap);
|
|
struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
|
|
struct dev_pagemap *pgmap);
|
|
bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn);
|
|
|
|
unsigned long vmem_altmap_offset(struct vmem_altmap *altmap);
|
|
void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns);
|
|
unsigned long memremap_compat_align(void);
|
|
#else
|
|
static inline void *devm_memremap_pages(struct device *dev,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
/*
|
|
* Fail attempts to call devm_memremap_pages() without
|
|
* ZONE_DEVICE support enabled, this requires callers to fall
|
|
* back to plain devm_memremap() based on config
|
|
*/
|
|
WARN_ON_ONCE(1);
|
|
return ERR_PTR(-ENXIO);
|
|
}
|
|
|
|
static inline void devm_memunmap_pages(struct device *dev,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
}
|
|
|
|
static inline struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
|
|
struct dev_pagemap *pgmap)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
static inline bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void vmem_altmap_free(struct vmem_altmap *altmap,
|
|
unsigned long nr_pfns)
|
|
{
|
|
}
|
|
|
|
/* when memremap_pages() is disabled all archs can remap a single page */
|
|
static inline unsigned long memremap_compat_align(void)
|
|
{
|
|
return PAGE_SIZE;
|
|
}
|
|
#endif /* CONFIG_ZONE_DEVICE */
|
|
|
|
static inline void put_dev_pagemap(struct dev_pagemap *pgmap)
|
|
{
|
|
if (pgmap)
|
|
percpu_ref_put(pgmap->ref);
|
|
}
|
|
|
|
#endif /* _LINUX_MEMREMAP_H_ */
|