dma-maping updates for Linux 6.6
- allow dynamic sizing of the swiotlb buffer, to cater for secure virtualization workloads that require all I/O to be bounce buffered (Petr Tesarik) - move a declaration to a header (Arnd Bergmann) - check for memory region overlap in dma-contiguous (Binglei Wang) - remove the somewhat dangerous runtime swiotlb-xen enablement and unexport is_swiotlb_active (Christoph Hellwig, Juergen Gross) - per-node CMA improvements (Yajun Deng) -----BEGIN PGP SIGNATURE----- iQI/BAABCgApFiEEgdbnc3r/njty3Iq9D55TZVIEUYMFAmTuDHkLHGhjaEBsc3Qu ZGUACgkQD55TZVIEUYOqvhAApMk2/ceTgVH17sXaKE822+xKvgv377O6TlggMeGG W4zA0KD69DNz0AfaaCc5U5f7n8Ld/YY1RsvkHW4b3jgw+KRTeQr0jjitBgP5kP2M A1+qxdyJpCTwiPt9s2+JFVPeyZ0s52V6OJODKRG3s0ore55R+U09VySKtASON+q3 GMKfWqQteKC+thg7NkrQ7JUixuo84oICws+rZn4K9ifsX2O0HYW6aMW0feRfZjJH r0TgqZc4RdPTSaF22oapR9Ls39+7hp/pBvoLm5sBNA3cl5C3X4VWo9ERMU1jW9h+ VYQv39NycUspgskWJmpbU06/+ooYqQlwHSR/vdNusmFIvxo4tf6/UX72YO5F8Dar ap0wYGauiEwTjSnhVxPTXk3obWyWEsgFAeRnPdTlH2CNmv38QZU2HLb8eU1pcXxX j+WI2Ewy9z22uBVYiPOKpdW1jkSfmlmfPp/8SbAdua7I3YQ90rQN6AvU06zAi/cL NQTgO81E4jPkygqAVgS/LeYziWAQ73yM7m9ExThtTgqFtHortwhJ4Fd8XKtvtvEb viXAZ/WZtQBv/CIKAW98NhgIDP/SPOT8ym6V35WK+kkNFMS6LMSQUfl9GgbHGyFa n9icMm7BmbDtT1+AKNafG9En4DtAf9M9QNidAVOyfrsIk6S0gZoZwvIStkA7on8a cNY= =kVVr -----END PGP SIGNATURE----- Merge tag 'dma-mapping-6.6-2023-08-29' of git://git.infradead.org/users/hch/dma-mapping Pull dma-maping updates from Christoph Hellwig: - allow dynamic sizing of the swiotlb buffer, to cater for secure virtualization workloads that require all I/O to be bounce buffered (Petr Tesarik) - move a declaration to a header (Arnd Bergmann) - check for memory region overlap in dma-contiguous (Binglei Wang) - remove the somewhat dangerous runtime swiotlb-xen enablement and unexport is_swiotlb_active (Christoph Hellwig, Juergen Gross) - per-node CMA improvements (Yajun Deng) * tag 'dma-mapping-6.6-2023-08-29' of git://git.infradead.org/users/hch/dma-mapping: swiotlb: optimize get_max_slots() swiotlb: move slot allocation explanation comment where it belongs swiotlb: search the software IO TLB only if the device makes use of it swiotlb: allocate a new memory pool when existing pools are full swiotlb: determine potential physical address limit swiotlb: if swiotlb is full, fall back to a transient memory pool swiotlb: add a flag whether SWIOTLB is allowed to grow swiotlb: separate memory pool data from other allocator data swiotlb: add documentation and rename swiotlb_do_find_slots() swiotlb: make io_tlb_default_mem local to swiotlb.c swiotlb: bail out of swiotlb_init_late() if swiotlb is already allocated dma-contiguous: check for memory region overlap dma-contiguous: support numa CMA for specified node dma-contiguous: support per-numa CMA for all architectures dma-mapping: move arch_dma_set_mask() declaration to header swiotlb: unexport is_swiotlb_active x86: always initialize xen-swiotlb when xen-pcifront is enabling xen/pci: add flag for PCI passthrough being possible
This commit is contained in:
commit
6c1b980a7e
|
@ -696,7 +696,7 @@
|
||||||
kernel/dma/contiguous.c
|
kernel/dma/contiguous.c
|
||||||
|
|
||||||
cma_pernuma=nn[MG]
|
cma_pernuma=nn[MG]
|
||||||
[ARM64,KNL,CMA]
|
[KNL,CMA]
|
||||||
Sets the size of kernel per-numa memory area for
|
Sets the size of kernel per-numa memory area for
|
||||||
contiguous memory allocations. A value of 0 disables
|
contiguous memory allocations. A value of 0 disables
|
||||||
per-numa CMA altogether. And If this option is not
|
per-numa CMA altogether. And If this option is not
|
||||||
|
@ -706,6 +706,17 @@
|
||||||
which is located in node nid, if the allocation fails,
|
which is located in node nid, if the allocation fails,
|
||||||
they will fallback to the global default memory area.
|
they will fallback to the global default memory area.
|
||||||
|
|
||||||
|
numa_cma=<node>:nn[MG][,<node>:nn[MG]]
|
||||||
|
[KNL,CMA]
|
||||||
|
Sets the size of kernel numa memory area for
|
||||||
|
contiguous memory allocations. It will reserve CMA
|
||||||
|
area for the specified node.
|
||||||
|
|
||||||
|
With numa CMA enabled, DMA users on node nid will
|
||||||
|
first try to allocate buffer from the numa area
|
||||||
|
which is located in node nid, if the allocation fails,
|
||||||
|
they will fallback to the global default memory area.
|
||||||
|
|
||||||
cmo_free_hint= [PPC] Format: { yes | no }
|
cmo_free_hint= [PPC] Format: { yes | no }
|
||||||
Specify whether pages are marked as being inactive
|
Specify whether pages are marked as being inactive
|
||||||
when they are freed. This is used in CMO environments
|
when they are freed. This is used in CMO environments
|
||||||
|
|
|
@ -125,12 +125,10 @@ static int __init xen_mm_init(void)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
/* we can work with the default swiotlb */
|
/* we can work with the default swiotlb */
|
||||||
if (!io_tlb_default_mem.nslabs) {
|
rc = swiotlb_init_late(swiotlb_size_or_default(),
|
||||||
rc = swiotlb_init_late(swiotlb_size_or_default(),
|
xen_swiotlb_gfp(), NULL);
|
||||||
xen_swiotlb_gfp(), NULL);
|
if (rc < 0)
|
||||||
if (rc < 0)
|
return rc;
|
||||||
return rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
cflush.op = 0;
|
cflush.op = 0;
|
||||||
cflush.a.dev_bus_addr = 0;
|
cflush.a.dev_bus_addr = 0;
|
||||||
|
|
|
@ -461,8 +461,6 @@ void __init bootmem_init(void)
|
||||||
arm64_hugetlb_cma_reserve();
|
arm64_hugetlb_cma_reserve();
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
dma_pernuma_cma_reserve();
|
|
||||||
|
|
||||||
kvm_hyp_reserve();
|
kvm_hyp_reserve();
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -664,7 +664,7 @@ static int __init octeon_pci_setup(void)
|
||||||
|
|
||||||
/* BAR1 movable regions contiguous to cover the swiotlb */
|
/* BAR1 movable regions contiguous to cover the swiotlb */
|
||||||
octeon_bar1_pci_phys =
|
octeon_bar1_pci_phys =
|
||||||
io_tlb_default_mem.start & ~((1ull << 22) - 1);
|
default_swiotlb_base() & ~((1ull << 22) - 1);
|
||||||
|
|
||||||
for (index = 0; index < 32; index++) {
|
for (index = 0; index < 32; index++) {
|
||||||
union cvmx_pci_bar1_indexx bar1_index;
|
union cvmx_pci_bar1_indexx bar1_index;
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
// SPDX-License-Identifier: GPL-2.0
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
|
||||||
#include <linux/dma-mapping.h>
|
#include <linux/dma-mapping.h>
|
||||||
|
#include <linux/dma-map-ops.h>
|
||||||
#include <linux/export.h>
|
#include <linux/export.h>
|
||||||
#include <asm/machdep.h>
|
#include <asm/machdep.h>
|
||||||
|
|
||||||
|
|
|
@ -2,12 +2,6 @@
|
||||||
#ifndef _ASM_X86_SWIOTLB_XEN_H
|
#ifndef _ASM_X86_SWIOTLB_XEN_H
|
||||||
#define _ASM_X86_SWIOTLB_XEN_H
|
#define _ASM_X86_SWIOTLB_XEN_H
|
||||||
|
|
||||||
#ifdef CONFIG_SWIOTLB_XEN
|
|
||||||
extern int pci_xen_swiotlb_init_late(void);
|
|
||||||
#else
|
|
||||||
static inline int pci_xen_swiotlb_init_late(void) { return -ENXIO; }
|
|
||||||
#endif
|
|
||||||
|
|
||||||
int xen_swiotlb_fixup(void *buf, unsigned long nslabs);
|
int xen_swiotlb_fixup(void *buf, unsigned long nslabs);
|
||||||
int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
|
int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order,
|
||||||
unsigned int address_bits,
|
unsigned int address_bits,
|
||||||
|
|
|
@ -72,9 +72,15 @@ static inline void __init pci_swiotlb_detect(void)
|
||||||
#endif /* CONFIG_SWIOTLB */
|
#endif /* CONFIG_SWIOTLB */
|
||||||
|
|
||||||
#ifdef CONFIG_SWIOTLB_XEN
|
#ifdef CONFIG_SWIOTLB_XEN
|
||||||
|
static bool xen_swiotlb_enabled(void)
|
||||||
|
{
|
||||||
|
return xen_initial_domain() || x86_swiotlb_enable ||
|
||||||
|
(IS_ENABLED(CONFIG_XEN_PCIDEV_FRONTEND) && xen_pv_pci_possible);
|
||||||
|
}
|
||||||
|
|
||||||
static void __init pci_xen_swiotlb_init(void)
|
static void __init pci_xen_swiotlb_init(void)
|
||||||
{
|
{
|
||||||
if (!xen_initial_domain() && !x86_swiotlb_enable)
|
if (!xen_swiotlb_enabled())
|
||||||
return;
|
return;
|
||||||
x86_swiotlb_enable = true;
|
x86_swiotlb_enable = true;
|
||||||
x86_swiotlb_flags |= SWIOTLB_ANY;
|
x86_swiotlb_flags |= SWIOTLB_ANY;
|
||||||
|
@ -83,27 +89,6 @@ static void __init pci_xen_swiotlb_init(void)
|
||||||
if (IS_ENABLED(CONFIG_PCI))
|
if (IS_ENABLED(CONFIG_PCI))
|
||||||
pci_request_acs();
|
pci_request_acs();
|
||||||
}
|
}
|
||||||
|
|
||||||
int pci_xen_swiotlb_init_late(void)
|
|
||||||
{
|
|
||||||
if (dma_ops == &xen_swiotlb_dma_ops)
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
/* we can work with the default swiotlb */
|
|
||||||
if (!io_tlb_default_mem.nslabs) {
|
|
||||||
int rc = swiotlb_init_late(swiotlb_size_or_default(),
|
|
||||||
GFP_KERNEL, xen_swiotlb_fixup);
|
|
||||||
if (rc < 0)
|
|
||||||
return rc;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* XXX: this switches the dma ops under live devices! */
|
|
||||||
dma_ops = &xen_swiotlb_dma_ops;
|
|
||||||
if (IS_ENABLED(CONFIG_PCI))
|
|
||||||
pci_request_acs();
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late);
|
|
||||||
#else
|
#else
|
||||||
static inline void __init pci_xen_swiotlb_init(void)
|
static inline void __init pci_xen_swiotlb_init(void)
|
||||||
{
|
{
|
||||||
|
|
|
@ -44,6 +44,9 @@ struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
|
||||||
/* Number of pages released from the initial allocation. */
|
/* Number of pages released from the initial allocation. */
|
||||||
unsigned long xen_released_pages;
|
unsigned long xen_released_pages;
|
||||||
|
|
||||||
|
/* Memory map would allow PCI passthrough. */
|
||||||
|
bool xen_pv_pci_possible;
|
||||||
|
|
||||||
/* E820 map used during setting up memory. */
|
/* E820 map used during setting up memory. */
|
||||||
static struct e820_table xen_e820_table __initdata;
|
static struct e820_table xen_e820_table __initdata;
|
||||||
|
|
||||||
|
@ -814,6 +817,9 @@ char * __init xen_memory_setup(void)
|
||||||
chunk_size = size;
|
chunk_size = size;
|
||||||
type = xen_e820_table.entries[i].type;
|
type = xen_e820_table.entries[i].type;
|
||||||
|
|
||||||
|
if (type == E820_TYPE_RESERVED)
|
||||||
|
xen_pv_pci_possible = true;
|
||||||
|
|
||||||
if (type == E820_TYPE_RAM) {
|
if (type == E820_TYPE_RAM) {
|
||||||
if (addr < mem_end) {
|
if (addr < mem_end) {
|
||||||
chunk_size = min(size, mem_end - addr);
|
chunk_size = min(size, mem_end - addr);
|
||||||
|
|
|
@ -3108,9 +3108,7 @@ void device_initialize(struct device *dev)
|
||||||
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
|
defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
|
||||||
dev->dma_coherent = dma_default_coherent;
|
dev->dma_coherent = dma_default_coherent;
|
||||||
#endif
|
#endif
|
||||||
#ifdef CONFIG_SWIOTLB
|
swiotlb_dev_init(dev);
|
||||||
dev->dma_io_tlb_mem = &io_tlb_default_mem;
|
|
||||||
#endif
|
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(device_initialize);
|
EXPORT_SYMBOL_GPL(device_initialize);
|
||||||
|
|
||||||
|
|
|
@ -22,7 +22,6 @@
|
||||||
#include <linux/bitops.h>
|
#include <linux/bitops.h>
|
||||||
#include <linux/time.h>
|
#include <linux/time.h>
|
||||||
#include <linux/ktime.h>
|
#include <linux/ktime.h>
|
||||||
#include <linux/swiotlb.h>
|
|
||||||
#include <xen/platform_pci.h>
|
#include <xen/platform_pci.h>
|
||||||
|
|
||||||
#include <asm/xen/swiotlb-xen.h>
|
#include <asm/xen/swiotlb-xen.h>
|
||||||
|
@ -669,11 +668,6 @@ static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
|
||||||
|
|
||||||
spin_unlock(&pcifront_dev_lock);
|
spin_unlock(&pcifront_dev_lock);
|
||||||
|
|
||||||
if (!err && !is_swiotlb_active(&pdev->xdev->dev)) {
|
|
||||||
err = pci_xen_swiotlb_init_late();
|
|
||||||
if (err)
|
|
||||||
dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
|
|
||||||
}
|
|
||||||
return err;
|
return err;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -381,7 +381,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
|
||||||
static int
|
static int
|
||||||
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
|
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
|
||||||
{
|
{
|
||||||
return xen_phys_to_dma(hwdev, io_tlb_default_mem.end - 1) <= mask;
|
return xen_phys_to_dma(hwdev, default_swiotlb_limit()) <= mask;
|
||||||
}
|
}
|
||||||
|
|
||||||
const struct dma_map_ops xen_swiotlb_dma_ops = {
|
const struct dma_map_ops xen_swiotlb_dma_ops = {
|
||||||
|
|
|
@ -625,7 +625,10 @@ struct device_physical_location {
|
||||||
* @dma_pools: Dma pools (if dma'ble device).
|
* @dma_pools: Dma pools (if dma'ble device).
|
||||||
* @dma_mem: Internal for coherent mem override.
|
* @dma_mem: Internal for coherent mem override.
|
||||||
* @cma_area: Contiguous memory area for dma allocations
|
* @cma_area: Contiguous memory area for dma allocations
|
||||||
* @dma_io_tlb_mem: Pointer to the swiotlb pool used. Not for driver use.
|
* @dma_io_tlb_mem: Software IO TLB allocator. Not for driver use.
|
||||||
|
* @dma_io_tlb_pools: List of transient swiotlb memory pools.
|
||||||
|
* @dma_io_tlb_lock: Protects changes to the list of active pools.
|
||||||
|
* @dma_uses_io_tlb: %true if device has used the software IO TLB.
|
||||||
* @archdata: For arch-specific additions.
|
* @archdata: For arch-specific additions.
|
||||||
* @of_node: Associated device tree node.
|
* @of_node: Associated device tree node.
|
||||||
* @fwnode: Associated device node supplied by platform firmware.
|
* @fwnode: Associated device node supplied by platform firmware.
|
||||||
|
@ -731,6 +734,11 @@ struct device {
|
||||||
#endif
|
#endif
|
||||||
#ifdef CONFIG_SWIOTLB
|
#ifdef CONFIG_SWIOTLB
|
||||||
struct io_tlb_mem *dma_io_tlb_mem;
|
struct io_tlb_mem *dma_io_tlb_mem;
|
||||||
|
#endif
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
struct list_head dma_io_tlb_pools;
|
||||||
|
spinlock_t dma_io_tlb_lock;
|
||||||
|
bool dma_uses_io_tlb;
|
||||||
#endif
|
#endif
|
||||||
/* arch specific additions */
|
/* arch specific additions */
|
||||||
struct dev_archdata archdata;
|
struct dev_archdata archdata;
|
||||||
|
|
|
@ -169,12 +169,6 @@ static inline void dma_free_contiguous(struct device *dev, struct page *page,
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_DMA_CMA*/
|
#endif /* CONFIG_DMA_CMA*/
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
|
||||||
void dma_pernuma_cma_reserve(void);
|
|
||||||
#else
|
|
||||||
static inline void dma_pernuma_cma_reserve(void) { }
|
|
||||||
#endif /* CONFIG_DMA_PERNUMA_CMA */
|
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_DECLARE_COHERENT
|
#ifdef CONFIG_DMA_DECLARE_COHERENT
|
||||||
int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
|
int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
|
||||||
dma_addr_t device_addr, size_t size);
|
dma_addr_t device_addr, size_t size);
|
||||||
|
@ -343,6 +337,12 @@ void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
|
||||||
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
||||||
dma_addr_t dma_addr, unsigned long attrs);
|
dma_addr_t dma_addr, unsigned long attrs);
|
||||||
|
|
||||||
|
#ifdef CONFIG_ARCH_HAS_DMA_SET_MASK
|
||||||
|
void arch_dma_set_mask(struct device *dev, u64 mask);
|
||||||
|
#else
|
||||||
|
#define arch_dma_set_mask(dev, mask) do { } while (0)
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_MMU
|
#ifdef CONFIG_MMU
|
||||||
/*
|
/*
|
||||||
* Page protection so that devices that can't snoop CPU caches can use the
|
* Page protection so that devices that can't snoop CPU caches can use the
|
||||||
|
|
|
@ -418,6 +418,8 @@ static inline void dma_sync_sgtable_for_device(struct device *dev,
|
||||||
#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
|
#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
|
||||||
#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
|
#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
|
||||||
|
|
||||||
|
bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
|
||||||
|
|
||||||
static inline void *dma_alloc_coherent(struct device *dev, size_t size,
|
static inline void *dma_alloc_coherent(struct device *dev, size_t size,
|
||||||
dma_addr_t *dma_handle, gfp_t gfp)
|
dma_addr_t *dma_handle, gfp_t gfp)
|
||||||
{
|
{
|
||||||
|
|
|
@ -8,6 +8,7 @@
|
||||||
#include <linux/types.h>
|
#include <linux/types.h>
|
||||||
#include <linux/limits.h>
|
#include <linux/limits.h>
|
||||||
#include <linux/spinlock.h>
|
#include <linux/spinlock.h>
|
||||||
|
#include <linux/workqueue.h>
|
||||||
|
|
||||||
struct device;
|
struct device;
|
||||||
struct page;
|
struct page;
|
||||||
|
@ -62,8 +63,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
|
||||||
#ifdef CONFIG_SWIOTLB
|
#ifdef CONFIG_SWIOTLB
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* struct io_tlb_mem - IO TLB Memory Pool Descriptor
|
* struct io_tlb_pool - IO TLB memory pool descriptor
|
||||||
*
|
|
||||||
* @start: The start address of the swiotlb memory pool. Used to do a quick
|
* @start: The start address of the swiotlb memory pool. Used to do a quick
|
||||||
* range check to see if the memory was in fact allocated by this
|
* range check to see if the memory was in fact allocated by this
|
||||||
* API.
|
* API.
|
||||||
|
@ -73,19 +73,48 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
|
||||||
* @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool
|
* @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool
|
||||||
* may be remapped in the memory encrypted case and store virtual
|
* may be remapped in the memory encrypted case and store virtual
|
||||||
* address for bounce buffer operation.
|
* address for bounce buffer operation.
|
||||||
* @nslabs: The number of IO TLB blocks (in groups of 64) between @start and
|
* @nslabs: The number of IO TLB slots between @start and @end. For the
|
||||||
* @end. For default swiotlb, this is command line adjustable via
|
* default swiotlb, this can be adjusted with a boot parameter,
|
||||||
* setup_io_tlb_npages.
|
* see setup_io_tlb_npages().
|
||||||
* @list: The free list describing the number of free entries available
|
* @late_alloc: %true if allocated using the page allocator.
|
||||||
* from each index.
|
* @nareas: Number of areas in the pool.
|
||||||
* @orig_addr: The original address corresponding to a mapped entry.
|
* @area_nslabs: Number of slots in each area.
|
||||||
* @alloc_size: Size of the allocated buffer.
|
* @areas: Array of memory area descriptors.
|
||||||
|
* @slots: Array of slot descriptors.
|
||||||
|
* @node: Member of the IO TLB memory pool list.
|
||||||
|
* @rcu: RCU head for swiotlb_dyn_free().
|
||||||
|
* @transient: %true if transient memory pool.
|
||||||
|
*/
|
||||||
|
struct io_tlb_pool {
|
||||||
|
phys_addr_t start;
|
||||||
|
phys_addr_t end;
|
||||||
|
void *vaddr;
|
||||||
|
unsigned long nslabs;
|
||||||
|
bool late_alloc;
|
||||||
|
unsigned int nareas;
|
||||||
|
unsigned int area_nslabs;
|
||||||
|
struct io_tlb_area *areas;
|
||||||
|
struct io_tlb_slot *slots;
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
struct list_head node;
|
||||||
|
struct rcu_head rcu;
|
||||||
|
bool transient;
|
||||||
|
#endif
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct io_tlb_mem - Software IO TLB allocator
|
||||||
|
* @defpool: Default (initial) IO TLB memory pool descriptor.
|
||||||
|
* @pool: IO TLB memory pool descriptor (if not dynamic).
|
||||||
|
* @nslabs: Total number of IO TLB slabs in all pools.
|
||||||
* @debugfs: The dentry to debugfs.
|
* @debugfs: The dentry to debugfs.
|
||||||
* @late_alloc: %true if allocated using the page allocator
|
|
||||||
* @force_bounce: %true if swiotlb bouncing is forced
|
* @force_bounce: %true if swiotlb bouncing is forced
|
||||||
* @for_alloc: %true if the pool is used for memory allocation
|
* @for_alloc: %true if the pool is used for memory allocation
|
||||||
* @nareas: The area number in the pool.
|
* @can_grow: %true if more pools can be allocated dynamically.
|
||||||
* @area_nslabs: The slot number in the area.
|
* @phys_limit: Maximum allowed physical address.
|
||||||
|
* @lock: Lock to synchronize changes to the list.
|
||||||
|
* @pools: List of IO TLB memory pool descriptors (if dynamic).
|
||||||
|
* @dyn_alloc: Dynamic IO TLB pool allocation work.
|
||||||
* @total_used: The total number of slots in the pool that are currently used
|
* @total_used: The total number of slots in the pool that are currently used
|
||||||
* across all areas. Used only for calculating used_hiwater in
|
* across all areas. Used only for calculating used_hiwater in
|
||||||
* debugfs.
|
* debugfs.
|
||||||
|
@ -93,30 +122,64 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
|
||||||
* in debugfs.
|
* in debugfs.
|
||||||
*/
|
*/
|
||||||
struct io_tlb_mem {
|
struct io_tlb_mem {
|
||||||
phys_addr_t start;
|
struct io_tlb_pool defpool;
|
||||||
phys_addr_t end;
|
|
||||||
void *vaddr;
|
|
||||||
unsigned long nslabs;
|
unsigned long nslabs;
|
||||||
struct dentry *debugfs;
|
struct dentry *debugfs;
|
||||||
bool late_alloc;
|
|
||||||
bool force_bounce;
|
bool force_bounce;
|
||||||
bool for_alloc;
|
bool for_alloc;
|
||||||
unsigned int nareas;
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
unsigned int area_nslabs;
|
bool can_grow;
|
||||||
struct io_tlb_area *areas;
|
u64 phys_limit;
|
||||||
struct io_tlb_slot *slots;
|
spinlock_t lock;
|
||||||
|
struct list_head pools;
|
||||||
|
struct work_struct dyn_alloc;
|
||||||
|
#endif
|
||||||
#ifdef CONFIG_DEBUG_FS
|
#ifdef CONFIG_DEBUG_FS
|
||||||
atomic_long_t total_used;
|
atomic_long_t total_used;
|
||||||
atomic_long_t used_hiwater;
|
atomic_long_t used_hiwater;
|
||||||
#endif
|
#endif
|
||||||
};
|
};
|
||||||
extern struct io_tlb_mem io_tlb_default_mem;
|
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
|
||||||
|
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
|
||||||
|
phys_addr_t paddr)
|
||||||
|
{
|
||||||
|
return &dev->dma_io_tlb_mem->defpool;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/**
|
||||||
|
* is_swiotlb_buffer() - check if a physical address belongs to a swiotlb
|
||||||
|
* @dev: Device which has mapped the buffer.
|
||||||
|
* @paddr: Physical address within the DMA buffer.
|
||||||
|
*
|
||||||
|
* Check if @paddr points into a bounce buffer.
|
||||||
|
*
|
||||||
|
* Return:
|
||||||
|
* * %true if @paddr points into a bounce buffer
|
||||||
|
* * %false otherwise
|
||||||
|
*/
|
||||||
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
|
|
||||||
return mem && paddr >= mem->start && paddr < mem->end;
|
if (!mem)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (IS_ENABLED(CONFIG_SWIOTLB_DYNAMIC)) {
|
||||||
|
/* Pairs with smp_wmb() in swiotlb_find_slots() and
|
||||||
|
* swiotlb_dyn_alloc(), which modify the RCU lists.
|
||||||
|
*/
|
||||||
|
smp_rmb();
|
||||||
|
return swiotlb_find_pool(dev, paddr);
|
||||||
|
}
|
||||||
|
return paddr >= mem->defpool.start && paddr < mem->defpool.end;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline bool is_swiotlb_force_bounce(struct device *dev)
|
static inline bool is_swiotlb_force_bounce(struct device *dev)
|
||||||
|
@ -128,13 +191,22 @@ static inline bool is_swiotlb_force_bounce(struct device *dev)
|
||||||
|
|
||||||
void swiotlb_init(bool addressing_limited, unsigned int flags);
|
void swiotlb_init(bool addressing_limited, unsigned int flags);
|
||||||
void __init swiotlb_exit(void);
|
void __init swiotlb_exit(void);
|
||||||
|
void swiotlb_dev_init(struct device *dev);
|
||||||
size_t swiotlb_max_mapping_size(struct device *dev);
|
size_t swiotlb_max_mapping_size(struct device *dev);
|
||||||
|
bool is_swiotlb_allocated(void);
|
||||||
bool is_swiotlb_active(struct device *dev);
|
bool is_swiotlb_active(struct device *dev);
|
||||||
void __init swiotlb_adjust_size(unsigned long size);
|
void __init swiotlb_adjust_size(unsigned long size);
|
||||||
|
phys_addr_t default_swiotlb_base(void);
|
||||||
|
phys_addr_t default_swiotlb_limit(void);
|
||||||
#else
|
#else
|
||||||
static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
|
static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void swiotlb_dev_init(struct device *dev)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
|
@ -151,6 +223,11 @@ static inline size_t swiotlb_max_mapping_size(struct device *dev)
|
||||||
return SIZE_MAX;
|
return SIZE_MAX;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline bool is_swiotlb_allocated(void)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static inline bool is_swiotlb_active(struct device *dev)
|
static inline bool is_swiotlb_active(struct device *dev)
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
|
@ -159,6 +236,16 @@ static inline bool is_swiotlb_active(struct device *dev)
|
||||||
static inline void swiotlb_adjust_size(unsigned long size)
|
static inline void swiotlb_adjust_size(unsigned long size)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline phys_addr_t default_swiotlb_base(void)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline phys_addr_t default_swiotlb_limit(void)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
#endif /* CONFIG_SWIOTLB */
|
#endif /* CONFIG_SWIOTLB */
|
||||||
|
|
||||||
extern void swiotlb_print_info(void);
|
extern void swiotlb_print_info(void);
|
||||||
|
|
|
@ -29,6 +29,12 @@ extern bool xen_pvh;
|
||||||
|
|
||||||
extern uint32_t xen_start_flags;
|
extern uint32_t xen_start_flags;
|
||||||
|
|
||||||
|
#ifdef CONFIG_XEN_PV
|
||||||
|
extern bool xen_pv_pci_possible;
|
||||||
|
#else
|
||||||
|
#define xen_pv_pci_possible 0
|
||||||
|
#endif
|
||||||
|
|
||||||
#include <xen/interface/hvm/start_info.h>
|
#include <xen/interface/hvm/start_info.h>
|
||||||
extern struct hvm_start_info pvh_start_info;
|
extern struct hvm_start_info pvh_start_info;
|
||||||
void xen_prepare_pvh(void);
|
void xen_prepare_pvh(void);
|
||||||
|
|
|
@ -90,6 +90,19 @@ config SWIOTLB
|
||||||
bool
|
bool
|
||||||
select NEED_DMA_MAP_STATE
|
select NEED_DMA_MAP_STATE
|
||||||
|
|
||||||
|
config SWIOTLB_DYNAMIC
|
||||||
|
bool "Dynamic allocation of DMA bounce buffers"
|
||||||
|
default n
|
||||||
|
depends on SWIOTLB
|
||||||
|
help
|
||||||
|
This enables dynamic resizing of the software IO TLB. The kernel
|
||||||
|
starts with one memory pool at boot and it will allocate additional
|
||||||
|
pools as needed. To reduce run-time kernel memory requirements, you
|
||||||
|
may have to specify a smaller size of the initial pool using
|
||||||
|
"swiotlb=" on the kernel command line.
|
||||||
|
|
||||||
|
If unsure, say N.
|
||||||
|
|
||||||
config DMA_BOUNCE_UNALIGNED_KMALLOC
|
config DMA_BOUNCE_UNALIGNED_KMALLOC
|
||||||
bool
|
bool
|
||||||
depends on SWIOTLB
|
depends on SWIOTLB
|
||||||
|
@ -145,15 +158,16 @@ config DMA_CMA
|
||||||
|
|
||||||
if DMA_CMA
|
if DMA_CMA
|
||||||
|
|
||||||
config DMA_PERNUMA_CMA
|
config DMA_NUMA_CMA
|
||||||
bool "Enable separate DMA Contiguous Memory Area for each NUMA Node"
|
bool "Enable separate DMA Contiguous Memory Area for NUMA Node"
|
||||||
default NUMA && ARM64
|
default NUMA
|
||||||
help
|
help
|
||||||
Enable this option to get pernuma CMA areas so that devices like
|
Enable this option to get numa CMA areas so that NUMA devices
|
||||||
ARM64 SMMU can get local memory by DMA coherent APIs.
|
can get local memory by DMA coherent APIs.
|
||||||
|
|
||||||
You can set the size of pernuma CMA by specifying "cma_pernuma=size"
|
You can set the size of pernuma CMA by specifying "cma_pernuma=size"
|
||||||
on the kernel's command line.
|
or set the node id and its size of CMA by specifying "numa_cma=
|
||||||
|
<node>:size[,<node>:size]" on the kernel's command line.
|
||||||
|
|
||||||
comment "Default contiguous memory area size:"
|
comment "Default contiguous memory area size:"
|
||||||
|
|
||||||
|
|
|
@ -50,6 +50,7 @@
|
||||||
#include <linux/sizes.h>
|
#include <linux/sizes.h>
|
||||||
#include <linux/dma-map-ops.h>
|
#include <linux/dma-map-ops.h>
|
||||||
#include <linux/cma.h>
|
#include <linux/cma.h>
|
||||||
|
#include <linux/nospec.h>
|
||||||
|
|
||||||
#ifdef CONFIG_CMA_SIZE_MBYTES
|
#ifdef CONFIG_CMA_SIZE_MBYTES
|
||||||
#define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES
|
#define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES
|
||||||
|
@ -96,11 +97,44 @@ static int __init early_cma(char *p)
|
||||||
}
|
}
|
||||||
early_param("cma", early_cma);
|
early_param("cma", early_cma);
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
#ifdef CONFIG_DMA_NUMA_CMA
|
||||||
|
|
||||||
|
static struct cma *dma_contiguous_numa_area[MAX_NUMNODES];
|
||||||
|
static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata;
|
||||||
static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES];
|
static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES];
|
||||||
static phys_addr_t pernuma_size_bytes __initdata;
|
static phys_addr_t pernuma_size_bytes __initdata;
|
||||||
|
|
||||||
|
static int __init early_numa_cma(char *p)
|
||||||
|
{
|
||||||
|
int nid, count = 0;
|
||||||
|
unsigned long tmp;
|
||||||
|
char *s = p;
|
||||||
|
|
||||||
|
while (*s) {
|
||||||
|
if (sscanf(s, "%lu%n", &tmp, &count) != 1)
|
||||||
|
break;
|
||||||
|
|
||||||
|
if (s[count] == ':') {
|
||||||
|
if (tmp >= MAX_NUMNODES)
|
||||||
|
break;
|
||||||
|
nid = array_index_nospec(tmp, MAX_NUMNODES);
|
||||||
|
|
||||||
|
s += count + 1;
|
||||||
|
tmp = memparse(s, &s);
|
||||||
|
numa_cma_size[nid] = tmp;
|
||||||
|
|
||||||
|
if (*s == ',')
|
||||||
|
s++;
|
||||||
|
else
|
||||||
|
break;
|
||||||
|
} else
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
early_param("numa_cma", early_numa_cma);
|
||||||
|
|
||||||
static int __init early_cma_pernuma(char *p)
|
static int __init early_cma_pernuma(char *p)
|
||||||
{
|
{
|
||||||
pernuma_size_bytes = memparse(p, &p);
|
pernuma_size_bytes = memparse(p, &p);
|
||||||
|
@ -127,32 +161,49 @@ static inline __maybe_unused phys_addr_t cma_early_percent_memory(void)
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
#ifdef CONFIG_DMA_NUMA_CMA
|
||||||
void __init dma_pernuma_cma_reserve(void)
|
static void __init dma_numa_cma_reserve(void)
|
||||||
{
|
{
|
||||||
int nid;
|
int nid;
|
||||||
|
|
||||||
if (!pernuma_size_bytes)
|
for_each_node(nid) {
|
||||||
return;
|
|
||||||
|
|
||||||
for_each_online_node(nid) {
|
|
||||||
int ret;
|
int ret;
|
||||||
char name[CMA_MAX_NAME];
|
char name[CMA_MAX_NAME];
|
||||||
struct cma **cma = &dma_contiguous_pernuma_area[nid];
|
struct cma **cma;
|
||||||
|
|
||||||
snprintf(name, sizeof(name), "pernuma%d", nid);
|
if (!node_online(nid)) {
|
||||||
ret = cma_declare_contiguous_nid(0, pernuma_size_bytes, 0, 0,
|
if (pernuma_size_bytes || numa_cma_size[nid])
|
||||||
0, false, name, cma, nid);
|
pr_warn("invalid node %d specified\n", nid);
|
||||||
if (ret) {
|
|
||||||
pr_warn("%s: reservation failed: err %d, node %d", __func__,
|
|
||||||
ret, nid);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
pr_debug("%s: reserved %llu MiB on node %d\n", __func__,
|
if (pernuma_size_bytes) {
|
||||||
(unsigned long long)pernuma_size_bytes / SZ_1M, nid);
|
|
||||||
|
cma = &dma_contiguous_pernuma_area[nid];
|
||||||
|
snprintf(name, sizeof(name), "pernuma%d", nid);
|
||||||
|
ret = cma_declare_contiguous_nid(0, pernuma_size_bytes, 0, 0,
|
||||||
|
0, false, name, cma, nid);
|
||||||
|
if (ret)
|
||||||
|
pr_warn("%s: reservation failed: err %d, node %d", __func__,
|
||||||
|
ret, nid);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (numa_cma_size[nid]) {
|
||||||
|
|
||||||
|
cma = &dma_contiguous_numa_area[nid];
|
||||||
|
snprintf(name, sizeof(name), "numa%d", nid);
|
||||||
|
ret = cma_declare_contiguous_nid(0, numa_cma_size[nid], 0, 0, 0, false,
|
||||||
|
name, cma, nid);
|
||||||
|
if (ret)
|
||||||
|
pr_warn("%s: reservation failed: err %d, node %d", __func__,
|
||||||
|
ret, nid);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#else
|
||||||
|
static inline void __init dma_numa_cma_reserve(void)
|
||||||
|
{
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -171,6 +222,8 @@ void __init dma_contiguous_reserve(phys_addr_t limit)
|
||||||
phys_addr_t selected_limit = limit;
|
phys_addr_t selected_limit = limit;
|
||||||
bool fixed = false;
|
bool fixed = false;
|
||||||
|
|
||||||
|
dma_numa_cma_reserve();
|
||||||
|
|
||||||
pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit);
|
pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit);
|
||||||
|
|
||||||
if (size_cmdline != -1) {
|
if (size_cmdline != -1) {
|
||||||
|
@ -303,7 +356,7 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp)
|
||||||
*/
|
*/
|
||||||
struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
|
struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
|
||||||
{
|
{
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
#ifdef CONFIG_DMA_NUMA_CMA
|
||||||
int nid = dev_to_node(dev);
|
int nid = dev_to_node(dev);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
@ -315,7 +368,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
|
||||||
if (size <= PAGE_SIZE)
|
if (size <= PAGE_SIZE)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
#ifdef CONFIG_DMA_NUMA_CMA
|
||||||
if (nid != NUMA_NO_NODE && !(gfp & (GFP_DMA | GFP_DMA32))) {
|
if (nid != NUMA_NO_NODE && !(gfp & (GFP_DMA | GFP_DMA32))) {
|
||||||
struct cma *cma = dma_contiguous_pernuma_area[nid];
|
struct cma *cma = dma_contiguous_pernuma_area[nid];
|
||||||
struct page *page;
|
struct page *page;
|
||||||
|
@ -325,6 +378,13 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
|
||||||
if (page)
|
if (page)
|
||||||
return page;
|
return page;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cma = dma_contiguous_numa_area[nid];
|
||||||
|
if (cma) {
|
||||||
|
page = cma_alloc_aligned(cma, size, gfp);
|
||||||
|
if (page)
|
||||||
|
return page;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
if (!dma_contiguous_default_area)
|
if (!dma_contiguous_default_area)
|
||||||
|
@ -356,10 +416,13 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
|
||||||
/*
|
/*
|
||||||
* otherwise, page is from either per-numa cma or default cma
|
* otherwise, page is from either per-numa cma or default cma
|
||||||
*/
|
*/
|
||||||
#ifdef CONFIG_DMA_PERNUMA_CMA
|
#ifdef CONFIG_DMA_NUMA_CMA
|
||||||
if (cma_release(dma_contiguous_pernuma_area[page_to_nid(page)],
|
if (cma_release(dma_contiguous_pernuma_area[page_to_nid(page)],
|
||||||
page, count))
|
page, count))
|
||||||
return;
|
return;
|
||||||
|
if (cma_release(dma_contiguous_numa_area[page_to_nid(page)],
|
||||||
|
page, count))
|
||||||
|
return;
|
||||||
#endif
|
#endif
|
||||||
if (cma_release(dma_contiguous_default_area, page, count))
|
if (cma_release(dma_contiguous_default_area, page, count))
|
||||||
return;
|
return;
|
||||||
|
@ -410,6 +473,11 @@ static int __init rmem_cma_setup(struct reserved_mem *rmem)
|
||||||
return -EBUSY;
|
return -EBUSY;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (memblock_is_region_reserved(rmem->base, rmem->size)) {
|
||||||
|
pr_info("Reserved memory: overlap with other memblock reserved region\n");
|
||||||
|
return -EBUSY;
|
||||||
|
}
|
||||||
|
|
||||||
if (!of_get_flat_dt_prop(node, "reusable", NULL) ||
|
if (!of_get_flat_dt_prop(node, "reusable", NULL) ||
|
||||||
of_get_flat_dt_prop(node, "no-map", NULL))
|
of_get_flat_dt_prop(node, "no-map", NULL))
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
|
@ -66,7 +66,7 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
|
bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
|
||||||
{
|
{
|
||||||
dma_addr_t dma_addr = phys_to_dma_direct(dev, phys);
|
dma_addr_t dma_addr = phys_to_dma_direct(dev, phys);
|
||||||
|
|
||||||
|
|
|
@ -760,12 +760,6 @@ bool dma_pci_p2pdma_supported(struct device *dev)
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported);
|
EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported);
|
||||||
|
|
||||||
#ifdef CONFIG_ARCH_HAS_DMA_SET_MASK
|
|
||||||
void arch_dma_set_mask(struct device *dev, u64 mask);
|
|
||||||
#else
|
|
||||||
#define arch_dma_set_mask(dev, mask) do { } while (0)
|
|
||||||
#endif
|
|
||||||
|
|
||||||
int dma_set_mask(struct device *dev, u64 mask)
|
int dma_set_mask(struct device *dev, u64 mask)
|
||||||
{
|
{
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -35,6 +35,7 @@
|
||||||
#include <linux/memblock.h>
|
#include <linux/memblock.h>
|
||||||
#include <linux/mm.h>
|
#include <linux/mm.h>
|
||||||
#include <linux/pfn.h>
|
#include <linux/pfn.h>
|
||||||
|
#include <linux/rculist.h>
|
||||||
#include <linux/scatterlist.h>
|
#include <linux/scatterlist.h>
|
||||||
#include <linux/set_memory.h>
|
#include <linux/set_memory.h>
|
||||||
#include <linux/spinlock.h>
|
#include <linux/spinlock.h>
|
||||||
|
@ -62,6 +63,13 @@
|
||||||
|
|
||||||
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
|
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* struct io_tlb_slot - IO TLB slot descriptor
|
||||||
|
* @orig_addr: The original address corresponding to a mapped entry.
|
||||||
|
* @alloc_size: Size of the allocated buffer.
|
||||||
|
* @list: The free list describing the number of free entries available
|
||||||
|
* from each index.
|
||||||
|
*/
|
||||||
struct io_tlb_slot {
|
struct io_tlb_slot {
|
||||||
phys_addr_t orig_addr;
|
phys_addr_t orig_addr;
|
||||||
size_t alloc_size;
|
size_t alloc_size;
|
||||||
|
@ -71,7 +79,22 @@ struct io_tlb_slot {
|
||||||
static bool swiotlb_force_bounce;
|
static bool swiotlb_force_bounce;
|
||||||
static bool swiotlb_force_disable;
|
static bool swiotlb_force_disable;
|
||||||
|
|
||||||
struct io_tlb_mem io_tlb_default_mem;
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
|
||||||
|
static void swiotlb_dyn_alloc(struct work_struct *work);
|
||||||
|
|
||||||
|
static struct io_tlb_mem io_tlb_default_mem = {
|
||||||
|
.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
|
||||||
|
.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
|
||||||
|
.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
|
||||||
|
swiotlb_dyn_alloc),
|
||||||
|
};
|
||||||
|
|
||||||
|
#else /* !CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
|
static struct io_tlb_mem io_tlb_default_mem;
|
||||||
|
|
||||||
|
#endif /* CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
|
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
|
||||||
static unsigned long default_nareas;
|
static unsigned long default_nareas;
|
||||||
|
@ -202,7 +225,7 @@ void __init swiotlb_adjust_size(unsigned long size)
|
||||||
|
|
||||||
void swiotlb_print_info(void)
|
void swiotlb_print_info(void)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
|
||||||
|
|
||||||
if (!mem->nslabs) {
|
if (!mem->nslabs) {
|
||||||
pr_warn("No low mem\n");
|
pr_warn("No low mem\n");
|
||||||
|
@ -231,7 +254,7 @@ static inline unsigned long nr_slots(u64 val)
|
||||||
*/
|
*/
|
||||||
void __init swiotlb_update_mem_attributes(void)
|
void __init swiotlb_update_mem_attributes(void)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
|
||||||
unsigned long bytes;
|
unsigned long bytes;
|
||||||
|
|
||||||
if (!mem->nslabs || mem->late_alloc)
|
if (!mem->nslabs || mem->late_alloc)
|
||||||
|
@ -240,9 +263,8 @@ void __init swiotlb_update_mem_attributes(void)
|
||||||
set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
|
set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
|
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
|
||||||
unsigned long nslabs, unsigned int flags,
|
unsigned long nslabs, bool late_alloc, unsigned int nareas)
|
||||||
bool late_alloc, unsigned int nareas)
|
|
||||||
{
|
{
|
||||||
void *vaddr = phys_to_virt(start);
|
void *vaddr = phys_to_virt(start);
|
||||||
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
|
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
|
||||||
|
@ -254,8 +276,6 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
|
||||||
mem->nareas = nareas;
|
mem->nareas = nareas;
|
||||||
mem->area_nslabs = nslabs / mem->nareas;
|
mem->area_nslabs = nslabs / mem->nareas;
|
||||||
|
|
||||||
mem->force_bounce = swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
|
|
||||||
|
|
||||||
for (i = 0; i < mem->nareas; i++) {
|
for (i = 0; i < mem->nareas; i++) {
|
||||||
spin_lock_init(&mem->areas[i].lock);
|
spin_lock_init(&mem->areas[i].lock);
|
||||||
mem->areas[i].index = 0;
|
mem->areas[i].index = 0;
|
||||||
|
@ -273,6 +293,23 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* add_mem_pool() - add a memory pool to the allocator
|
||||||
|
* @mem: Software IO TLB allocator.
|
||||||
|
* @pool: Memory pool to be added.
|
||||||
|
*/
|
||||||
|
static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
spin_lock(&mem->lock);
|
||||||
|
list_add_rcu(&pool->node, &mem->pools);
|
||||||
|
mem->nslabs += pool->nslabs;
|
||||||
|
spin_unlock(&mem->lock);
|
||||||
|
#else
|
||||||
|
mem->nslabs = pool->nslabs;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
|
static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
|
||||||
unsigned int flags,
|
unsigned int flags,
|
||||||
int (*remap)(void *tlb, unsigned long nslabs))
|
int (*remap)(void *tlb, unsigned long nslabs))
|
||||||
|
@ -312,7 +349,7 @@ static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
|
||||||
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
||||||
int (*remap)(void *tlb, unsigned long nslabs))
|
int (*remap)(void *tlb, unsigned long nslabs))
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
|
||||||
unsigned long nslabs;
|
unsigned long nslabs;
|
||||||
unsigned int nareas;
|
unsigned int nareas;
|
||||||
size_t alloc_size;
|
size_t alloc_size;
|
||||||
|
@ -323,6 +360,18 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
||||||
if (swiotlb_force_disable)
|
if (swiotlb_force_disable)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
io_tlb_default_mem.force_bounce =
|
||||||
|
swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
if (!remap)
|
||||||
|
io_tlb_default_mem.can_grow = true;
|
||||||
|
if (flags & SWIOTLB_ANY)
|
||||||
|
io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
|
||||||
|
else
|
||||||
|
io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
|
||||||
|
#endif
|
||||||
|
|
||||||
if (!default_nareas)
|
if (!default_nareas)
|
||||||
swiotlb_adjust_nareas(num_possible_cpus());
|
swiotlb_adjust_nareas(num_possible_cpus());
|
||||||
|
|
||||||
|
@ -356,8 +405,9 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, flags, false,
|
swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false,
|
||||||
default_nareas);
|
default_nareas);
|
||||||
|
add_mem_pool(&io_tlb_default_mem, mem);
|
||||||
|
|
||||||
if (flags & SWIOTLB_VERBOSE)
|
if (flags & SWIOTLB_VERBOSE)
|
||||||
swiotlb_print_info();
|
swiotlb_print_info();
|
||||||
|
@ -376,7 +426,7 @@ void __init swiotlb_init(bool addressing_limit, unsigned int flags)
|
||||||
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
|
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
|
||||||
int (*remap)(void *tlb, unsigned long nslabs))
|
int (*remap)(void *tlb, unsigned long nslabs))
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
|
||||||
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
|
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
|
||||||
unsigned int nareas;
|
unsigned int nareas;
|
||||||
unsigned char *vstart = NULL;
|
unsigned char *vstart = NULL;
|
||||||
|
@ -384,9 +434,25 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
|
||||||
bool retried = false;
|
bool retried = false;
|
||||||
int rc = 0;
|
int rc = 0;
|
||||||
|
|
||||||
|
if (io_tlb_default_mem.nslabs)
|
||||||
|
return 0;
|
||||||
|
|
||||||
if (swiotlb_force_disable)
|
if (swiotlb_force_disable)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
if (!remap)
|
||||||
|
io_tlb_default_mem.can_grow = true;
|
||||||
|
if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
|
||||||
|
io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
|
||||||
|
else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
|
||||||
|
io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
|
||||||
|
else
|
||||||
|
io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
|
||||||
|
#endif
|
||||||
|
|
||||||
if (!default_nareas)
|
if (!default_nareas)
|
||||||
swiotlb_adjust_nareas(num_possible_cpus());
|
swiotlb_adjust_nareas(num_possible_cpus());
|
||||||
|
|
||||||
|
@ -438,8 +504,9 @@ retry:
|
||||||
|
|
||||||
set_memory_decrypted((unsigned long)vstart,
|
set_memory_decrypted((unsigned long)vstart,
|
||||||
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
|
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
|
||||||
swiotlb_init_io_tlb_mem(mem, virt_to_phys(vstart), nslabs, 0, true,
|
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
|
||||||
nareas);
|
nareas);
|
||||||
|
add_mem_pool(&io_tlb_default_mem, mem);
|
||||||
|
|
||||||
swiotlb_print_info();
|
swiotlb_print_info();
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -453,7 +520,7 @@ error_area:
|
||||||
|
|
||||||
void __init swiotlb_exit(void)
|
void __init swiotlb_exit(void)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = &io_tlb_default_mem;
|
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
|
||||||
unsigned long tbl_vaddr;
|
unsigned long tbl_vaddr;
|
||||||
size_t tbl_size, slots_size;
|
size_t tbl_size, slots_size;
|
||||||
unsigned int area_order;
|
unsigned int area_order;
|
||||||
|
@ -486,6 +553,265 @@ void __init swiotlb_exit(void)
|
||||||
memset(mem, 0, sizeof(*mem));
|
memset(mem, 0, sizeof(*mem));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
|
||||||
|
/**
|
||||||
|
* alloc_dma_pages() - allocate pages to be used for DMA
|
||||||
|
* @gfp: GFP flags for the allocation.
|
||||||
|
* @bytes: Size of the buffer.
|
||||||
|
*
|
||||||
|
* Allocate pages from the buddy allocator. If successful, make the allocated
|
||||||
|
* pages decrypted that they can be used for DMA.
|
||||||
|
*
|
||||||
|
* Return: Decrypted pages, or %NULL on failure.
|
||||||
|
*/
|
||||||
|
static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes)
|
||||||
|
{
|
||||||
|
unsigned int order = get_order(bytes);
|
||||||
|
struct page *page;
|
||||||
|
void *vaddr;
|
||||||
|
|
||||||
|
page = alloc_pages(gfp, order);
|
||||||
|
if (!page)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
vaddr = page_address(page);
|
||||||
|
if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
|
||||||
|
goto error;
|
||||||
|
return page;
|
||||||
|
|
||||||
|
error:
|
||||||
|
__free_pages(page, order);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
|
||||||
|
* @dev: Device for which a memory pool is allocated.
|
||||||
|
* @bytes: Size of the buffer.
|
||||||
|
* @phys_limit: Maximum allowed physical address of the buffer.
|
||||||
|
* @gfp: GFP flags for the allocation.
|
||||||
|
*
|
||||||
|
* Return: Allocated pages, or %NULL on allocation failure.
|
||||||
|
*/
|
||||||
|
static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
|
||||||
|
u64 phys_limit, gfp_t gfp)
|
||||||
|
{
|
||||||
|
struct page *page;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Allocate from the atomic pools if memory is encrypted and
|
||||||
|
* the allocation is atomic, because decrypting may block.
|
||||||
|
*/
|
||||||
|
if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
|
||||||
|
void *vaddr;
|
||||||
|
|
||||||
|
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
|
||||||
|
dma_coherent_ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
gfp &= ~GFP_ZONEMASK;
|
||||||
|
if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
|
||||||
|
gfp |= __GFP_DMA;
|
||||||
|
else if (phys_limit <= DMA_BIT_MASK(32))
|
||||||
|
gfp |= __GFP_DMA32;
|
||||||
|
|
||||||
|
while ((page = alloc_dma_pages(gfp, bytes)) &&
|
||||||
|
page_to_phys(page) + bytes - 1 > phys_limit) {
|
||||||
|
/* allocated, but too high */
|
||||||
|
__free_pages(page, get_order(bytes));
|
||||||
|
|
||||||
|
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
|
||||||
|
phys_limit < DMA_BIT_MASK(64) &&
|
||||||
|
!(gfp & (__GFP_DMA32 | __GFP_DMA)))
|
||||||
|
gfp |= __GFP_DMA32;
|
||||||
|
else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
|
||||||
|
!(gfp & __GFP_DMA))
|
||||||
|
gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
|
||||||
|
else
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
return page;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
|
||||||
|
* @vaddr: Virtual address of the buffer.
|
||||||
|
* @bytes: Size of the buffer.
|
||||||
|
*/
|
||||||
|
static void swiotlb_free_tlb(void *vaddr, size_t bytes)
|
||||||
|
{
|
||||||
|
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
|
||||||
|
dma_free_from_pool(NULL, vaddr, bytes))
|
||||||
|
return;
|
||||||
|
|
||||||
|
/* Intentional leak if pages cannot be encrypted again. */
|
||||||
|
if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
|
||||||
|
__free_pages(virt_to_page(vaddr), get_order(bytes));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
|
||||||
|
* @dev: Device for which a memory pool is allocated.
|
||||||
|
* @minslabs: Minimum number of slabs.
|
||||||
|
* @nslabs: Desired (maximum) number of slabs.
|
||||||
|
* @nareas: Number of areas.
|
||||||
|
* @phys_limit: Maximum DMA buffer physical address.
|
||||||
|
* @gfp: GFP flags for the allocations.
|
||||||
|
*
|
||||||
|
* Allocate and initialize a new IO TLB memory pool. The actual number of
|
||||||
|
* slabs may be reduced if allocation of @nslabs fails. If even
|
||||||
|
* @minslabs cannot be allocated, this function fails.
|
||||||
|
*
|
||||||
|
* Return: New memory pool, or %NULL on allocation failure.
|
||||||
|
*/
|
||||||
|
static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
|
||||||
|
unsigned long minslabs, unsigned long nslabs,
|
||||||
|
unsigned int nareas, u64 phys_limit, gfp_t gfp)
|
||||||
|
{
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
unsigned int slot_order;
|
||||||
|
struct page *tlb;
|
||||||
|
size_t pool_size;
|
||||||
|
size_t tlb_size;
|
||||||
|
|
||||||
|
pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
|
||||||
|
pool = kzalloc(pool_size, gfp);
|
||||||
|
if (!pool)
|
||||||
|
goto error;
|
||||||
|
pool->areas = (void *)pool + sizeof(*pool);
|
||||||
|
|
||||||
|
tlb_size = nslabs << IO_TLB_SHIFT;
|
||||||
|
while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
|
||||||
|
if (nslabs <= minslabs)
|
||||||
|
goto error_tlb;
|
||||||
|
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
|
||||||
|
nareas = limit_nareas(nareas, nslabs);
|
||||||
|
tlb_size = nslabs << IO_TLB_SHIFT;
|
||||||
|
}
|
||||||
|
|
||||||
|
slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
|
||||||
|
pool->slots = (struct io_tlb_slot *)
|
||||||
|
__get_free_pages(gfp, slot_order);
|
||||||
|
if (!pool->slots)
|
||||||
|
goto error_slots;
|
||||||
|
|
||||||
|
swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
|
||||||
|
return pool;
|
||||||
|
|
||||||
|
error_slots:
|
||||||
|
swiotlb_free_tlb(page_address(tlb), tlb_size);
|
||||||
|
error_tlb:
|
||||||
|
kfree(pool);
|
||||||
|
error:
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_dyn_alloc() - dynamic memory pool allocation worker
|
||||||
|
* @work: Pointer to dyn_alloc in struct io_tlb_mem.
|
||||||
|
*/
|
||||||
|
static void swiotlb_dyn_alloc(struct work_struct *work)
|
||||||
|
{
|
||||||
|
struct io_tlb_mem *mem =
|
||||||
|
container_of(work, struct io_tlb_mem, dyn_alloc);
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
|
||||||
|
pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
|
||||||
|
default_nareas, mem->phys_limit, GFP_KERNEL);
|
||||||
|
if (!pool) {
|
||||||
|
pr_warn_ratelimited("Failed to allocate new pool");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
add_mem_pool(mem, pool);
|
||||||
|
|
||||||
|
/* Pairs with smp_rmb() in is_swiotlb_buffer(). */
|
||||||
|
smp_wmb();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_dyn_free() - RCU callback to free a memory pool
|
||||||
|
* @rcu: RCU head in the corresponding struct io_tlb_pool.
|
||||||
|
*/
|
||||||
|
static void swiotlb_dyn_free(struct rcu_head *rcu)
|
||||||
|
{
|
||||||
|
struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
|
||||||
|
size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
|
||||||
|
size_t tlb_size = pool->end - pool->start;
|
||||||
|
|
||||||
|
free_pages((unsigned long)pool->slots, get_order(slots_size));
|
||||||
|
swiotlb_free_tlb(pool->vaddr, tlb_size);
|
||||||
|
kfree(pool);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_find_pool() - find the IO TLB pool for a physical address
|
||||||
|
* @dev: Device which has mapped the DMA buffer.
|
||||||
|
* @paddr: Physical address within the DMA buffer.
|
||||||
|
*
|
||||||
|
* Find the IO TLB memory pool descriptor which contains the given physical
|
||||||
|
* address, if any.
|
||||||
|
*
|
||||||
|
* Return: Memory pool which contains @paddr, or %NULL if none.
|
||||||
|
*/
|
||||||
|
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
|
||||||
|
{
|
||||||
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
|
||||||
|
rcu_read_lock();
|
||||||
|
list_for_each_entry_rcu(pool, &mem->pools, node) {
|
||||||
|
if (paddr >= pool->start && paddr < pool->end)
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
|
||||||
|
list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
|
||||||
|
if (paddr >= pool->start && paddr < pool->end)
|
||||||
|
goto out;
|
||||||
|
}
|
||||||
|
pool = NULL;
|
||||||
|
out:
|
||||||
|
rcu_read_unlock();
|
||||||
|
return pool;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_del_pool() - remove an IO TLB pool from a device
|
||||||
|
* @dev: Owning device.
|
||||||
|
* @pool: Memory pool to be removed.
|
||||||
|
*/
|
||||||
|
static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
|
||||||
|
{
|
||||||
|
unsigned long flags;
|
||||||
|
|
||||||
|
spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
|
||||||
|
list_del_rcu(&pool->node);
|
||||||
|
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
|
||||||
|
|
||||||
|
call_rcu(&pool->rcu, swiotlb_dyn_free);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_dev_init() - initialize swiotlb fields in &struct device
|
||||||
|
* @dev: Device to be initialized.
|
||||||
|
*/
|
||||||
|
void swiotlb_dev_init(struct device *dev)
|
||||||
|
{
|
||||||
|
dev->dma_io_tlb_mem = &io_tlb_default_mem;
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
INIT_LIST_HEAD(&dev->dma_io_tlb_pools);
|
||||||
|
spin_lock_init(&dev->dma_io_tlb_lock);
|
||||||
|
dev->dma_uses_io_tlb = false;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Return the offset into a iotlb slot required to keep the device happy.
|
* Return the offset into a iotlb slot required to keep the device happy.
|
||||||
*/
|
*/
|
||||||
|
@ -500,7 +826,7 @@ static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
|
||||||
static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
|
static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
|
||||||
enum dma_data_direction dir)
|
enum dma_data_direction dir)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
|
||||||
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
|
int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
|
||||||
phys_addr_t orig_addr = mem->slots[index].orig_addr;
|
phys_addr_t orig_addr = mem->slots[index].orig_addr;
|
||||||
size_t alloc_size = mem->slots[index].alloc_size;
|
size_t alloc_size = mem->slots[index].alloc_size;
|
||||||
|
@ -577,12 +903,10 @@ static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
|
||||||
*/
|
*/
|
||||||
static inline unsigned long get_max_slots(unsigned long boundary_mask)
|
static inline unsigned long get_max_slots(unsigned long boundary_mask)
|
||||||
{
|
{
|
||||||
if (boundary_mask == ~0UL)
|
return (boundary_mask >> IO_TLB_SHIFT) + 1;
|
||||||
return 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
|
|
||||||
return nr_slots(boundary_mask + 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index)
|
static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
|
||||||
{
|
{
|
||||||
if (index >= mem->area_nslabs)
|
if (index >= mem->area_nslabs)
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -623,19 +947,30 @@ static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
|
||||||
}
|
}
|
||||||
#endif /* CONFIG_DEBUG_FS */
|
#endif /* CONFIG_DEBUG_FS */
|
||||||
|
|
||||||
/*
|
/**
|
||||||
* Find a suitable number of IO TLB entries size that will fit this request and
|
* swiotlb_area_find_slots() - search for slots in one IO TLB memory area
|
||||||
* allocate a buffer from that IO TLB pool.
|
* @dev: Device which maps the buffer.
|
||||||
|
* @pool: Memory pool to be searched.
|
||||||
|
* @area_index: Index of the IO TLB memory area to be searched.
|
||||||
|
* @orig_addr: Original (non-bounced) IO buffer address.
|
||||||
|
* @alloc_size: Total requested size of the bounce buffer,
|
||||||
|
* including initial alignment padding.
|
||||||
|
* @alloc_align_mask: Required alignment of the allocated buffer.
|
||||||
|
*
|
||||||
|
* Find a suitable sequence of IO TLB entries for the request and allocate
|
||||||
|
* a buffer from the given IO TLB memory area.
|
||||||
|
* This function takes care of locking.
|
||||||
|
*
|
||||||
|
* Return: Index of the first allocated slot, or -1 on error.
|
||||||
*/
|
*/
|
||||||
static int swiotlb_do_find_slots(struct device *dev, int area_index,
|
static int swiotlb_area_find_slots(struct device *dev, struct io_tlb_pool *pool,
|
||||||
phys_addr_t orig_addr, size_t alloc_size,
|
int area_index, phys_addr_t orig_addr, size_t alloc_size,
|
||||||
unsigned int alloc_align_mask)
|
unsigned int alloc_align_mask)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_area *area = pool->areas + area_index;
|
||||||
struct io_tlb_area *area = mem->areas + area_index;
|
|
||||||
unsigned long boundary_mask = dma_get_seg_boundary(dev);
|
unsigned long boundary_mask = dma_get_seg_boundary(dev);
|
||||||
dma_addr_t tbl_dma_addr =
|
dma_addr_t tbl_dma_addr =
|
||||||
phys_to_dma_unencrypted(dev, mem->start) & boundary_mask;
|
phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
|
||||||
unsigned long max_slots = get_max_slots(boundary_mask);
|
unsigned long max_slots = get_max_slots(boundary_mask);
|
||||||
unsigned int iotlb_align_mask =
|
unsigned int iotlb_align_mask =
|
||||||
dma_get_min_align_mask(dev) | alloc_align_mask;
|
dma_get_min_align_mask(dev) | alloc_align_mask;
|
||||||
|
@ -647,7 +982,7 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
|
||||||
unsigned int slot_index;
|
unsigned int slot_index;
|
||||||
|
|
||||||
BUG_ON(!nslots);
|
BUG_ON(!nslots);
|
||||||
BUG_ON(area_index >= mem->nareas);
|
BUG_ON(area_index >= pool->nareas);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* For allocations of PAGE_SIZE or larger only look for page aligned
|
* For allocations of PAGE_SIZE or larger only look for page aligned
|
||||||
|
@ -664,35 +999,30 @@ static int swiotlb_do_find_slots(struct device *dev, int area_index,
|
||||||
stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
|
stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
|
||||||
|
|
||||||
spin_lock_irqsave(&area->lock, flags);
|
spin_lock_irqsave(&area->lock, flags);
|
||||||
if (unlikely(nslots > mem->area_nslabs - area->used))
|
if (unlikely(nslots > pool->area_nslabs - area->used))
|
||||||
goto not_found;
|
goto not_found;
|
||||||
|
|
||||||
slot_base = area_index * mem->area_nslabs;
|
slot_base = area_index * pool->area_nslabs;
|
||||||
index = area->index;
|
index = area->index;
|
||||||
|
|
||||||
for (slots_checked = 0; slots_checked < mem->area_nslabs; ) {
|
for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
|
||||||
slot_index = slot_base + index;
|
slot_index = slot_base + index;
|
||||||
|
|
||||||
if (orig_addr &&
|
if (orig_addr &&
|
||||||
(slot_addr(tbl_dma_addr, slot_index) &
|
(slot_addr(tbl_dma_addr, slot_index) &
|
||||||
iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
|
iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
|
||||||
index = wrap_area_index(mem, index + 1);
|
index = wrap_area_index(pool, index + 1);
|
||||||
slots_checked++;
|
slots_checked++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
|
||||||
* If we find a slot that indicates we have 'nslots' number of
|
|
||||||
* contiguous buffers, we allocate the buffers from that slot
|
|
||||||
* and mark the entries as '0' indicating unavailable.
|
|
||||||
*/
|
|
||||||
if (!iommu_is_span_boundary(slot_index, nslots,
|
if (!iommu_is_span_boundary(slot_index, nslots,
|
||||||
nr_slots(tbl_dma_addr),
|
nr_slots(tbl_dma_addr),
|
||||||
max_slots)) {
|
max_slots)) {
|
||||||
if (mem->slots[slot_index].list >= nslots)
|
if (pool->slots[slot_index].list >= nslots)
|
||||||
goto found;
|
goto found;
|
||||||
}
|
}
|
||||||
index = wrap_area_index(mem, index + stride);
|
index = wrap_area_index(pool, index + stride);
|
||||||
slots_checked += stride;
|
slots_checked += stride;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -701,48 +1031,159 @@ not_found:
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
found:
|
found:
|
||||||
|
/*
|
||||||
|
* If we find a slot that indicates we have 'nslots' number of
|
||||||
|
* contiguous buffers, we allocate the buffers from that slot onwards
|
||||||
|
* and set the list of free entries to '0' indicating unavailable.
|
||||||
|
*/
|
||||||
for (i = slot_index; i < slot_index + nslots; i++) {
|
for (i = slot_index; i < slot_index + nslots; i++) {
|
||||||
mem->slots[i].list = 0;
|
pool->slots[i].list = 0;
|
||||||
mem->slots[i].alloc_size = alloc_size - (offset +
|
pool->slots[i].alloc_size = alloc_size - (offset +
|
||||||
((i - slot_index) << IO_TLB_SHIFT));
|
((i - slot_index) << IO_TLB_SHIFT));
|
||||||
}
|
}
|
||||||
for (i = slot_index - 1;
|
for (i = slot_index - 1;
|
||||||
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
|
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
|
||||||
mem->slots[i].list; i--)
|
pool->slots[i].list; i--)
|
||||||
mem->slots[i].list = ++count;
|
pool->slots[i].list = ++count;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Update the indices to avoid searching in the next round.
|
* Update the indices to avoid searching in the next round.
|
||||||
*/
|
*/
|
||||||
area->index = wrap_area_index(mem, index + nslots);
|
area->index = wrap_area_index(pool, index + nslots);
|
||||||
area->used += nslots;
|
area->used += nslots;
|
||||||
spin_unlock_irqrestore(&area->lock, flags);
|
spin_unlock_irqrestore(&area->lock, flags);
|
||||||
|
|
||||||
inc_used_and_hiwater(mem, nslots);
|
inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots);
|
||||||
return slot_index;
|
return slot_index;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
|
/**
|
||||||
size_t alloc_size, unsigned int alloc_align_mask)
|
* swiotlb_pool_find_slots() - search for slots in one memory pool
|
||||||
|
* @dev: Device which maps the buffer.
|
||||||
|
* @pool: Memory pool to be searched.
|
||||||
|
* @orig_addr: Original (non-bounced) IO buffer address.
|
||||||
|
* @alloc_size: Total requested size of the bounce buffer,
|
||||||
|
* including initial alignment padding.
|
||||||
|
* @alloc_align_mask: Required alignment of the allocated buffer.
|
||||||
|
*
|
||||||
|
* Search through one memory pool to find a sequence of slots that match the
|
||||||
|
* allocation constraints.
|
||||||
|
*
|
||||||
|
* Return: Index of the first allocated slot, or -1 on error.
|
||||||
|
*/
|
||||||
|
static int swiotlb_pool_find_slots(struct device *dev, struct io_tlb_pool *pool,
|
||||||
|
phys_addr_t orig_addr, size_t alloc_size,
|
||||||
|
unsigned int alloc_align_mask)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
int start = raw_smp_processor_id() & (pool->nareas - 1);
|
||||||
int start = raw_smp_processor_id() & (mem->nareas - 1);
|
|
||||||
int i = start, index;
|
int i = start, index;
|
||||||
|
|
||||||
do {
|
do {
|
||||||
index = swiotlb_do_find_slots(dev, i, orig_addr, alloc_size,
|
index = swiotlb_area_find_slots(dev, pool, i, orig_addr,
|
||||||
alloc_align_mask);
|
alloc_size, alloc_align_mask);
|
||||||
if (index >= 0)
|
if (index >= 0)
|
||||||
return index;
|
return index;
|
||||||
if (++i >= mem->nareas)
|
if (++i >= pool->nareas)
|
||||||
i = 0;
|
i = 0;
|
||||||
} while (i != start);
|
} while (i != start);
|
||||||
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_find_slots() - search for slots in the whole swiotlb
|
||||||
|
* @dev: Device which maps the buffer.
|
||||||
|
* @orig_addr: Original (non-bounced) IO buffer address.
|
||||||
|
* @alloc_size: Total requested size of the bounce buffer,
|
||||||
|
* including initial alignment padding.
|
||||||
|
* @alloc_align_mask: Required alignment of the allocated buffer.
|
||||||
|
* @retpool: Used memory pool, updated on return.
|
||||||
|
*
|
||||||
|
* Search through the whole software IO TLB to find a sequence of slots that
|
||||||
|
* match the allocation constraints.
|
||||||
|
*
|
||||||
|
* Return: Index of the first allocated slot, or -1 on error.
|
||||||
|
*/
|
||||||
|
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
|
||||||
|
size_t alloc_size, unsigned int alloc_align_mask,
|
||||||
|
struct io_tlb_pool **retpool)
|
||||||
|
{
|
||||||
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
unsigned long nslabs;
|
||||||
|
unsigned long flags;
|
||||||
|
u64 phys_limit;
|
||||||
|
int index;
|
||||||
|
|
||||||
|
rcu_read_lock();
|
||||||
|
list_for_each_entry_rcu(pool, &mem->pools, node) {
|
||||||
|
index = swiotlb_pool_find_slots(dev, pool, orig_addr,
|
||||||
|
alloc_size, alloc_align_mask);
|
||||||
|
if (index >= 0) {
|
||||||
|
rcu_read_unlock();
|
||||||
|
goto found;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rcu_read_unlock();
|
||||||
|
if (!mem->can_grow)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
schedule_work(&mem->dyn_alloc);
|
||||||
|
|
||||||
|
nslabs = nr_slots(alloc_size);
|
||||||
|
phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
|
||||||
|
pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
|
||||||
|
GFP_NOWAIT | __GFP_NOWARN);
|
||||||
|
if (!pool)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
index = swiotlb_pool_find_slots(dev, pool, orig_addr,
|
||||||
|
alloc_size, alloc_align_mask);
|
||||||
|
if (index < 0) {
|
||||||
|
swiotlb_dyn_free(&pool->rcu);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
pool->transient = true;
|
||||||
|
spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
|
||||||
|
list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);
|
||||||
|
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
|
||||||
|
|
||||||
|
found:
|
||||||
|
dev->dma_uses_io_tlb = true;
|
||||||
|
/* Pairs with smp_rmb() in is_swiotlb_buffer() */
|
||||||
|
smp_wmb();
|
||||||
|
|
||||||
|
*retpool = pool;
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
#else /* !CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
|
static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
|
||||||
|
size_t alloc_size, unsigned int alloc_align_mask,
|
||||||
|
struct io_tlb_pool **retpool)
|
||||||
|
{
|
||||||
|
*retpool = &dev->dma_io_tlb_mem->defpool;
|
||||||
|
return swiotlb_pool_find_slots(dev, *retpool,
|
||||||
|
orig_addr, alloc_size, alloc_align_mask);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_FS
|
#ifdef CONFIG_DEBUG_FS
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mem_used() - get number of used slots in an allocator
|
||||||
|
* @mem: Software IO TLB allocator.
|
||||||
|
*
|
||||||
|
* The result is accurate in this version of the function, because an atomic
|
||||||
|
* counter is available if CONFIG_DEBUG_FS is set.
|
||||||
|
*
|
||||||
|
* Return: Number of used slots.
|
||||||
|
*/
|
||||||
static unsigned long mem_used(struct io_tlb_mem *mem)
|
static unsigned long mem_used(struct io_tlb_mem *mem)
|
||||||
{
|
{
|
||||||
return atomic_long_read(&mem->total_used);
|
return atomic_long_read(&mem->total_used);
|
||||||
|
@ -750,16 +1191,50 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
|
||||||
|
|
||||||
#else /* !CONFIG_DEBUG_FS */
|
#else /* !CONFIG_DEBUG_FS */
|
||||||
|
|
||||||
static unsigned long mem_used(struct io_tlb_mem *mem)
|
/**
|
||||||
|
* mem_pool_used() - get number of used slots in a memory pool
|
||||||
|
* @pool: Software IO TLB memory pool.
|
||||||
|
*
|
||||||
|
* The result is not accurate, see mem_used().
|
||||||
|
*
|
||||||
|
* Return: Approximate number of used slots.
|
||||||
|
*/
|
||||||
|
static unsigned long mem_pool_used(struct io_tlb_pool *pool)
|
||||||
{
|
{
|
||||||
int i;
|
int i;
|
||||||
unsigned long used = 0;
|
unsigned long used = 0;
|
||||||
|
|
||||||
for (i = 0; i < mem->nareas; i++)
|
for (i = 0; i < pool->nareas; i++)
|
||||||
used += mem->areas[i].used;
|
used += pool->areas[i].used;
|
||||||
return used;
|
return used;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* mem_used() - get number of used slots in an allocator
|
||||||
|
* @mem: Software IO TLB allocator.
|
||||||
|
*
|
||||||
|
* The result is not accurate, because there is no locking of individual
|
||||||
|
* areas.
|
||||||
|
*
|
||||||
|
* Return: Approximate number of used slots.
|
||||||
|
*/
|
||||||
|
static unsigned long mem_used(struct io_tlb_mem *mem)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
unsigned long used = 0;
|
||||||
|
|
||||||
|
rcu_read_lock();
|
||||||
|
list_for_each_entry_rcu(pool, &mem->pools, node)
|
||||||
|
used += mem_pool_used(pool);
|
||||||
|
rcu_read_unlock();
|
||||||
|
|
||||||
|
return used;
|
||||||
|
#else
|
||||||
|
return mem_pool_used(&mem->defpool);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
#endif /* CONFIG_DEBUG_FS */
|
#endif /* CONFIG_DEBUG_FS */
|
||||||
|
|
||||||
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
||||||
|
@ -769,6 +1244,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
|
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
unsigned int i;
|
unsigned int i;
|
||||||
int index;
|
int index;
|
||||||
phys_addr_t tlb_addr;
|
phys_addr_t tlb_addr;
|
||||||
|
@ -789,7 +1265,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
||||||
}
|
}
|
||||||
|
|
||||||
index = swiotlb_find_slots(dev, orig_addr,
|
index = swiotlb_find_slots(dev, orig_addr,
|
||||||
alloc_size + offset, alloc_align_mask);
|
alloc_size + offset, alloc_align_mask, &pool);
|
||||||
if (index == -1) {
|
if (index == -1) {
|
||||||
if (!(attrs & DMA_ATTR_NO_WARN))
|
if (!(attrs & DMA_ATTR_NO_WARN))
|
||||||
dev_warn_ratelimited(dev,
|
dev_warn_ratelimited(dev,
|
||||||
|
@ -804,8 +1280,8 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
||||||
* needed.
|
* needed.
|
||||||
*/
|
*/
|
||||||
for (i = 0; i < nr_slots(alloc_size + offset); i++)
|
for (i = 0; i < nr_slots(alloc_size + offset); i++)
|
||||||
mem->slots[index + i].orig_addr = slot_addr(orig_addr, i);
|
pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
|
||||||
tlb_addr = slot_addr(mem->start, index) + offset;
|
tlb_addr = slot_addr(pool->start, index) + offset;
|
||||||
/*
|
/*
|
||||||
* When dir == DMA_FROM_DEVICE we could omit the copy from the orig
|
* When dir == DMA_FROM_DEVICE we could omit the copy from the orig
|
||||||
* to the tlb buffer, if we knew for sure the device will
|
* to the tlb buffer, if we knew for sure the device will
|
||||||
|
@ -819,7 +1295,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
|
||||||
|
|
||||||
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
|
static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
|
||||||
unsigned long flags;
|
unsigned long flags;
|
||||||
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
|
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
|
||||||
int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
|
int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
|
||||||
|
@ -863,9 +1339,44 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
|
||||||
area->used -= nslots;
|
area->used -= nslots;
|
||||||
spin_unlock_irqrestore(&area->lock, flags);
|
spin_unlock_irqrestore(&area->lock, flags);
|
||||||
|
|
||||||
dec_used(mem, nslots);
|
dec_used(dev->dma_io_tlb_mem, nslots);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
|
||||||
|
/**
|
||||||
|
* swiotlb_del_transient() - delete a transient memory pool
|
||||||
|
* @dev: Device which mapped the buffer.
|
||||||
|
* @tlb_addr: Physical address within a bounce buffer.
|
||||||
|
*
|
||||||
|
* Check whether the address belongs to a transient SWIOTLB memory pool.
|
||||||
|
* If yes, then delete the pool.
|
||||||
|
*
|
||||||
|
* Return: %true if @tlb_addr belonged to a transient pool that was released.
|
||||||
|
*/
|
||||||
|
static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
|
||||||
|
{
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
|
||||||
|
pool = swiotlb_find_pool(dev, tlb_addr);
|
||||||
|
if (!pool->transient)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
dec_used(dev->dma_io_tlb_mem, pool->nslabs);
|
||||||
|
swiotlb_del_pool(dev, pool);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
#else /* !CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
|
static inline bool swiotlb_del_transient(struct device *dev,
|
||||||
|
phys_addr_t tlb_addr)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif /* CONFIG_SWIOTLB_DYNAMIC */
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* tlb_addr is the physical address of the bounce buffer to unmap.
|
* tlb_addr is the physical address of the bounce buffer to unmap.
|
||||||
*/
|
*/
|
||||||
|
@ -880,6 +1391,8 @@ void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
|
||||||
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
|
(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
|
||||||
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
|
swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_FROM_DEVICE);
|
||||||
|
|
||||||
|
if (swiotlb_del_transient(dev, tlb_addr))
|
||||||
|
return;
|
||||||
swiotlb_release_slots(dev, tlb_addr);
|
swiotlb_release_slots(dev, tlb_addr);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -950,13 +1463,47 @@ size_t swiotlb_max_mapping_size(struct device *dev)
|
||||||
return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
|
return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* is_swiotlb_allocated() - check if the default software IO TLB is initialized
|
||||||
|
*/
|
||||||
|
bool is_swiotlb_allocated(void)
|
||||||
|
{
|
||||||
|
return io_tlb_default_mem.nslabs;
|
||||||
|
}
|
||||||
|
|
||||||
bool is_swiotlb_active(struct device *dev)
|
bool is_swiotlb_active(struct device *dev)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
|
|
||||||
return mem && mem->nslabs;
|
return mem && mem->nslabs;
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL_GPL(is_swiotlb_active);
|
|
||||||
|
/**
|
||||||
|
* default_swiotlb_base() - get the base address of the default SWIOTLB
|
||||||
|
*
|
||||||
|
* Get the lowest physical address used by the default software IO TLB pool.
|
||||||
|
*/
|
||||||
|
phys_addr_t default_swiotlb_base(void)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
io_tlb_default_mem.can_grow = false;
|
||||||
|
#endif
|
||||||
|
return io_tlb_default_mem.defpool.start;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* default_swiotlb_limit() - get the address limit of the default SWIOTLB
|
||||||
|
*
|
||||||
|
* Get the highest physical address used by the default software IO TLB pool.
|
||||||
|
*/
|
||||||
|
phys_addr_t default_swiotlb_limit(void)
|
||||||
|
{
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
return io_tlb_default_mem.phys_limit;
|
||||||
|
#else
|
||||||
|
return io_tlb_default_mem.defpool.end - 1;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
#ifdef CONFIG_DEBUG_FS
|
#ifdef CONFIG_DEBUG_FS
|
||||||
|
|
||||||
|
@ -1031,17 +1578,18 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
|
||||||
struct page *swiotlb_alloc(struct device *dev, size_t size)
|
struct page *swiotlb_alloc(struct device *dev, size_t size)
|
||||||
{
|
{
|
||||||
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
phys_addr_t tlb_addr;
|
phys_addr_t tlb_addr;
|
||||||
int index;
|
int index;
|
||||||
|
|
||||||
if (!mem)
|
if (!mem)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
index = swiotlb_find_slots(dev, 0, size, 0);
|
index = swiotlb_find_slots(dev, 0, size, 0, &pool);
|
||||||
if (index == -1)
|
if (index == -1)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
tlb_addr = slot_addr(mem->start, index);
|
tlb_addr = slot_addr(pool->start, index);
|
||||||
|
|
||||||
return pfn_to_page(PFN_DOWN(tlb_addr));
|
return pfn_to_page(PFN_DOWN(tlb_addr));
|
||||||
}
|
}
|
||||||
|
@ -1078,29 +1626,37 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
|
||||||
* to it.
|
* to it.
|
||||||
*/
|
*/
|
||||||
if (!mem) {
|
if (!mem) {
|
||||||
|
struct io_tlb_pool *pool;
|
||||||
|
|
||||||
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
|
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
|
||||||
if (!mem)
|
if (!mem)
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
|
pool = &mem->defpool;
|
||||||
|
|
||||||
mem->slots = kcalloc(nslabs, sizeof(*mem->slots), GFP_KERNEL);
|
pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);
|
||||||
if (!mem->slots) {
|
if (!pool->slots) {
|
||||||
kfree(mem);
|
kfree(mem);
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
mem->areas = kcalloc(nareas, sizeof(*mem->areas),
|
pool->areas = kcalloc(nareas, sizeof(*pool->areas),
|
||||||
GFP_KERNEL);
|
GFP_KERNEL);
|
||||||
if (!mem->areas) {
|
if (!pool->areas) {
|
||||||
kfree(mem->slots);
|
kfree(pool->slots);
|
||||||
kfree(mem);
|
kfree(mem);
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
|
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
|
||||||
rmem->size >> PAGE_SHIFT);
|
rmem->size >> PAGE_SHIFT);
|
||||||
swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, SWIOTLB_FORCE,
|
swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
|
||||||
false, nareas);
|
false, nareas);
|
||||||
|
mem->force_bounce = true;
|
||||||
mem->for_alloc = true;
|
mem->for_alloc = true;
|
||||||
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
||||||
|
spin_lock_init(&mem->lock);
|
||||||
|
#endif
|
||||||
|
add_mem_pool(mem, pool);
|
||||||
|
|
||||||
rmem->priv = mem;
|
rmem->priv = mem;
|
||||||
|
|
||||||
|
|
10
mm/cma.c
10
mm/cma.c
|
@ -267,6 +267,9 @@ int __init cma_declare_contiguous_nid(phys_addr_t base,
|
||||||
if (alignment && !is_power_of_2(alignment))
|
if (alignment && !is_power_of_2(alignment))
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
|
if (!IS_ENABLED(CONFIG_NUMA))
|
||||||
|
nid = NUMA_NO_NODE;
|
||||||
|
|
||||||
/* Sanitise input arguments. */
|
/* Sanitise input arguments. */
|
||||||
alignment = max_t(phys_addr_t, alignment, CMA_MIN_ALIGNMENT_BYTES);
|
alignment = max_t(phys_addr_t, alignment, CMA_MIN_ALIGNMENT_BYTES);
|
||||||
if (fixed && base & (alignment - 1)) {
|
if (fixed && base & (alignment - 1)) {
|
||||||
|
@ -372,14 +375,15 @@ int __init cma_declare_contiguous_nid(phys_addr_t base,
|
||||||
if (ret)
|
if (ret)
|
||||||
goto free_mem;
|
goto free_mem;
|
||||||
|
|
||||||
pr_info("Reserved %ld MiB at %pa\n", (unsigned long)size / SZ_1M,
|
pr_info("Reserved %ld MiB at %pa on node %d\n", (unsigned long)size / SZ_1M,
|
||||||
&base);
|
&base, nid);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
free_mem:
|
free_mem:
|
||||||
memblock_phys_free(base, size);
|
memblock_phys_free(base, size);
|
||||||
err:
|
err:
|
||||||
pr_err("Failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M);
|
pr_err("Failed to reserve %ld MiB on node %d\n", (unsigned long)size / SZ_1M,
|
||||||
|
nid);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -895,10 +895,9 @@ void __init setup_kmalloc_cache_index_table(void)
|
||||||
|
|
||||||
static unsigned int __kmalloc_minalign(void)
|
static unsigned int __kmalloc_minalign(void)
|
||||||
{
|
{
|
||||||
#ifdef CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC
|
if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&
|
||||||
if (io_tlb_default_mem.nslabs)
|
is_swiotlb_allocated())
|
||||||
return ARCH_KMALLOC_MINALIGN;
|
return ARCH_KMALLOC_MINALIGN;
|
||||||
#endif
|
|
||||||
return dma_get_cache_alignment();
|
return dma_get_cache_alignment();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue