mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-28 20:02:17 +00:00
![Petr Tesarik](/assets/img/avatar_default.png)
When CONFIG_SWIOTLB_DYNAMIC=y, devices which do not use the software IO TLB can avoid swiotlb lookup. A flag is added by commit1395706a14
("swiotlb: search the software IO TLB only if the device makes use of it"), the flag is correctly set, but it is then never checked. Add the actual check here. Note that this code is an alternative to the default pool check, not an additional check, because: 1. swiotlb_find_pool() also searches the default pool; 2. if dma_uses_io_tlb is false, the default swiotlb pool is not used. Tested in a KVM guest against a QEMU RAM-backed SATA disk over virtio and *not* using software IO TLB, this patch increases IOPS by approx 2% for 4-way parallel I/O. The write memory barrier in swiotlb_dyn_alloc() is not needed, because a newly allocated pool must always be observed by swiotlb_find_slots() before an address from that pool is passed to is_swiotlb_buffer(). Correctness was verified using the following litmus test: C swiotlb-new-pool (* * Result: Never * * Check that a newly allocated pool is always visible when the * corresponding swiotlb buffer is visible. *) { mem_pools = default; } P0(int **mem_pools, int *pool) { /* add_mem_pool() */ WRITE_ONCE(*pool, 999); rcu_assign_pointer(*mem_pools, pool); } P1(int **mem_pools, int *flag, int *buf) { /* swiotlb_find_slots() */ int *r0; int r1; rcu_read_lock(); r0 = READ_ONCE(*mem_pools); r1 = READ_ONCE(*r0); rcu_read_unlock(); if (r1) { WRITE_ONCE(*flag, 1); smp_mb(); } /* device driver (presumed) */ WRITE_ONCE(*buf, r1); } P2(int **mem_pools, int *flag, int *buf) { /* device driver (presumed) */ int r0 = READ_ONCE(*buf); /* is_swiotlb_buffer() */ int r1; int *r2; int r3; smp_rmb(); r1 = READ_ONCE(*flag); if (r1) { /* swiotlb_find_pool() */ rcu_read_lock(); r2 = READ_ONCE(*mem_pools); r3 = READ_ONCE(*r2); rcu_read_unlock(); } } exists (2:r0<>0 /\ 2:r3=0) (* Not found. *) Fixes:1395706a14
("swiotlb: search the software IO TLB only if the device makes use of it") Reported-by: Jonathan Corbet <corbet@lwn.net> Closes: https://lore.kernel.org/linux-iommu/87a5uz3ob8.fsf@meer.lwn.net/ Signed-off-by: Petr Tesarik <petr@tesarici.cz> Reviewed-by: Catalin Marinas <catalin.marinas@arm.com> Signed-off-by: Christoph Hellwig <hch@lst.de>
286 lines
8.1 KiB
C
286 lines
8.1 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __LINUX_SWIOTLB_H
|
|
#define __LINUX_SWIOTLB_H
|
|
|
|
#include <linux/device.h>
|
|
#include <linux/dma-direction.h>
|
|
#include <linux/init.h>
|
|
#include <linux/types.h>
|
|
#include <linux/limits.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/workqueue.h>
|
|
|
|
struct device;
|
|
struct page;
|
|
struct scatterlist;
|
|
|
|
#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */
|
|
#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */
|
|
#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */
|
|
|
|
/*
|
|
* Maximum allowable number of contiguous slabs to map,
|
|
* must be a power of 2. What is the appropriate value ?
|
|
* The complexity of {map,unmap}_single is linearly dependent on this value.
|
|
*/
|
|
#define IO_TLB_SEGSIZE 128
|
|
|
|
/*
|
|
* log of the size of each IO TLB slab. The number of slabs is command line
|
|
* controllable.
|
|
*/
|
|
#define IO_TLB_SHIFT 11
|
|
#define IO_TLB_SIZE (1 << IO_TLB_SHIFT)
|
|
|
|
/* default to 64MB */
|
|
#define IO_TLB_DEFAULT_SIZE (64UL<<20)
|
|
|
|
unsigned long swiotlb_size_or_default(void);
|
|
void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
|
|
int (*remap)(void *tlb, unsigned long nslabs));
|
|
int swiotlb_init_late(size_t size, gfp_t gfp_mask,
|
|
int (*remap)(void *tlb, unsigned long nslabs));
|
|
extern void __init swiotlb_update_mem_attributes(void);
|
|
|
|
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
|
|
size_t mapping_size, size_t alloc_size,
|
|
unsigned int alloc_aligned_mask, enum dma_data_direction dir,
|
|
unsigned long attrs);
|
|
|
|
extern void swiotlb_tbl_unmap_single(struct device *hwdev,
|
|
phys_addr_t tlb_addr,
|
|
size_t mapping_size,
|
|
enum dma_data_direction dir,
|
|
unsigned long attrs);
|
|
|
|
void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
|
|
size_t size, enum dma_data_direction dir);
|
|
void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
|
|
size_t size, enum dma_data_direction dir);
|
|
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
|
|
size_t size, enum dma_data_direction dir, unsigned long attrs);
|
|
|
|
#ifdef CONFIG_SWIOTLB
|
|
|
|
/**
|
|
* struct io_tlb_pool - IO TLB memory pool descriptor
|
|
* @start: The start address of the swiotlb memory pool. Used to do a quick
|
|
* range check to see if the memory was in fact allocated by this
|
|
* API.
|
|
* @end: The end address of the swiotlb memory pool. Used to do a quick
|
|
* range check to see if the memory was in fact allocated by this
|
|
* API.
|
|
* @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool
|
|
* may be remapped in the memory encrypted case and store virtual
|
|
* address for bounce buffer operation.
|
|
* @nslabs: The number of IO TLB slots between @start and @end. For the
|
|
* default swiotlb, this can be adjusted with a boot parameter,
|
|
* see setup_io_tlb_npages().
|
|
* @late_alloc: %true if allocated using the page allocator.
|
|
* @nareas: Number of areas in the pool.
|
|
* @area_nslabs: Number of slots in each area.
|
|
* @areas: Array of memory area descriptors.
|
|
* @slots: Array of slot descriptors.
|
|
* @node: Member of the IO TLB memory pool list.
|
|
* @rcu: RCU head for swiotlb_dyn_free().
|
|
* @transient: %true if transient memory pool.
|
|
*/
|
|
struct io_tlb_pool {
|
|
phys_addr_t start;
|
|
phys_addr_t end;
|
|
void *vaddr;
|
|
unsigned long nslabs;
|
|
bool late_alloc;
|
|
unsigned int nareas;
|
|
unsigned int area_nslabs;
|
|
struct io_tlb_area *areas;
|
|
struct io_tlb_slot *slots;
|
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
|
struct list_head node;
|
|
struct rcu_head rcu;
|
|
bool transient;
|
|
#endif
|
|
};
|
|
|
|
/**
|
|
* struct io_tlb_mem - Software IO TLB allocator
|
|
* @defpool: Default (initial) IO TLB memory pool descriptor.
|
|
* @pool: IO TLB memory pool descriptor (if not dynamic).
|
|
* @nslabs: Total number of IO TLB slabs in all pools.
|
|
* @debugfs: The dentry to debugfs.
|
|
* @force_bounce: %true if swiotlb bouncing is forced
|
|
* @for_alloc: %true if the pool is used for memory allocation
|
|
* @can_grow: %true if more pools can be allocated dynamically.
|
|
* @phys_limit: Maximum allowed physical address.
|
|
* @lock: Lock to synchronize changes to the list.
|
|
* @pools: List of IO TLB memory pool descriptors (if dynamic).
|
|
* @dyn_alloc: Dynamic IO TLB pool allocation work.
|
|
* @total_used: The total number of slots in the pool that are currently used
|
|
* across all areas. Used only for calculating used_hiwater in
|
|
* debugfs.
|
|
* @used_hiwater: The high water mark for total_used. Used only for reporting
|
|
* in debugfs.
|
|
*/
|
|
struct io_tlb_mem {
|
|
struct io_tlb_pool defpool;
|
|
unsigned long nslabs;
|
|
struct dentry *debugfs;
|
|
bool force_bounce;
|
|
bool for_alloc;
|
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
|
bool can_grow;
|
|
u64 phys_limit;
|
|
spinlock_t lock;
|
|
struct list_head pools;
|
|
struct work_struct dyn_alloc;
|
|
#endif
|
|
#ifdef CONFIG_DEBUG_FS
|
|
atomic_long_t total_used;
|
|
atomic_long_t used_hiwater;
|
|
#endif
|
|
};
|
|
|
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
|
|
|
struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
|
|
|
|
#else
|
|
|
|
static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
|
|
phys_addr_t paddr)
|
|
{
|
|
return &dev->dma_io_tlb_mem->defpool;
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* is_swiotlb_buffer() - check if a physical address belongs to a swiotlb
|
|
* @dev: Device which has mapped the buffer.
|
|
* @paddr: Physical address within the DMA buffer.
|
|
*
|
|
* Check if @paddr points into a bounce buffer.
|
|
*
|
|
* Return:
|
|
* * %true if @paddr points into a bounce buffer
|
|
* * %false otherwise
|
|
*/
|
|
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
|
|
|
if (!mem)
|
|
return false;
|
|
|
|
#ifdef CONFIG_SWIOTLB_DYNAMIC
|
|
/*
|
|
* All SWIOTLB buffer addresses must have been returned by
|
|
* swiotlb_tbl_map_single() and passed to a device driver.
|
|
* If a SWIOTLB address is checked on another CPU, then it was
|
|
* presumably loaded by the device driver from an unspecified private
|
|
* data structure. Make sure that this load is ordered before reading
|
|
* dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().
|
|
*
|
|
* This barrier pairs with smp_mb() in swiotlb_find_slots().
|
|
*/
|
|
smp_rmb();
|
|
return READ_ONCE(dev->dma_uses_io_tlb) &&
|
|
swiotlb_find_pool(dev, paddr);
|
|
#else
|
|
return paddr >= mem->defpool.start && paddr < mem->defpool.end;
|
|
#endif
|
|
}
|
|
|
|
static inline bool is_swiotlb_force_bounce(struct device *dev)
|
|
{
|
|
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
|
|
|
|
return mem && mem->force_bounce;
|
|
}
|
|
|
|
void swiotlb_init(bool addressing_limited, unsigned int flags);
|
|
void __init swiotlb_exit(void);
|
|
void swiotlb_dev_init(struct device *dev);
|
|
size_t swiotlb_max_mapping_size(struct device *dev);
|
|
bool is_swiotlb_allocated(void);
|
|
bool is_swiotlb_active(struct device *dev);
|
|
void __init swiotlb_adjust_size(unsigned long size);
|
|
phys_addr_t default_swiotlb_base(void);
|
|
phys_addr_t default_swiotlb_limit(void);
|
|
#else
|
|
static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
|
|
{
|
|
}
|
|
|
|
static inline void swiotlb_dev_init(struct device *dev)
|
|
{
|
|
}
|
|
|
|
static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
|
|
{
|
|
return false;
|
|
}
|
|
static inline bool is_swiotlb_force_bounce(struct device *dev)
|
|
{
|
|
return false;
|
|
}
|
|
static inline void swiotlb_exit(void)
|
|
{
|
|
}
|
|
static inline size_t swiotlb_max_mapping_size(struct device *dev)
|
|
{
|
|
return SIZE_MAX;
|
|
}
|
|
|
|
static inline bool is_swiotlb_allocated(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline bool is_swiotlb_active(struct device *dev)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline void swiotlb_adjust_size(unsigned long size)
|
|
{
|
|
}
|
|
|
|
static inline phys_addr_t default_swiotlb_base(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline phys_addr_t default_swiotlb_limit(void)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_SWIOTLB */
|
|
|
|
extern void swiotlb_print_info(void);
|
|
|
|
#ifdef CONFIG_DMA_RESTRICTED_POOL
|
|
struct page *swiotlb_alloc(struct device *dev, size_t size);
|
|
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
|
|
|
|
static inline bool is_swiotlb_for_alloc(struct device *dev)
|
|
{
|
|
return dev->dma_io_tlb_mem->for_alloc;
|
|
}
|
|
#else
|
|
static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
|
|
{
|
|
return NULL;
|
|
}
|
|
static inline bool swiotlb_free(struct device *dev, struct page *page,
|
|
size_t size)
|
|
{
|
|
return false;
|
|
}
|
|
static inline bool is_swiotlb_for_alloc(struct device *dev)
|
|
{
|
|
return false;
|
|
}
|
|
#endif /* CONFIG_DMA_RESTRICTED_POOL */
|
|
|
|
#endif /* __LINUX_SWIOTLB_H */
|