mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

dma-mapping fixes for Linux 6.9 

- fix up swiotlb buffer padding even more (Petr Tesarik)
  - fix for partial dma_sync on swiotlb (Michael Kelley)
  - swiotlb debugfs fix (Dexuan Cui)
 -----BEGIN PGP SIGNATURE-----
 
 iQI/BAABCgApFiEEgdbnc3r/njty3Iq9D55TZVIEUYMFAmYbZMsLHGhjaEBsc3Qu
 ZGUACgkQD55TZVIEUYMzpw/9EA2Hfo0QRokyvYzHA9ks1JeMZMeV5fzJEb/NYb+V
 6cX2U0jRsdRqsZZXKl9c/oXwhiGLnQtuA9pIsMc18hqZooWxzoFWuc438UUihfvk
 6BVmciBl13gm0Addd5ajDVV7tPLWGmFY/rSNyD0yhPRlRy4PmVYMtaWpX+ec0jnW
 YUTSI+UTjknl7H5TXVDg/JmyHB7xTjGCAI75kMgxNK4PCX5oF5uqVpXZcF1YH5sj
 STQuUrLU/jE54G6X3sM44GJntDWq3RBRI2pTsgQC70MAdrrPm2g60BorccA8qx38
 Kz5CII8KuUvkIOtvncNmEzlGlfwflx91nF5ItQtJNobrd5GMSNtzaSLDR0LyfzLF
 w+awHYZyReEwE5oYcN8VQQR4mOZnqi3VRK+vTlTjcHXyXY3k0LdW+Jxzd6TkAeqr
 49ECkVHpw4QaQBKH6RWkED1Xd0CDAABx2HJ2QkHYEwwLL/KPvaehExY9LjDpQut4
 qJaAqLPKzxvAJdFjvH4HlJDhpw1BH0Pj3H6i4y0UgEzOKReMGun1FoqCseSJhhGc
 laFtFrIpooLqOD/h0e21XpVfEcRDkLwEjAtpkZXrfBZenZF8rFDm+vTOpBiyAK2a
 2KiU0+z/f1Flx5E0gLtxn5yPsWo2cuIjKfsVSewIwjrh6p/XFBF+iQi4dwISFxKz
 JrE=
 =U10z
 -----END PGP SIGNATURE-----

Merge tag 'dma-maping-6.9-2024-04-14' of git://git.infradead.org/users/hch/dma-mapping

Pull dma-mapping fixes from Christoph Hellwig:

 - fix up swiotlb buffer padding even more (Petr Tesarik)

 - fix for partial dma_sync on swiotlb (Michael Kelley)

 - swiotlb debugfs fix (Dexuan Cui)

* tag 'dma-maping-6.9-2024-04-14' of git://git.infradead.org/users/hch/dma-mapping:
  swiotlb: do not set total_used to 0 in swiotlb_create_debugfs_files()
  swiotlb: fix swiotlb_bounce() to do partial sync's correctly
  swiotlb: extend buffer pre-padding to alloc_align_mask if necessary
This commit is contained in:
Linus Torvalds 2024-04-14 10:02:40 -07:00
parent 7efd0a7403 a1255ccab8
commit ddd7ad5cf1
1 changed files with 58 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
kernel/dma/swiotlb.c
View File

@ -69,11 +69,14 @@
* @alloc_size: Size of the allocated buffer.
* @list: The free list describing the number of free entries available
* from each index.
* @pad_slots: Number of preceding padding slots. Valid only in the first
* allocated non-padding slot.
*/
struct io_tlb_slot {
phys_addr_t orig_addr;
size_t alloc_size;
unsigned int list;
unsigned short list;
unsigned short pad_slots;
};
static bool swiotlb_force_bounce;
@ -287,6 +290,7 @@ static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
mem->nslabs - i);
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
mem->slots[i].pad_slots = 0;
}
memset(vaddr, 0, bytes);
@ -821,12 +825,30 @@ void swiotlb_dev_init(struct device *dev)
#endif
}
/*
* Return the offset into a iotlb slot required to keep the device happy.
/**
* swiotlb_align_offset() - Get required offset into an IO TLB allocation.
* @dev: Owning device.
* @align_mask: Allocation alignment mask.
* @addr: DMA address.
*
* Return the minimum offset from the start of an IO TLB allocation which is
* required for a given buffer address and allocation alignment to keep the
* device happy.
*
* First, the address bits covered by min_align_mask must be identical in the
* original address and the bounce buffer address. High bits are preserved by
* choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
* padding bytes before the bounce buffer.
*
* Second, @align_mask specifies which bits of the first allocated slot must
* be zero. This may require allocating additional padding slots, and then the
* offset (in bytes) from the first such padding slot is returned.
*/
static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
static unsigned int swiotlb_align_offset(struct device *dev,
unsigned int align_mask, u64 addr)
{
return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1);
return addr & dma_get_min_align_mask(dev) &
(align_mask | (IO_TLB_SIZE - 1));
}
/*
@ -841,27 +863,23 @@ static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size
size_t alloc_size = mem->slots[index].alloc_size;
unsigned long pfn = PFN_DOWN(orig_addr);
unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
unsigned int tlb_offset, orig_addr_offset;
int tlb_offset;
if (orig_addr == INVALID_PHYS_ADDR)
return;
tlb_offset = tlb_addr & (IO_TLB_SIZE - 1);
orig_addr_offset = swiotlb_align_offset(dev, orig_addr);
if (tlb_offset < orig_addr_offset) {
dev_WARN_ONCE(dev, 1,
"Access before mapping start detected. orig offset %u, requested offset %u.\n",
orig_addr_offset, tlb_offset);
return;
}
tlb_offset -= orig_addr_offset;
if (tlb_offset > alloc_size) {
dev_WARN_ONCE(dev, 1,
"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu+%u.\n",
alloc_size, size, tlb_offset);
return;
}
/*
* It's valid for tlb_offset to be negative. This can happen when the
* "offset" returned by swiotlb_align_offset() is non-zero, and the
* tlb_addr is pointing within the first "offset" bytes of the second
* or subsequent slots of the allocated swiotlb area. While it's not
* valid for tlb_addr to be pointing within the first "offset" bytes
* of the first slot, there's no way to check for such an error since
* this function can't distinguish the first slot from the second and
* subsequent slots.
*/
tlb_offset = (tlb_addr & (IO_TLB_SIZE - 1)) -
swiotlb_align_offset(dev, 0, orig_addr);
orig_addr += tlb_offset;
alloc_size -= tlb_offset;
@ -1005,7 +1023,7 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
unsigned int nslots = nr_slots(alloc_size), stride;
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
unsigned int offset = swiotlb_align_offset(dev, 0, orig_addr);
unsigned int index, slots_checked, count = 0, i;
unsigned long flags;
unsigned int slot_base;
@ -1328,11 +1346,12 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
unsigned long attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset = swiotlb_align_offset(dev, orig_addr);
unsigned int offset;
struct io_tlb_pool *pool;
unsigned int i;
int index;
phys_addr_t tlb_addr;
unsigned short pad_slots;
if (!mem || !mem->nslabs) {
dev_warn_ratelimited(dev,
@ -1349,6 +1368,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
return (phys_addr_t)DMA_MAPPING_ERROR;
}
offset = swiotlb_align_offset(dev, alloc_align_mask, orig_addr);
index = swiotlb_find_slots(dev, orig_addr,
alloc_size + offset, alloc_align_mask, &pool);
if (index == -1) {
@ -1364,6 +1384,10 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
* This is needed when we sync the memory. Then we sync the buffer if
* needed.
*/
pad_slots = offset >> IO_TLB_SHIFT;
offset &= (IO_TLB_SIZE - 1);
index += pad_slots;
pool->slots[index].pad_slots = pad_slots;
for (i = 0; i < nr_slots(alloc_size + offset); i++)
pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);
tlb_addr = slot_addr(pool->start, index) + offset;
@ -1384,13 +1408,17 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
{
struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr);
unsigned long flags;
unsigned int offset = swiotlb_align_offset(dev, tlb_addr);
int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
int nslots = nr_slots(mem->slots[index].alloc_size + offset);
int aindex = index / mem->area_nslabs;
struct io_tlb_area *area = &mem->areas[aindex];
unsigned int offset = swiotlb_align_offset(dev, 0, tlb_addr);
int index, nslots, aindex;
struct io_tlb_area *area;
int count, i;
index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
index -= mem->slots[index].pad_slots;
nslots = nr_slots(mem->slots[index].alloc_size + offset);
aindex = index / mem->area_nslabs;
area = &mem->areas[aindex];
/*
* Return the buffer to the free list by setting the corresponding
* entries to indicate the number of contiguous entries available.
@ -1413,6 +1441,7 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
mem->slots[i].list = ++count;
mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
mem->slots[i].alloc_size = 0;
mem->slots[i].pad_slots = 0;
}
/*
@ -1647,9 +1676,6 @@ DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
const char *dirname)
{
atomic_long_set(&mem->total_used, 0);
atomic_long_set(&mem->used_hiwater, 0);
mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
if (!mem->nslabs)
return;
@ -1660,7 +1686,6 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
&fops_io_tlb_hiwater);
#ifdef CONFIG_SWIOTLB_DYNAMIC
atomic_long_set(&mem->transient_nslabs, 0);
debugfs_create_file("io_tlb_transient_nslabs", 0400, mem->debugfs,
mem, &fops_io_tlb_transient_used);
#endif