2019-05-29 23:57:35 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2007-10-21 23:41:48 +00:00
|
|
|
/*
|
2009-07-01 17:49:06 +00:00
|
|
|
* Copyright © 2006-2009, Intel Corporation.
|
2007-10-21 23:41:48 +00:00
|
|
|
*
|
2008-02-23 23:23:35 +00:00
|
|
|
* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
|
2007-10-21 23:41:48 +00:00
|
|
|
*/
|
|
|
|
|
2008-09-09 15:37:29 +00:00
|
|
|
#include <linux/iova.h>
|
2015-07-13 11:31:30 +00:00
|
|
|
#include <linux/module.h>
|
2015-01-12 17:51:14 +00:00
|
|
|
#include <linux/slab.h>
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/bitops.h>
|
2017-06-27 16:16:47 +00:00
|
|
|
#include <linux/cpu.h>
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
|
2017-09-21 15:52:46 +00:00
|
|
|
/* The anchor node sits above the top of the usable address space */
|
|
|
|
#define IOVA_ANCHOR ~0UL
|
|
|
|
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
static bool iova_rcache_insert(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn,
|
|
|
|
unsigned long size);
|
|
|
|
static unsigned long iova_rcache_get(struct iova_domain *iovad,
|
|
|
|
unsigned long size,
|
|
|
|
unsigned long limit_pfn);
|
|
|
|
static void init_iova_rcaches(struct iova_domain *iovad);
|
2021-03-25 12:30:00 +00:00
|
|
|
static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
static void free_iova_rcaches(struct iova_domain *iovad);
|
2021-03-25 12:29:58 +00:00
|
|
|
|
|
|
|
static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node)
|
|
|
|
{
|
|
|
|
struct iova_domain *iovad;
|
|
|
|
|
|
|
|
iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead);
|
|
|
|
|
|
|
|
free_cpu_cached_iovas(cpu, iovad);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-09-30 07:44:24 +00:00
|
|
|
static void free_global_cached_iovas(struct iova_domain *iovad);
|
2015-01-12 17:51:14 +00:00
|
|
|
|
2021-03-05 16:35:22 +00:00
|
|
|
static struct iova *to_iova(struct rb_node *node)
|
|
|
|
{
|
|
|
|
return rb_entry(node, struct iova, node);
|
|
|
|
}
|
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
void
|
2015-01-12 17:51:16 +00:00
|
|
|
init_iova_domain(struct iova_domain *iovad, unsigned long granule,
|
2017-09-21 15:52:45 +00:00
|
|
|
unsigned long start_pfn)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
2015-01-12 17:51:16 +00:00
|
|
|
/*
|
|
|
|
* IOVA granularity will normally be equal to the smallest
|
|
|
|
* supported IOMMU page size; both *must* be capable of
|
|
|
|
* representing individual CPU pages exactly.
|
|
|
|
*/
|
|
|
|
BUG_ON((granule > PAGE_SIZE) || !is_power_of_2(granule));
|
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
spin_lock_init(&iovad->iova_rbtree_lock);
|
|
|
|
iovad->rbroot = RB_ROOT;
|
2017-09-21 15:52:47 +00:00
|
|
|
iovad->cached_node = &iovad->anchor.node;
|
|
|
|
iovad->cached32_node = &iovad->anchor.node;
|
2015-01-12 17:51:16 +00:00
|
|
|
iovad->granule = granule;
|
2015-01-12 17:51:15 +00:00
|
|
|
iovad->start_pfn = start_pfn;
|
2017-09-21 15:52:45 +00:00
|
|
|
iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad));
|
2018-09-05 04:27:36 +00:00
|
|
|
iovad->max32_alloc_size = iovad->dma_32bit_pfn;
|
2017-09-21 15:52:46 +00:00
|
|
|
iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
|
|
|
|
rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
|
|
|
|
rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
|
2021-03-25 12:29:58 +00:00
|
|
|
cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
init_iova_rcaches(iovad);
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(init_iova_domain);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
static struct rb_node *
|
2017-09-21 15:52:47 +00:00
|
|
|
__get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
2017-09-21 15:52:47 +00:00
|
|
|
if (limit_pfn <= iovad->dma_32bit_pfn)
|
|
|
|
return iovad->cached32_node;
|
2017-09-21 15:52:44 +00:00
|
|
|
|
2017-09-21 15:52:47 +00:00
|
|
|
return iovad->cached_node;
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2017-09-21 15:52:44 +00:00
|
|
|
__cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
2017-09-21 15:52:44 +00:00
|
|
|
if (new->pfn_hi < iovad->dma_32bit_pfn)
|
|
|
|
iovad->cached32_node = &new->node;
|
|
|
|
else
|
|
|
|
iovad->cached_node = &new->node;
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
__cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
|
|
|
|
{
|
|
|
|
struct iova *cached_iova;
|
|
|
|
|
2021-03-05 16:35:22 +00:00
|
|
|
cached_iova = to_iova(iovad->cached32_node);
|
2019-07-20 18:08:48 +00:00
|
|
|
if (free == cached_iova ||
|
|
|
|
(free->pfn_hi < iovad->dma_32bit_pfn &&
|
2022-03-03 14:40:08 +00:00
|
|
|
free->pfn_lo >= cached_iova->pfn_lo))
|
2017-09-21 15:52:44 +00:00
|
|
|
iovad->cached32_node = rb_next(&free->node);
|
2022-03-03 14:40:08 +00:00
|
|
|
|
|
|
|
if (free->pfn_lo < iovad->dma_32bit_pfn)
|
2018-09-05 04:27:36 +00:00
|
|
|
iovad->max32_alloc_size = iovad->dma_32bit_pfn;
|
2017-09-21 15:52:44 +00:00
|
|
|
|
2021-03-05 16:35:22 +00:00
|
|
|
cached_iova = to_iova(iovad->cached_node);
|
2017-09-21 15:52:47 +00:00
|
|
|
if (free->pfn_lo >= cached_iova->pfn_lo)
|
2017-09-21 15:52:44 +00:00
|
|
|
iovad->cached_node = rb_next(&free->node);
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
|
|
|
|
2021-03-05 16:35:23 +00:00
|
|
|
static struct rb_node *iova_find_limit(struct iova_domain *iovad, unsigned long limit_pfn)
|
|
|
|
{
|
|
|
|
struct rb_node *node, *next;
|
|
|
|
/*
|
|
|
|
* Ideally what we'd like to judge here is whether limit_pfn is close
|
|
|
|
* enough to the highest-allocated IOVA that starting the allocation
|
|
|
|
* walk from the anchor node will be quicker than this initial work to
|
|
|
|
* find an exact starting point (especially if that ends up being the
|
|
|
|
* anchor node anyway). This is an incredibly crude approximation which
|
|
|
|
* only really helps the most likely case, but is at least trivially easy.
|
|
|
|
*/
|
|
|
|
if (limit_pfn > iovad->dma_32bit_pfn)
|
|
|
|
return &iovad->anchor.node;
|
|
|
|
|
|
|
|
node = iovad->rbroot.rb_node;
|
|
|
|
while (to_iova(node)->pfn_hi < limit_pfn)
|
|
|
|
node = node->rb_right;
|
|
|
|
|
|
|
|
search_left:
|
|
|
|
while (node->rb_left && to_iova(node->rb_left)->pfn_lo >= limit_pfn)
|
|
|
|
node = node->rb_left;
|
|
|
|
|
|
|
|
if (!node->rb_left)
|
|
|
|
return node;
|
|
|
|
|
|
|
|
next = node->rb_left;
|
|
|
|
while (next->rb_right) {
|
|
|
|
next = next->rb_right;
|
|
|
|
if (to_iova(next)->pfn_lo >= limit_pfn) {
|
|
|
|
node = next;
|
|
|
|
goto search_left;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return node;
|
|
|
|
}
|
|
|
|
|
2017-02-24 11:13:37 +00:00
|
|
|
/* Insert the iova into domain rbtree by holding writer lock */
|
|
|
|
static void
|
|
|
|
iova_insert_rbtree(struct rb_root *root, struct iova *iova,
|
|
|
|
struct rb_node *start)
|
|
|
|
{
|
|
|
|
struct rb_node **new, *parent = NULL;
|
|
|
|
|
|
|
|
new = (start) ? &start : &(root->rb_node);
|
|
|
|
/* Figure out where to put new node */
|
|
|
|
while (*new) {
|
2021-03-05 16:35:22 +00:00
|
|
|
struct iova *this = to_iova(*new);
|
2017-02-24 11:13:37 +00:00
|
|
|
|
|
|
|
parent = *new;
|
|
|
|
|
|
|
|
if (iova->pfn_lo < this->pfn_lo)
|
|
|
|
new = &((*new)->rb_left);
|
|
|
|
else if (iova->pfn_lo > this->pfn_lo)
|
|
|
|
new = &((*new)->rb_right);
|
|
|
|
else {
|
|
|
|
WARN_ON(1); /* this should not happen */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
/* Add new node and rebalance tree. */
|
|
|
|
rb_link_node(&iova->node, parent, new);
|
|
|
|
rb_insert_color(&iova->node, root);
|
|
|
|
}
|
|
|
|
|
2008-03-04 23:22:04 +00:00
|
|
|
static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
|
|
|
|
unsigned long size, unsigned long limit_pfn,
|
|
|
|
struct iova *new, bool size_aligned)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
2017-09-21 15:52:47 +00:00
|
|
|
struct rb_node *curr, *prev;
|
|
|
|
struct iova *curr_iova;
|
2007-10-21 23:41:48 +00:00
|
|
|
unsigned long flags;
|
2020-09-30 07:44:23 +00:00
|
|
|
unsigned long new_pfn, retry_pfn;
|
2017-09-21 15:52:43 +00:00
|
|
|
unsigned long align_mask = ~0UL;
|
2020-09-30 07:44:23 +00:00
|
|
|
unsigned long high_pfn = limit_pfn, low_pfn = iovad->start_pfn;
|
2017-09-21 15:52:43 +00:00
|
|
|
|
|
|
|
if (size_aligned)
|
|
|
|
align_mask <<= fls_long(size - 1);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
/* Walk the tree backwards */
|
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
2018-09-05 04:27:36 +00:00
|
|
|
if (limit_pfn <= iovad->dma_32bit_pfn &&
|
|
|
|
size >= iovad->max32_alloc_size)
|
|
|
|
goto iova32_full;
|
|
|
|
|
2017-09-21 15:52:47 +00:00
|
|
|
curr = __get_cached_rbnode(iovad, limit_pfn);
|
2021-03-05 16:35:22 +00:00
|
|
|
curr_iova = to_iova(curr);
|
2020-09-30 07:44:23 +00:00
|
|
|
retry_pfn = curr_iova->pfn_hi + 1;
|
|
|
|
|
|
|
|
retry:
|
2017-09-21 15:52:47 +00:00
|
|
|
do {
|
2020-09-30 07:44:23 +00:00
|
|
|
high_pfn = min(high_pfn, curr_iova->pfn_lo);
|
|
|
|
new_pfn = (high_pfn - size) & align_mask;
|
2008-03-04 23:22:04 +00:00
|
|
|
prev = curr;
|
2007-10-21 23:41:48 +00:00
|
|
|
curr = rb_prev(curr);
|
2021-03-05 16:35:22 +00:00
|
|
|
curr_iova = to_iova(curr);
|
2020-09-30 07:44:23 +00:00
|
|
|
} while (curr && new_pfn <= curr_iova->pfn_hi && new_pfn >= low_pfn);
|
|
|
|
|
|
|
|
if (high_pfn < size || new_pfn < low_pfn) {
|
|
|
|
if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) {
|
|
|
|
high_pfn = limit_pfn;
|
|
|
|
low_pfn = retry_pfn;
|
2021-03-05 16:35:23 +00:00
|
|
|
curr = iova_find_limit(iovad, limit_pfn);
|
2021-03-05 16:35:22 +00:00
|
|
|
curr_iova = to_iova(curr);
|
2020-09-30 07:44:23 +00:00
|
|
|
goto retry;
|
|
|
|
}
|
2019-03-20 18:57:23 +00:00
|
|
|
iovad->max32_alloc_size = size;
|
2018-09-05 04:27:36 +00:00
|
|
|
goto iova32_full;
|
2019-03-20 18:57:23 +00:00
|
|
|
}
|
2007-10-21 23:41:58 +00:00
|
|
|
|
|
|
|
/* pfn_lo will point to size aligned address if size_aligned is set */
|
2017-09-21 15:52:43 +00:00
|
|
|
new->pfn_lo = new_pfn;
|
2007-10-21 23:41:58 +00:00
|
|
|
new->pfn_hi = new->pfn_lo + size - 1;
|
2007-10-21 23:41:48 +00:00
|
|
|
|
2017-02-24 11:13:37 +00:00
|
|
|
/* If we have 'prev', it's a valid place to start the insertion. */
|
|
|
|
iova_insert_rbtree(&iovad->rbroot, new, prev);
|
2017-09-21 15:52:44 +00:00
|
|
|
__cached_rbnode_insert_update(iovad, new);
|
2008-03-04 23:22:04 +00:00
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
|
|
|
return 0;
|
2018-09-05 04:27:36 +00:00
|
|
|
|
|
|
|
iova32_full:
|
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
|
|
|
return -ENOMEM;
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
|
|
|
|
2015-07-13 11:31:28 +00:00
|
|
|
static struct kmem_cache *iova_cache;
|
|
|
|
static unsigned int iova_cache_users;
|
|
|
|
static DEFINE_MUTEX(iova_cache_mutex);
|
|
|
|
|
2020-12-03 18:34:51 +00:00
|
|
|
static struct iova *alloc_iova_mem(void)
|
2015-07-13 11:31:28 +00:00
|
|
|
{
|
iommu/iova: Silence warnings under memory pressure
When running heavy memory pressure workloads, this 5+ old system is
throwing endless warnings below because disk IO is too slow to recover
from swapping. Since the volume from alloc_iova_fast() could be large,
once it calls printk(), it will trigger disk IO (writing to the log
files) and pending softirqs which could cause an infinite loop and make
no progress for days by the ongoimng memory reclaim. This is the counter
part for Intel where the AMD part has already been merged. See the
commit 3d708895325b ("iommu/amd: Silence warnings under memory
pressure"). Since the allocation failure will be reported in
intel_alloc_iova(), so just call dev_err_once() there because even the
"ratelimited" is too much, and silence the one in alloc_iova_mem() to
avoid the expensive warn_alloc().
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
slab_out_of_memory: 66 callbacks suppressed
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
cache: iommu_iova, object size: 40, buffer size: 448, default order:
0, min order: 0
node 0: slabs: 1822, objs: 16398, free: 0
node 1: slabs: 2051, objs: 18459, free: 31
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
cache: iommu_iova, object size: 40, buffer size: 448, default order:
0, min order: 0
node 0: slabs: 1822, objs: 16398, free: 0
node 1: slabs: 2051, objs: 18459, free: 31
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
cache: iommu_iova, object size: 40, buffer size: 448, default order:
0, min order: 0
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
node 0: slabs: 697, objs: 4182, free: 0
node 0: slabs: 697, objs: 4182, free: 0
node 0: slabs: 697, objs: 4182, free: 0
node 0: slabs: 697, objs: 4182, free: 0
node 1: slabs: 381, objs: 2286, free: 27
node 1: slabs: 381, objs: 2286, free: 27
node 1: slabs: 381, objs: 2286, free: 27
node 1: slabs: 381, objs: 2286, free: 27
node 0: slabs: 1822, objs: 16398, free: 0
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
node 1: slabs: 2051, objs: 18459, free: 31
node 0: slabs: 697, objs: 4182, free: 0
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
node 1: slabs: 381, objs: 2286, free: 27
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
node 0: slabs: 697, objs: 4182, free: 0
node 1: slabs: 381, objs: 2286, free: 27
hpsa 0000:03:00.0: DMAR: Allocating 1-page iova failed
warn_alloc: 96 callbacks suppressed
kworker/11:1H: page allocation failure: order:0,
mode:0xa20(GFP_ATOMIC), nodemask=(null),cpuset=/,mems_allowed=0-1
CPU: 11 PID: 1642 Comm: kworker/11:1H Tainted: G B
Hardware name: HP ProLiant XL420 Gen9/ProLiant XL420 Gen9, BIOS U19
12/27/2015
Workqueue: kblockd blk_mq_run_work_fn
Call Trace:
dump_stack+0xa0/0xea
warn_alloc.cold.94+0x8a/0x12d
__alloc_pages_slowpath+0x1750/0x1870
__alloc_pages_nodemask+0x58a/0x710
alloc_pages_current+0x9c/0x110
alloc_slab_page+0xc9/0x760
allocate_slab+0x48f/0x5d0
new_slab+0x46/0x70
___slab_alloc+0x4ab/0x7b0
__slab_alloc+0x43/0x70
kmem_cache_alloc+0x2dd/0x450
SLUB: Unable to allocate memory on node -1, gfp=0xa20(GFP_ATOMIC)
alloc_iova+0x33/0x210
cache: skbuff_head_cache, object size: 208, buffer size: 640, default
order: 0, min order: 0
node 0: slabs: 697, objs: 4182, free: 0
alloc_iova_fast+0x62/0x3d1
node 1: slabs: 381, objs: 2286, free: 27
intel_alloc_iova+0xce/0xe0
intel_map_sg+0xed/0x410
scsi_dma_map+0xd7/0x160
scsi_queue_rq+0xbf7/0x1310
blk_mq_dispatch_rq_list+0x4d9/0xbc0
blk_mq_sched_dispatch_requests+0x24a/0x300
__blk_mq_run_hw_queue+0x156/0x230
blk_mq_run_work_fn+0x3b/0x40
process_one_work+0x579/0xb90
worker_thread+0x63/0x5b0
kthread+0x1e6/0x210
ret_from_fork+0x3a/0x50
Mem-Info:
active_anon:2422723 inactive_anon:361971 isolated_anon:34403
active_file:2285 inactive_file:1838 isolated_file:0
unevictable:0 dirty:1 writeback:5 unstable:0
slab_reclaimable:13972 slab_unreclaimable:453879
mapped:2380 shmem:154 pagetables:6948 bounce:0
free:19133 free_pcp:7363 free_cma:0
Signed-off-by: Qian Cai <cai@lca.pw>
Signed-off-by: Joerg Roedel <jroedel@suse.de>
2019-11-22 19:16:54 +00:00
|
|
|
return kmem_cache_zalloc(iova_cache, GFP_ATOMIC | __GFP_NOWARN);
|
2015-07-13 11:31:28 +00:00
|
|
|
}
|
|
|
|
|
2020-12-03 18:34:52 +00:00
|
|
|
static void free_iova_mem(struct iova *iova)
|
2015-07-13 11:31:28 +00:00
|
|
|
{
|
2017-09-21 15:52:46 +00:00
|
|
|
if (iova->pfn_lo != IOVA_ANCHOR)
|
|
|
|
kmem_cache_free(iova_cache, iova);
|
2015-07-13 11:31:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int iova_cache_get(void)
|
|
|
|
{
|
|
|
|
mutex_lock(&iova_cache_mutex);
|
|
|
|
if (!iova_cache_users) {
|
2021-03-25 12:29:58 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = cpuhp_setup_state_multi(CPUHP_IOMMU_IOVA_DEAD, "iommu/iova:dead", NULL,
|
|
|
|
iova_cpuhp_dead);
|
|
|
|
if (ret) {
|
|
|
|
mutex_unlock(&iova_cache_mutex);
|
|
|
|
pr_err("Couldn't register cpuhp handler\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-07-13 11:31:28 +00:00
|
|
|
iova_cache = kmem_cache_create(
|
|
|
|
"iommu_iova", sizeof(struct iova), 0,
|
|
|
|
SLAB_HWCACHE_ALIGN, NULL);
|
|
|
|
if (!iova_cache) {
|
2021-03-25 12:29:58 +00:00
|
|
|
cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD);
|
2015-07-13 11:31:28 +00:00
|
|
|
mutex_unlock(&iova_cache_mutex);
|
2020-05-07 16:18:03 +00:00
|
|
|
pr_err("Couldn't create iova cache\n");
|
2015-07-13 11:31:28 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
iova_cache_users++;
|
|
|
|
mutex_unlock(&iova_cache_mutex);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(iova_cache_get);
|
2015-07-13 11:31:28 +00:00
|
|
|
|
|
|
|
void iova_cache_put(void)
|
|
|
|
{
|
|
|
|
mutex_lock(&iova_cache_mutex);
|
|
|
|
if (WARN_ON(!iova_cache_users)) {
|
|
|
|
mutex_unlock(&iova_cache_mutex);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
iova_cache_users--;
|
2021-03-25 12:29:58 +00:00
|
|
|
if (!iova_cache_users) {
|
|
|
|
cpuhp_remove_multi_state(CPUHP_IOMMU_IOVA_DEAD);
|
2015-07-13 11:31:28 +00:00
|
|
|
kmem_cache_destroy(iova_cache);
|
2021-03-25 12:29:58 +00:00
|
|
|
}
|
2015-07-13 11:31:28 +00:00
|
|
|
mutex_unlock(&iova_cache_mutex);
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(iova_cache_put);
|
2015-07-13 11:31:28 +00:00
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
/**
|
|
|
|
* alloc_iova - allocates an iova
|
2012-07-21 17:21:32 +00:00
|
|
|
* @iovad: - iova domain in question
|
|
|
|
* @size: - size of page frames to allocate
|
|
|
|
* @limit_pfn: - max limit address
|
|
|
|
* @size_aligned: - set if size_aligned address range is required
|
2015-01-12 17:51:15 +00:00
|
|
|
* This function allocates an iova in the range iovad->start_pfn to limit_pfn,
|
|
|
|
* searching top-down from limit_pfn to iovad->start_pfn. If the size_aligned
|
2007-10-21 23:41:58 +00:00
|
|
|
* flag is set then the allocated address iova->pfn_lo will be naturally
|
|
|
|
* aligned on roundup_power_of_two(size).
|
2007-10-21 23:41:48 +00:00
|
|
|
*/
|
|
|
|
struct iova *
|
|
|
|
alloc_iova(struct iova_domain *iovad, unsigned long size,
|
2007-10-21 23:41:58 +00:00
|
|
|
unsigned long limit_pfn,
|
|
|
|
bool size_aligned)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
|
|
|
struct iova *new_iova;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
new_iova = alloc_iova_mem();
|
|
|
|
if (!new_iova)
|
|
|
|
return NULL;
|
|
|
|
|
2017-05-16 11:26:48 +00:00
|
|
|
ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn + 1,
|
2008-03-04 23:22:04 +00:00
|
|
|
new_iova, size_aligned);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
if (ret) {
|
|
|
|
free_iova_mem(new_iova);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return new_iova;
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(alloc_iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
static struct iova *
|
|
|
|
private_find_iova(struct iova_domain *iovad, unsigned long pfn)
|
2007-10-21 23:41:48 +00:00
|
|
|
{
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
struct rb_node *node = iovad->rbroot.rb_node;
|
|
|
|
|
|
|
|
assert_spin_locked(&iovad->iova_rbtree_lock);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
while (node) {
|
2021-03-05 16:35:22 +00:00
|
|
|
struct iova *iova = to_iova(node);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
if (pfn < iova->pfn_lo)
|
|
|
|
node = node->rb_left;
|
2017-09-21 15:52:42 +00:00
|
|
|
else if (pfn > iova->pfn_hi)
|
2007-10-21 23:41:48 +00:00
|
|
|
node = node->rb_right;
|
2017-09-21 15:52:42 +00:00
|
|
|
else
|
|
|
|
return iova; /* pfn falls within iova's range */
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
|
2021-05-10 11:53:02 +00:00
|
|
|
static void remove_iova(struct iova_domain *iovad, struct iova *iova)
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
{
|
|
|
|
assert_spin_locked(&iovad->iova_rbtree_lock);
|
|
|
|
__cached_rbnode_delete_update(iovad, iova);
|
|
|
|
rb_erase(&iova->node, &iovad->rbroot);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* find_iova - finds an iova for a given pfn
|
|
|
|
* @iovad: - iova domain in question.
|
|
|
|
* @pfn: - page frame number
|
|
|
|
* This function finds and returns an iova belonging to the
|
2020-12-22 16:42:32 +00:00
|
|
|
* given domain which matches the given pfn.
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
*/
|
|
|
|
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
struct iova *iova;
|
|
|
|
|
|
|
|
/* Take the lock so that no other thread is manipulating the rbtree */
|
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
|
|
|
iova = private_find_iova(iovad, pfn);
|
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
|
|
|
return iova;
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(find_iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* __free_iova - frees the given iova
|
|
|
|
* @iovad: iova domain in question.
|
|
|
|
* @iova: iova in question.
|
|
|
|
* Frees the given iova belonging to the giving domain
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
__free_iova(struct iova_domain *iovad, struct iova *iova)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
2021-05-10 11:53:02 +00:00
|
|
|
remove_iova(iovad, iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
2021-05-10 11:53:02 +00:00
|
|
|
free_iova_mem(iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(__free_iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* free_iova - finds and frees the iova for a given pfn
|
|
|
|
* @iovad: - iova domain in question.
|
|
|
|
* @pfn: - pfn that is allocated previously
|
|
|
|
* This functions finds an iova for a given pfn and then
|
|
|
|
* frees the iova from that domain.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
free_iova(struct iova_domain *iovad, unsigned long pfn)
|
|
|
|
{
|
2020-11-17 10:25:34 +00:00
|
|
|
unsigned long flags;
|
|
|
|
struct iova *iova;
|
2015-04-17 04:32:47 +00:00
|
|
|
|
2020-11-17 10:25:34 +00:00
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
|
|
|
iova = private_find_iova(iovad, pfn);
|
2021-05-10 11:53:02 +00:00
|
|
|
if (!iova) {
|
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
remove_iova(iovad, iova);
|
2020-11-17 10:25:34 +00:00
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
2021-05-10 11:53:02 +00:00
|
|
|
free_iova_mem(iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(free_iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
/**
|
|
|
|
* alloc_iova_fast - allocates an iova from rcache
|
|
|
|
* @iovad: - iova domain in question
|
|
|
|
* @size: - size of page frames to allocate
|
|
|
|
* @limit_pfn: - max limit address
|
2017-09-20 08:52:02 +00:00
|
|
|
* @flush_rcache: - set to flush rcache on regular allocation failure
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
* This function tries to satisfy an iova allocation from the rcache,
|
2017-09-20 08:52:02 +00:00
|
|
|
* and falls back to regular allocation on failure. If regular allocation
|
|
|
|
* fails too and the flush_rcache flag is set then the rcache will be flushed.
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
*/
|
|
|
|
unsigned long
|
|
|
|
alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
|
2017-09-20 08:52:02 +00:00
|
|
|
unsigned long limit_pfn, bool flush_rcache)
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
{
|
|
|
|
unsigned long iova_pfn;
|
|
|
|
struct iova *new_iova;
|
|
|
|
|
2021-12-07 11:17:26 +00:00
|
|
|
/*
|
|
|
|
* Freeing non-power-of-two-sized allocations back into the IOVA caches
|
|
|
|
* will come back to bite us badly, so we have to waste a bit of space
|
|
|
|
* rounding up anything cacheable to make sure that can't happen. The
|
|
|
|
* order of the unadjusted size will still match upon freeing.
|
|
|
|
*/
|
|
|
|
if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
|
|
|
|
size = roundup_pow_of_two(size);
|
|
|
|
|
2017-09-19 13:48:40 +00:00
|
|
|
iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
if (iova_pfn)
|
|
|
|
return iova_pfn;
|
|
|
|
|
|
|
|
retry:
|
|
|
|
new_iova = alloc_iova(iovad, size, limit_pfn, true);
|
|
|
|
if (!new_iova) {
|
|
|
|
unsigned int cpu;
|
|
|
|
|
2017-09-20 08:52:02 +00:00
|
|
|
if (!flush_rcache)
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Try replenishing IOVAs by flushing rcache. */
|
2017-09-20 08:52:02 +00:00
|
|
|
flush_rcache = false;
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
free_cpu_cached_iovas(cpu, iovad);
|
2020-09-30 07:44:24 +00:00
|
|
|
free_global_cached_iovas(iovad);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
goto retry;
|
|
|
|
}
|
|
|
|
|
|
|
|
return new_iova->pfn_lo;
|
|
|
|
}
|
2021-08-31 10:36:22 +00:00
|
|
|
EXPORT_SYMBOL_GPL(alloc_iova_fast);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* free_iova_fast - free iova pfn range into rcache
|
|
|
|
* @iovad: - iova domain in question.
|
|
|
|
* @pfn: - pfn that is allocated previously
|
|
|
|
* @size: - # of pages in range
|
|
|
|
* This functions frees an iova range by trying to put it into the rcache,
|
|
|
|
* falling back to regular iova deallocation via free_iova() if this fails.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
|
|
|
|
{
|
|
|
|
if (iova_rcache_insert(iovad, pfn, size))
|
|
|
|
return;
|
|
|
|
|
|
|
|
free_iova(iovad, pfn);
|
|
|
|
}
|
2021-08-31 10:36:22 +00:00
|
|
|
EXPORT_SYMBOL_GPL(free_iova_fast);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
/**
|
2020-12-22 16:42:32 +00:00
|
|
|
* put_iova_domain - destroys the iova domain
|
2007-10-21 23:41:48 +00:00
|
|
|
* @iovad: - iova domain in question.
|
|
|
|
* All the iova's in that domain are destroyed.
|
|
|
|
*/
|
|
|
|
void put_iova_domain(struct iova_domain *iovad)
|
|
|
|
{
|
2017-09-19 13:48:39 +00:00
|
|
|
struct iova *iova, *tmp;
|
2007-10-21 23:41:48 +00:00
|
|
|
|
2021-03-25 12:29:58 +00:00
|
|
|
cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
|
|
|
|
&iovad->cpuhp_dead);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
free_iova_rcaches(iovad);
|
2017-09-19 13:48:39 +00:00
|
|
|
rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
|
2007-10-21 23:41:48 +00:00
|
|
|
free_iova_mem(iova);
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(put_iova_domain);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
static int
|
|
|
|
__is_range_overlap(struct rb_node *node,
|
|
|
|
unsigned long pfn_lo, unsigned long pfn_hi)
|
|
|
|
{
|
2021-03-05 16:35:22 +00:00
|
|
|
struct iova *iova = to_iova(node);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))
|
|
|
|
return 1;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-02-19 06:07:37 +00:00
|
|
|
static inline struct iova *
|
|
|
|
alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi)
|
|
|
|
{
|
|
|
|
struct iova *iova;
|
|
|
|
|
|
|
|
iova = alloc_iova_mem();
|
|
|
|
if (iova) {
|
|
|
|
iova->pfn_lo = pfn_lo;
|
|
|
|
iova->pfn_hi = pfn_hi;
|
|
|
|
}
|
|
|
|
|
|
|
|
return iova;
|
|
|
|
}
|
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
static struct iova *
|
|
|
|
__insert_new_range(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn_lo, unsigned long pfn_hi)
|
|
|
|
{
|
|
|
|
struct iova *iova;
|
|
|
|
|
2014-02-19 06:07:37 +00:00
|
|
|
iova = alloc_and_init_iova(pfn_lo, pfn_hi);
|
|
|
|
if (iova)
|
2017-02-24 11:13:37 +00:00
|
|
|
iova_insert_rbtree(&iovad->rbroot, iova, NULL);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
return iova;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
__adjust_overlap_range(struct iova *iova,
|
|
|
|
unsigned long *pfn_lo, unsigned long *pfn_hi)
|
|
|
|
{
|
|
|
|
if (*pfn_lo < iova->pfn_lo)
|
|
|
|
iova->pfn_lo = *pfn_lo;
|
|
|
|
if (*pfn_hi > iova->pfn_hi)
|
|
|
|
*pfn_lo = iova->pfn_hi + 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* reserve_iova - reserves an iova in the given range
|
|
|
|
* @iovad: - iova domain pointer
|
|
|
|
* @pfn_lo: - lower page frame address
|
|
|
|
* @pfn_hi:- higher pfn adderss
|
|
|
|
* This function allocates reserves the address range from pfn_lo to pfn_hi so
|
|
|
|
* that this address is not dished out as part of alloc_iova.
|
|
|
|
*/
|
|
|
|
struct iova *
|
|
|
|
reserve_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn_lo, unsigned long pfn_hi)
|
|
|
|
{
|
|
|
|
struct rb_node *node;
|
|
|
|
unsigned long flags;
|
|
|
|
struct iova *iova;
|
|
|
|
unsigned int overlap = 0;
|
|
|
|
|
2017-09-21 15:52:46 +00:00
|
|
|
/* Don't allow nonsensical pfns */
|
|
|
|
if (WARN_ON((pfn_hi | pfn_lo) > (ULLONG_MAX >> iova_shift(iovad))))
|
|
|
|
return NULL;
|
|
|
|
|
2009-07-08 14:23:30 +00:00
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
2007-10-21 23:41:48 +00:00
|
|
|
for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) {
|
|
|
|
if (__is_range_overlap(node, pfn_lo, pfn_hi)) {
|
2021-03-05 16:35:22 +00:00
|
|
|
iova = to_iova(node);
|
2007-10-21 23:41:48 +00:00
|
|
|
__adjust_overlap_range(iova, &pfn_lo, &pfn_hi);
|
|
|
|
if ((pfn_lo >= iova->pfn_lo) &&
|
|
|
|
(pfn_hi <= iova->pfn_hi))
|
|
|
|
goto finish;
|
|
|
|
overlap = 1;
|
|
|
|
|
|
|
|
} else if (overlap)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2011-03-31 01:57:33 +00:00
|
|
|
/* We are here either because this is the first reserver node
|
2007-10-21 23:41:48 +00:00
|
|
|
* or need to insert remaining non overlap addr range
|
|
|
|
*/
|
|
|
|
iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
|
|
|
|
finish:
|
|
|
|
|
2009-07-08 14:23:30 +00:00
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
2007-10-21 23:41:48 +00:00
|
|
|
return iova;
|
|
|
|
}
|
2015-07-13 11:31:29 +00:00
|
|
|
EXPORT_SYMBOL_GPL(reserve_iova);
|
2007-10-21 23:41:48 +00:00
|
|
|
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
/*
|
|
|
|
* Magazine caches for IOVA ranges. For an introduction to magazines,
|
|
|
|
* see the USENIX 2001 paper "Magazines and Vmem: Extending the Slab
|
|
|
|
* Allocator to Many CPUs and Arbitrary Resources" by Bonwick and Adams.
|
|
|
|
* For simplicity, we use a static magazine size and don't implement the
|
|
|
|
* dynamic size tuning described in the paper.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define IOVA_MAG_SIZE 128
|
|
|
|
|
|
|
|
struct iova_magazine {
|
|
|
|
unsigned long size;
|
|
|
|
unsigned long pfns[IOVA_MAG_SIZE];
|
|
|
|
};
|
|
|
|
|
|
|
|
struct iova_cpu_rcache {
|
|
|
|
spinlock_t lock;
|
|
|
|
struct iova_magazine *loaded;
|
|
|
|
struct iova_magazine *prev;
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct iova_magazine *iova_magazine_alloc(gfp_t flags)
|
|
|
|
{
|
|
|
|
return kzalloc(sizeof(struct iova_magazine), flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void iova_magazine_free(struct iova_magazine *mag)
|
|
|
|
{
|
|
|
|
kfree(mag);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!mag)
|
|
|
|
return;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
|
|
|
|
|
|
|
|
for (i = 0 ; i < mag->size; ++i) {
|
|
|
|
struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
|
|
|
|
|
2020-06-02 13:08:18 +00:00
|
|
|
if (WARN_ON(!iova))
|
|
|
|
continue;
|
|
|
|
|
2021-05-10 11:53:02 +00:00
|
|
|
remove_iova(iovad, iova);
|
|
|
|
free_iova_mem(iova);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
|
|
|
|
|
|
|
|
mag->size = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool iova_magazine_full(struct iova_magazine *mag)
|
|
|
|
{
|
|
|
|
return (mag && mag->size == IOVA_MAG_SIZE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool iova_magazine_empty(struct iova_magazine *mag)
|
|
|
|
{
|
|
|
|
return (!mag || mag->size == 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned long iova_magazine_pop(struct iova_magazine *mag,
|
|
|
|
unsigned long limit_pfn)
|
|
|
|
{
|
2017-09-28 10:31:23 +00:00
|
|
|
int i;
|
|
|
|
unsigned long pfn;
|
|
|
|
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
BUG_ON(iova_magazine_empty(mag));
|
|
|
|
|
2017-09-28 10:31:23 +00:00
|
|
|
/* Only fall back to the rbtree if we have no suitable pfns at all */
|
|
|
|
for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--)
|
|
|
|
if (i == 0)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Swap it to pop it */
|
|
|
|
pfn = mag->pfns[i];
|
|
|
|
mag->pfns[i] = mag->pfns[--mag->size];
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
|
2017-09-28 10:31:23 +00:00
|
|
|
return pfn;
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
|
|
|
|
{
|
|
|
|
BUG_ON(iova_magazine_full(mag));
|
|
|
|
|
|
|
|
mag->pfns[mag->size++] = pfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void init_iova_rcaches(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
struct iova_cpu_rcache *cpu_rcache;
|
|
|
|
struct iova_rcache *rcache;
|
|
|
|
unsigned int cpu;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
|
|
|
|
rcache = &iovad->rcaches[i];
|
|
|
|
spin_lock_init(&rcache->lock);
|
|
|
|
rcache->depot_size = 0;
|
|
|
|
rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
|
|
|
|
if (WARN_ON(!rcache->cpu_rcaches))
|
|
|
|
continue;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
|
|
|
|
spin_lock_init(&cpu_rcache->lock);
|
|
|
|
cpu_rcache->loaded = iova_magazine_alloc(GFP_KERNEL);
|
|
|
|
cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Try inserting IOVA range starting with 'iova_pfn' into 'rcache', and
|
|
|
|
* return true on success. Can fail if rcache is full and we can't free
|
|
|
|
* space, and free_iova() (our only caller) will then return the IOVA
|
|
|
|
* range to the rbtree instead.
|
|
|
|
*/
|
|
|
|
static bool __iova_rcache_insert(struct iova_domain *iovad,
|
|
|
|
struct iova_rcache *rcache,
|
|
|
|
unsigned long iova_pfn)
|
|
|
|
{
|
|
|
|
struct iova_magazine *mag_to_free = NULL;
|
|
|
|
struct iova_cpu_rcache *cpu_rcache;
|
|
|
|
bool can_insert = false;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2017-06-27 16:16:47 +00:00
|
|
|
cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
spin_lock_irqsave(&cpu_rcache->lock, flags);
|
|
|
|
|
|
|
|
if (!iova_magazine_full(cpu_rcache->loaded)) {
|
|
|
|
can_insert = true;
|
|
|
|
} else if (!iova_magazine_full(cpu_rcache->prev)) {
|
|
|
|
swap(cpu_rcache->prev, cpu_rcache->loaded);
|
|
|
|
can_insert = true;
|
|
|
|
} else {
|
|
|
|
struct iova_magazine *new_mag = iova_magazine_alloc(GFP_ATOMIC);
|
|
|
|
|
|
|
|
if (new_mag) {
|
|
|
|
spin_lock(&rcache->lock);
|
|
|
|
if (rcache->depot_size < MAX_GLOBAL_MAGS) {
|
|
|
|
rcache->depot[rcache->depot_size++] =
|
|
|
|
cpu_rcache->loaded;
|
|
|
|
} else {
|
|
|
|
mag_to_free = cpu_rcache->loaded;
|
|
|
|
}
|
|
|
|
spin_unlock(&rcache->lock);
|
|
|
|
|
|
|
|
cpu_rcache->loaded = new_mag;
|
|
|
|
can_insert = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (can_insert)
|
|
|
|
iova_magazine_push(cpu_rcache->loaded, iova_pfn);
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
|
|
|
|
|
|
|
|
if (mag_to_free) {
|
|
|
|
iova_magazine_free_pfns(mag_to_free, iovad);
|
|
|
|
iova_magazine_free(mag_to_free);
|
|
|
|
}
|
|
|
|
|
|
|
|
return can_insert;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
|
|
|
|
unsigned long size)
|
|
|
|
{
|
|
|
|
unsigned int log_size = order_base_2(size);
|
|
|
|
|
|
|
|
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Caller wants to allocate a new IOVA range from 'rcache'. If we can
|
|
|
|
* satisfy the request, return a matching non-NULL range and remove
|
|
|
|
* it from the 'rcache'.
|
|
|
|
*/
|
|
|
|
static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
|
|
|
|
unsigned long limit_pfn)
|
|
|
|
{
|
|
|
|
struct iova_cpu_rcache *cpu_rcache;
|
|
|
|
unsigned long iova_pfn = 0;
|
|
|
|
bool has_pfn = false;
|
|
|
|
unsigned long flags;
|
|
|
|
|
2017-06-27 16:16:47 +00:00
|
|
|
cpu_rcache = raw_cpu_ptr(rcache->cpu_rcaches);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
spin_lock_irqsave(&cpu_rcache->lock, flags);
|
|
|
|
|
|
|
|
if (!iova_magazine_empty(cpu_rcache->loaded)) {
|
|
|
|
has_pfn = true;
|
|
|
|
} else if (!iova_magazine_empty(cpu_rcache->prev)) {
|
|
|
|
swap(cpu_rcache->prev, cpu_rcache->loaded);
|
|
|
|
has_pfn = true;
|
|
|
|
} else {
|
|
|
|
spin_lock(&rcache->lock);
|
|
|
|
if (rcache->depot_size > 0) {
|
|
|
|
iova_magazine_free(cpu_rcache->loaded);
|
|
|
|
cpu_rcache->loaded = rcache->depot[--rcache->depot_size];
|
|
|
|
has_pfn = true;
|
|
|
|
}
|
|
|
|
spin_unlock(&rcache->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (has_pfn)
|
|
|
|
iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
|
|
|
|
|
|
|
|
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
|
|
|
|
|
|
|
|
return iova_pfn;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Try to satisfy IOVA allocation range from rcache. Fail if requested
|
|
|
|
* size is too big or the DMA limit we are given isn't satisfied by the
|
|
|
|
* top element in the magazine.
|
|
|
|
*/
|
|
|
|
static unsigned long iova_rcache_get(struct iova_domain *iovad,
|
|
|
|
unsigned long size,
|
|
|
|
unsigned long limit_pfn)
|
|
|
|
{
|
|
|
|
unsigned int log_size = order_base_2(size);
|
|
|
|
|
|
|
|
if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
|
|
|
|
return 0;
|
|
|
|
|
2017-09-19 13:48:40 +00:00
|
|
|
return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* free rcache data structures.
|
|
|
|
*/
|
|
|
|
static void free_iova_rcaches(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
struct iova_rcache *rcache;
|
2017-09-19 13:48:39 +00:00
|
|
|
struct iova_cpu_rcache *cpu_rcache;
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
unsigned int cpu;
|
|
|
|
int i, j;
|
|
|
|
|
|
|
|
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
|
|
|
|
rcache = &iovad->rcaches[i];
|
2017-09-19 13:48:39 +00:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
|
|
|
|
iova_magazine_free(cpu_rcache->loaded);
|
|
|
|
iova_magazine_free(cpu_rcache->prev);
|
|
|
|
}
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
free_percpu(rcache->cpu_rcaches);
|
2017-09-19 13:48:39 +00:00
|
|
|
for (j = 0; j < rcache->depot_size; ++j)
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
iova_magazine_free(rcache->depot[j]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* free all the IOVA ranges cached by a cpu (used when cpu is unplugged)
|
|
|
|
*/
|
2021-03-25 12:30:00 +00:00
|
|
|
static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
{
|
|
|
|
struct iova_cpu_rcache *cpu_rcache;
|
|
|
|
struct iova_rcache *rcache;
|
|
|
|
unsigned long flags;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
|
|
|
|
rcache = &iovad->rcaches[i];
|
|
|
|
cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
|
|
|
|
spin_lock_irqsave(&cpu_rcache->lock, flags);
|
|
|
|
iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
|
|
|
|
iova_magazine_free_pfns(cpu_rcache->prev, iovad);
|
|
|
|
spin_unlock_irqrestore(&cpu_rcache->lock, flags);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-30 07:44:24 +00:00
|
|
|
/*
|
|
|
|
* free all the IOVA ranges of global cache
|
|
|
|
*/
|
|
|
|
static void free_global_cached_iovas(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
struct iova_rcache *rcache;
|
|
|
|
unsigned long flags;
|
|
|
|
int i, j;
|
|
|
|
|
|
|
|
for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
|
|
|
|
rcache = &iovad->rcaches[i];
|
|
|
|
spin_lock_irqsave(&rcache->lock, flags);
|
|
|
|
for (j = 0; j < rcache->depot_size; ++j) {
|
|
|
|
iova_magazine_free_pfns(rcache->depot[j], iovad);
|
|
|
|
iova_magazine_free(rcache->depot[j]);
|
|
|
|
}
|
|
|
|
rcache->depot_size = 0;
|
|
|
|
spin_unlock_irqrestore(&rcache->lock, flags);
|
|
|
|
}
|
|
|
|
}
|
2015-07-13 11:31:30 +00:00
|
|
|
MODULE_AUTHOR("Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>");
|
|
|
|
MODULE_LICENSE("GPL");
|