mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-25 02:09:42 +00:00
iommu/amd: Add per-domain timer to flush per-cpu queues
Add a timer to each dma_ops domain so that we flush unused IOTLB entries regularily, even if the queues don't get full all the time. Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
parent
a6e3f6f030
commit
fca6af6a59
1 changed files with 67 additions and 17 deletions
|
@ -182,6 +182,13 @@ struct dma_ops_domain {
|
|||
* flushed out of the IOTLB of the domain.
|
||||
*/
|
||||
atomic64_t flush_finish_cnt;
|
||||
|
||||
/*
|
||||
* Timer to make sure we don't keep IOVAs around unflushed
|
||||
* for too long
|
||||
*/
|
||||
struct timer_list flush_timer;
|
||||
atomic_t flush_timer_on;
|
||||
};
|
||||
|
||||
static struct iova_domain reserved_iova_ranges;
|
||||
|
@ -1834,6 +1841,14 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void dma_ops_domain_flush_tlb(struct dma_ops_domain *dom)
|
||||
{
|
||||
atomic64_inc(&dom->flush_start_cnt);
|
||||
domain_flush_tlb(&dom->domain);
|
||||
domain_flush_complete(&dom->domain);
|
||||
atomic64_inc(&dom->flush_finish_cnt);
|
||||
}
|
||||
|
||||
static inline bool queue_ring_full(struct flush_queue *queue)
|
||||
{
|
||||
assert_spin_locked(&queue->lock);
|
||||
|
@ -1875,22 +1890,12 @@ static inline void queue_ring_remove_head(struct flush_queue *queue)
|
|||
queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
|
||||
}
|
||||
|
||||
static void queue_add(struct dma_ops_domain *dom,
|
||||
unsigned long address, unsigned long pages)
|
||||
static void queue_ring_free_flushed(struct dma_ops_domain *dom,
|
||||
struct flush_queue *queue)
|
||||
{
|
||||
struct flush_queue *queue;
|
||||
unsigned long flags;
|
||||
u64 counter;
|
||||
u64 counter = atomic64_read(&dom->flush_finish_cnt);
|
||||
int idx;
|
||||
|
||||
pages = __roundup_pow_of_two(pages);
|
||||
address >>= PAGE_SHIFT;
|
||||
|
||||
counter = atomic64_read(&dom->flush_finish_cnt);
|
||||
|
||||
queue = get_cpu_ptr(dom->flush_queue);
|
||||
spin_lock_irqsave(&queue->lock, flags);
|
||||
|
||||
queue_ring_for_each(idx, queue) {
|
||||
/*
|
||||
* This assumes that counter values in the ring-buffer are
|
||||
|
@ -1905,12 +1910,25 @@ static void queue_add(struct dma_ops_domain *dom,
|
|||
|
||||
queue_ring_remove_head(queue);
|
||||
}
|
||||
}
|
||||
|
||||
static void queue_add(struct dma_ops_domain *dom,
|
||||
unsigned long address, unsigned long pages)
|
||||
{
|
||||
struct flush_queue *queue;
|
||||
unsigned long flags;
|
||||
int idx;
|
||||
|
||||
pages = __roundup_pow_of_two(pages);
|
||||
address >>= PAGE_SHIFT;
|
||||
|
||||
queue = get_cpu_ptr(dom->flush_queue);
|
||||
spin_lock_irqsave(&queue->lock, flags);
|
||||
|
||||
queue_ring_free_flushed(dom, queue);
|
||||
|
||||
if (queue_ring_full(queue)) {
|
||||
atomic64_inc(&dom->flush_start_cnt);
|
||||
domain_flush_tlb(&dom->domain);
|
||||
domain_flush_complete(&dom->domain);
|
||||
atomic64_inc(&dom->flush_finish_cnt);
|
||||
dma_ops_domain_flush_tlb(dom);
|
||||
queue_release(dom, queue);
|
||||
}
|
||||
|
||||
|
@ -1921,9 +1939,33 @@ static void queue_add(struct dma_ops_domain *dom,
|
|||
queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
|
||||
|
||||
spin_unlock_irqrestore(&queue->lock, flags);
|
||||
|
||||
if (atomic_cmpxchg(&dom->flush_timer_on, 0, 1) == 0)
|
||||
mod_timer(&dom->flush_timer, jiffies + msecs_to_jiffies(10));
|
||||
|
||||
put_cpu_ptr(dom->flush_queue);
|
||||
}
|
||||
|
||||
static void queue_flush_timeout(unsigned long data)
|
||||
{
|
||||
struct dma_ops_domain *dom = (struct dma_ops_domain *)data;
|
||||
int cpu;
|
||||
|
||||
atomic_set(&dom->flush_timer_on, 0);
|
||||
|
||||
dma_ops_domain_flush_tlb(dom);
|
||||
|
||||
for_each_possible_cpu(cpu) {
|
||||
struct flush_queue *queue;
|
||||
unsigned long flags;
|
||||
|
||||
queue = per_cpu_ptr(dom->flush_queue, cpu);
|
||||
spin_lock_irqsave(&queue->lock, flags);
|
||||
queue_ring_free_flushed(dom, queue);
|
||||
spin_unlock_irqrestore(&queue->lock, flags);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Free a domain, only used if something went wrong in the
|
||||
* allocation path and we need to free an already allocated page table
|
||||
|
@ -1935,6 +1977,9 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
|
|||
|
||||
del_domain_from_list(&dom->domain);
|
||||
|
||||
if (timer_pending(&dom->flush_timer))
|
||||
del_timer(&dom->flush_timer);
|
||||
|
||||
dma_ops_domain_free_flush_queue(dom);
|
||||
|
||||
put_iova_domain(&dom->iovad);
|
||||
|
@ -1978,6 +2023,11 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
|
|||
if (dma_ops_domain_alloc_flush_queue(dma_dom))
|
||||
goto free_dma_dom;
|
||||
|
||||
setup_timer(&dma_dom->flush_timer, queue_flush_timeout,
|
||||
(unsigned long)dma_dom);
|
||||
|
||||
atomic_set(&dma_dom->flush_timer_on, 0);
|
||||
|
||||
add_domain_to_list(&dma_dom->domain);
|
||||
|
||||
return dma_dom;
|
||||
|
|
Loading…
Reference in a new issue