mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-25 02:09:42 +00:00
iommu/amd: Add flush counters to struct dma_ops_domain
The counters are increased every time the TLB for a given domain is flushed. We also store the current value of that counter into newly added entries of the flush-queue, so that we can tell whether this entry is already flushed. Signed-off-by: Joerg Roedel <jroedel@suse.de>
This commit is contained in:
parent
e241f8e76c
commit
a6e3f6f030
1 changed files with 52 additions and 0 deletions
|
@ -141,6 +141,7 @@ static void detach_device(struct device *dev);
|
|||
struct flush_queue_entry {
|
||||
unsigned long iova_pfn;
|
||||
unsigned long pages;
|
||||
u64 counter; /* Flush counter when this entry was added to the queue */
|
||||
};
|
||||
|
||||
struct flush_queue {
|
||||
|
@ -160,6 +161,27 @@ struct dma_ops_domain {
|
|||
struct iova_domain iovad;
|
||||
|
||||
struct flush_queue __percpu *flush_queue;
|
||||
|
||||
/*
|
||||
* We need two counter here to be race-free wrt. IOTLB flushing and
|
||||
* adding entries to the flush queue.
|
||||
*
|
||||
* The flush_start_cnt is incremented _before_ the IOTLB flush starts.
|
||||
* New entries added to the flush ring-buffer get their 'counter' value
|
||||
* from here. This way we can make sure that entries added to the queue
|
||||
* (or other per-cpu queues of the same domain) while the TLB is about
|
||||
* to be flushed are not considered to be flushed already.
|
||||
*/
|
||||
atomic64_t flush_start_cnt;
|
||||
|
||||
/*
|
||||
* The flush_finish_cnt is incremented when an IOTLB flush is complete.
|
||||
* This value is always smaller than flush_start_cnt. The queue_add
|
||||
* function frees all IOVAs that have a counter value smaller than
|
||||
* flush_finish_cnt. This makes sure that we only free IOVAs that are
|
||||
* flushed out of the IOTLB of the domain.
|
||||
*/
|
||||
atomic64_t flush_finish_cnt;
|
||||
};
|
||||
|
||||
static struct iova_domain reserved_iova_ranges;
|
||||
|
@ -1777,6 +1799,9 @@ static int dma_ops_domain_alloc_flush_queue(struct dma_ops_domain *dom)
|
|||
{
|
||||
int cpu;
|
||||
|
||||
atomic64_set(&dom->flush_start_cnt, 0);
|
||||
atomic64_set(&dom->flush_finish_cnt, 0);
|
||||
|
||||
dom->flush_queue = alloc_percpu(struct flush_queue);
|
||||
if (!dom->flush_queue)
|
||||
return -ENOMEM;
|
||||
|
@ -1844,22 +1869,48 @@ static inline unsigned queue_ring_add(struct flush_queue *queue)
|
|||
return idx;
|
||||
}
|
||||
|
||||
static inline void queue_ring_remove_head(struct flush_queue *queue)
|
||||
{
|
||||
assert_spin_locked(&queue->lock);
|
||||
queue->head = (queue->head + 1) % FLUSH_QUEUE_SIZE;
|
||||
}
|
||||
|
||||
static void queue_add(struct dma_ops_domain *dom,
|
||||
unsigned long address, unsigned long pages)
|
||||
{
|
||||
struct flush_queue *queue;
|
||||
unsigned long flags;
|
||||
u64 counter;
|
||||
int idx;
|
||||
|
||||
pages = __roundup_pow_of_two(pages);
|
||||
address >>= PAGE_SHIFT;
|
||||
|
||||
counter = atomic64_read(&dom->flush_finish_cnt);
|
||||
|
||||
queue = get_cpu_ptr(dom->flush_queue);
|
||||
spin_lock_irqsave(&queue->lock, flags);
|
||||
|
||||
queue_ring_for_each(idx, queue) {
|
||||
/*
|
||||
* This assumes that counter values in the ring-buffer are
|
||||
* monotonously rising.
|
||||
*/
|
||||
if (queue->entries[idx].counter >= counter)
|
||||
break;
|
||||
|
||||
free_iova_fast(&dom->iovad,
|
||||
queue->entries[idx].iova_pfn,
|
||||
queue->entries[idx].pages);
|
||||
|
||||
queue_ring_remove_head(queue);
|
||||
}
|
||||
|
||||
if (queue_ring_full(queue)) {
|
||||
atomic64_inc(&dom->flush_start_cnt);
|
||||
domain_flush_tlb(&dom->domain);
|
||||
domain_flush_complete(&dom->domain);
|
||||
atomic64_inc(&dom->flush_finish_cnt);
|
||||
queue_release(dom, queue);
|
||||
}
|
||||
|
||||
|
@ -1867,6 +1918,7 @@ static void queue_add(struct dma_ops_domain *dom,
|
|||
|
||||
queue->entries[idx].iova_pfn = address;
|
||||
queue->entries[idx].pages = pages;
|
||||
queue->entries[idx].counter = atomic64_read(&dom->flush_start_cnt);
|
||||
|
||||
spin_unlock_irqrestore(&queue->lock, flags);
|
||||
put_cpu_ptr(dom->flush_queue);
|
||||
|
|
Loading…
Reference in a new issue