2019-06-01 08:08:42 +00:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2007-10-21 23:41:48 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2006, Intel Corporation.
|
|
|
|
*
|
2008-02-23 23:23:35 +00:00
|
|
|
* Copyright (C) 2006-2008 Intel Corporation
|
|
|
|
* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
|
2007-10-21 23:41:48 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef _IOVA_H_
|
|
|
|
#define _IOVA_H_
|
|
|
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/rbtree.h>
|
2017-08-10 14:14:59 +00:00
|
|
|
#include <linux/atomic.h>
|
2007-10-21 23:41:48 +00:00
|
|
|
#include <linux/dma-mapping.h>
|
|
|
|
|
|
|
|
/* iova structure */
|
|
|
|
struct iova {
|
|
|
|
struct rb_node node;
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
unsigned long pfn_hi; /* Highest allocated pfn */
|
|
|
|
unsigned long pfn_lo; /* Lowest allocated pfn */
|
|
|
|
};
|
|
|
|
|
|
|
|
struct iova_magazine;
|
|
|
|
struct iova_cpu_rcache;
|
|
|
|
|
|
|
|
#define IOVA_RANGE_CACHE_MAX_SIZE 6 /* log of max cached IOVA range size (in pages) */
|
|
|
|
#define MAX_GLOBAL_MAGS 32 /* magazines per bin */
|
|
|
|
|
|
|
|
struct iova_rcache {
|
|
|
|
spinlock_t lock;
|
|
|
|
unsigned long depot_size;
|
|
|
|
struct iova_magazine *depot[MAX_GLOBAL_MAGS];
|
|
|
|
struct iova_cpu_rcache __percpu *cpu_rcaches;
|
2007-10-21 23:41:48 +00:00
|
|
|
};
|
|
|
|
|
2017-08-10 12:44:28 +00:00
|
|
|
struct iova_domain;
|
|
|
|
|
|
|
|
/* Call-Back from IOVA code into IOMMU drivers */
|
|
|
|
typedef void (* iova_flush_cb)(struct iova_domain *domain);
|
|
|
|
|
|
|
|
/* Destructor for per-entry data */
|
|
|
|
typedef void (* iova_entry_dtor)(unsigned long data);
|
|
|
|
|
|
|
|
/* Number of entries per Flush Queue */
|
|
|
|
#define IOVA_FQ_SIZE 256
|
|
|
|
|
2017-08-10 14:58:18 +00:00
|
|
|
/* Timeout (in ms) after which entries are flushed from the Flush-Queue */
|
|
|
|
#define IOVA_FQ_TIMEOUT 10
|
|
|
|
|
2017-08-10 12:44:28 +00:00
|
|
|
/* Flush Queue entry for defered flushing */
|
|
|
|
struct iova_fq_entry {
|
|
|
|
unsigned long iova_pfn;
|
|
|
|
unsigned long pages;
|
|
|
|
unsigned long data;
|
2017-08-10 14:14:59 +00:00
|
|
|
u64 counter; /* Flush counter when this entrie was added */
|
2017-08-10 12:44:28 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Per-CPU Flush Queue structure */
|
|
|
|
struct iova_fq {
|
|
|
|
struct iova_fq_entry entries[IOVA_FQ_SIZE];
|
|
|
|
unsigned head, tail;
|
2017-08-10 14:31:17 +00:00
|
|
|
spinlock_t lock;
|
2017-08-10 12:44:28 +00:00
|
|
|
};
|
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
/* holds all the iova translations for a domain */
|
|
|
|
struct iova_domain {
|
|
|
|
spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */
|
|
|
|
struct rb_root rbroot; /* iova domain rbtree root */
|
2017-09-21 15:52:44 +00:00
|
|
|
struct rb_node *cached_node; /* Save last alloced node */
|
|
|
|
struct rb_node *cached32_node; /* Save last 32-bit alloced node */
|
2015-01-12 17:51:16 +00:00
|
|
|
unsigned long granule; /* pfn granularity for this domain */
|
2015-01-12 17:51:15 +00:00
|
|
|
unsigned long start_pfn; /* Lower limit for this domain */
|
2008-02-06 09:36:23 +00:00
|
|
|
unsigned long dma_32bit_pfn;
|
2018-09-05 04:27:36 +00:00
|
|
|
unsigned long max32_alloc_size; /* Size of last failed allocation */
|
2019-04-03 08:35:21 +00:00
|
|
|
struct iova_fq __percpu *fq; /* Flush Queue */
|
|
|
|
|
|
|
|
atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that
|
|
|
|
have been started */
|
|
|
|
|
|
|
|
atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that
|
|
|
|
have been finished */
|
|
|
|
|
2017-09-21 15:52:46 +00:00
|
|
|
struct iova anchor; /* rbtree lookup anchor */
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
|
2017-08-10 12:44:28 +00:00
|
|
|
|
|
|
|
iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU
|
|
|
|
TLBs */
|
|
|
|
|
|
|
|
iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
|
|
|
|
iova entry */
|
|
|
|
|
2017-08-10 14:58:18 +00:00
|
|
|
struct timer_list fq_timer; /* Timer to regularily empty the
|
|
|
|
flush-queues */
|
|
|
|
atomic_t fq_timer_on; /* 1 when timer is active, 0
|
|
|
|
when not */
|
2007-10-21 23:41:48 +00:00
|
|
|
};
|
|
|
|
|
2014-07-11 06:19:36 +00:00
|
|
|
static inline unsigned long iova_size(struct iova *iova)
|
|
|
|
{
|
|
|
|
return iova->pfn_hi - iova->pfn_lo + 1;
|
|
|
|
}
|
|
|
|
|
2015-01-12 17:51:16 +00:00
|
|
|
static inline unsigned long iova_shift(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
return __ffs(iovad->granule);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long iova_mask(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
return iovad->granule - 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova)
|
|
|
|
{
|
|
|
|
return iova & iova_mask(iovad);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline size_t iova_align(struct iova_domain *iovad, size_t size)
|
|
|
|
{
|
|
|
|
return ALIGN(size, iovad->granule);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova)
|
|
|
|
{
|
|
|
|
return (dma_addr_t)iova->pfn_lo << iova_shift(iovad);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova)
|
|
|
|
{
|
|
|
|
return iova >> iova_shift(iovad);
|
|
|
|
}
|
|
|
|
|
2017-03-22 23:06:17 +00:00
|
|
|
#if IS_ENABLED(CONFIG_IOMMU_IOVA)
|
2015-07-13 11:31:28 +00:00
|
|
|
int iova_cache_get(void);
|
|
|
|
void iova_cache_put(void);
|
2015-01-12 17:51:14 +00:00
|
|
|
|
2007-10-21 23:41:48 +00:00
|
|
|
void free_iova(struct iova_domain *iovad, unsigned long pfn);
|
|
|
|
void __free_iova(struct iova_domain *iovad, struct iova *iova);
|
|
|
|
struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
|
2007-10-21 23:41:58 +00:00
|
|
|
unsigned long limit_pfn,
|
|
|
|
bool size_aligned);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
|
|
|
|
unsigned long size);
|
2017-08-10 13:49:44 +00:00
|
|
|
void queue_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn, unsigned long pages,
|
|
|
|
unsigned long data);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
|
2017-09-20 08:52:02 +00:00
|
|
|
unsigned long limit_pfn, bool flush_rcache);
|
2007-10-21 23:41:48 +00:00
|
|
|
struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
|
|
|
|
unsigned long pfn_hi);
|
2015-01-12 17:51:16 +00:00
|
|
|
void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
|
2017-09-21 15:52:45 +00:00
|
|
|
unsigned long start_pfn);
|
2017-08-10 12:44:28 +00:00
|
|
|
int init_iova_flush_queue(struct iova_domain *iovad,
|
|
|
|
iova_flush_cb flush_cb, iova_entry_dtor entry_dtor);
|
2007-10-21 23:41:48 +00:00
|
|
|
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
|
|
|
|
void put_iova_domain(struct iova_domain *iovad);
|
iommu/iova: introduce per-cpu caching to iova allocation
IOVA allocation has two problems that impede high-throughput I/O.
First, it can do a linear search over the allocated IOVA ranges.
Second, the rbtree spinlock that serializes IOVA allocations becomes
contended.
Address these problems by creating an API for caching allocated IOVA
ranges, so that the IOVA allocator isn't accessed frequently. This
patch adds a per-CPU cache, from which CPUs can alloc/free IOVAs
without taking the rbtree spinlock. The per-CPU caches are backed by
a global cache, to avoid invoking the (linear-time) IOVA allocator
without needing to make the per-CPU cache size excessive. This design
is based on magazines, as described in "Magazines and Vmem: Extending
the Slab Allocator to Many CPUs and Arbitrary Resources" (currently
available at https://www.usenix.org/legacy/event/usenix01/bonwick.html)
Adding caching on top of the existing rbtree allocator maintains the
property that IOVAs are densely packed in the IO virtual address space,
which is important for keeping IOMMU page table usage low.
To keep the cache size reasonable, we bound the IOVA space a CPU can
cache by 32 MiB (we cache a bounded number of IOVA ranges, and only
ranges of size <= 128 KiB). The shared global cache is bounded at
4 MiB of IOVA space.
Signed-off-by: Omer Peleg <omer@cs.technion.ac.il>
[mad@cs.technion.ac.il: rebased, cleaned up and reworded the commit message]
Signed-off-by: Adam Morrison <mad@cs.technion.ac.il>
Reviewed-by: Shaohua Li <shli@fb.com>
Reviewed-by: Ben Serebrin <serebrin@google.com>
[dwmw2: split out VT-d part into a separate patch]
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>
2016-04-20 08:34:11 +00:00
|
|
|
void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
|
2017-03-20 19:11:28 +00:00
|
|
|
#else
|
|
|
|
static inline int iova_cache_get(void)
|
|
|
|
{
|
|
|
|
return -ENOTSUPP;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void iova_cache_put(void)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void free_iova(struct iova_domain *iovad, unsigned long pfn)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void __free_iova(struct iova_domain *iovad, struct iova *iova)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct iova *alloc_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long size,
|
|
|
|
unsigned long limit_pfn,
|
|
|
|
bool size_aligned)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void free_iova_fast(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn,
|
|
|
|
unsigned long size)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2017-08-10 13:49:44 +00:00
|
|
|
static inline void queue_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn, unsigned long pages,
|
|
|
|
unsigned long data)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2017-03-20 19:11:28 +00:00
|
|
|
static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
|
|
|
|
unsigned long size,
|
2017-09-20 08:52:02 +00:00
|
|
|
unsigned long limit_pfn,
|
|
|
|
bool flush_rcache)
|
2017-03-20 19:11:28 +00:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct iova *reserve_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn_lo,
|
|
|
|
unsigned long pfn_hi)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void init_iova_domain(struct iova_domain *iovad,
|
|
|
|
unsigned long granule,
|
2017-09-21 15:52:45 +00:00
|
|
|
unsigned long start_pfn)
|
2017-03-20 19:11:28 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2017-08-10 12:44:28 +00:00
|
|
|
static inline int init_iova_flush_queue(struct iova_domain *iovad,
|
|
|
|
iova_flush_cb flush_cb,
|
|
|
|
iova_entry_dtor entry_dtor)
|
|
|
|
{
|
|
|
|
return -ENODEV;
|
|
|
|
}
|
|
|
|
|
2017-03-20 19:11:28 +00:00
|
|
|
static inline struct iova *find_iova(struct iova_domain *iovad,
|
|
|
|
unsigned long pfn)
|
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void put_iova_domain(struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void free_cpu_cached_iovas(unsigned int cpu,
|
|
|
|
struct iova_domain *iovad)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
#endif
|
2007-10-21 23:41:48 +00:00
|
|
|
|
|
|
|
#endif
|